CI - Apply (Terraform) #89
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI - Apply (Terraform) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: "Pull request number to test (leave empty to use current branch)" | |
| required: false | |
| type: number | |
| topology: | |
| description: "Terraform topology to test" | |
| required: true | |
| type: choice | |
| default: "public-fss-monitoring-tf" | |
| options: | |
| - public-base-tf | |
| - public-bastion-operator-tf | |
| - public-fss-monitoring-tf | |
| - public-lustre-tf | |
| - public-fss-lustre-monitoring-tf | |
| - private-base-tf | |
| - private-fss-monitoring-tf | |
| - private-fss-lustre-monitoring-tf | |
| overrides: | |
| description: 'Extra variable overrides as JSON (e.g. {"worker_gpu_enabled":"true"})' | |
| required: false | |
| type: string | |
| default: '{}' | |
| issue_comment: | |
| types: [created] | |
| workflow_call: | |
| inputs: | |
| topology: | |
| type: string | |
| default: 'public-fss-monitoring-tf' | |
| overrides: | |
| type: string | |
| default: '{}' | |
| pr_number: | |
| type: number | |
| default: 0 | |
| concurrency: | |
| group: >- | |
| ${{ | |
| github.event_name == 'issue_comment' && format('ci-apply-tf-pr-{0}', github.event.issue.number) || | |
| github.event_name == 'workflow_dispatch' && github.event.inputs.pr_number != '' && format('ci-apply-tf-pr-{0}', github.event.inputs.pr_number) || | |
| github.event_name == 'workflow_call' && inputs.pr_number != 0 && format('ci-apply-tf-pr-{0}', inputs.pr_number) || | |
| format('ci-apply-tf-{0}', inputs.topology || github.run_id) | |
| }} | |
| cancel-in-progress: false | |
| env: | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true | |
| GIT_TEMPLATE_DIR: "" | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| # --------------------------------------------------------------------------- | |
| # Resolve PR context for PR-linked runs and verify /ok-to-run-apply-tf | |
| # comments are authorized. | |
| # --------------------------------------------------------------------------- | |
| check-tf-trigger: | |
| name: Check TF trigger | |
| if: >- | |
| (github.event_name == 'issue_comment' && | |
| github.event.issue.pull_request && | |
| startsWith(github.event.comment.body, '/ok-to-run-apply-tf') && | |
| contains(fromJSON('["OguzPastirmaci","arnaudfroidmont","robo-cap"]'), github.event.comment.user.login)) || | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'workflow_call' && inputs.pr_number != 0) | |
| runs-on: ubuntu-latest | |
| outputs: | |
| topology: ${{ steps.parse.outputs.topology || steps.manual.outputs.topology }} | |
| overrides: ${{ steps.parse.outputs.overrides || steps.manual.outputs.overrides }} | |
| pr_number: ${{ steps.pr-comment.outputs.pr_number || steps.pr-dispatch.outputs.pr_number || steps.pr-call.outputs.pr_number }} | |
| pr_head_sha: ${{ steps.pr-comment.outputs.head_sha || steps.pr-dispatch.outputs.head_sha || steps.pr-call.outputs.head_sha }} | |
| base_ref: ${{ steps.pr-comment.outputs.base_ref || steps.pr-dispatch.outputs.base_ref || steps.pr-call.outputs.base_ref }} | |
| steps: | |
| - name: React to trigger comment | |
| if: github.event_name == 'issue_comment' | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 | |
| with: | |
| script: | | |
| await github.rest.reactions.createForIssueComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, | |
| content: 'rocket' | |
| }) | |
| - name: Resolve PR context from comment | |
| id: pr-comment | |
| if: github.event_name == 'issue_comment' | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 | |
| env: | |
| PR_NUMBER: ${{ github.event.issue.number }} | |
| with: | |
| script: | | |
| const prNumber = Number(process.env.PR_NUMBER); | |
| const pr = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: prNumber, | |
| }); | |
| core.setOutput('pr_number', String(prNumber)); | |
| core.setOutput('head_sha', pr.data.head.sha); | |
| core.setOutput('base_ref', pr.data.base.ref); | |
| - name: Resolve PR context from workflow dispatch | |
| id: pr-dispatch | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_number != '' | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 | |
| env: | |
| PR_NUMBER: ${{ github.event.inputs.pr_number }} | |
| with: | |
| script: | | |
| const prNumber = Number(process.env.PR_NUMBER); | |
| const pr = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: prNumber, | |
| }); | |
| core.setOutput('pr_number', String(prNumber)); | |
| core.setOutput('head_sha', pr.data.head.sha); | |
| core.setOutput('base_ref', pr.data.base.ref); | |
| - name: Resolve PR context from workflow call | |
| id: pr-call | |
| if: github.event_name == 'workflow_call' | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 | |
| env: | |
| PR_NUMBER: ${{ inputs.pr_number }} | |
| with: | |
| script: | | |
| const prNumber = Number(process.env.PR_NUMBER); | |
| const pr = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: prNumber, | |
| }); | |
| core.setOutput('pr_number', String(prNumber)); | |
| core.setOutput('head_sha', pr.data.head.sha); | |
| core.setOutput('base_ref', pr.data.base.ref); | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| if: github.event_name == 'issue_comment' | |
| - name: Parse topology and overrides from comment | |
| id: parse | |
| if: github.event_name == 'issue_comment' | |
| env: | |
| COMMENT: ${{ github.event.comment.body }} | |
| run: .github/scripts/parse-comment.sh "/ok-to-run-apply-tf" "public-base-tf|public-bastion-operator-tf|public-fss-monitoring-tf|public-lustre-tf|public-fss-lustre-monitoring-tf|private-base-tf|private-fss-monitoring-tf|private-fss-lustre-monitoring-tf" "public-fss-monitoring-tf" | |
| - name: Use workflow inputs | |
| id: manual | |
| if: github.event_name != 'issue_comment' | |
| run: | | |
| echo "topology=${{ inputs.topology }}" >> "$GITHUB_OUTPUT" | |
| echo "overrides=${{ inputs.overrides }}" >> "$GITHUB_OUTPUT" | |
| # --------------------------------------------------------------------------- | |
| # Apply: init, apply, assert outputs, health checks, then always destroy. | |
| # --------------------------------------------------------------------------- | |
| apply-tf: | |
| name: Terraform / ${{ inputs.topology || needs.check-tf-trigger.outputs.topology || 'public-fss-monitoring-tf' }} | |
| needs: [check-tf-trigger] | |
| if: >- | |
| always() && ( | |
| (github.event_name == 'release' && inputs.pr_number == 0) || | |
| needs.check-tf-trigger.result == 'success' | |
| ) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| env: | |
| TOPOLOGY: ${{ inputs.topology || needs.check-tf-trigger.outputs.topology || 'public-fss-monitoring-tf' }} | |
| OVERRIDES_JSON: ${{ inputs.overrides || needs.check-tf-trigger.outputs.overrides || '{}' }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| with: | |
| ref: >- | |
| ${{ | |
| needs.check-tf-trigger.outputs.pr_number != '' && format('refs/pull/{0}/head', needs.check-tf-trigger.outputs.pr_number) || | |
| github.ref | |
| }} | |
| fetch-depth: 0 | |
| - name: Verify checked-out PR SHA | |
| if: needs.check-tf-trigger.outputs.pr_head_sha != '' | |
| env: | |
| EXPECTED_SHA: ${{ needs.check-tf-trigger.outputs.pr_head_sha }} | |
| run: | | |
| ACTUAL_SHA=$(git rev-parse HEAD) | |
| if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then | |
| echo "Expected PR head SHA $EXPECTED_SHA but checked out $ACTUAL_SHA." | |
| echo "The PR changed after this run started. Re-run against the latest commit." | |
| exit 1 | |
| fi | |
| - name: Squash PR changes and rebase onto base branch | |
| if: needs.check-tf-trigger.outputs.base_ref != '' | |
| env: | |
| BASE_REF: ${{ needs.check-tf-trigger.outputs.base_ref }} | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git fetch origin "${BASE_REF}:refs/remotes/origin/${BASE_REF}" | |
| MERGE_BASE=$(git merge-base HEAD "origin/${BASE_REF}") | |
| git reset --soft "$MERGE_BASE" | |
| git commit -m "Squashed PR changes" | |
| git rebase "origin/${BASE_REF}" | |
| - name: Install Terraform | |
| run: | | |
| TF_VERSION="1.5.7" | |
| curl -fsSL "https://releases.hashicorp.com/terraform/${TF_VERSION}/terraform_${TF_VERSION}_linux_amd64.zip" -o tf.zip | |
| unzip -o tf.zip -d /usr/local/bin | |
| rm tf.zip | |
| terraform version | |
| - name: Install OCI CLI | |
| run: pip install oci-cli --quiet | |
| - name: Set up OCI credentials | |
| env: | |
| OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }} | |
| OCI_API_KEY_FINGERPRINT: ${{ secrets.OCI_API_KEY_FINGERPRINT }} | |
| OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }} | |
| OCI_REGION: ${{ vars.OCI_REGION }} | |
| OCI_API_KEY_PRIVATE_KEY: ${{ secrets.OCI_API_KEY_PRIVATE_KEY }} | |
| run: .github/scripts/setup-oci-credentials.sh | |
| - name: Generate ephemeral SSH key for bastion tunnel | |
| if: startsWith(env.TOPOLOGY, 'private') | |
| run: | | |
| mkdir -p ~/.ssh | |
| ssh-keygen -t rsa -b 4096 -f ~/.ssh/bastion_key -N "" -q | |
| - name: Cache Terraform providers | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 | |
| with: | |
| path: terraform/.terraform | |
| key: terraform-${{ hashFiles('terraform/versions.tf', 'terraform/.terraform.lock.hcl') }} | |
| restore-keys: terraform- | |
| - name: Build variable files | |
| env: | |
| OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }} | |
| OCI_REGION: ${{ vars.OCI_REGION }} | |
| OCI_COMPARTMENT_OCID: ${{ secrets.OCI_COMPARTMENT_OCID }} | |
| OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }} | |
| WORKER_OPS_AD: ${{ vars.WORKER_OPS_AD }} | |
| WORKER_OPS_IMAGE_CUSTOM_ID: ${{ vars.WORKER_OPS_IMAGE_CUSTOM_ID }} | |
| WORKER_CPU_AD: ${{ vars.WORKER_CPU_AD }} | |
| WORKER_CPU_IMAGE_CUSTOM_ID: ${{ vars.WORKER_CPU_IMAGE_CUSTOM_ID }} | |
| WORKER_GPU_AD: ${{ vars.WORKER_GPU_AD }} | |
| WORKER_GPU_IMAGE_CUSTOM_ID: ${{ vars.WORKER_GPU_IMAGE_CUSTOM_ID }} | |
| SSH_PUBLIC_KEY: ${{ vars.SSH_PUBLIC_KEY }} | |
| run: .github/scripts/build-variables-tf.sh | |
| - name: Terraform init | |
| run: terraform -chdir=terraform init | |
| - name: Apply | |
| env: | |
| OCI_COMPARTMENT_OCID: ${{ secrets.OCI_COMPARTMENT_OCID }} | |
| run: | | |
| # For lustre topologies, fetch ADs dynamically and retry on capacity errors. | |
| # For all other topologies, a single apply with no AD override. | |
| if [[ "$TOPOLOGY" == *lustre* ]]; then | |
| mapfile -t ADS < <(oci iam availability-domain list \ | |
| --compartment-id "$OCI_COMPARTMENT_OCID" \ | |
| --query 'data[*].name' \ | |
| --raw-output | jq -r '.[]') | |
| else | |
| ADS=("") | |
| fi | |
| APPLY_SUCCESS=false | |
| for AD in "${ADS[@]}"; do | |
| if [ -n "$AD" ]; then | |
| echo "Setting lustre_ad=$AD" | |
| jq --arg ad "$AD" '. + {lustre_ad: $ad}' overrides.tfvars.json > overrides_ad.tfvars.json | |
| OVERRIDE_FILE="overrides_ad.tfvars.json" | |
| else | |
| OVERRIDE_FILE="overrides.tfvars.json" | |
| fi | |
| set +e | |
| terraform -chdir=terraform apply -auto-approve \ | |
| -var-file="../secrets.tfvars.json" \ | |
| -var-file="../test/tfvars/tf/${TOPOLOGY}.tfvars" \ | |
| -var-file="../${OVERRIDE_FILE}" 2>&1 | tee apply.log | |
| APPLY_EXIT=${PIPESTATUS[0]} | |
| set -e | |
| if [ "$APPLY_EXIT" -eq 0 ]; then | |
| APPLY_SUCCESS=true | |
| break | |
| fi | |
| if grep -q "Not enough capacity to create Lustre file system" apply.log; then | |
| echo "No Lustre capacity in AD '${AD:-default}', trying next..." | |
| continue | |
| fi | |
| cat apply.log | |
| echo "Apply failed" | |
| exit 1 | |
| done | |
| if ! $APPLY_SUCCESS; then | |
| echo "Apply failed: no Lustre capacity available in any AD." | |
| exit 1 | |
| fi | |
| - name: Capture outputs | |
| run: terraform -chdir=terraform output -json > state.json | |
| - name: Assert outputs | |
| run: .github/scripts/assert-outputs.sh state.json "" | |
| - name: Generate kubeconfig | |
| env: | |
| OCI_REGION: ${{ vars.OCI_REGION }} | |
| run: .github/scripts/generate-kubeconfig.sh state.json "" | |
| - name: Cluster health checks | |
| run: .github/scripts/check-cluster-health.sh state.json "" | |
| - name: Network health checks | |
| run: .github/scripts/check-network-health.sh state.json "" | |
| - name: GPU resource health checks | |
| run: .github/scripts/check-gpu-health.sh state.json "" | |
| - name: FSS health checks | |
| if: contains(env.TOPOLOGY, 'fss') | |
| run: .github/scripts/check-fss-health.sh state.json "" | |
| - name: Lustre health checks | |
| if: contains(env.TOPOLOGY, 'lustre') | |
| run: .github/scripts/check-lustre-health.sh state.json "" | |
| - name: Monitoring health checks | |
| if: contains(env.TOPOLOGY, 'monitoring') | |
| run: .github/scripts/check-monitoring-health.sh state.json "" | |
| - name: Clean up test resources (pre-reboot) | |
| run: .github/scripts/cleanup-test-resources.sh | |
| - name: Verify test resources deleted | |
| run: | | |
| for i in $(seq 1 24); do | |
| REMAINING=$(kubectl get pods net-server net-client dns-checker fss-writer fss-reader fss-hostpath-reader lustre-writer lustre-reader lustre-hostpath-reader --ignore-not-found --no-headers 2>/dev/null | wc -l | tr -d ' ') | |
| PVC_REMAINING=$(kubectl get pvc fss-test-pvc lustre-test-pvc --ignore-not-found --no-headers 2>/dev/null | wc -l | tr -d ' ') | |
| if [ "$REMAINING" -eq 0 ] && [ "$PVC_REMAINING" -eq 0 ]; then | |
| echo "All test resources verified deleted" | |
| break | |
| fi | |
| if [ "$i" -eq 24 ]; then | |
| echo "FAIL: test resources still present after 2 minutes" | |
| exit 1 | |
| fi | |
| echo " [$i/24] Waiting for test resources to be fully deleted..." | |
| sleep 5 | |
| done | |
| - name: Reboot nodes | |
| run: .github/scripts/reboot-nodes.sh | |
| - name: Cluster health checks (post-reboot) | |
| run: .github/scripts/check-cluster-health.sh state.json "" | |
| - name: Network health checks (post-reboot) | |
| run: .github/scripts/check-network-health.sh state.json "" | |
| - name: GPU resource health checks (post-reboot) | |
| run: .github/scripts/check-gpu-health.sh state.json "" | |
| - name: FSS health checks (post-reboot) | |
| if: contains(env.TOPOLOGY, 'fss') | |
| run: .github/scripts/check-fss-health.sh state.json "" | |
| - name: Lustre health checks (post-reboot) | |
| if: contains(env.TOPOLOGY, 'lustre') | |
| run: .github/scripts/check-lustre-health.sh state.json "" | |
| - name: Monitoring health checks (post-reboot) | |
| if: contains(env.TOPOLOGY, 'monitoring') | |
| run: .github/scripts/check-monitoring-health.sh state.json "" | |
| - name: Clean up test resources | |
| if: always() | |
| run: .github/scripts/cleanup-test-resources.sh | |
| - name: Tear down bastion tunnel | |
| if: always() && startsWith(env.TOPOLOGY, 'private') | |
| run: .github/scripts/teardown-bastion-tunnel.sh | |
| - name: Destroy | |
| if: always() | |
| run: | | |
| OVERRIDE_FILE="overrides.tfvars.json" | |
| if [ -f "overrides_ad.tfvars.json" ]; then | |
| OVERRIDE_FILE="overrides_ad.tfvars.json" | |
| fi | |
| terraform -chdir=terraform destroy -auto-approve \ | |
| -var-file="../secrets.tfvars.json" \ | |
| -var-file="../test/tfvars/tf/${TOPOLOGY}.tfvars" \ | |
| -var-file="../${OVERRIDE_FILE}" | |
| - name: Show state on destroy failure | |
| if: always() && failure() | |
| run: terraform -chdir=terraform show 2>/dev/null | tail -50 || true | |
| - name: Post result comment | |
| if: always() && needs.check-tf-trigger.outputs.pr_number != '' | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 | |
| env: | |
| JOB_STATUS: ${{ job.status }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| PR_NUMBER: ${{ needs.check-tf-trigger.outputs.pr_number }} | |
| OVERRIDES_JSON: ${{ env.OVERRIDES_JSON }} | |
| with: | |
| script: | | |
| const status = process.env.JOB_STATUS | |
| const icon = status === 'success' ? '✅' : '❌' | |
| const topology = process.env.TOPOLOGY | |
| const overrides = process.env.OVERRIDES_JSON | |
| const runUrl = process.env.RUN_URL | |
| const rows = [ | |
| `| Topology | \`${topology}\` |`, | |
| `| Result | ${icon} ${status} |`, | |
| ] | |
| if (overrides && overrides !== '{}') { | |
| rows.push(`| Overrides | \`${overrides}\` |`) | |
| } | |
| const body = [ | |
| `### ${icon} CI - Apply (Terraform) results`, | |
| '', | |
| `| | |`, | |
| `|---|---|`, | |
| ...rows, | |
| '', | |
| `[View full run log](${runUrl})`, | |
| ].join('\n') | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: Number(process.env.PR_NUMBER), | |
| body, | |
| }) |