ci: gate staging on extension tests (+ build-time caching) (#205) #69
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Deploy CKAN (staging) | |
| on: | |
| push: | |
| branches: [master, ckan211-prod-deploy-pr] | |
| pull_request: | |
| # Runs the test job only (build/deploy are gated off pull_request below), | |
| # so PRs get a visible test status check without deploying. | |
| branches: [master, ckan211-prod-deploy-pr] | |
| workflow_dispatch: | |
| inputs: | |
| image_tag: | |
| description: "Image tag to deploy (e.g., sha-abc1234). Leave blank to build from the workflow's ref." | |
| required: false | |
| type: string | |
| concurrency: | |
| group: deploy-staging-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| ACR_NAME: adracr | |
| IMAGE_NAME: ckan | |
| NAMESPACE: adr-s | |
| URL: https://dev.adr.fjelltopp.org | |
| jobs: | |
| test: | |
| name: Extension tests | |
| # Skip on a pure redeploy (workflow_dispatch with an existing image_tag); | |
| # that image was already tested when it was built. | |
| if: github.event_name != 'workflow_dispatch' || inputs.image_tag == '' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| env: | |
| # Fresh runner DB — no need to restart the db container during testsetup. | |
| SKIP_DB_RESTART: "True" | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| with: | |
| submodules: recursive | |
| # The dominant cost is `pipenv sync --dev` at bootstrap (CKAN + ~19 | |
| # extensions), which lands in the bind-mounted .adxvenv. Caching it on | |
| # Pipfile.lock makes warm runs a near no-op. .adxvenv is gitignored and | |
| # written by root in-container, so there are no ownership issues here. | |
| # Restore + save are split (rather than the combined cache action) so the | |
| # venv is saved even when the test step fails — otherwise the expensive | |
| # `pipenv sync` is repeated cold on every triage run. Saved at job end. | |
| - name: Restore Python venv | |
| id: venv-cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .adxvenv | |
| key: adxvenv-${{ runner.os }}-${{ hashFiles('Pipfile.lock') }} | |
| restore-keys: | | |
| adxvenv-${{ runner.os }}- | |
| # Second cost: the entrypoint runs `yarn install` + build for the unaids | |
| # React app on every boot. Cache its node_modules on the React yarn.lock. | |
| - name: Restore unaids React node_modules | |
| id: react-cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: submodules/ckanext-unaids/ckanext/unaids/react/node_modules | |
| key: react-nm-${{ runner.os }}-${{ hashFiles('submodules/ckanext-unaids/ckanext/unaids/react/yarn.lock') }} | |
| restore-keys: | | |
| react-nm-${{ runner.os }}- | |
| - name: Prepare .env | |
| run: cp dev.env .env | |
| - name: Build images and start the dev stack | |
| run: | | |
| ./adx build | |
| ./adx up | |
| - name: Wait for CKAN bootstrap | |
| run: | | |
| for i in $(seq 1 90); do | |
| if docker logs ckan 2>&1 | grep -q 'CKAN bootstrapping finished, environment ready'; then | |
| echo "CKAN ready after ${i} checks" | |
| exit 0 | |
| fi | |
| echo "Waiting for CKAN bootstrap (${i}/90)…" | |
| sleep 10 | |
| done | |
| echo "::error::CKAN did not finish bootstrapping in time" | |
| docker logs ckan | |
| exit 1 | |
| - name: Create test databases | |
| run: ./adx testsetup | |
| - name: Run extension tests | |
| run: | | |
| # Run every suite (don't fail-fast) so one run shows the full picture, | |
| # then fail at the end if any suite failed. | |
| failed=() | |
| for ext in unaids validation scheming dhis2harvester emailasusername; do | |
| echo "::group::ckanext-${ext}" | |
| if ./adx test "${ext}" --no-interaction; then | |
| echo "ckanext-${ext}: PASS" | |
| else | |
| echo "ckanext-${ext}: FAIL" | |
| failed+=("${ext}") | |
| fi | |
| echo "::endgroup::" | |
| done | |
| echo "--- summary ---" | |
| if [ ${#failed[@]} -ne 0 ]; then | |
| echo "::error::Failing suites: ${failed[*]}" | |
| exit 1 | |
| fi | |
| echo "All extension suites passed" | |
| - name: Dump CKAN logs on failure | |
| if: failure() | |
| run: docker logs ckan || true | |
| - name: Save Python venv | |
| if: always() && steps.venv-cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: .adxvenv | |
| key: ${{ steps.venv-cache.outputs.cache-primary-key }} | |
| - name: Save unaids React node_modules | |
| if: always() && steps.react-cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: submodules/ckanext-unaids/ckanext/unaids/react/node_modules | |
| key: ${{ steps.react-cache.outputs.cache-primary-key }} | |
| build: | |
| needs: test | |
| # Run when tests pass (success) or were skipped (redeploy), and we're | |
| # actually building (not a redeploy of an existing tag). | |
| if: >- | |
| always() | |
| && github.event_name != 'pull_request' | |
| && needs.test.result != 'failure' | |
| && needs.test.result != 'cancelled' | |
| && (github.event_name != 'workflow_dispatch' || inputs.image_tag == '') | |
| runs-on: ubuntu-latest | |
| outputs: | |
| image_tag: ${{ steps.meta.outputs.version }} | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| with: | |
| submodules: recursive | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.6.1 | |
| with: | |
| images: ${{ env.ACR_NAME }}.azurecr.io/${{ env.IMAGE_NAME }} | |
| tags: | | |
| type=sha,prefix=sha-,format=short | |
| - name: Login to ACR | |
| uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 | |
| with: | |
| registry: ${{ env.ACR_NAME }}.azurecr.io | |
| username: ${{ secrets.ACR_USERNAME }} | |
| password: ${{ secrets.ACR_PASSWORD }} | |
| - name: Build and push | |
| uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.10.0 | |
| with: | |
| context: . | |
| file: deploy/Dockerfile.prod | |
| push: true | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| deploy: | |
| needs: [test, build] | |
| # Never deploy if tests failed/cancelled. Otherwise deploy when the image | |
| # was built (success) or already exists (build skipped on redeploy). | |
| if: >- | |
| always() | |
| && github.event_name != 'pull_request' | |
| && needs.test.result != 'failure' | |
| && needs.test.result != 'cancelled' | |
| && (needs.build.result == 'success' || needs.build.result == 'skipped') | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: staging | |
| url: ${{ env.URL }} | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| - name: Resolve image tag | |
| id: params | |
| run: | | |
| if [[ -n "${{ inputs.image_tag }}" ]]; then | |
| echo "image_tag=${{ inputs.image_tag }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "image_tag=${{ needs.build.outputs.image_tag }}" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Setup kubeconfig | |
| run: | | |
| mkdir -p ~/.kube | |
| echo "${{ secrets.KUBECONFIG_BASE64 }}" | base64 -d > ~/.kube/config | |
| chmod 600 ~/.kube/config | |
| - name: Deploy to AKS | |
| run: | | |
| kubectl create configmap ckan-env-config \ | |
| --from-file=env.ini=deploy/staging.ini \ | |
| -n ${{ env.NAMESPACE }} \ | |
| --dry-run=client -o yaml | kubectl apply -f - | |
| kubectl set image deployment/ckan \ | |
| ckan=${{ env.ACR_NAME }}.azurecr.io/${{ env.IMAGE_NAME }}:${{ steps.params.outputs.image_tag }} \ | |
| -n ${{ env.NAMESPACE }} | |
| kubectl rollout status deployment/ckan -n ${{ env.NAMESPACE }} --timeout=10m |