dbt_pipeline_CICD #40
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: dbt_pipeline_CICD | |
| env: | |
| MOTHERDUCK_TOKEN: ${{ secrets.MOTHERDUCK_TOKEN }} | |
| DBT_PROJECT_DIR: ${{ github.workspace }}/pipeline | |
| on: | |
| push: | |
| branches: [ "main" ] | |
| paths: | |
| - 'pipeline/**' | |
| - 'reports/**' | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| actions: write | |
| jobs: | |
| detect: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pipeline_changed: ${{ steps.filter.outputs.pipeline }} | |
| reports_changed: ${{ steps.filter.outputs.reports }} | |
| steps: | |
| - name: Checkout Code | |
| if: github.event_name == 'push' | |
| uses: actions/checkout@v4 | |
| - name: Detect changes | |
| id: filter | |
| if: github.event_name == 'push' | |
| uses: dorny/paths-filter@v3 | |
| with: | |
| filters: | | |
| pipeline: | |
| - 'pipeline/**' | |
| reports: | |
| - 'reports/**' | |
| - name: Set default outputs | |
| run: | | |
| echo "pipeline_changed=false" >> $GITHUB_OUTPUT | |
| echo "reports_changed=false" >> $GITHUB_OUTPUT | |
| run-dbt-pipeline: | |
| needs: detect | |
| if: | | |
| github.event_name == 'workflow_dispatch' || | |
| needs.detect.outputs.pipeline_changed == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: Install dbt dependencies | |
| run: pip install dbt-duckdb | |
| - name: Set dbt target output | |
| run: | | |
| echo "DBT_TARGET=prod" >> $GITHUB_ENV | |
| working-directory: ${{ env.DBT_PROJECT_DIR }} | |
| - name: Download previous manifest | |
| if: vars.MANIFEST_RUN_ID | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: previous-manifest | |
| path: state | |
| run-id: ${{ vars.MANIFEST_RUN_ID }} | |
| github-token: ${{ secrets.MY_PAT }} | |
| continue-on-error: true | |
| # - name: dbt deps | |
| # run: dbt deps --target $DBT_TARGET | |
| # working-directory: ${{ env.DBT_PROJECT_DIR }} | |
| - name: dbt build - modified | |
| run: | | |
| if [ -f state/manifest.json ]; then | |
| dbt build --target $DBT_TARGET --select state:modified+ --state state | |
| else | |
| dbt build --target $DBT_TARGET --exclude tag:static --exclude tag:dim | |
| fi | |
| working-directory: ${{ env.DBT_PROJECT_DIR }} | |
| # - name: Generate dbt doc | |
| # run: dbt docs generate | |
| # working-directory: ${{ env.DBT_PROJECT_DIR }} | |
| # Upload dbt state manifest | |
| - name: Upload manifest | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: previous-manifest | |
| path: pipeline/target/manifest.json | |
| - name: Save manifest run_id | |
| if: vars.MANIFEST_RUN_ID | |
| env: | |
| GH_TOKEN: ${{ secrets.MY_PAT }} | |
| run: | | |
| gh api \ | |
| --method PATCH \ | |
| -H "Accept: application/vnd.github+json" \ | |
| /repos/${{ github.repository }}/actions/variables/MANIFEST_RUN_ID \ | |
| -f value=${{ github.run_id }} | |
| trigger-reports-workflow: | |
| needs: [detect, run-dbt-pipeline] | |
| if: | | |
| github.event_name == 'push' && | |
| ( | |
| needs.detect.outputs.pipeline_changed == 'true' || | |
| needs.detect.outputs.reports_changed == 'true' | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Dispatch reports workflow | |
| uses: peter-evans/repository-dispatch@v3 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| event-type: run-reports-workflow |