Merge pull request #15 from bchaoss/dev

update

Author: bchaoss

.devcontainer/devcontainer.json

@@ -18,11 +18,9 @@
       ]
     }
   },
-  "initializeCommand": {
-    "bi local env setup": "echo \"EVIDENCE_SOURCE__us_tariff_dataset__token=${MOTHERDUCK_TOKEN}\" > reports/.env"
+  "containerEnv": {
+    "MOTHERDUCK_TOKEN": "${localEnv:MOTHERDUCK_TOKEN}"
   },
-  "postCreateCommand": {
-    "dbt dependencies": "pip install -r requirements.txt",
-    "bi dependencies": "cd reports/ && npm install && npm run sources"
-  }
+  "forwardPorts": [3000],
+  "postCreateCommand": "bash ./.devcontainer/setup.sh"
 }

.devcontainer/setup.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e # stop when error
+set -x
+
+# dbt install and deps
+pip install -r requirements.txt
+
+# bi install and setup
+cd reports
+npm install
+if [ -z "${MOTHERDUCK_TOKEN}" ]; then
+    echo "MOTHERDUCK_TOKEN is not setup."
+else
+    echo "EVIDENCE_SOURCE__us_tariff_dataset__token=${MOTHERDUCK_TOKEN}" >> .env
+    npm run sources
+fi
+
+echo "post setup finished."

.github/workflows/dbt-CICD.yml

@@ -9,14 +9,32 @@ on:
     branches: [ "main" ]
     paths:
       - 'pipeline/**'
+      - 'reports/**'
 
   workflow_dispatch:
 
 permissions:
   contents: read
 
 jobs:
+  detect:
+    runs-on: ubuntu-latest
+    outputs:
+      pipeline_changed: ${{ steps.filter.outputs.pipeline }}
+      reports_changed: ${{ steps.filter.outputs.reports }}
+    steps:
+      - id: filter
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            pipeline:
+              - 'pipeline/**'
+            reports:
+              - 'reports/**'
+
   run-dbt-pipeline:
+    needs: detect
+    if: needs.detect.outputs.pipeline_changed == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
@@ -64,3 +82,16 @@ jobs:
         with:
           name: previous-manifest
           path: target/manifest.json
+
+  trigger-reports-workflow:
+    needs: [detect, run-dbt-pipeline]
+    if: |
+      needs.detect.outputs.pipeline_changed == 'true' ||
+      needs.detect.outputs.reports_changed == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Dispatch reports workflow
+        uses: peter-evans/repository-dispatch@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          event-type: run-reports-workflow

.github/workflows/reports-CICD.yml

@@ -6,15 +6,8 @@ env:
   PAGE_DEPLOY_DIR: ${{ github.workspace }}/reports/build/US-Tariff-What-Didnt-and-What-Did
 
 on:
-  workflow_run:
-    workflows: ["dbt_pipeline_cicd"]
-    types:
-      - completed
-
-  push:
-    branches: [ "main" ]
-    paths:
-      - 'reports/**'
+  repository_dispatch:
+    types: [run-reports-workflow]
 
   workflow_dispatch:
 

ingest/trade.py

@@ -5,14 +5,17 @@
 import pyarrow as pa
 
 # Configuration
-target_monthly_start = "2023-01"
+target_monthly_start = "2022-01"
 DATABASE_NAME = "us_tariff"
 SCHEMA = "raw"
 
-target_trade_type_list = ["exports"]
 # target_trade_type_list = ["imports", "exports"]
+# BY_COUNTRY = False
+# BY_ENDUSE = True
+
+target_trade_type_list = ["exports"]
 BY_COUNTRY = True
-BY_ENDUSE = True
+BY_ENDUSE = False
 
 
 def calc_timeout(monthly_start, by_country=False):

pipeline/dbt_project.yml

@@ -22,9 +22,9 @@ models:
       +schema: marts
       +materialized: table
 
-seeds:
-  seeds_file_name:
-    +quote_columns: false
+# seeds:
+#   seeds_file_name:
+#     +quote_columns: false
 
 vars:
   raw_data_path: "/workspaces/US-Tariff-What-Didnt-and-What-Did/data"

pipeline/models/marts/dim_region_lookup.sql

@@ -0,0 +1,8 @@
+from {{ source('raw', 'us_trade_exports_country_monthly') }}
+
+select 
+    SUBSTR(cty_code,1,1) AS region_code,
+    cty_name AS region_name,
+    
+where cty_code like '%XXX'
+group by all

pipeline/models/marts/fact_us_trade_exports_country_monthly.sql

@@ -0,0 +1,32 @@
+-- depends_on: {{ ref('dim_region_lookup') }}
+
+from {{ source('raw', 'us_trade_exports_country_monthly') }} a
+left join {{ ref('dim_region_lookup') }} c
+  on SUBSTR(a.cty_code,1,1) = c.region_code
+
+select
+  DF AS export_code,
+  case when DF = '1' then 'Domestic Export'
+    when DF = '2' then 'Foreign Export'
+  end AS export_type,
+
+  a.CTY_CODE,
+  a.CTY_NAME,
+  c.REGION_NAME,
+
+  YEAR(STRPTIME(time, '%Y-%m'))  as year,
+  SUBSTR(time, 6, 2) as month,
+  STRPTIME(time || '-01', '%Y-%m-%d')::DATE as month_begin_date,
+
+  {% for column in ['ALL_VAL_MO'] %}
+  {{ column }}::DECIMAL(18,2) AS {{ column }},
+  {% endfor %}
+
+where not (
+  a.cty_code='-'         -- Total
+  OR
+  a.cty_code like '00%'  -- Orgs
+  OR
+  a.cty_code like '%XXX' -- Regions
+)
+and DF <> '-'

pipeline/models/marts/fact_us_trade_exports_enduse_country_monthly.sql

@@ -19,7 +19,7 @@ select
 
   YEAR(STRPTIME(time, '%Y-%m'))  as year,
   substr(time, 6, 2) as month,
-  STRPTIME(time || '-01', '%Y-%m-%d') as month_begin_date,
+  STRPTIME(time || '-01', '%Y-%m-%d')::DATE as month_begin_date,
 
   {% for column in ['ALL_VAL_MO'] %}
   a.{{ column }}::DECIMAL(18,2) AS {{ column }},

pipeline/models/marts/fact_us_trade_exports_enduse_monthly.sql

@@ -16,7 +16,7 @@ select
 
   YEAR(STRPTIME(time, '%Y-%m'))  as year,
   substr(time, 6, 2) as month,
-  STRPTIME(time || '-01', '%Y-%m-%d') as month_begin_date,
+  STRPTIME(time || '-01', '%Y-%m-%d')::DATE as month_begin_date,
 
   {% for column in ['ALL_VAL_MO'] %}
   a.{{ column }}::DECIMAL(18,2) AS {{ column }},