Demo/bigquery pr

.github/workflows/pr_job.yml

@@ -0,0 +1,65 @@
+name: dbt pull request job
+
+on:
+  pull_request:
+    types:
+    - opened
+    - reopened
+    - synchronize
+    - ready_for_review
+  push:
+    branches:
+      - '!master' # TODO: change to your main branch name
+
+env:
+  #TODO(developer): update PROJECT_ID
+  PROJECT_ID: "dbt-demo-386220"
+  DBT_PROFILES_DIR: ./
+  GOOGLE_APPLICATION_CREDENTIALS: ./service_account.json
+
+jobs:
+  run:
+    runs-on: ubuntu-20.04
+    if: ${{ !github.event.pull_request.draft }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+
+      - name: install requirements
+        run: pip install -q -r requirements.txt
+
+      - name: install datafold-sdk
+        run: pip install -q datafold-sdk
+
+      - name: Authenticate dbt to BigQuery
+        run: 'echo "$KEYFILE" > ./service_account.json' # persist file in root dir
+        shell: bash
+        env:
+          KEYFILE: ${{secrets.GOOGLE_APPLICATION_CREDENTIALS}} # TODO: add a github secret
+
+      - name: dbt deps
+        run: dbt deps
+
+      - name: Find Current Pull Request
+        uses: jwalton/[email protected]
+        id: findPR
+
+      - name: dbt build
+        run: dbt build --full-refresh --profiles-dir ./
+
+      # TODO: update the --config-id to your config id
+      - name: submit artifacts to datafold to compare to the production manifest
+        run: |
+          set -ex
+          datafold dbt upload --ci-config-id 272 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA}
+        env:
+          DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }}
+          DATAFOLD_RUN_TYPE: "${{ 'pull_request' }}"
+          GIT_SHA: "${{ github.event.pull_request.head.sha }}"

.github/workflows/prod_job.yml

@@ -49,4 +49,4 @@ jobs:
         env:
           DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }}
           DATAFOLD_RUN_TYPE: "${{ github.ref == 'refs/heads/master' && 'production' || 'pull_request' }}"
-          GIT_SHA: "${{ github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}"
+          GIT_SHA: "${{ github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}" # TODO: update refs/heads/master for your main branch name

dbt_project.yml

@@ -31,6 +31,10 @@ quoting:
   identifier: true
   schema: true # translates to gcp dataset
 
+vars:
+  data_diff:
+    datasource_id: 6575
+
 seeds:
   jaffle_shop: # you must include the project name
     # +project: lumpy-space-prince # if you want to create a model in a separate project

models/core/fct_orders.sql

@@ -41,4 +41,4 @@ final as (
 
 )
 
-select * from final
+select * from final limit 10

profiles.yml

@@ -9,7 +9,7 @@ jaffle_shop:
       type: bigquery
       method: service-account # for production runs: https://docs.getdbt.com/docs/profile-bigquery#section-service-account-file-authentication
       project: "{{ env_var('PROJECT_ID', 'dbt-demo-386220') }}" # parameterize project for dbt runs based on environment variable with default project
-      dataset: dbt_sung # You can also use "schema" here
+      dataset: "{{ env_var('SCHEMA_OVERRIDE', 'dbt_sung') }}" # You can also use "schema" here
       threads: 24 # https://docs.getdbt.com/docs/configure-your-profile#section-understanding-threads
       keyfile: "{{ env_var('GOOGLE_APPLICATION_CREDENTIALS', 'service_account.json') }}"
       timeout_seconds: 300

requirements.txt

@@ -1 +1,2 @@
-dbt-bigquery==1.6.5
+dbt-bigquery==1.6.5
+data-diff==0.8.4