diff --git a/.github/workflows/pr_job.yml b/.github/workflows/pr_job.yml new file mode 100644 index 0000000..a00960b --- /dev/null +++ b/.github/workflows/pr_job.yml @@ -0,0 +1,65 @@ +name: dbt pull request job + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + push: + branches: + - '!master' # TODO: change to your main branch name + +env: + #TODO(developer): update PROJECT_ID + PROJECT_ID: "dbt-demo-386220" + DBT_PROFILES_DIR: ./ + GOOGLE_APPLICATION_CREDENTIALS: ./service_account.json + +jobs: + run: + runs-on: ubuntu-20.04 + if: ${{ !github.event.pull_request.draft }} + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + steps: + - name: checkout + uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: install requirements + run: pip install -q -r requirements.txt + + - name: install datafold-sdk + run: pip install -q datafold-sdk + + - name: Authenticate dbt to BigQuery + run: 'echo "$KEYFILE" > ./service_account.json' # persist file in root dir + shell: bash + env: + KEYFILE: ${{secrets.GOOGLE_APPLICATION_CREDENTIALS}} # TODO: add a github secret + + - name: dbt deps + run: dbt deps + + - name: Find Current Pull Request + uses: jwalton/gh-find-current-pr@v1.3.0 + id: findPR + + - name: dbt build + run: dbt build --full-refresh --profiles-dir ./ + + # TODO: update the --config-id to your config id + - name: submit artifacts to datafold to compare to the production manifest + run: | + set -ex + datafold dbt upload --ci-config-id 272 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} + env: + DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }} + DATAFOLD_RUN_TYPE: "${{ 'pull_request' }}" + GIT_SHA: "${{ github.event.pull_request.head.sha }}" diff --git a/.github/workflows/prod_job.yml b/.github/workflows/prod_job.yml index 05d65fb..5d716d3 100644 --- a/.github/workflows/prod_job.yml +++ b/.github/workflows/prod_job.yml @@ -49,4 +49,4 @@ jobs: env: DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }} DATAFOLD_RUN_TYPE: "${{ github.ref == 'refs/heads/master' && 'production' || 'pull_request' }}" - GIT_SHA: "${{ github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}" + GIT_SHA: "${{ github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}" # TODO: update refs/heads/master for your main branch name diff --git a/dbt_project.yml b/dbt_project.yml index 316286e..4b7ea37 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -31,6 +31,10 @@ quoting: identifier: true schema: true # translates to gcp dataset +vars: + data_diff: + datasource_id: 6575 + seeds: jaffle_shop: # you must include the project name # +project: lumpy-space-prince # if you want to create a model in a separate project diff --git a/models/core/fct_orders.sql b/models/core/fct_orders.sql index b534278..a3ab5f5 100644 --- a/models/core/fct_orders.sql +++ b/models/core/fct_orders.sql @@ -41,4 +41,4 @@ final as ( ) -select * from final +select * from final limit 10 \ No newline at end of file diff --git a/profiles.yml b/profiles.yml index 7a09028..fa9bfd9 100644 --- a/profiles.yml +++ b/profiles.yml @@ -9,7 +9,7 @@ jaffle_shop: type: bigquery method: service-account # for production runs: https://docs.getdbt.com/docs/profile-bigquery#section-service-account-file-authentication project: "{{ env_var('PROJECT_ID', 'dbt-demo-386220') }}" # parameterize project for dbt runs based on environment variable with default project - dataset: dbt_sung # You can also use "schema" here + dataset: "{{ env_var('SCHEMA_OVERRIDE', 'dbt_sung') }}" # You can also use "schema" here threads: 24 # https://docs.getdbt.com/docs/configure-your-profile#section-understanding-threads keyfile: "{{ env_var('GOOGLE_APPLICATION_CREDENTIALS', 'service_account.json') }}" timeout_seconds: 300 diff --git a/requirements.txt b/requirements.txt index 856690a..e7e34f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -dbt-bigquery==1.6.5 \ No newline at end of file +dbt-bigquery==1.6.5 +data-diff==0.8.4 \ No newline at end of file