Merge pull request #32 from cagov/ci-checks-on-warehouse

CI checks on warehouse

transform/.github/workflows/pre-commit.yml

@@ -5,10 +5,34 @@ on:
   push:
     branches: [main]
 
+defaults:
+  run:
+    shell: bash -l {0}
+
 jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - id: auth
+        name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v1
+        with:
+          # The credentials here can read metadata only, which makes them
+          # good enough for `dbt compile`, but not good enough for
+          # `dbt docs generate`. Should we want to do more detailed checks
+          # at some point, we should revisit the service account permissions.
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
+          export_environment_variables: true
       - uses: actions/setup-python@v3
+      - name: Install Conda environment with Micromamba
+        uses: mamba-org/provision-with-micromamba@v15
+        with:
+          environment-file: ci/environment.yml
+      - name: Build manifest and catalog
+        env:
+          DBT_PROFILES_DIR: ci
+        run: dbt deps && dbt compile
       - uses: pre-commit/[email protected]
+        env:
+          DBT_PROFILES_DIR: ci

transform/.gitignore

@@ -1,4 +1,4 @@
-
 target/
 dbt_packages/
 logs/
+.vscode/

transform/.pre-commit-config.yaml

@@ -27,8 +27,31 @@ repos:
     rev: 1.4.2
     hooks:
       - id: sqlfluff-lint
-        args: ["--config", ".sqlfluff-ci"]
-        additional_dependencies: ["dbt-bigquery", "sqlfluff-templater-dbt"]
+        additional_dependencies:
+          ["dbt-bigquery==1.3", "sqlfluff-templater-dbt", "dbt-core==1.3.2"]
       - id: sqlfluff-fix
-        args: ["--config", ".sqlfluff-ci"]
-        additional_dependencies: ["dbt-bigquery", "sqlfluff-templater-dbt"]
+        additional_dependencies:
+          ["dbt-bigquery==1.3", "sqlfluff-templater-dbt", "dbt-core==1.3.2"]
+  - repo: https://github.com/dbt-checkpoint/dbt-checkpoint
+    # Last tagged release is old and unsupported, but the most recent refs
+    # have some broken fixes from Datacoves... This is just a known
+    # fair-to-decent commit.
+    rev: 9097e05
+    hooks:
+      # There are a ton of checks in this repo, these are the ones that
+      # seemed most immediately useful.
+      # https://github.com/dbt-checkpoint/dbt-checkpoint/blob/main/HOOKS.md
+      - id: check-model-columns-have-desc
+      - id: check-model-has-description
+      - id: check-model-has-properties-file
+      - id: check-script-ref-and-source
+      - id: check-script-has-no-table-name
+      - id: check-source-columns-have-desc
+      - id: check-macro-has-description
+      - id: check-macro-arguments-have-desc
+      # It would be great to run these source checks, but there is no way to exclude
+      # souces that come from third-party packages, and we shouldn't be in the business
+      # of maintaining those.
+      # - id: check-model-has-all-columns
+      # - id: check-source-has-all-columns
+      # - id: check-source-table-has-description

transform/ci/environment.yml

@@ -0,0 +1,6 @@
+name: ci
+channels:
+  - conda-forge
+dependencies:
+  - dbt-core=1.3.2
+  - dbt-bigquery=1.3

transform/ci/profiles.yml

@@ -0,0 +1,15 @@
+config:
+  send_anonymous_usage_stats: false
+  use_colors: true
+  warn_error: true
+
+default:
+  target: dev
+  outputs:
+    dev:
+      type: bigquery
+      method: oauth
+      project: "{{ env_var('GCP_PROJECT') }}"
+      # Should not be created in CI as the CI service account is read-only!
+      dataset: dbt_ci_should_not_create_this_dataset
+      threads: 4

transform/requirements.txt

@@ -1,3 +1,3 @@
-dbt-core==1.3.1
-dbt-bigquery==1.3.0
+dbt-core<1.4
+dbt-bigquery<1.4
 pre-commit