From 212bc9fe2f28797e9c97f685ddb09a5497b84f4d Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Thu, 26 Jan 2023 12:55:10 -0800 Subject: [PATCH 1/5] Add some dbt pre-commit hooks --- transform/.gitignore | 2 +- transform/.pre-commit-config.yaml | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/transform/.gitignore b/transform/.gitignore index 49f147cb..7f6bec61 100644 --- a/transform/.gitignore +++ b/transform/.gitignore @@ -1,4 +1,4 @@ - target/ dbt_packages/ logs/ +.vscode/ diff --git a/transform/.pre-commit-config.yaml b/transform/.pre-commit-config.yaml index 163770b3..8d1ab505 100644 --- a/transform/.pre-commit-config.yaml +++ b/transform/.pre-commit-config.yaml @@ -32,3 +32,26 @@ repos: - id: sqlfluff-fix args: ["--config", ".sqlfluff-ci"] additional_dependencies: ["dbt-bigquery", "sqlfluff-templater-dbt"] + - repo: https://github.com/dbt-checkpoint/dbt-checkpoint + # Last tagged release is old and unsupported, but the most recent refs + # have some broken fixes from Datacoves... This is just a known + # fair-to-decent commit. + rev: 9097e05 + hooks: + # There are a ton of checks in this repo, these are the ones that + # seemed most immediately useful. + # https://github.com/dbt-checkpoint/dbt-checkpoint/blob/main/HOOKS.md + - id: check-model-columns-have-desc + - id: check-model-has-description + - id: check-model-has-properties-file + - id: check-script-ref-and-source + - id: check-script-has-no-table-name + - id: check-source-columns-have-desc + - id: check-macro-has-description + - id: check-macro-arguments-have-desc + # It would be great to run these source checks, but there is no way to exclude + # souces that come from third-party packages, and we shouldn't be in the business + # of maintaining those. + # - id: check-model-has-all-columns + # - id: check-source-has-all-columns + # - id: check-source-table-has-description From 096f1e418da0a0943d60cf3ed00e9be0e0240cbc Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Thu, 26 Jan 2023 13:15:19 -0800 Subject: [PATCH 2/5] WIP try to run dbt compile in CI Try relaxing requirements Forgot to set profiles dbt deps Do we need docs? Sigh Try using micromamba Try with env.yml More version tweaking Pin for now --- transform/.github/workflows/pre-commit.yml | 22 ++++++++++++++++++++++ transform/ci/environment.yml | 6 ++++++ transform/ci/profiles.yml | 15 +++++++++++++++ transform/requirements.txt | 4 ++-- 4 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 transform/ci/environment.yml create mode 100644 transform/ci/profiles.yml diff --git a/transform/.github/workflows/pre-commit.yml b/transform/.github/workflows/pre-commit.yml index ec4d9b3a..3f7410a2 100644 --- a/transform/.github/workflows/pre-commit.yml +++ b/transform/.github/workflows/pre-commit.yml @@ -5,10 +5,32 @@ on: push: branches: [main] +defaults: + run: + shell: bash -l {0} + jobs: pre-commit: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v1 + with: + # The credentials here can read metadata only, which makes them + # good enough for `dbt compile`, but not good enough for + # `dbt docs generate`. Should we want to do more detailed checks + # at some point, we should revisit the service account permissions. + credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} + export_environment_variables: true - uses: actions/setup-python@v3 + - name: Install Conda environment with Micromamba + uses: mamba-org/provision-with-micromamba@v15 + with: + environment-file: ci/environment.yml + - name: Build manifest and catalog + env: + DBT_PROFILES_DIR: ci + run: dbt deps && dbt compile - uses: pre-commit/action@v3.0.0 diff --git a/transform/ci/environment.yml b/transform/ci/environment.yml new file mode 100644 index 00000000..f5f3d981 --- /dev/null +++ b/transform/ci/environment.yml @@ -0,0 +1,6 @@ +name: ci +channels: + - conda-forge +dependencies: + - dbt-core=1.3.2 + - dbt-bigquery=1.3 diff --git a/transform/ci/profiles.yml b/transform/ci/profiles.yml new file mode 100644 index 00000000..e020ce50 --- /dev/null +++ b/transform/ci/profiles.yml @@ -0,0 +1,15 @@ +config: + send_anonymous_usage_stats: false + use_colors: true + warn_error: true + +default: + target: dev + outputs: + dev: + type: bigquery + method: oauth + project: "{{ env_var('GCP_PROJECT') }}" + # Should not be created in CI as the CI service account is read-only! + dataset: dbt_ci_should_not_create_this_dataset + threads: 4 diff --git a/transform/requirements.txt b/transform/requirements.txt index aecdd337..3c1f42de 100644 --- a/transform/requirements.txt +++ b/transform/requirements.txt @@ -1,3 +1,3 @@ -dbt-core==1.3.1 -dbt-bigquery==1.3.0 +dbt-core<1.4 +dbt-bigquery<1.4 pre-commit From c5e6f5d1eaeac38921200be8eebed707cf41b9e1 Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Thu, 26 Jan 2023 14:23:55 -0800 Subject: [PATCH 3/5] Try specifying pre-commit additional deps --- transform/.pre-commit-config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/transform/.pre-commit-config.yaml b/transform/.pre-commit-config.yaml index 8d1ab505..e28e2e8f 100644 --- a/transform/.pre-commit-config.yaml +++ b/transform/.pre-commit-config.yaml @@ -28,10 +28,12 @@ repos: hooks: - id: sqlfluff-lint args: ["--config", ".sqlfluff-ci"] - additional_dependencies: ["dbt-bigquery", "sqlfluff-templater-dbt"] + additional_dependencies: + ["dbt-bigquery==1.3", "sqlfluff-templater-dbt", "dbt-core==1.3.2"] - id: sqlfluff-fix args: ["--config", ".sqlfluff-ci"] - additional_dependencies: ["dbt-bigquery", "sqlfluff-templater-dbt"] + additional_dependencies: + ["dbt-bigquery==1.3", "sqlfluff-templater-dbt", "dbt-core==1.3.2"] - repo: https://github.com/dbt-checkpoint/dbt-checkpoint # Last tagged release is old and unsupported, but the most recent refs # have some broken fixes from Datacoves... This is just a known From aedf98a54d9d9f05e814da232565402298ecbecd Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Thu, 26 Jan 2023 14:38:33 -0800 Subject: [PATCH 4/5] Try using the dbt templater in CI now that we can compile --- transform/.pre-commit-config.yaml | 4 ++-- transform/.sqlfluff-ci | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) delete mode 100644 transform/.sqlfluff-ci diff --git a/transform/.pre-commit-config.yaml b/transform/.pre-commit-config.yaml index e28e2e8f..1cb115e4 100644 --- a/transform/.pre-commit-config.yaml +++ b/transform/.pre-commit-config.yaml @@ -27,11 +27,11 @@ repos: rev: 1.4.2 hooks: - id: sqlfluff-lint - args: ["--config", ".sqlfluff-ci"] + args: ["--profiles-dir", "ci"] additional_dependencies: ["dbt-bigquery==1.3", "sqlfluff-templater-dbt", "dbt-core==1.3.2"] - id: sqlfluff-fix - args: ["--config", ".sqlfluff-ci"] + args: ["--profiles-dir", "ci"] additional_dependencies: ["dbt-bigquery==1.3", "sqlfluff-templater-dbt", "dbt-core==1.3.2"] - repo: https://github.com/dbt-checkpoint/dbt-checkpoint diff --git a/transform/.sqlfluff-ci b/transform/.sqlfluff-ci deleted file mode 100644 index 0737b613..00000000 --- a/transform/.sqlfluff-ci +++ /dev/null @@ -1,7 +0,0 @@ -[sqlfluff] -# The dbt templater is gives a more accurate picture for the linter. However, -# the dbt templater requires an actual connection to the warehouse to work. -# Eventually, consider pointing this at a test instance for CI, but for now, -# just use the jinja templater and ignore any templating errors that occur. -templater = jinja -ignore = templating From 081f885c9b0647e5706b31622ef13d13a9eb3ce1 Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Thu, 26 Jan 2023 14:55:08 -0800 Subject: [PATCH 5/5] Try setting profiles dir for pre-commit step --- transform/.github/workflows/pre-commit.yml | 2 ++ transform/.pre-commit-config.yaml | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/transform/.github/workflows/pre-commit.yml b/transform/.github/workflows/pre-commit.yml index 3f7410a2..c72d3b73 100644 --- a/transform/.github/workflows/pre-commit.yml +++ b/transform/.github/workflows/pre-commit.yml @@ -34,3 +34,5 @@ jobs: DBT_PROFILES_DIR: ci run: dbt deps && dbt compile - uses: pre-commit/action@v3.0.0 + env: + DBT_PROFILES_DIR: ci diff --git a/transform/.pre-commit-config.yaml b/transform/.pre-commit-config.yaml index 1cb115e4..e8933869 100644 --- a/transform/.pre-commit-config.yaml +++ b/transform/.pre-commit-config.yaml @@ -27,11 +27,9 @@ repos: rev: 1.4.2 hooks: - id: sqlfluff-lint - args: ["--profiles-dir", "ci"] additional_dependencies: ["dbt-bigquery==1.3", "sqlfluff-templater-dbt", "dbt-core==1.3.2"] - id: sqlfluff-fix - args: ["--profiles-dir", "ci"] additional_dependencies: ["dbt-bigquery==1.3", "sqlfluff-templater-dbt", "dbt-core==1.3.2"] - repo: https://github.com/dbt-checkpoint/dbt-checkpoint