From 0a44d27efa90ae0c7900fd37e4c4b6575b3d1469 Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Mon, 10 Apr 2023 16:04:45 -0700 Subject: [PATCH 1/3] Add frontmatter for dbt docs --- .gitignore | 1 + .pre-commit-config.yaml | 2 +- mkdocs.yml | 1 + transform/models/overview.md | 37 ++++++++++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 transform/models/overview.md diff --git a/.gitignore b/.gitignore index b7636e6a..3cf7c71f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.swp +docs/dbt_docs # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b9308656..00334737 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,7 +76,7 @@ repos: id: dbt-deps language: python entry: dbt deps --project-dir=transform - always_run: true + always_run: false pass_filenames: false types: [sql] additional_dependencies: diff --git a/mkdocs.yml b/mkdocs.yml index a11780f4..505d3074 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,6 +31,7 @@ nav: - Security Guidelines: security.md - Cloud Infrastructure: cloud-infrastructure.md - Writing Documentation: writing-documentation.md + - dbt Project: /dbt_docs/index.html - Learning: - git: learning/git.md - dbt: learning/dbt.md diff --git a/transform/models/overview.md b/transform/models/overview.md new file mode 100644 index 00000000..c8e89aa8 --- /dev/null +++ b/transform/models/overview.md @@ -0,0 +1,37 @@ +{% docs __overview__ %} + +# CalData dbt Documentation + +Welcome to the CalData Data Services and Engineering `dbt` docs. +To go back to the top-level docs, follow [this link](../) + +## Navigation + +You can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models in your project. + +### Project Tab + +The Project tab mirrors the directory structure of your dbt project. +In this tab, you can see all of the models defined in your dbt project, as well as models imported from dbt packages. + +### Database Tab + +The Database tab also exposes your models, but in a format that looks more like a database explorer. +This view shows relations (tables and views) grouped into database schemas. +Note that ephemeral models are not shown in this interface, as they do not exist in the database. + +## Graph Exploration + +You can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models. + +On model pages, you'll see the immediate parents and children of the model you're exploring. +By clicking the Expand button at the top-right of this lineage pane, +you'll be able to see all of the models that are used to build, or are built from, +the model you're exploring. + +Once expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the models in the graph. +For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/reference/node-selection/syntax). + +Note that you can also right-click on models to interactively filter and explore the graph. + +{% enddocs %} From e6e1cb19d494738655ae57540a5e103591007423 Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Mon, 10 Apr 2023 16:08:17 -0700 Subject: [PATCH 2/3] Work on deploy --- .github/workflows/docs.yml | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 59122d11..9ac37ac1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,6 +3,7 @@ on: push: branches: - main + - dbt-docs permissions: contents: write jobs: @@ -13,9 +14,30 @@ jobs: - uses: actions/setup-python@v4 with: python-version: 3.x + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v1 + with: + # The credentials here can read metadata only, which makes them + # good enough for `dbt compile`, but not good enough for + # `dbt docs generate`. Should we want to do more detailed checks + # at some point, we should revisit the service account permissions. + credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} + export_environment_variables: true - uses: actions/cache@v2 with: key: ${{ github.ref }} path: .cache - - run: pip install mkdocs-material - - run: mkdocs gh-deploy --force + - name: Install dependencies + run: | + pip install -r transform/requirements.txt + pip install mkdocs-material + - name: Build dbt docs + env: + DBT_PROFILES_DIR: transform/ci + run: | + dbt deps --project-dir=transform + dbt docs generate --project-dir=transform + cp -r transform/target docs/dbt_docs + - name: Deploy 🚀 + run: mkdocs gh-deploy --force From 8ed97cc3928ea6ad70f28b16d0bfa3506d083cd6 Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Tue, 11 Apr 2023 14:55:39 -0700 Subject: [PATCH 3/3] Disable building on branch --- .github/workflows/docs.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9ac37ac1..fc40f26d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,7 +3,6 @@ on: push: branches: - main - - dbt-docs permissions: contents: write jobs: @@ -18,10 +17,8 @@ jobs: name: Authenticate to Google Cloud uses: google-github-actions/auth@v1 with: - # The credentials here can read metadata only, which makes them - # good enough for `dbt compile`, but not good enough for - # `dbt docs generate`. Should we want to do more detailed checks - # at some point, we should revisit the service account permissions. + # The credentials here can read metadata, data, and create jobs (execute + # queries), which allows it to compile the dbt project and generate docs. credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} export_environment_variables: true - uses: actions/cache@v2