diff --git a/.github/ISSUE_TEMPLATE/a-improve-docs.yml b/.github/ISSUE_TEMPLATE/a-improve-docs.yml index 70b173e49a4..c9030bc227b 100644 --- a/.github/ISSUE_TEMPLATE/a-improve-docs.yml +++ b/.github/ISSUE_TEMPLATE/a-improve-docs.yml @@ -5,7 +5,7 @@ body: - type: markdown attributes: value: | - * You can ask questions or submit ideas for the dbt docs in [Discussions](https://github.com/dbt-labs/docs.getdbt.com/discussions) + * You can ask questions or submit ideas for the dbt docs in [Issues](https://github.com/dbt-labs/docs.getdbt.com/issues/new/choose) * Before you file an issue read the [Contributing guide](https://github.com/dbt-labs/docs.getdbt.com#contributing). * Check to make sure someone hasn't already opened a similar [issue](https://github.com/dbt-labs/docs.getdbt.com/issues). diff --git a/.github/ISSUE_TEMPLATE/contribute-to-developer-blog.yml b/.github/ISSUE_TEMPLATE/contribute-to-developer-blog.yml index f138b9e4e06..037da98dc6f 100644 --- a/.github/ISSUE_TEMPLATE/contribute-to-developer-blog.yml +++ b/.github/ISSUE_TEMPLATE/contribute-to-developer-blog.yml @@ -1,4 +1,4 @@ -name: Contribute to the dbt Developer Blog +name: Propose a dbt Developer Blog idea description: > For proposing a new post on the dbt Developer Blog. labels: ["content","developer blog"] diff --git a/.github/ISSUE_TEMPLATE/improve-the-site.yml b/.github/ISSUE_TEMPLATE/improve-the-site.yml index dd585324f89..01ebdea711a 100644 --- a/.github/ISSUE_TEMPLATE/improve-the-site.yml +++ b/.github/ISSUE_TEMPLATE/improve-the-site.yml @@ -5,7 +5,7 @@ body: - type: markdown attributes: value: | - * You can ask questions or submit ideas for the dbt docs in [Discussions](https://github.com/dbt-labs/docs.getdbt.com/discussions) + * You can ask questions or submit ideas for the dbt docs in [Issues](https://github.com/dbt-labs/docs.getdbt.com/issues/new/choose) * Before you file an issue read the [Contributing guide](https://github.com/dbt-labs/docs.getdbt.com#contributing). * Check to make sure someone hasn't already opened a similar [issue](https://github.com/dbt-labs/docs.getdbt.com/issues). diff --git a/.github/labeler.yml b/.github/labeler.yml index 176f1874009..316098eb51c 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -3,6 +3,7 @@ developer blog: guides: - website/docs/guides/**/* +- website/docs/quickstarts/**/* content: - website/docs/**/* diff --git a/.github/workflows/label.yml b/.github/workflows/label.yml index 5ebef4f88ca..48615e60b9e 100644 --- a/.github/workflows/label.yml +++ b/.github/workflows/label.yml @@ -2,37 +2,45 @@ name: Add/Remove Labels on: pull_request_target: - types: [ opened, closed ] + types: [opened] jobs: add_new_contributor_label: if: github.event.action == 'opened' - permissions: - contents: read - pull-requests: write runs-on: ubuntu-latest steps: - - uses: actions/github-script@v6 - with: - script: | - const creator = context.payload.sender.login + - name: Add new contributor label + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.DOCS_SECRET }} + script: | + const creator = context.payload.sender.login; const opts = github.rest.issues.listForRepo.endpoint.merge({ ...context.issue, creator, - state: 'all' - }) - const issues = await github.paginate(opts) + state: 'all', + }); + + const issues = await github.paginate(opts); + + let isAlreadyContributor = false; + for (const issue of issues) { if (issue.number === context.issue.number) { - continue + continue; } - if (issue.pull_request) { - return // creator is already a contributor + if (issue.pull_request && issue.user.login === creator) { + isAlreadyContributor = true; + break; } } - await github.rest.issues.addLabels({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: ['new contributor'] - }) + + if (!isAlreadyContributor) { + console.log('Adding label: new contributor'); + await github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['new contributor'], + }); + } diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 7e4bb5c268a..cc231cdcde3 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -5,8 +5,8 @@ name: "Pull Request Labeler" on: -- pull_request_target - + pull_request_target: + types: [opened] jobs: triage: permissions: diff --git a/README.md b/README.md index da82ab45fd6..c749fedf95a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Creating an inclusive and equitable environment for our documents is more import We welcome contributions from community members to this repo: - **Fixes**: When you notice an error, you can use the `Edit this page` button at the bottom of each page to suggest a change. - **New documentation**: If you contributed code in [dbt-core](https://github.com/dbt-labs/dbt-core), we encourage you to also write the docs here! Please reach out in the dbt community if you need help finding a place for these docs. -- **Major rewrites**: You can [file an issue](https://github.com/dbt-labs/docs.getdbt.com/issues/new?assignees=&labels=content%2Cimprovement&template=improve-docs.yml) or [start a discussion](https://github.com/dbt-labs/docs.getdbt.com/discussions) to propose ideas for a content area that requires attention. +- **Major rewrites**: You can [file an issue](https://github.com/dbt-labs/docs.getdbt.com/issues/new/choose) to propose ideas for a content area that requires attention. You can use components documented in the [docusaurus library](https://v2.docusaurus.io/docs/markdown-features/). diff --git a/website/blog/2020-07-01-how-to-create-near-real-time-models-with-just-dbt-sql.md b/website/blog/2020-07-01-how-to-create-near-real-time-models-with-just-dbt-sql.md index 944d6fdd3f9..cdfd4da5f5d 100644 --- a/website/blog/2020-07-01-how-to-create-near-real-time-models-with-just-dbt-sql.md +++ b/website/blog/2020-07-01-how-to-create-near-real-time-models-with-just-dbt-sql.md @@ -13,6 +13,13 @@ date: 2020-07-01 is_featured: false --- +:::caution More up-to-date information available + +Since this blog post was first published, many data platforms have added support for [materialized views](/blog/announcing-materialized-views), which are a superior way to achieve the goals outlined here. We recommend them over the below approach. + + +::: + Before I dive into how to create this, I have to say this. **You probably don’t need this**. I, along with my other Fishtown colleagues, have spent countless hours working with clients that ask for near-real-time streaming data. However, when we start digging into the project, it is often realized that the use case is not there. There are a variety of reasons why near real-time streaming is not a good fit. Two key ones are: 1. The source data isn’t updating frequently enough. diff --git a/website/blog/ctas.yml b/website/blog/ctas.yml index 1b3fad79b80..2e3170faae4 100644 --- a/website/blog/ctas.yml +++ b/website/blog/ctas.yml @@ -10,3 +10,8 @@ subheader: Check out guides on getting your warehouse set up and connected to dbt Cloud. button_text: Learn more url: https://docs.getdbt.com/quickstarts +- name: coalesce_2023_signup + header: Join data practitioners worldwide at Coalesce 2023 + subheader: Kicking off on October 16th, both online and in-person (Sydney, London, and San Diego) + button_text: Register now + url: https://coalesce.getdbt.com/?utm_medium=internal&utm_source=docs&utm_campaign=q3-2024_coalesce-2023_aw&utm_content=coalesce____&utm_term=all___ \ No newline at end of file diff --git a/website/blog/metadata.yml b/website/blog/metadata.yml index 20fa93f8d7e..a5afa86e667 100644 --- a/website/blog/metadata.yml +++ b/website/blog/metadata.yml @@ -2,7 +2,7 @@ featured_image: "" # This CTA lives in right sidebar on blog index -featured_cta: "staging" +featured_cta: "coalesce_2023_signup" # Show or hide hero title, description, cta from blog index show_title: true diff --git a/website/dbt-versions.js b/website/dbt-versions.js index 3eff99e7f98..910c4a6ef4d 100644 --- a/website/dbt-versions.js +++ b/website/dbt-versions.js @@ -27,6 +27,10 @@ exports.versions = [ ] exports.versionedPages = [ + { + "page": "reference/resource-configs/store_failures_as", + "firstVersion": "1.7", + }, { "page": "docs/build/build-metrics-intro", "firstVersion": "1.6", @@ -170,6 +174,10 @@ exports.versionedPages = [ { "page": "reference/resource-configs/grants", "firstVersion": "1.2", + }, + { + "page": "docs/build/saved-queries", + "firstVersion": "1.7", } ] diff --git a/website/docs/community/resources/oss-expectations.md b/website/docs/community/resources/oss-expectations.md index 649a9dea94f..9c916de1240 100644 --- a/website/docs/community/resources/oss-expectations.md +++ b/website/docs/community/resources/oss-expectations.md @@ -4,7 +4,7 @@ title: "Expectations for OSS contributors" Whether it's a dbt package, a plugin, `dbt-core`, or this very documentation site, contributing to the open source code that supports the dbt ecosystem is a great way to level yourself up as a developer, and to give back to the community. The goal of this page is to help you understand what to expect when contributing to dbt open source software (OSS). While we can only speak for our own experience as open source maintainers, many of these guidelines apply when contributing to other open source projects, too. -Have you seen things in other OSS projects that you quite like, and think we could learn from? [Open a discussion on the Developer Hub](https://github.com/dbt-labs/docs.getdbt.com/discussions/new), or start a conversation in the dbt Community Slack (for example: `#community-strategy`, `#dbt-core-development`, `#package-ecosystem`, `#adapter-ecosystem`). We always appreciate hearing from you! +Have you seen things in other OSS projects that you quite like, and think we could learn from? [Open a discussion on the dbt Community Forum](https://discourse.getdbt.com), or start a conversation in the dbt Community Slack (for example: `#community-strategy`, `#dbt-core-development`, `#package-ecosystem`, `#adapter-ecosystem`). We always appreciate hearing from you! ## Principles @@ -51,7 +51,7 @@ An issue could be a bug you’ve identified while using the product or reading t ### Best practices for issues -- Issues are **not** for support / troubleshooting / debugging help. Please [open a discussion on the Developer Hub](https://github.com/dbt-labs/docs.getdbt.com/discussions/new), so other future users can find and read proposed solutions. If you need help formulating your question, you can post in the `#advice-dbt-help` channel in the [dbt Community Slack](https://www.getdbt.com/community/). +- Issues are **not** for support / troubleshooting / debugging help. Please [open a discussion on the dbt Community Forum](https://discourse.getdbt.com), so other future users can find and read proposed solutions. If you need help formulating your question, you can post in the `#advice-dbt-help` channel in the [dbt Community Slack](https://www.getdbt.com/community/). - Always search existing issues first, to see if someone else had the same idea / found the same bug you did. - Many repositories offer templates for creating issues, such as when reporting a bug or requesting a new feature. If available, please select the relevant template and fill it out to the best of your ability. This will help other people understand your issue and respond. diff --git a/website/docs/community/resources/viewpoint.md b/website/docs/community/resources/viewpoint.md index e159c6178a3..5c3f80555c5 100644 --- a/website/docs/community/resources/viewpoint.md +++ b/website/docs/community/resources/viewpoint.md @@ -7,7 +7,7 @@ id: "viewpoint" In 2015-2016, a team of folks at RJMetrics had the opportunity to observe, and participate in, a significant evolution of the analytics ecosystem. The seeds of dbt were conceived in this environment, and the viewpoint below was written to reflect what we had learned and how we believed the world should be different. **dbt is our attempt to address the workflow challenges we observed, and as such, this viewpoint is the most foundational statement of the dbt project's goals.** -The remainder of this document is largely unedited from [the original post](https://blog.getdbt.com/building-a-mature-analytics-workflow/). +The remainder of this document is largely unedited from [the original post](https://getdbt.com/blog/building-a-mature-analytics-workflow). ::: diff --git a/website/docs/docs/about-setup.md b/website/docs/docs/about-setup.md index 3fb868b8448..ceb34a5ccbb 100644 --- a/website/docs/docs/about-setup.md +++ b/website/docs/docs/about-setup.md @@ -3,11 +3,13 @@ title: About dbt setup id: about-setup description: "About setup of dbt Core and Cloud" sidebar_label: "About dbt setup" +pagination_next: "docs/environments-in-dbt" +pagination_prev: null --- dbt compiles and runs your analytics code against your data platform, enabling you and your team to collaborate on a single source of truth for metrics, insights, and business definitions. There are two options for deploying dbt: -**dbt Cloud** runs dbt Core in a hosted (single or multi-tenant) environment with a browser-based interface. The intuitive UI will aid you in setting up the various components. dbt Cloud comes equipped with turnkey support for scheduling jobs, CI/CD, hosting documentation, monitoring & alerting, and an integrated developer environment (IDE). +**dbt Cloud** runs dbt Core in a hosted (single or multi-tenant) environment with a browser-based interface. The intuitive user interface aids you in setting up the various components. dbt Cloud comes equipped with turnkey support for scheduling jobs, CI/CD, hosting documentation, monitoring, and alerting. It also offers an integrated development environment (IDE) and allows you to develop and run dbt commands from your local command line (CLI) or code editor. **dbt Core** is an open-source command line tool that can be installed locally in your environment, and communication with databases is facilitated through adapters. @@ -19,7 +21,7 @@ To begin configuring dbt now, select the option that is right for you. diff --git a/website/docs/docs/build/about-metricflow.md b/website/docs/docs/build/about-metricflow.md index 68879911597..4910f12a792 100644 --- a/website/docs/docs/build/about-metricflow.md +++ b/website/docs/docs/build/about-metricflow.md @@ -4,38 +4,56 @@ id: about-metricflow description: "Learn more about MetricFlow and its key concepts" sidebar_label: About MetricFlow tags: [Metrics, Semantic Layer] +pagination_next: "docs/build/join-logic" +pagination_prev: null --- -This guide introduces MetricFlow's fundamental ideas for new users. MetricFlow, which powers the dbt Semantic Layer, helps you define and manage the logic for your company's metrics. It's an opinionated set of abstractions and helps data consumers retrieve metric datasets from a data platform quickly and efficiently. +This guide introduces MetricFlow's fundamental ideas for people new to this feature. MetricFlow, which powers the dbt Semantic Layer, helps you define and manage the logic for your company's metrics. It's an opinionated set of abstractions and helps data consumers retrieve metric datasets from a data platform quickly and efficiently. -:::info +MetricFlow handles SQL query construction and defines the specification for dbt semantic models and metrics. It allows you to define metrics in your dbt project and query them with [MetricFlow commands](/docs/build/metricflow-commands) whether in dbt Cloud or dbt Core. -MetricFlow is a new way to define metrics and one of the key components of the [dbt Semantic Layer](/docs/use-dbt-semantic-layer/dbt-sl). It handles SQL query construction and defines the specification for dbt semantic models and metrics. +Before you start, consider the following guidelines: -MetricFlow is currently available on dbt v1.6 or higher for all users. dbt Core users can use the MetricFlow CLI to define metrics in their local dbt Core project. However, to experience the power of the universal [dbt Semantic Layer](/docs/use-dbt-semantic-layer/dbt-sl) and query those metrics in downstream tools, you'll need a dbt Cloud [Team or Enterprise](https://www.getdbt.com/pricing/) account. +- Define metrics in YAML and query them using these [new metric specifications](https://github.com/dbt-labs/dbt-core/discussions/7456). +- You must be on [dbt version](/docs/dbt-versions/upgrade-core-in-cloud) 1.6 or higher to use MetricFlow. +- Use MetricFlow with Snowflake, BigQuery, Databricks, Postgres (dbt Core only), or Redshift. +- Discover insights and query your metrics using the [dbt Semantic Layer](/docs/use-dbt-semantic-layer/dbt-sl) and its diverse range of [available integrations](/docs/use-dbt-semantic-layer/avail-sl-integrations). You must have a dbt Cloud account on the [Team or Enterprise plan](https://www.getdbt.com/pricing/). -::: +## MetricFlow -There are a few key principles: +MetricFlow is a SQL query generation tool designed to streamline metric creation across different data dimensions for diverse business needs. +- It operates through YAML files, where a semantic graph links language to data. This graph comprises [semantic models](/docs/build/semantic-models) (data entry points) and [metrics](/docs/build/metrics-overview) (functions for creating quantitative indicators). +- MetricFlow is a [BSL package](https://github.com/dbt-labs/metricflow) with code source available, and compatible with dbt version 1.6 and higher. Data practitioners and enthusiasts are highly encouraged to contribute. +- As a part of the dbt Semantic Layer, MetricFlow empowers organizations to define metrics using YAML abstractions. +- To query metric dimensions, dimension values, and validate configurations, use [MetricFlow commands](/docs/build/metricflow-commands). -- **Flexible, but complete** — Ability to create any metric on any data model by defining logic in flexible abstractions. -- **Don't Repeat Yourself (DRY)** — Avoid repetition by allowing metric definitions to be enabled whenever possible. -- **Simple with progressive complexity** — Make MetricFlow approachable by relying on known concepts and structures in data modeling. -- **Performant and efficient** — Allow for performance optimizations in centralized data engineering while still enabling distributed definition and ownership of logic. -## MetricFlow +**Note** — MetricFlow doesn't support dbt [builtin functions or packages](/reference/dbt-jinja-functions/builtins) at this time, however, support is planned for the future. + +MetricFlow abides by these principles: -- MetricFlow is a SQL query generation engine that helps you create metrics by constructing appropriate queries for different granularities and dimensions that are useful for various business applications. +- **Flexibility with completeness**: Define metric logic using flexible abstractions on any data model. +- **DRY (Don't Repeat Yourself)**: Minimize redundancy by enabling metric definitions whenever possible. +- **Simplicity with gradual complexity:** Approach MetricFlow using familiar data modeling concepts. +- **Performance and efficiency**: Optimize performance while supporting centralized data engineering and distributed logic ownership. -- It uses YAML files to define a semantic graph, which maps language to data. This graph consists of [semantic models](/docs/build/semantic-models), which serve as data entry points, and [metrics](/docs/build/metrics-overview), which are functions used to create new quantitative indicators. + ### Semantic graph @@ -112,8 +130,6 @@ group by 1, 2 -> Introducing MetricFlow, a key component of the dbt Semantic Layer 🤩 - simplifying data collaboration and governance. - In the following three example tabs, use MetricFlow to define a semantic model that uses order_total as a metric and a sample schema to create consistent and accurate results — eliminating confusion, code duplication, and streamlining your workflow. diff --git a/website/docs/docs/build/analyses.md b/website/docs/docs/build/analyses.md index cd74c2e052d..74b138ac67a 100644 --- a/website/docs/docs/build/analyses.md +++ b/website/docs/docs/build/analyses.md @@ -2,6 +2,7 @@ title: "Analyses" description: "Read this tutorial to learn how to use custom analyses when building in dbt." id: "analyses" +pagination_next: null --- ## Overview diff --git a/website/docs/docs/build/build-metrics-intro.md b/website/docs/docs/build/build-metrics-intro.md index a6fab61d576..cdac51224ed 100644 --- a/website/docs/docs/build/build-metrics-intro.md +++ b/website/docs/docs/build/build-metrics-intro.md @@ -5,27 +5,28 @@ description: "Learn about MetricFlow and build your metrics with semantic models sidebar_label: Build your metrics tags: [Metrics, Semantic Layer, Governance] hide_table_of_contents: true +pagination_next: "docs/build/sl-getting-started" +pagination_prev: null --- -Use MetricFlow in dbt to centrally define your metrics. As a key component of the [dbt Semantic Layer](/docs/use-dbt-semantic-layer/dbt-sl), MetricFlow is responsible for SQL query construction and defining specifications for dbt semantic models and metrics. +Use MetricFlow in dbt to centrally define your metrics. As a key component of the [dbt Semantic Layer](/docs/use-dbt-semantic-layer/dbt-sl), MetricFlow is responsible for SQL query construction and defining specifications for dbt semantic models and metrics. It uses familiar constructs like semantic models and metrics to avoid duplicative coding, optimize your development workflow, ensure data governance for company metrics, and guarantee consistency for data consumers. -Use familiar constructs like semantic models and metrics to avoid duplicative coding, optimize your development workflow, ensure data governance for company metrics, and guarantee consistency for data consumers. -:::info -MetricFlow is currently available on dbt v1.6 or higher and allows users to define metrics in their dbt project whether in dbt Cloud or dbt Core. dbt Core users can use the MetricFlow CLI to define metrics in their local dbt Core project. However, to experience the power of the universal [dbt Semantic Layer](/docs/use-dbt-semantic-layer/dbt-sl) and query those metrics in downstream tools, you'll need a dbt Cloud [Team or Enterprise](https://www.getdbt.com/pricing/) account. - -::: - -Before you start, consider the following guidelines: - -- Define metrics in YAML and query them using these [new metric specifications](https://github.com/dbt-labs/dbt-core/discussions/7456). -- You must be on dbt v1.6 or higher to use MetricFlow. [Upgrade your dbt version](/docs/dbt-versions/upgrade-core-in-cloud) to get started. -- Use MetricFlow with Snowflake, BigQuery, Databricks, Postgres (CLI only), or Redshift. (dbt Cloud Postgres support coming soon) -- Unlock insights and query your metrics using the [dbt Semantic Layer](/docs/use-dbt-semantic-layer/dbt-sl) and its diverse range of [available integrations](/docs/use-dbt-semantic-layer/avail-sl-integrations). +MetricFlow allows you to: +- Intuitively define metrics in your dbt project +- Develop from your preferred environment, whether that's the [dbt Cloud CLI](/docs/cloud/cloud-cli-installation), [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud), or [dbt Core](/docs/core/installation) +- Use [MetricFlow commands](/docs/build/metricflow-commands) to query and test those metrics in your development environment +- Harness the true magic of the universal dbt Semantic Layer and dynamically query these metrics in downstream tools (Available for dbt Cloud [Team or Enterprise](https://www.getdbt.com/pricing/) accounts only).
+ + - - @@ -121,9 +128,13 @@ Use `is_partition: True` to show that a dimension exists over a specific time wi You can also use `is_partition` for [categorical](#categorical) dimensions as well. -MetricFlow enables metric aggregation during query time. For example, you can aggregate the `messages_per_month` measure. If you originally had a `time_granularity` for the time dimensions `metric_time`, you can specify a yearly granularity for aggregation in your CLI query: +MetricFlow enables metric aggregation during query time. For example, you can aggregate the `messages_per_month` measure. If you originally had a `time_granularity` for the time dimensions `metric_time`, you can specify a yearly granularity for aggregation in your query: ```bash +# dbt Cloud users +dbt sl query --metrics messages_per_month --dimensions metric_time --order metric_time --time-granularity year + +# dbt Core users mf query --metrics messages_per_month --dimensions metric_time --order metric_time --time-granularity year ``` @@ -344,7 +355,11 @@ In the sales tier example, if sales_person_id 456 is Tier 2 from 2022-03-08 onwa The following command or code represents how to return the count of transactions generated by each sales tier per month: -``` +```bash +# dbt Cloud users +dbt sl query --metrics transactions --dimensions metric_time__month,sales_person__tier --order metric_time__month --order sales_person__tier + +# dbt Core users mf query --metrics transactions --dimensions metric_time__month,sales_person__tier --order metric_time__month --order sales_person__tier ``` diff --git a/website/docs/docs/build/enhance-your-code.md b/website/docs/docs/build/enhance-your-code.md new file mode 100644 index 00000000000..5f2d48f6f5a --- /dev/null +++ b/website/docs/docs/build/enhance-your-code.md @@ -0,0 +1,38 @@ +--- +title: "Enhance your code" +description: "Learn how you can enhance your code" +pagination_next: "docs/build/project-variables" +pagination_prev: null +--- + +
+ + + + + +
+
+
+ + + + + +
\ No newline at end of file diff --git a/website/docs/docs/build/enhance-your-models.md b/website/docs/docs/build/enhance-your-models.md new file mode 100644 index 00000000000..46e7fa74353 --- /dev/null +++ b/website/docs/docs/build/enhance-your-models.md @@ -0,0 +1,23 @@ +--- +title: "Enhance your models" +description: "Learn how you can enhance your models" +pagination_next: "docs/build/materializations" +pagination_prev: null +--- + +
+ + + + + +
+
\ No newline at end of file diff --git a/website/docs/docs/build/groups.md b/website/docs/docs/build/groups.md index 7ac5337ba0d..d4fda045277 100644 --- a/website/docs/docs/build/groups.md +++ b/website/docs/docs/build/groups.md @@ -19,7 +19,7 @@ This functionality is new in v1.5. ## About groups -A group is a collection of nodes within a dbt DAG. Groups are named, and every group has an `owner`. They enable intentional collaboration within and across teams by restricting [access to private](/reference/resource-properties/access) models. +A group is a collection of nodes within a dbt DAG. Groups are named, and every group has an `owner`. They enable intentional collaboration within and across teams by restricting [access to private](/reference/resource-configs/access) models. Group members may include models, tests, seeds, snapshots, analyses, and metrics. (Not included: sources and exposures.) Each node may belong to only one group. @@ -94,7 +94,7 @@ select ... ### Referencing a model in a group -By default, all models within a group have the `protected` [access modifier](/reference/resource-properties/access). This means they can be referenced by downstream resources in _any_ group in the same project, using the [`ref`](/reference/dbt-jinja-functions/ref) function. If a grouped model's `access` property is set to `private`, only resources within its group can reference it. +By default, all models within a group have the `protected` [access modifier](/reference/resource-configs/access). This means they can be referenced by downstream resources in _any_ group in the same project, using the [`ref`](/reference/dbt-jinja-functions/ref) function. If a grouped model's `access` property is set to `private`, only resources within its group can reference it. diff --git a/website/docs/docs/build/hooks-operations.md b/website/docs/docs/build/hooks-operations.md index effbebb3c37..85378498a36 100644 --- a/website/docs/docs/build/hooks-operations.md +++ b/website/docs/docs/build/hooks-operations.md @@ -4,6 +4,8 @@ description: "Read this tutorial to learn how to use hooks and operations when b id: "hooks-operations" --- +import OnRunCommands from '/snippets/_onrunstart-onrunend-commands.md'; + ## Related documentation * [pre-hook & post-hook](/reference/resource-configs/pre-hook-post-hook) * [on-run-start & on-run-end](/reference/project-configs/on-run-start-on-run-end) @@ -33,8 +35,8 @@ dbt provides hooks and operations so you can version control and execute these s Hooks are snippets of SQL that are executed at different times: * `pre-hook`: executed _before_ a model, seed or snapshot is built. * `post-hook`: executed _after_ a model, seed or snapshot is built. - * `on-run-start`: executed at the _start_ of `dbt run`, `dbt test`, `dbt seed` or `dbt snapshot` - * `on-run-end`: executed at the _end_ of `dbt run`, `dbt test`, `dbt seed` or `dbt snapshot` + * `on-run-start`: executed at the _start_ of + * `on-run-end`: executed at the _end_ of Hooks are a more-advanced capability that enable you to run custom SQL, and leverage database-specific actions, beyond what dbt makes available out-of-the-box with standard materializations and configurations. diff --git a/website/docs/docs/build/incremental-models.md b/website/docs/docs/build/incremental-models.md index 07a571cd4db..3a597499f04 100644 --- a/website/docs/docs/build/incremental-models.md +++ b/website/docs/docs/build/incremental-models.md @@ -390,7 +390,7 @@ models: # `DBT_INTERNAL_DEST` and `DBT_INTERNAL_SOURCE` are the standard aliases for the target table and temporary table, respectively, during an incremental run using the merge strategy. ``` -Alternatively, here are the same same configurations configured within a model file: +Alternatively, here are the same configurations configured within a model file: ```sql -- in models/my_incremental_model.sql diff --git a/website/docs/docs/build/materializations.md b/website/docs/docs/build/materializations.md index 463651ccc77..79fe2e1b7c5 100644 --- a/website/docs/docs/build/materializations.md +++ b/website/docs/docs/build/materializations.md @@ -2,6 +2,7 @@ title: "Materializations" description: "Read this tutorial to learn how to use materializations when building in dbt." id: "materializations" +pagination_next: "docs/build/incremental-models" --- ## Overview @@ -140,8 +141,7 @@ required with incremental materializations less configuration options available, see your database platform's docs for more details * Materialized views may not be supported by every database platform * **Advice:** - * Consider materialized views for use cases where incremental models are sufficient, -but you would like the data platform to manage the incremental logic and refresh. + * Consider materialized views for use cases where incremental models are sufficient, but you would like the data platform to manage the incremental logic and refresh. ## Python materializations diff --git a/website/docs/docs/build/measures.md b/website/docs/docs/build/measures.md index ba82a4aa4a5..e06b5046976 100644 --- a/website/docs/docs/build/measures.md +++ b/website/docs/docs/build/measures.md @@ -234,6 +234,15 @@ metrics: We can query the semi-additive metrics using the following syntax: +For dbt Cloud: + +```bash +dbt sl query --metrics mrr_by_end_of_month --dimensions metric_time__month --order metric_time__month +dbt sl query --metrics mrr_by_end_of_month --dimensions metric_time__week --order metric_time__week +``` + +For dbt Core: + ```bash mf query --metrics mrr_by_end_of_month --dimensions metric_time__month --order metric_time__month mf query --metrics mrr_by_end_of_month --dimensions metric_time__week --order metric_time__week diff --git a/website/docs/docs/build/metricflow-cli.md b/website/docs/docs/build/metricflow-commands.md similarity index 62% rename from website/docs/docs/build/metricflow-cli.md rename to website/docs/docs/build/metricflow-commands.md index 2650b2215ae..049b415d40e 100644 --- a/website/docs/docs/build/metricflow-cli.md +++ b/website/docs/docs/build/metricflow-commands.md @@ -1,69 +1,142 @@ --- -title: MetricFlow CLI -id: metricflow-cli -description: "Query metrics and metadata in your dbt project with the metricflow cli" -sidebar_label: "MetricFlow CLI commands" +title: MetricFlow commands +id: metricflow-commands +description: "Query metrics and metadata in your dbt project with the MetricFlow commands." +sidebar_label: "MetricFlow commands" tags: [Metrics, Semantic Layer] --- -Once you define metrics in your dbt project, you can query metrics, dimensions, dimension values, and validate your configs using the MetricFlow command line (CLI). +Once you define metrics in your dbt project, you can query metrics, dimensions, dimension values, and validate your configs using the MetricFlow commands. -# Installation +MetricFlow allows you to define and query metrics in your dbt project in the [dbt Cloud CLI](/docs/cloud/cloud-cli-installation), [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud), or [dbt Core](/docs/core/installation). To experience the power of the universal [dbt Semantic Layer](/docs/use-dbt-semantic-layer/dbt-sl) and dynamically query those metrics in downstream tools, you'll need a dbt Cloud [Team or Enterprise](https://www.getdbt.com/pricing/) account. -You can install the [MetricFlow CLI](https://github.com/dbt-labs/metricflow#getting-started) from [PyPI](https://pypi.org/project/dbt-metricflow/). You need to use `pip` to install the MetricFlow CLI on Windows or Linux operating systems: +MetricFlow is compatible with Python versions 3.8, 3.9, 3.10 and 3.11. -1. Create or activate your virtual environment.`python -m venv venv` + +## MetricFlow + +MetricFlow is a dbt package that allows you to define and query metrics in your dbt project. You can use MetricFlow to query metrics in your dbt project in the dbt Cloud CLI, dbt Cloud IDE, or dbt Core. + +**Note** — MetricFlow commands aren't supported in dbt Cloud jobs yet. However, you can add MetricFlow validations with your git provider (such as GitHub Actions) by installing MetricFlow (`pip install metricflow`). This allows you to run MetricFlow commands as part of your continuous integration checks on PRs. + + + + + +MetricFlow commands are embedded in the dbt Cloud CLI, which means you can immediately run them once you install the dbt Cloud CLI. + +A benefit to using the dbt Cloud CLI or dbt Cloud IDE is that you won't need to manage versioning — your dbt Cloud account will automatically manage the versioning for you. + + + + + +:::info +You can create metrics using MetricFlow in the dbt Cloud IDE. However, support for running MetricFlow commands in the IDE will be available soon. +::: + + + + + + +:::info Use dbt Cloud CLI for semantic layer development + +Use the dbt Cloud CLI for the experience in defining and querying metrics in your dbt project on dbt Cloud or dbt Core with MetricFlow. + +A benefit to using the dbt Cloud CLI or dbt Cloud IDE is that you won't need to manage versioning — your dbt Cloud account will automatically manage the versioning for you. +::: + + +You can install [MetricFlow](https://github.com/dbt-labs/metricflow#getting-started) from [PyPI](https://pypi.org/project/dbt-metricflow/). You need to use `pip` to instal MetricFlow on Windows or Linux operating systems: + +1. Create or activate your virtual environment`python -m venv venv` 2. Run `pip install dbt-metricflow` + * You can install MetricFlow using PyPI as an extension of your dbt adapter in the command line. To install the adapter, run `pip install "dbt-metricflow[your_adapter_name]"` and add the adapter name at the end of the command. For example, for a Snowflake adapter run `pip install "dbt-metricflow[snowflake]"` - * You can install MetricFlow using PyPI as an extension of your dbt adapter in the CLI. To install the adapter, run `pip install "dbt-metricflow[your_adapter_name]"` and add the adapter name at the end of the command. For example, for a Snowflake adapter run `pip install "dbt-metricflow[snowflake]"` +**Note**, you'll need to manage versioning between dbt Core, your adapter, and MetricFlow. -The MetricFlow CLI is compatible with Python versions 3.8, 3.9, 3.10 and 3.11 + + + -# CLI commands -The MetricFlow CLI provides the following commands to retrieve metadata and query metrics. +## MetricFlow commands -To execute the commands, use the `mf` prefix before the command name. For example, to list all metrics, run `mf list metrics`: +MetricFlow provides the following commands to retrieve metadata and query metrics. + + + + +Use the `dbt sl` prefix before the command name to execute them in dbt Cloud. For example, to list all metrics, run `dbt sl list metrics`. - [`list`](#list) — Retrieves metadata values. - [`list metrics`](#list-metrics) — Lists metrics with dimensions. - [`list dimensions`](#list) — Lists unique dimensions for metrics. - [`list dimension-values`](#list-dimension-values) — List dimensions with metrics. - [`list entities`](#list-entities) — Lists all unique entities. +- [`query`](#query) — Query metrics and dimensions you want to see in the command line interface. Refer to [query examples](#query-examples) to help you get started. + + + + -## List + + +Use the `mf` prefix before the command name to execute them in dbt Core. For example, to list all metrics, run `mf list metrics`. + +- [`list`](#list) — Retrieves metadata values. +- [`list metrics`](#list-metrics) — Lists metrics with dimensions. +- [`list dimensions`](#list) — Lists unique dimensions for metrics. +- [`list dimension-values`](#list-dimension-values) — List dimensions with metrics. +- [`list entities`](#list-entities) — Lists all unique entities. +- [`validate-configs`](#validate-configs) — Validates semantic model configurations. +- [`health-checks`](#health-checks) — Performs data platform health check. +- [`tutorial`](#tutorial) — Dedicated MetricFlow tutorial to help get you started. +- [`query`](#query) — Query metrics and dimensions you want to see in the command line interface. Refer to [query examples](#query-examples) to help you get started. + + + + +### List This command retrieves metadata values related to [Metrics](/docs/build/metrics-overview), [Dimensions](/docs/build/dimensions), and [Entities](/docs/build/entities) values. -## List metrics +### List metrics ```bash -mf list - +dbt sl list # In dbt Cloud +mf list # In dbt Core +``` This command lists the metrics with their available dimensions: ```bash -mf list metrics +dbt sl list metrics # In dbt Cloud + +mf list metrics # In dbt Core + Options: --search TEXT Filter available metrics by this search term --show-all-dimensions Show all dimensions associated with a metric. --help Show this message and exit. ``` -## List dimensions +### List dimensions This command lists all unique dimensions for a metric or multiple metrics. It displays only common dimensions when querying multiple metrics: ```bash -mf list dimensions --metrics +dbt sl list dimensions --metrics # In dbt Cloud + +mf list dimensions --metrics # In dbt Core + Options: - --metrics SEQUENCE List dimensions by given metrics (intersection). Ex. - --metrics bookings,messages + --metrics SEQUENCE List dimensions by given metrics (intersection). Ex. --metrics bookings,messages --help Show this message and exit. ``` @@ -72,7 +145,10 @@ Options: This command lists all dimension values with the corresponding metric: ```bash -mf list dimension-values --metrics --dimension +dbt sl list dimension-values --metrics --dimension # In dbt Cloud + +mf list dimension-values --metrics --dimension # In dbt Core + Options: --dimension TEXT Dimension to query values from [required] --metrics SEQUENCE Metrics that are associated with the dimension @@ -83,24 +159,30 @@ Options: of the data (inclusive) --help Show this message and exit. ``` -## List entities + +### List entities This command lists all unique entities: ```bash -mf list entities --metrics +dbt sl list entities --metrics # In dbt Cloud + +mf list entities --metrics # In dbt Core + Options: - --metrics SEQUENCE List entities by given metrics (intersection). Ex. - --metrics bookings,messages + --metrics SEQUENCE List entities by given metrics (intersection). Ex. --metrics bookings,messages --help Show this message and exit. ``` -## Validate-configs +### Validate-configs This command performs validations against the defined semantic model configurations: ```bash -mf validate-configs +dbt sl validate-configs # In dbt Cloud + +mf validate-configs # In dbt Core + Options: --dw-timeout INTEGER Optional timeout for data warehouse validation steps. Default None. @@ -118,28 +200,34 @@ Options: --help Show this message and exit. ``` -## Health checks +### Health checks This command performs a health check against the data platform you provided in the configs: ```bash -mf health-checks +dbt sl health-checks #in dbt Cloud + +mf health-checks #in dbt Core ``` -## Tutorial +### Tutorial Follow the dedicated MetricFlow tutorial to help you get started: ```bash -mf tutorial +dbt sl tutorial # In dbt Cloud + +mf tutorial # In dbt Core ``` -## Query +### Query Create a new query with MetricFlow, execute that query against the user's data platform, and return the result: ```bash -mf query --metrics --group-by +dbt sl query --metrics --group-by # In dbt Cloud + +mf query --metrics --group-by # In dbt Core Options: @@ -170,8 +258,9 @@ Options: --csv FILENAME Provide filepath for data frame output to csv - --explain In the query output, show the query that was - executed against the data warehouse + --compile (dbt Cloud) In the query output, show the query that was + --explain (dbt Core) executed against the data warehouse + --show-dataflow-plan Display dataflow plan in explain output @@ -186,7 +275,7 @@ Options: ``` -## Query examples +### Query examples The following tabs present various different types of query examples that you can use to query metrics and dimensions. Select the tab that best suits your needs: @@ -198,7 +287,9 @@ Use the example to query metrics by dimension and return the `order_total` metri **Query** ```bash -mf query --metrics order_total --group-by metric_time +dbt sl query --metrics order_total --group-by metric_time # In dbt Cloud + +mf query --metrics order_total --group-by metric_time # In dbt Core ``` **Result** @@ -221,7 +312,9 @@ You can include multiple dimensions in a query. For example, you can group by th **Query** ```bash -mf query --metrics order_total --group-by metric_time, is_food_order +dbt sl query --metrics order_total --group-by metric_time, is_food_order # In dbt Cloud + +mf query --metrics order_total --group-by metric_time, is_food_order # In dbt Core ``` **Result** @@ -248,7 +341,11 @@ You can add order and limit functions to filter and present the data in a readab **Query** ```bash -mf query --metrics order_total --group-by metric_time,is_food_order --limit 10 --order -metric_time +# In dbt Cloud +dbt sl query --metrics order_total --group-by metric_time,is_food_order --limit 10 --order -metric_time + +# In dbt Core +mf query --metrics order_total --group-by metric_time,is_food_order --limit 10 --order -metric_time ``` **Result** @@ -273,7 +370,11 @@ You can further filter the data set by adding a `where` clause to your query. **Query** ```bash -mf query --metrics order_total --group-by metric_time --where "{{Dimension('order_id__is_food_order')}} = True" +# In dbt Cloud +dbt sl query --metrics order_total --group-by metric_time --where "{{ Dimension('order_id__is_food_order') }} = True" + +# In dbt Core +mf query --metrics order_total --group-by metric_time --where "{{ Dimension('order_id__is_food_order') }} = True" ``` **Result** @@ -301,7 +402,12 @@ To filter by time, there are dedicated start and end time options. Using these o **Query** ```bash - mf query --metrics order_total --group-by metric_time,is_food_order --limit 10 --order -metric_time --where "is_food_order = True" --start-time '2017-08-22' --end-time '2017-08-27' + +# In dbt Cloud +dbt sl query --metrics order_total --group-by metric_time,is_food_order --limit 10 --order -metric_time --where "is_food_order = True" --start-time '2017-08-22' --end-time '2017-08-27' + +# In dbt Core +mf query --metrics order_total --group-by metric_time,is_food_order --limit 10 --order -metric_time --where "is_food_order = True" --start-time '2017-08-22' --end-time '2017-08-27' ``` **Result** @@ -331,20 +437,24 @@ The following tabs present additional query examples, like exporting to a CSV. S - + -Add `--explain` to your query to view the SQL generated by MetricFlow. +Add `--compile` (or `--explain` for dbt Core users) to your query to view the SQL generated by MetricFlow. **Query** ```bash - mf query --metrics order_total --group-by metric_time,is_food_order --limit 10 --order -metric_time --where "is_food_order = True" --start-time '2017-08-22' --end-time '2017-08-27' --explain +# In dbt Cloud +dbt sl query --metrics order_total --group-by metric_time,is_food_order --limit 10 --order -metric_time --where "is_food_order = True" --start-time '2017-08-22' --end-time '2017-08-27' --compile + +# In dbt Core +mf query --metrics order_total --group-by metric_time,is_food_order --limit 10 --order -metric_time --where "is_food_order = True" --start-time '2017-08-22' --end-time '2017-08-27' --explain ``` **Result** ```bash ✔ Success 🦄 - query completed after 0.28 seconds -🔎 SQL (remove --explain to see data or add --show-dataflow-plan to see the generated dataflow plan): +🔎 SQL (remove --compile to see data or add --show-dataflow-plan to see the generated dataflow plan): SELECT metric_time , is_food_order @@ -374,6 +484,10 @@ Add the `--csv file_name.csv` flag to export the results of your query to a csv. **Query** ```bash +# In dbt Cloud +dbt sl query --metrics order_total --group-by metric_time,is_food_order --limit 10 --order -metric_time --where "is_food_order = True" --start-time '2017-08-22' --end-time '2017-08-27' --csv query_example.csv + +# In dbt Core mf query --metrics order_total --group-by metric_time,is_food_order --limit 10 --order -metric_time --where "is_food_order = True" --start-time '2017-08-22' --end-time '2017-08-27' --csv query_example.csv ``` @@ -386,14 +500,16 @@ mf query --metrics order_total --group-by metric_time,is_food_order --limit 10 - -## Time granularity +### Time granularity Optionally, you can specify the time granularity you want your data to be aggregated at by appending two underscores and the unit of granularity you want to `metric_time`, the global time dimension. You can group the granularity by: `day`, `week`, `month`, `quarter`, and `year`. Below is an example for querying metric data at a monthly grain: ```bash -mf query --metrics revenue --group-by metric_time__month +dbt sl query --metrics revenue --group-by metric_time__month # In dbt Cloud + +mf query --metrics revenue --group-by metric_time__month # In dbt Core ``` ## FAQs @@ -403,7 +519,7 @@ mf query --metrics revenue --group-by metric_time__month To add a dimension filter to a where filter, you have to indicate that the filter item is part of your model and use a template wrapper: {{Dimension('primary_entity__dimension_name')}}. -Here's an example query: mf query --metrics order_total --group-by metric_time --where "{{Dimension('order_id__is_food_order')}} = True".

Before using the template wrapper, however, you will need to set up your terminal to escape curly braces for the filter template to work. +Here's an example query: dbt sl query --metrics order_total --group-by metric_time --where "{{Dimension('order_id__is_food_order')}} = True".

Before using the template wrapper, however, you will need to set up your terminal to escape curly braces for the filter template to work.
How to set up your terminal to escape curly braces? diff --git a/website/docs/docs/build/metricflow-time-spine.md b/website/docs/docs/build/metricflow-time-spine.md index 254fa3cc5f0..997d85e38a8 100644 --- a/website/docs/docs/build/metricflow-time-spine.md +++ b/website/docs/docs/build/metricflow-time-spine.md @@ -12,6 +12,8 @@ To create this table, you need to create a model in your dbt project called `met + + ```sql {{ config( @@ -38,8 +40,44 @@ final as ( select * from final ``` + + + + + +```sql +{{ + config( + materialized = 'table', + ) +}} + +with days as ( + + {{ + dbt.date_spine( + 'day', + "to_date('01/01/2000','mm/dd/yyyy')", + "to_date('01/01/2027','mm/dd/yyyy')" + ) + }} + +), + +final as ( + select cast(date_day as date) as date_day + from days +) + +select * from final +``` + + + + + ```sql -- filename: metricflow_time_spine.sql -- BigQuery supports DATE() instead of TO_DATE(). Use this model if you're using BigQuery @@ -61,4 +99,33 @@ final as ( select * from final ``` + + + + + +```sql +-- filename: metricflow_time_spine.sql +-- BigQuery supports DATE() instead of TO_DATE(). Use this model if you're using BigQuery +{{config(materialized='table')}} +with days as ( + {{dbt.date_spine( + 'day', + "DATE(2000,01,01)", + "DATE(2030,01,01)" + ) + }} +), + +final as ( + select cast(date_day as date) as date_day + from days +) + +select * +from final +``` + + + You only need to include the `date_day` column in the table. MetricFlow can handle broader levels of detail, but it doesn't currently support finer grains. diff --git a/website/docs/docs/build/metrics-overview.md b/website/docs/docs/build/metrics-overview.md index e6d875386ee..81af149a7d9 100644 --- a/website/docs/docs/build/metrics-overview.md +++ b/website/docs/docs/build/metrics-overview.md @@ -4,6 +4,7 @@ id: metrics-overview description: "Metrics can be defined in the same or separate YAML files from semantic models within the same dbt project repo." sidebar_label: "Creating metrics" tags: [Metrics, Semantic Layer] +pagination_next: "docs/build/cumulative" --- Once you've created your semantic models, it's time to start adding metrics! Metrics can be defined in the same YAML files as your semantic models, or split into separate YAML files into any other subdirectories (provided that these subdirectories are also within the same dbt project repo) @@ -16,7 +17,7 @@ The keys for metrics definitions are: | `description` | Provide the description for your metric. | Optional | | `type` | Define the type of metric, which can be `simple`, `ratio`, `cumulative`, or `derived`. | Required | | `type_params` | Additional parameters used to configure metrics. `type_params` are different for each metric type. | Required | -| `configs` | Provide the specific configurations for your metric. | Optional | +| `config` | Provide the specific configurations for your metric. | Optional | | `label` | The display name for your metric. This value will be shown in downstream tools. | Required | | `filter` | You can optionally add a filter string to any metric type, applying filters to dimensions, entities, or time dimensions during metric computation. Consider it as your WHERE clause. | Optional | | `meta` | Additional metadata you want to add to your metric. | Optional | @@ -31,10 +32,10 @@ metrics: type: the type of the metric ## Required type_params: ## Required - specific properties for the metric type - configs: here for `enabled` ## Optional + config: here for `enabled` ## Optional label: The display name for your metric. This value will be shown in downstream tools. ## Required filter: | ## Optional - {{ Dimension('entity__name') }} > 0 and {{ Dimension(' entity__another name') }} is not + {{ Dimension('entity__name') }} > 0 and {{ Dimension(' entity__another_name') }} is not null ``` diff --git a/website/docs/docs/build/models.md b/website/docs/docs/build/models.md index e0683158e6d..d10eb5ed01a 100644 --- a/website/docs/docs/build/models.md +++ b/website/docs/docs/build/models.md @@ -2,6 +2,8 @@ title: "About dbt models" description: "Read this tutorial to learn how to use models when building in dbt." id: "models" +pagination_next: "docs/build/sql-models" +pagination_prev: null --- ## Overview diff --git a/website/docs/docs/build/organize-your-outputs.md b/website/docs/docs/build/organize-your-outputs.md new file mode 100644 index 00000000000..ad5efeda1c7 --- /dev/null +++ b/website/docs/docs/build/organize-your-outputs.md @@ -0,0 +1,38 @@ +--- +title: "Organize your outputs" +description: "Learn how you can organize your outputs" +pagination_next: "docs/build/custom-schemas" +pagination_prev: null +--- + +
+ + + + + +
+
+
+ + + + + +
\ No newline at end of file diff --git a/website/docs/docs/build/packages.md b/website/docs/docs/build/packages.md index 74e25262994..8d18a55e949 100644 --- a/website/docs/docs/build/packages.md +++ b/website/docs/docs/build/packages.md @@ -3,7 +3,7 @@ title: "Packages" id: "packages" --- -## What is a package? + Software engineers frequently modularize code into libraries. These libraries help programmers operate with leverage: they can spend more time focusing on their unique business logic, and less time implementing code that someone else has already spent the time perfecting. In dbt, libraries like these are called _packages_. dbt's packages are so powerful because so many of the analytic problems we encountered are shared across organizations, for example: @@ -22,13 +22,19 @@ dbt _packages_ are in fact standalone dbt projects, with models and macros that * Models in the package will be materialized when you `dbt run`. * You can use `ref` in your own models to refer to models from the package. * You can use macros in the package in your own project. +* It's important to note that defining and installing dbt packages is different from [defining and installing Python packages](/docs/build/python-models#using-pypi-packages) -:::note Using Python packages -Defining and installing dbt packages is different from [defining and installing Python packages](/docs/build/python-models#using-pypi-packages). +:::info `dependencies.yml` has replaced `packages.yml` +Starting from dbt v1.6, `dependencies.yml` has replaced `packages.yml`. This file can now contain both types of dependencies: "package" and "project" dependencies. +- "Package" dependencies lets you add source code from someone else's dbt project into your own, like a library. +- "Project" dependencies provide a different way to build on top of someone else's work in dbt. Refer to [Project dependencies](/docs/collaborate/govern/project-dependencies) for more info. +- +You can rename `packages.yml` to `dependencies.yml`, _unless_ you need to use Jinja within your packages specification. This could be necessary, for example, if you want to add an environment variable with a git token in a private git package specification. ::: + ## How do I add a package to my project? 1. Add a file named `dependencies.yml` or `packages.yml` to your dbt project. This should be at the same level as your `dbt_project.yml` file. 2. Specify the package(s) you wish to add using one of the supported syntaxes, for example: @@ -366,3 +372,4 @@ packages: ``` + diff --git a/website/docs/docs/build/project-variables.md b/website/docs/docs/build/project-variables.md index a69132d6a3b..1fe45b18ac0 100644 --- a/website/docs/docs/build/project-variables.md +++ b/website/docs/docs/build/project-variables.md @@ -1,6 +1,7 @@ --- title: "Project variables" id: "project-variables" +pagination_next: "docs/build/environment-variables" --- dbt provides a mechanism, [variables](/reference/dbt-jinja-functions/var), to provide data to models for diff --git a/website/docs/docs/build/projects.md b/website/docs/docs/build/projects.md index 0d7dd889fa6..b4b04e3334d 100644 --- a/website/docs/docs/build/projects.md +++ b/website/docs/docs/build/projects.md @@ -1,6 +1,8 @@ --- title: "About dbt projects" id: "projects" +pagination_next: null +pagination_prev: null --- A dbt project informs dbt about the context of your project and how to transform your data (build your data sets). By design, dbt enforces the top-level structure of a dbt project such as the `dbt_project.yml` file, the `models` directory, the `snapshots` directory, and so on. Within the directories of the top-level, you can organize your project in any way that meets the needs of your organization and data pipeline. diff --git a/website/docs/docs/build/saved-queries.md b/website/docs/docs/build/saved-queries.md new file mode 100644 index 00000000000..39a4b2e52fd --- /dev/null +++ b/website/docs/docs/build/saved-queries.md @@ -0,0 +1,43 @@ +--- +title: Saved queries +id: saved-queries +description: "Saved queries are a way to save commonly used queries in MetricFlow. They can be used to save time and avoid writing the same query over and over again." +sidebar_label: "Saved queries" +tags: [Metrics, Semantic Layer] +--- + +:::info Saved queries coming soon +Saved queries isn't currently available in MetricFlow but support is coming soon. +::: + +Saved queries are a way to save commonly used queries in MetricFlow. You can group metrics, dimensions, and filters that are logically related into a saved query. + +To define a saved query, refer to the following specification: + + Parameter | Description | Type | +| --------- | ----------- | ---- | +| `name` | The name of the metric. | Required | +| `description` | The description of the metric. | Optional | +| `metrics` | The metrics included in the saved query. | Required | +| `group_bys` | The value displayed in downstream tools. | Required | +| `where` | Filter applied to the query. | Optional | + +The following is an example of a saved query: + +```yaml +saved_query: + name: p0_booking + description: Booking-related metrics that are of the highest priority. + metrics: + - bookings + - instant_bookings + group_bys: + - TimeDimension('metric_time', 'day') + - Dimension('listing__capacity_latest') + where: + - "{{ Dimension('listing__capacity_latest') }} > 3" +``` + +### FAQs + +* All metrics in a saved query need to use the same dimensions in the `group_by` or `where` clauses. diff --git a/website/docs/docs/build/semantic-models.md b/website/docs/docs/build/semantic-models.md index bb56bd212e6..226c138b545 100644 --- a/website/docs/docs/build/semantic-models.md +++ b/website/docs/docs/build/semantic-models.md @@ -6,19 +6,23 @@ keywords: - dbt metrics layer sidebar_label: Semantic models tags: [Metrics, Semantic Layer] +pagination_next: "docs/build/dimensions" --- -Semantic models serve as the foundation for defining data in MetricFlow, which powers the dbt Semantic Layer. You can think of semantic models as nodes in your semantic graph, connected via entities as edges. MetricFlow takes semantic models defined in YAML configuration files as inputs and creates a semantic graph that can be used to query metrics. +Semantic models are the foundation for data definition in MetricFlow, which powers the dbt Semantic Layer: -Each semantic model corresponds to a dbt model in your DAG. Therefore you will have one YAML config for each semantic model in your dbt project. You can create multiple semantic models out of a single dbt model, as long as you give each semantic model a unique name. - -You can configure semantic models in your dbt project directory in a `YAML` file. Depending on your project structure, you can nest semantic models under a `metrics:` folder or organize them under project sources. +- Think of semantic models as nodes connected by entities in a semantic graph. +- MetricFlow uses YAML configuration files to create this graph for querying metrics. +- Each semantic model corresponds to a dbt model in your DAG, requiring a unique YAML configuration for each semantic model. +- You can create multiple semantic models from a single dbt model, as long as you give each semantic model a unique name. +- Configure semantic models in a YAML file within your dbt project directory. +- Organize them under a `metrics:` folder or within project sources as needed. Semantic models have 6 components and this page explains the definitions with some examples: | Component | Description | Type | | --------- | ----------- | ---- | -| [Name](#name) | Unique name for the semantic model | Required | +| [Name](#name) | Choose a unique name for the semantic model. Avoid using double underscores (__) in the name as they're not supported. | Required | | [Description](#description) | Includes important details in the description | Optional | | [Model](#model) | Specifies the dbt model for the semantic model using the `ref` function | Required | | [Defaults](#defaults) | The defaults for the model, currently only `agg_time_dimension` is supported. | Required | @@ -26,6 +30,7 @@ Semantic models have 6 components and this page explains the definitions with so | [Primary Entity](#primary-entity) | If a primary entity exists, this component is Optional. If the semantic model has no primary entity, then this property is required. | Optional | | [Dimensions](#dimensions) | Different ways to group or slice data for a metric, they can be `time` or `categorical` | Required | | [Measures](#measures) | Aggregations applied to columns in your data model. They can be the final metric or used as building blocks for more complex metrics | Optional | +| Label | The display name for your semantic model `node`, `dimension`, `entity`, and/or `measures` | Optional | ## Semantic models components @@ -105,9 +110,32 @@ semantic_models: type: categorical ``` + + +Semantic models support configs in either the schema file or at the project level. + +Semantic model config in `models/semantic.yml`: +```yml +semantic_models: + - name: orders + config: + enabled: true | false + group: some_group +``` + +Semantic model config in `dbt_project.yml`: +```yml +semantic_models: + my_project_name: + +enabled: true | false + +group: some_group +``` + + + ### Name -Define the name of the semantic model. You must define a unique name for the semantic model. The semantic graph will use this name to identify the model, and you can update it at any time. +Define the name of the semantic model. You must define a unique name for the semantic model. The semantic graph will use this name to identify the model, and you can update it at any time. Avoid using double underscores (__) in the name as they're not supported. ### Description @@ -205,8 +233,7 @@ For semantic models with a measure, you must have a [primary time group](/docs/b | `agg` | dbt supports the following aggregations: `sum`, `max`, `min`, `count_distinct`, and `sum_boolean`. | Required | | `expr` | You can either reference an existing column in the table or use a SQL expression to create or derive a new one. | Optional | | `non_additive_dimension` | Non-additive dimensions can be specified for measures that cannot be aggregated over certain dimensions, such as bank account balances, to avoid producing incorrect results. | Optional | -| `create_metric`* | You can create a metric directly from a measure with create_metric: True and specify its display name with create_metric_display_name. | Optional | -_*Coming soon_ +| `create_metric` | You can create a metric directly from a measure with `create_metric: True` and specify its display name with create_metric_display_name. Default is false. | Optional | import SetUpPages from '/snippets/_metrics-dependencies.md'; diff --git a/website/docs/docs/build/simple.md b/website/docs/docs/build/simple.md index 7022ca9d007..1803e952a69 100644 --- a/website/docs/docs/build/simple.md +++ b/website/docs/docs/build/simple.md @@ -4,6 +4,7 @@ id: simple description: "Use simple metrics to directly reference a single measure." sidebar_label: Simple tags: [Metrics, Semantic Layer] +pagination_next: null --- Simple metrics are metrics that directly reference a single measure, without any additional measures involved. They are aggregations over a column in your data platform and can be filtered by one or multiple dimensions. diff --git a/website/docs/docs/build/sl-getting-started.md b/website/docs/docs/build/sl-getting-started.md index f070bc27538..34c3479cbc0 100644 --- a/website/docs/docs/build/sl-getting-started.md +++ b/website/docs/docs/build/sl-getting-started.md @@ -5,41 +5,36 @@ description: "Learn how to create your first semantic model and metric." sidebar_label: Get started with MetricFlow tags: [Metrics, Semantic Layer] meta: - api_name: dbt Semantic Layer API + api_name: dbt Semantic Layer APIs --- -import InstallMetricFlow from '/snippets/_sl-install-metricflow.md'; import CreateModel from '/snippets/_sl-create-semanticmodel.md'; import DefineMetrics from '/snippets/_sl-define-metrics.md'; import ConfigMetric from '/snippets/_sl-configure-metricflow.md'; import TestQuery from '/snippets/_sl-test-and-query-metrics.md'; +import ConnectQueryAPI from '/snippets/_sl-connect-and-query-api.md'; +import RunProdJob from '/snippets/_sl-run-prod-job.md'; -This getting started page presents a sample workflow to help you create your first metrics in dbt Cloud or the command-line interface (CLI). It uses the [Jaffle shop example project](https://github.com/dbt-labs/jaffle-sl-template) as the project data source and is available for you to use. +This getting started page presents a sample workflow to help you create your first metrics in dbt Cloud or the command line interface (CLI). It uses the [Jaffle shop example project](https://github.com/dbt-labs/jaffle-sl-template) as the project data source and is available for you to use. If you prefer, you can create semantic models and metrics for your own dbt project. This page will guide you on how to: - [Create a semantic model](#create-a-semantic-model) using MetricFlow - [Define metrics](#define-metrics) using MetricFlow -- [Test and query metrics locally](#test-and-query-metrics) using MetricFlow +- [Test and query metrics](#test-and-query-metrics) using MetricFlow - [Run a production job](#run-a-production-job) in dbt Cloud - [Set up dbt Semantic Layer](#set-up-dbt-semantic-layer) in dbt Cloud - [Connect to and query the API](#connect-and-query-api) with dbt Cloud - -MetricFlow allows users to define metrics in their dbt project whether in dbt Cloud or in dbt Core. dbt Core users can use the [MetricFlow CLI](/docs/build/metricflow-cli) to define metrics in their local dbt Core project. +MetricFlow allows you to define metrics in your dbt project and query them whether in dbt Cloud or dbt Core with [MetricFlow commands](/docs/build/metricflow-commands). However, to experience the power of the universal [dbt Semantic Layer](/docs/use-dbt-semantic-layer/dbt-sl) and query those metrics in downstream tools, you'll need a dbt Cloud [Team or Enterprise](https://www.getdbt.com/pricing/) account. ## Prerequisites -- Have an understanding of key concepts in [MetricFlow](/docs/build/about-metricflow), which powers the revamped dbt Semantic Layer. -- Have both your production and development environments running dbt version 1.6 or higher. Refer to [upgrade in dbt Cloud](/docs/dbt-versions/upgrade-core-in-cloud) for more info. -- Use Snowflake, BigQuery, Databricks, Redshift, or Postgres (CLI only. dbt Cloud support coming soon). -- Create a successful run in the environment where you configure the Semantic Layer. - - **Note:** Semantic Layer currently supports the Deployment environment for querying. (_development querying experience coming soon_) -- Set up the [Semantic Layer API](/docs/dbt-cloud-apis/sl-api-overview) in the integrated tool to import metric definitions. - - **Note:** To access the API and query metrics in downstream tools, you must have a dbt Cloud [Team or Enterprise](https://www.getdbt.com/pricing/) account. dbt Core or Developer accounts can define metrics using [MetricFlow CLI](/docs/build/metricflow-cli) or the [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud).
-- Understand [MetricFlow's](/docs/build/about-metricflow) key concepts, which powers the revamped dbt Semantic Layer. +import SetUp from '/snippets/_v2-sl-prerequisites.md'; + + :::tip New to dbt or metrics? Try our [Jaffle shop example project](https://github.com/dbt-labs/jaffle-sl-template) to help you get started! @@ -63,15 +58,7 @@ New to dbt or metrics? Try our [Jaffle shop example project](https://github.com/ ## Run a production job -Before you begin, you must have a dbt Cloud Team or Enterprise [multi-tenant](/docs/cloud/about-cloud/regions-ip-addresses) deployment, hosted in North America (cloud.getdbt.com login URL). - -Once you’ve defined metrics in your dbt project, you can perform a job run in your dbt Cloud deployment environment to materialize your metrics. Only the deployment environment is supported for the dbt Semantic Layer at this moment. - -1. Go to **Deploy** in the menu bar -2. Select **Jobs** to re-run the job with the most recent code in the deployment environment. -3. Your metric should appear as a red node in the dbt Cloud IDE and dbt directed acyclic graphs (DAG). - - + ## Set up dbt Semantic Layer @@ -81,16 +68,7 @@ import SlSetUp from '/snippets/_new-sl-setup.md'; ## Connect and query API -You can query your metrics in a JDBC-enabled tool or use existing first-class integrations with the dbt Semantic Layer. - -You must have a dbt Cloud Team or Enterprise [multi-tenant](/docs/cloud/about-cloud/regions-ip-addresses) deployment, hosted in North America. (Additional region support coming soon) - -- To learn how to use the JDBC API and what tools you can query it with, refer to the {frontMatter.meta.api_name}.
- - * To authenticate, you need to [generate a service token](/docs/dbt-cloud-apis/service-tokens) with Semantic Layer Only and Metadata Only permissions. - * Refer to the [SQL query syntax](/docs/dbt-cloud-apis/sl-jdbc#querying-the-api-for-metric-metadata) to query metrics using the API. - -- To learn more about the sophisticated integrations that connect to the dbt Semantic Layer, refer to [Available integrations](/docs/use-dbt-semantic-layer/avail-sl-integrations) for more info. + ## FAQs diff --git a/website/docs/docs/build/tests.md b/website/docs/docs/build/tests.md index fa78d0df905..75ee5992a76 100644 --- a/website/docs/docs/build/tests.md +++ b/website/docs/docs/build/tests.md @@ -241,7 +241,7 @@ where {{ column_name }} is null ## Storing test failures -Normally, a test query will calculate failures as part of its execution. If you set the optional `--store-failures` flag or [`store_failures` config](/reference/resource-configs/store_failures), dbt will first save the results of a test query to a table in the database, and then query that table to calculate the number of failures. +Normally, a test query will calculate failures as part of its execution. If you set the optional `--store-failures` flag, the [`store_failures`](/reference/resource-configs/store_failures), or the [`store_failures_as`](/reference/resource-configs/store_failures_as) configs, dbt will first save the results of a test query to a table in the database, and then query that table to calculate the number of failures. This workflow allows you to query and examine failing records much more quickly in development: diff --git a/website/docs/docs/build/validation.md b/website/docs/docs/build/validation.md index ad485850d23..02ce48729a4 100644 --- a/website/docs/docs/build/validation.md +++ b/website/docs/docs/build/validation.md @@ -12,16 +12,14 @@ These validations ensure that configuration files follow the expected schema, th The code that handles validation [can be found here](https://github.com/dbt-labs/dbt-semantic-interfaces/tree/main/dbt_semantic_interfaces/validations) for those who want to dive deeper into this topic. -## Prerequisites - -- You have installed the [MetricFlow CLI package](https://github.com/dbt-labs/metricflow) ## Validations command -You can run validations from the CLI with the following [MetricFlow commands](/docs/build/metricflow-cli): +You can run validations from dbt Cloud or the command line with the following [MetricFlow commands](/docs/build/metricflow-commands): ```bash -mf validate-configs +dbt sl validate-configs # dbt Cloud users +mf validate-configs # dbt Core users ``` ## Parsing diff --git a/website/docs/docs/cloud/about-cloud-develop-defer.md b/website/docs/docs/cloud/about-cloud-develop-defer.md new file mode 100644 index 00000000000..1c0b316f885 --- /dev/null +++ b/website/docs/docs/cloud/about-cloud-develop-defer.md @@ -0,0 +1,55 @@ +--- +title: Using defer in dbt Cloud +id: about-cloud-develop-defer +description: "Learn how to leverage defer to prod when developing with dbt Cloud." +sidebar_label: "Using defer in dbt Cloud" +pagination_next: "docs/cloud/cloud-cli-installation" +--- + + +[Defer](/reference/node-selection/defer) is a powerful feature that allows developers to only build and run and test models they've edited without having to first run and build all the models that come before them (upstream parents). This is powered by using a production manifest for comparison, and dbt will resolve the `{{ ref() }}` function with upstream production artifacts. + +Both the dbt Cloud IDE and the dbt Cloud CLI allow users to natively defer to production metadata directly in their development workflows, dramatically reducing development time and warehouse spend by preventing unnecessary model builds. + +## Required setup + +- You must select the **[Production environment](/docs/deploy/deploy-environments#set-as-production-environment-beta)** checkbox in the **Environment Settings** page. + - This can be set for one deployment environment per dbt Cloud project. +- You must have a successful job run first. + +When using 'defer', it compares artifacts from the most recent successful production job, excluding CI jobs. + +### Defer in the dbt Cloud IDE + +To enable 'Defer' in the dbt Cloud IDE, toggle the **Defer to production** button on the command bar. Once enabled, dbt Cloud will: + +1. Pull down the most recent manifest from the Production environment for comparison +2. Pass the `--defer` flag to the command (for any command that accepts the flag) + +For example, if you were to start developing on a new branch with [nothing in your development schema](/reference/node-selection/defer#usage), edit a single model, and run `dbt build -s state:modified` — only the edited model would run. Any `{{ ref() }}` functions will point to the production location of the referenced models. + + + +### Defer in dbt Cloud CLI + +One key difference between using `--defer` in the dbt Cloud CLI and the dbt Cloud IDE is that `--defer` is *automatically* enabled in the dbt Cloud CLI for all invocations, comparing with production artifacts. You can disable it with the `--no-defer` flag. + +The dbt Cloud CLI offers additional flexibility by letting you choose the source environment for deferral artifacts. You can set a `defer-env-id` key in either your `dbt_project.yml` or `dbt_cloud.yml` file. If you do not provide a `defer-env-id` setting, the dbt Cloud CLI will use artifacts from your dbt Cloud environment marked 'Production'. + + + + ```yml +dever-env-id: '123456' +``` + + + + + + +```yml +dbt_cloud: + dever-env-id: '123456' +``` + + diff --git a/website/docs/docs/cloud/about-cloud-develop.md b/website/docs/docs/cloud/about-cloud-develop.md new file mode 100644 index 00000000000..9f864ede5ca --- /dev/null +++ b/website/docs/docs/cloud/about-cloud-develop.md @@ -0,0 +1,33 @@ +--- +title: About developing in dbt Cloud +id: about-cloud-develop +description: "Learn how to develop your dbt projects using dbt Cloud." +sidebar_label: "About developing in dbt Cloud" +pagination_next: "docs/cloud/cloud-cli-installation" +hide_table_of_contents: true +--- + +dbt Cloud offers a fast and reliable way to work on your dbt project. It runs dbt Core in a hosted (single or multi-tenant) environment. You can develop in your browser using an integrated development environment (IDE) or in a dbt Cloud-powered command line interface (CLI): + +
+ + + + + +

+ +The following sections provide detailed instructions on setting up the dbt Cloud CLI and dbt Cloud IDE. To get started with dbt development, you'll need a [developer](/docs/cloud/manage-access/seats-and-users) account. For a more comprehensive guide about developing in dbt, refer to our [quickstart guides](/quickstarts). + + +--------- +**Note**: The dbt Cloud CLI and the open-sourced dbt Core are both command line tools that let you run dbt commands. The key distinction is the dbt Cloud CLI is tailored for dbt Cloud's infrastructure and integrates with all its [features](/docs/cloud/about-cloud/dbt-cloud-features). + diff --git a/website/docs/docs/cloud/about-cloud-setup.md b/website/docs/docs/cloud/about-cloud-setup.md index baa2465472e..7b68b52a45a 100644 --- a/website/docs/docs/cloud/about-cloud-setup.md +++ b/website/docs/docs/cloud/about-cloud-setup.md @@ -3,6 +3,8 @@ title: About dbt Cloud setup id: about-cloud-setup description: "Configuration settings for dbt Cloud." sidebar_label: "About dbt Cloud setup" +pagination_next: "docs/dbt-cloud-environments" +pagination_prev: null --- dbt Cloud is the fastest and most reliable way to deploy your dbt jobs. It contains a myriad of settings that can be configured by admins, from the necessities (data platform integration) to security enhancements (SSO) and quality-of-life features (RBAC). This portion of our documentation will take you through the various settings found by clicking on the gear icon in the dbt Cloud UI, including: @@ -11,6 +13,8 @@ dbt Cloud is the fastest and most reliable way to deploy your dbt jobs. It conta - Configuring access to [GitHub](/docs/cloud/git/connect-github), [GitLab](/docs/cloud/git/connect-gitlab), or your own [git repo URL](/docs/cloud/git/import-a-project-by-git-url). - [Managing users and licenses](/docs/cloud/manage-access/seats-and-users) - [Configuring secure access](/docs/cloud/manage-access/about-user-access) +- Configuring the [dbt Cloud IDE](/docs/cloud/about-cloud-develop) +- Installing and configuring the [dbt Cloud CLI](/docs/cloud/cloud-cli-installation) These settings are intended for dbt Cloud administrators. If you need a more detailed first-time setup guide for specific data platforms, read our [quickstart guides](/quickstarts). diff --git a/website/docs/docs/cloud/about-cloud/about-cloud-ide.md b/website/docs/docs/cloud/about-cloud/about-cloud-ide.md index 923212c1260..7643928feec 100644 --- a/website/docs/docs/cloud/about-cloud/about-cloud-ide.md +++ b/website/docs/docs/cloud/about-cloud/about-cloud-ide.md @@ -5,7 +5,7 @@ description: "about dbt Cloud Integrated Development Environment" sidebar_label: About dbt Cloud IDE --- -The dbt Cloud integrated development environment (IDE) is a single interface for building, testing, running, and version-controlling dbt projects from your browser. With the Cloud IDE, you can compile dbt code into SQL and run it against your database directly. The IDE leverages the open-source [dbt-rpc](/reference/commands/rpc) plugin to recompile only the changes made in your project. +The dbt Cloud integrated development environment (IDE) is a single interface for building, testing, running, and version-controlling dbt projects from your browser. With the Cloud IDE, you can compile dbt code into SQL and run it against your database directly. With the Cloud IDE, you can: @@ -25,7 +25,7 @@ With the Cloud IDE, you can: For more information, read the complete [Cloud IDE guide](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud). -## Relatd docs +## Related docs - [IDE user interface](/docs/cloud/dbt-cloud-ide/ide-user-interface) - [Tips and tricks](/docs/cloud/dbt-cloud-ide/dbt-cloud-tips) diff --git a/website/docs/docs/cloud/about-cloud/dbt-cloud-features.md b/website/docs/docs/cloud/about-cloud/about-dbt-cloud.md similarity index 78% rename from website/docs/docs/cloud/about-cloud/dbt-cloud-features.md rename to website/docs/docs/cloud/about-cloud/about-dbt-cloud.md index f1d8b32cdb1..71f3175a108 100644 --- a/website/docs/docs/cloud/about-cloud/dbt-cloud-features.md +++ b/website/docs/docs/cloud/about-cloud/about-dbt-cloud.md @@ -4,86 +4,95 @@ id: "dbt-cloud-features" sidebar_label: "dbt Cloud features" description: "Explore dbt Cloud's features and learn why dbt Cloud is the fastest way to deploy dbt" hide_table_of_contents: true +pagination_next: "docs/cloud/about-cloud/architecture" +pagination_prev: null --- -dbt Cloud is the fastest and most reliable way to deploy dbt. Develop, test, schedule, document, and investigate data models all in one browser-based UI. In addition to providing a hosted architecture for running dbt across your organization, dbt Cloud comes equipped with turnkey support for scheduling jobs, CI/CD, hosting documentation, monitoring & alerting, and an integrated development environment (IDE). +dbt Cloud is the fastest and most reliable way to deploy dbt. Develop, test, schedule, document, and investigate data models all in one browser-based UI. + +In addition to providing a hosted architecture for running dbt across your organization, dbt Cloud comes equipped with turnkey support for scheduling jobs, CI/CD, hosting documentation, monitoring and alerting, an integrated development environment (IDE), and allows you to develop and run dbt commands from your local command line interface (CLI) or code editor. dbt Cloud's [flexible plans](https://www.getdbt.com/pricing/) and features make it well-suited for data teams of any size — sign up for your [free 14-day trial](https://www.getdbt.com/signup/)!
+ + + link="/docs/cloud/dbt-cloud-ide/develop-in-the-cloud" + icon="dbt-bit"/> + icon="dbt-bit"/> + icon="dbt-bit"/> + icon="dbt-bit"/> + + + icon="dbt-bit"/> + icon="dbt-bit"/> + icon="dbt-bit"/> + icon="dbt-bit"/> + icon="dbt-bit"/> + icon="dbt-bit"/> - - - - + icon="dbt-bit"/>

*These features are available on [selected plans](https://www.getdbt.com/pricing/). diff --git a/website/docs/docs/cloud/about-cloud/architecture.md b/website/docs/docs/cloud/about-cloud/architecture.md index 4ad016f4007..52614f0cbcd 100644 --- a/website/docs/docs/cloud/about-cloud/architecture.md +++ b/website/docs/docs/cloud/about-cloud/architecture.md @@ -42,7 +42,7 @@ Some data warehouse providers offer advanced security features that can be lever ### Git sync -dbt Cloud can sync with a variety of git providers, including [Github](/docs/cloud/git/connect-github), [Gitlab](/docs/cloud/git/connect-gitlab), and [Azure DevOps](/docs/cloud/git/connect-azure-devops) within its integrated development environment ([IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud). Communication takes place over HTTPS rather than SSH and is protected using the TLS 1.2 protocol for data in transit. +dbt Cloud can sync with a variety of git providers, including [Github](/docs/cloud/git/connect-github), [Gitlab](/docs/cloud/git/connect-gitlab), and [Azure DevOps](/docs/cloud/git/connect-azure-devops) within its integrated development environment ([IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud)). Communication takes place over HTTPS rather than SSH and is protected using the TLS 1.2 protocol for data in transit. The git repo information is stored on dbt Cloud servers to make it accessible during the IDE sessions. When the git sync is disabled, you must [contact support](mailto:support@getdbt.com) to request the deletion of the synced data. diff --git a/website/docs/docs/cloud/about-cloud/browsers.md b/website/docs/docs/cloud/about-cloud/browsers.md index 2fc5a8b4b4d..12665bc7b72 100644 --- a/website/docs/docs/cloud/about-cloud/browsers.md +++ b/website/docs/docs/cloud/about-cloud/browsers.md @@ -2,6 +2,7 @@ title: "Supported browsers" id: "browsers" description: "dbt Cloud supports the latest browsers like Chrome and Firefox." +pagination_next: null --- To have the best experience with dbt Cloud, we recommend using the latest versions of the following browsers: diff --git a/website/docs/docs/cloud/billing.md b/website/docs/docs/cloud/billing.md index 61251f6e41d..1d71d33e9a1 100644 --- a/website/docs/docs/cloud/billing.md +++ b/website/docs/docs/cloud/billing.md @@ -3,28 +3,81 @@ title: "Billing" id: billing description: "dbt Cloud billing information." sidebar_label: Billing +pagination_next: null +pagination_prev: null --- dbt Cloud offers a variety of [plans and pricing](https://www.getdbt.com/pricing/) to fit your organization’s needs. With flexible billing options that appeal to large enterprises and small businesses and [server availability](/docs/cloud/about-cloud/regions-ip-addresses) worldwide, dbt Cloud is the fastest and easiest way to begin transforming your data. ## How does dbt Cloud pricing work? -As a customer, you pay for the number of seats you have and the amount of usage consumed each month. Usage is based on the number of Successful Models Built, and seats are billed primarily on the amount of Developer licenses purchased. All billing computations are conducted in Coordinated Universal Time (UTC). +As a customer, you pay for the number of seats you have and the amount of usage consumed each month. Seats are billed primarily on the amount of Developer and Read licenses purchased. Usage is based on the number of [Successful Models Built](#what-counts-as-a-successful-model-built) and, if purchased and used, Semantic Layer Query Units subject to reasonable usage. All billing computations are conducted in Coordinated Universal Time (UTC). + +### What counts as a seat license? + +There are three types of possible seat licenses: + +* **Developer** — for roles and permissions that require interaction with the dbt Cloud environment day-to-day. +* **Read-Only** — for access to view certain documents and reports. +* **IT** — for access to specific features related to account management (for example, configuring git integration). ### What counts as a Successful Model Built? -dbt Cloud considers a Successful Model Built as any model that is successfully built via a run through dbt Cloud’s orchestration functionality in a dbt Cloud deployment environment. Models are counted when built and run. This includes any jobs run via dbt Cloud's scheduler, CI builds (jobs triggered by pull requests), runs kicked off via the dbt Cloud API, and any successor dbt Cloud tools with similar functionality. This also includes models that are successfully built even when a run may fail to complete. For example, you may have a job that contains 100 models and on one of its runs, 51 models are successfully built and then the job fails. In this situation, only 51 models would be counted. +dbt Cloud considers a Successful Model Built as any model that is successfully built via a run through dbt Cloud’s orchestration functionality in a dbt Cloud deployment environment. Models are counted when built and run. This includes any jobs run via dbt Cloud's scheduler, CI builds (jobs triggered by pull requests), runs kicked off via the dbt Cloud API, and any successor dbt Cloud tools with similar functionality. This also includes models that are successfully built even when a run may fail to complete. For example, you may have a job that contains 100 models and on one of its runs, 51 models are successfully built and then the job fails. In this situation, only 51 models would be counted. Any models built in a dbt Cloud development environment (for example, via the IDE) do not count towards your usage. Tests, seeds, ephemeral models, and snapshots also do not count. +| What counts towards Successful Models Built | | +|---------------------------------------------|---------------------| +| View | ✅ | +| Table | ✅ | +| Incremental | ✅ | +| Ephemeral Models | ❌ | +| Tests | ❌ | +| Seeds | ❌ | +| Snapshots | ❌ | -### What counts as a seat license? +### What counts as a Query Unit?​ -There are three types of possible seat licenses: +The dbt Semantic Layer, powered by MetricFlow, measures usage in distinct query units. Every successful request you make to render or run SQL to the Semantic Layer API counts as at least one query unit, even if no data is returned. If the query calculates or renders SQL for multiple metrics, each calculated metric will be counted as a query unit. +If a request to run a query is not executed successfully in the data platform or if a query results in an error without completion, it is not counted as a query unit. Requests for metadata from the Semantic Layer are also not counted as query units. -* **Developer** — for roles and permissions that require interaction with the dbt Cloud environment day-to-day. -* **Read-Only** — for access to view certain documents and reports. -* **IT** — for access to specific features related to account management (for example, configuring git integration). +Examples of query units include: + +Querying one metric, grouping by one dimension → 1 query unit + +```shell +dbt sl query --metrics revenue --group_by metric_time +``` +Querying one metric, grouping by two dimensions → 1 query unit + +```shell +dbt sl query --metrics revenue --group_by metric_time,user__country +``` + +Querying two metrics, grouping by two dimensions → 2 query units + +```shell +dbt sl query --metrics revenue,gross_sales --group_by metric_time,user__country +``` + +Running an explain for one metric → 1 query unit + +```shell +dbt sl query --metrics revenue --group_by metric_time --explain +``` + +Running an explain for two metrics → 2 query units + +```shell +dbt sl query --metrics revenue,gross_sales --group_by metric_time --explain +``` + +Running a query for only dimensions such as dimension_values or a query with no metrics → 1 query unit + +```shell +bt sl list dimension-values --dimension user__country +``` ### Viewing usage in the product @@ -59,7 +112,7 @@ All included successful models built numbers above reflect our most current pric Team customers pay monthly via credit card for seats and usage, and accounts include 15,000 models monthly. Seats are charged upfront at the beginning of the month. If you add seats during the month, seats will be prorated and charged on the same day. Seats removed during the month will be reflected on the next invoice and are not eligible for refunds. You can change the credit card information and the number of seats from the billings section anytime. Accounts will receive one monthly invoice that includes the upfront charge for the seats and the usage charged in arrears from the previous month. -Usage is calculated and charged in arrears for the previous month. If you exceed 15,000 models in any month, you will be billed for additional usage on your next invoice. Additional use is billed at the rates on our [pricing page](https://www.getdbt.com/pricing). +Usage is calculated and charged in arrears for the previous month. If you exceed 15,000 models in any month, you will be billed for additional usage on your next invoice. Additional usage is billed at the rates on our [pricing page](https://www.getdbt.com/pricing). Included models that are not consumed do not roll over to future months. You can estimate your bill with a simple formula: @@ -68,15 +121,22 @@ Included models that are not consumed do not roll over to future months. You can All included successful models built numbers above reflect our most current pricing and packaging. Based on your usage terms when you signed up for the Team Plan, the included model entitlements may be different from what’s reflected above. -:::note Legacy pricing plans - -Customers who purchased the dbt Cloud Team plan before August 11, 2023, remain on a legacy pricing plan as long as their account is in good standing. The legacy pricing plan is based on seats and includes unlimited models subject to reasonable use. dbt Labs may institute use limits if reasonable use is exceeded. Additional features, upgrades, or updates may be subject to separate charges. Any changes to your current plan pricing will be communicated in advance according to our Terms of Use. +### Enterprise plan billing + +As an Enterprise customer, you pay annually via invoice, monthly in arrears for additional usage (if applicable), and may benefit from negotiated usage rates. Please refer to your order form or contract for your specific pricing details, or [contact the account team](https://www.getdbt.com/contact-demo) with any questions. + +### Legacy plans + +Customers who purchased the dbt Cloud Team plan before August 11, 2023, remain on a legacy pricing plan as long as your account is in good standing. The legacy pricing plan is based on seats and includes unlimited models, subject to reasonable use. + +:::note Legacy Semantic Layer + +For customers using the legacy Semantic Layer with dbt_metrics package, this product will be deprecated in December 2023. Legacy users may choose to upgrade at any time to the revamped version, Semantic Layer powered by MetricFlow. The revamped version is available to most customers (see [prerequisites](/docs/use-dbt-semantic-layer/quickstart-sl#prerequisites)) for a limited time on a free trial basis, subject to reasonable use. ::: -### Enterprise plan billing +dbt Labs may institute use limits if reasonable use is exceeded. Additional features, upgrades, or updates may be subject to separate charges. Any changes to your current plan pricing will be communicated in advance according to our Terms of Use. -As an Enterprise customer, you pay annually via invoice, monthly in arrears for additional usage (if applicable), and may benefit from negotiated usage rates. Please refer to your order form or contract for your specific pricing details, or [contact the account team](https://www.getdbt.com/contact-demo) with any questions. ## Managing usage @@ -191,3 +251,10 @@ _Yes. Your dbt Cloud account will be upgraded without impacting your existing pr * How do I determine the right plan for me? _The best option is to consult with our sales team. They'll help you figure out what is right for your needs. We also offer a free two-week trial on the Team plan._ + +* What are the Semantic Layer trial terms? +_Team and Enterprise customers can sign up for a free trial of the dbt Semantic Layer, powered by MetricFlow, for use of up to 1,000 query units per month. The trial will be available at least through January 2024. dbt Labs may extend the trial period in its sole discretion. During the trial period, we may reach out to discuss pricing options or ask for feedback. At the end of the trial, free access may be removed and a purchase may be required to continue use. dbt Labs reserves the right to change limits in a free trial or institute pricing when required or at any time in its sole discretion._ + +* What is the reasonable use limitation for the dbt Semantic Layer powered by MetricFlow during the trial? +_Each account will be limited to 1,000 Queried Metrics per month during the trial period and may be changed at the sole discretion of dbt Labs._ + diff --git a/website/docs/docs/cloud/cloud-cli-installation.md b/website/docs/docs/cloud/cloud-cli-installation.md index 68a8ef365d6..5c0d6ad543e 100644 --- a/website/docs/docs/cloud/cloud-cli-installation.md +++ b/website/docs/docs/cloud/cloud-cli-installation.md @@ -1,110 +1,233 @@ --- -title: Installing the dbt Cloud CLI (Alpha) +title: Install dbt Cloud CLI +sidebar_label: "Install dbt Cloud CLI" id: cloud-cli-installation description: "Instructions for installing and configuring dbt Cloud CLI" +pagination_next: "docs/cloud/configure-cloud-cli" --- -:::warning Alpha functionality +import CloudCLIFlag from '/snippets/_cloud-cli-flag.md'; -The following installation instructions are for the dbt Cloud CLI, currently in Alpha (actively in development and being tested). + -These instructions are not intended for general audiences at this time. + +dbt Cloud natively supports developing using a command line (CLI), empowering team members to contribute with enhanced flexibility and collaboration. The dbt Cloud CLI allows you to run dbt commands against your dbt Cloud development environment from your local command line. + +dbt commands are run against dbt Cloud's infrastructure and benefit from: + +* Secure credential storage in the dbt Cloud platform. +* [Automatic deferral](/docs/cloud/about-cloud-develop-defer) of build artifacts to your Cloud project's production environment. +* Speedier, lower-cost builds. +* Support for dbt Mesh ([cross-project `ref`](/docs/collaborate/govern/project-dependencies)), +* Significant platform improvements, to be released over the coming months. + + +## Prerequisites +The dbt Cloud CLI is available in all [deployment regions](/docs/cloud/about-cloud/regions-ip-addresses) and and for both multi-tenant and single-tenant accounts (Azure single-tenant not supported at this time). + +You must be on dbt version 1.5 or higher. Refer to [dbt Cloud versions](/docs/dbt-versions/upgrade-core-in-cloud) to upgrade. + +## Install dbt Cloud CLI + +You can install the dbt Cloud CLI on the command line by using one of these methods: + + + + + +Before you begin, make sure you have [Homebrew installed](http://brew.sh/) in your code editor or command line terminal. Refer to the [FAQs](#faqs) if your operating system runs into path conflicts. + + +1. Run the following command to verify that there is no conflict with a dbt Core installation on your system: + + ```bash + which dbt + ``` + - This should return a `dbt not found`. If the dbt help text appears, use `pip uninstall dbt` to deactivate dbt Core from your machine. + +2. Install the dbt Cloud CLI with Homebrew: + + ```bash + brew untap dbt-labs/dbt + brew tap dbt-labs/dbt-cli + brew install dbt + ``` + +3. Verify the installation by running `dbt --help` from the command line. If the help text doesn't indicate that you're using the dbt Cloud CLI, make sure you've deactivated your pyenv or venv and don't have a version of dbt globally installed. + * You don't have to run the `dbt deps` command when your environment starts. Previously, you had to do it during initialization. However, you'll still need to run `dbt deps` if you make changes to your `packages.yml` file. + + + + + +Refer to the [FAQs](#faqs) if your operating system runs into path conflicts. + +1. Download the latest Windows release for your platform from [GitHub](https://github.com/dbt-labs/dbt-cli/releases). + +2. Extract the `dbt.exe` executable into the same folder as your dbt project. + +:::info + +Advanced users can configure multiple projects to use the same dbt Cloud CLI by placing the executable in the Program Files folder and [adding it to their Windows PATH environment variable](https://medium.com/@kevinmarkvi/how-to-add-executables-to-your-path-in-windows-5ffa4ce61a53). + +Note that if you are using VS Code, you must restart it to pick up modified environment variables. +::: + +3. Verify the installation by running `./dbt --help` from the command line. If the help text doesn't indicate that you're using the dbt Cloud CLI, make sure you've deactivated your pyenv or venv and don't have a version of dbt globally installed. + * You don't have to run the `dbt deps` command when your environment starts. Previously, you had to do it during initialization. However, you'll still need to run `dbt deps` if you make changes to your `packages.yml` file. + + + + + +Refer to the [FAQs](#faqs) if your operating system runs into path conflicts. + +1. Download the latest Linux release for your platform from [GitHub](https://github.com/dbt-labs/dbt-cli/releases). (Pick the file based on your CPU architecture) + +2. Extract the `dbt-cloud-cli` binary to the same folder as your dbt project. + + ```bash + tar -xf dbt_0.29.9_linux_amd64.tar.gz + ./dbt --version + ``` + +:::info + +Advanced users can configure multiple projects to use the same Cloud CLI executable by adding it to their PATH environment variable in their shell profile. + +::: + +3. Verify the installation by running `./dbt --help` from the command line. If the help text doesn't indicate that you're using the dbt Cloud CLI, make sure you've deactivated your pyenv or venv and don't have a version of dbt globally installed. + * You don't have to run the `dbt deps` command when your environment starts. Previously, you had to do it during initialization. However, you'll still need to run `dbt deps` if you make changes to your `packages.yml` file. + + + + + +:::info Use native packages or a virtual environment to avoid overriding dbt Core + +Installing the dbt Cloud CLI with pip replaces dbt Core. This change can be avoided by using the native install method and configuring your PATH or by creating a new virtual environment. + +Otherwise, to switch back to dbt Core, uninstall the dbt Cloud CLI and follow the dbt Core installation instructions. ::: -## Installing dbt Cloud CLI +Before installing the dbt Cloud CLI, make sure you have Python installed and your virtual environment venv or pyenv . If you already have a Python environment configured, you can skip to the [pip installation step](#install-dbt-cloud-cli-in-pip). + +### Install a virtual environment + +We recommend using virtual environments (venv) to namespace `cloud-cli`. + +1. Create a new venv: + ```shell + python3 -m venv dbt-cloud + ``` + +2. Activate the virtual environment each time you create a shell window or session: + ```shell + source dbt-cloud/bin/activate # activate the environment for Mac and Linux OR + dbt-env\Scripts\activate # activate the environment for Windows + ``` + +3. (Mac and Linux only) Create an alias to activate your dbt environment with every new shell window or session. You can add the following to your shell's configuration file (for example, $HOME/.bashrc, $HOME/.zshrc) while replacing `` with the path to your virtual environment configuration: + ```shell + alias env_dbt='source /bin/activate' + ``` + +### Install dbt Cloud CLI in pip + +1. (Optional) If you already have dbt Core installed, this installation will override that package. Note your dbt Core version in case you need to reinstall it later: + + ```bash + dbt --version + ``` + +2. Make sure you're in your virtual environment and run the following command to install the dbt Cloud CLI: + + ```bash + pip3 install dbt + ``` + +3. (Optional) To revert back to dbt Core, first uninstall both the dbt Cloud CLI and dbt Core +4. Reinstall dbt Core using the version from Step 2. + + ```bash + pip3 uninstall dbt-core dbt + pip3 install dbt-core==VERSION + ``` + + + + + -### Install and update with Brew on MacOS (recommended) +## Update dbt Cloud CLI -1. Install the dbt Cloud CLI: +The following instructions explain how to update the dbt CLoud CLI to the latest version depending on your operating system. -```bash -brew tap dbt-labs/dbt-cli -brew install dbt-cloud-cli -``` +During the public preview period, we recommend updating before filing a bug report. This is because the API is subject to breaking changes. -2. Verify the installation by requesting your homebrew installation path (not your dbt core installs). If the `which dbt` command returns nothing, then you should modify your PATH in `~.zshrc` or create an alias. -```bash -which dbt -dbt --help -``` + -### Manually install (Windows and Linux) + -1. Download the latest release for your platform from [GitHub](https://github.com/dbt-labs/dbt-cli/releases). -2. Add the `dbt` executable to your path. -3. Move to a directory with a dbt project, and create a `dbt_cloud.yml` file containing your `project-id` from dbt Cloud. -4. Invoke `dbt --help` from your terminal to see a list of supported commands. +To update the dbt Cloud CLI, run `brew upgrade dbt`. (You can also use `brew install dbt`). -#### Updating your dbt Cloud installation (Windows + Linux) + -Follow the same process in [Installing dbt Cloud CLI](#manually-install-windows-only) and replace the existing `dbt` executable with the new one. You should not have to go through the security steps again. + -## Setting up the CLI +To update, follow the same process explained in [Windows](/docs/cloud/cloud-cli-installation?install=windows#install-dbt-cloud-cli) and replace the existing `dbt.exe` executable with the new one. -The following instructions are for setting up the dbt Cloud CLI. + -1. Ensure that you have created a project in [dbt Cloud](https://cloud.getdbt.com/). + -2. Ensure that your personal [development credentials](https://cloud.getdbt.com/settings/profile/credentials) are set on the project. +To update, follow the same process explained in [Windows](/docs/cloud/cloud-cli-installation?install=linux#install-dbt-cloud-cli) and replace the existing `dbt` executable with the new one. -3. Navigate to [your profile](https://cloud.getdbt.com/settings/profile) and enable the **Beta** flag under **Experimental Features.** + -4. Create an environment variable with your [dbt Cloud API key](https://cloud.getdbt.com/settings/profile#api-access): + -```bash -vi ~/.zshrc +To update: +- Make sure you're in your virtual environment +- Run `pip install --upgrade dbt`. + + -# dbt Cloud CLI -export DBT_CLOUD_API_KEY="1234" # Replace "1234" with your API key -``` + -5. Load the new environment variable. Note: You may need to reactivate your Python virtual environment after sourcing your shell's dot file. Alternatively, restart your shell instead of sourcing the shell's dot file -```bash -source ~/.zshrc -``` +## Next steps -6. Navigate to a dbt project +After installation, you can [configure](/docs/cloud/configure-cloud-cli) the dbt Cloud CLI for your dbt Cloud project and use it to run [dbt commands](/reference/dbt-commands) similar to dbt Core. -```bash -cd ~/dbt-projects/jaffle_shop -``` +For example, you can execute `dbt compile` to compile a project using dbt Cloud. -7. Create a `dbt_cloud.yml` in the root project directory. The file is required to have a `project-id` field with a valid [project ID](#glossary). Enter the following commands: +Note, that if you're using the dbt Cloud CLI, you can connect to your data platform directly in the dbt Cloud interface and don't need a [`profiles.yml`](/docs/core/connect-data-platform/profiles.yml) file. -```bash -pwd # Input -/Users/user/dbt-projects/jaffle_shop # Output -``` -```bash -echo "project-id: ''" > dbt_cloud.yml # Input -``` +## FAQs -```bash -cat dbt_cloud.yml # Input -project-id: '123456' # Output -``` +
-You can find your project ID by selecting your project and clicking on **Develop** in the navigation bar. Your project ID is the number in the URL: https://cloud.getdbt.com/develop/26228/projects/PROJECT_ID. +What's the difference between the dbt Cloud CLI and dbt Core? +The dbt Cloud CLI and dbt Core, an open-source project, are both command line tools that enable you to run dbt commands. The key distinction is the dbt Cloud CLI is tailored for dbt Cloud's infrastructure and integrates with all its features. -If `dbt_cloud.yml` already exists, edit the file, and verify the project ID field uses a valid project ID. +
-#### Upgrade the CLI with Brew +
+How do I run both the dbt Cloud CLI and dbt Core? +For compatibility, both the dbt Cloud CLI and dbt Core are invoked by running dbt. This can create path conflicts if your operating system selects one over the other based on your $PATH environment variable (settings). -```bash -brew update -brew upgrade dbt-cloud-cli -``` +If you have dbt Core installed locally, either: -## Using dbt Cloud CLI +1. Install using [pip](/docs/cloud/cloud-cli-installation?install=pip#install-dbt-cloud-cli). -**Coming soon** +2. Install natively, but ensure that you deactivate your Python environment or uninstall it using `pip uninstall dbt` before proceeding. -## Glossary +3. (Advanced users) Install natively, but modify the $PATH environment variable to correctly point to the dbt Cloud CLI binary to use both dbt Cloud CLI and dbt Core together. -- **dbt cloud API key:** Your API key found by navigating to the **gear icon**, clicking **Profile Settings**, and scrolling down to **API**. -- **Project ID:** The ID of the dbt project you're working with. Can be retrieved from the dbt Cloud URL after a project has been selected, for example, `https://cloud.getdbt.com/deploy/{accountID}/projects/{projectID}` -- **Development credentials:** Your personal warehouse credentials for the project you’re working with. They can be set by selecting the project and entering them in dbt Cloud. Navigate to the **gear icon**, click **Profile Settings**, and click **Credentials** from the left-side menu. +You can always uninstall the dbt Cloud CLI to return to using dbt Core. +
diff --git a/website/docs/docs/cloud/configure-cloud-cli.md b/website/docs/docs/cloud/configure-cloud-cli.md new file mode 100644 index 00000000000..c03f638c9b4 --- /dev/null +++ b/website/docs/docs/cloud/configure-cloud-cli.md @@ -0,0 +1,101 @@ +--- +title: Configure dbt Cloud CLI +id: configure-cloud-cli +description: "Instructions on how to configure the dbt Cloud CLI" +sidebar_label: "Configure dbt Cloud CLI" +pagination_next: null +--- + +import CloudCLIFlag from '/snippets/_cloud-cli-flag.md'; + + + + +## Prerequisites + +- You must set up a project in dbt Cloud. + - **Note** — If you're using the dbt Cloud CLI, you can connect to your data platform directly in the dbt Cloud interface and don't need a [`profiles.yml`](/docs/core/connect-data-platform/profiles.yml) file. +- You must have your [personal development credentials](/docs/dbt-cloud-environments#set-developer-credentials) set for that project. The dbt Cloud CLI will use these credentials, stored securely in dbt Cloud, to communicate with your data platform. +- You must [enroll](/docs/dbt-versions/experimental-features) in the dbt Cloud beta features. + - To enroll, navigate to your **Profile Settings** and enable the **Beta** flag under **Experimental Features**. +- You must be on dbt version 1.5 or higher. Refer to [dbt Cloud versions](/docs/dbt-versions/upgrade-core-in-cloud) to upgrade. + +## Configure the dbt Cloud CLI + +Once you install the dbt Cloud CLI, you need to configure it to connect to a dbt Cloud project. + +1. Ensure you meet the prerequisites above. + +2. Download your credentials from dbt Cloud by clicking on the **Try the dbt Cloud CLI** banner on the dbt Cloud homepage. Alternatively, if you're in dbt Cloud, you can download the credentials from the links provided based on your region: + + - North America: https://cloud.getdbt.com/cloud-cli + - EMEA: https://emea.dbt.com/cloud-cli + - APAC: https://apac.dbt.com/cloud-cli + - North American Cell 1: `https:/ACCOUNT_PREFIX.us1.dbt.com/cloud-cli` + - Single-tenant: `https://YOUR_ACCESS_URL/cloud-cli` + +3. Follow the banner instructions and download the config file to: + - Mac or Linux: `~/.dbt/dbt_cloud.yml` + - Windows: `C:\Users\yourusername\.dbt\dbt_cloud.yml` + + The config file looks like this: + + ```yaml + version: "1" + context: + active-project: "" + active-host: "" + defer-env-id: "" + projects: + - project-id: "" + account-host: "" + api-key: "" + + - project-id: "" + account-host: "" + api-key: "" + + ``` + +4. After downloading the config file, navigate to a dbt project in your terminal: + + ```bash + cd ~/dbt-projects/jaffle_shop + ``` + +5. In your `dbt_project.yml` file, ensure you have or include a `dbt-cloud` section with a `project-id` field. The `project-id` field contains the dbt Cloud project ID you want to use. + + ```yaml + # dbt_project.yml + name: + + version: + ... + + dbt-cloud: + project-id: PROJECT_ID + ``` + + - To find your project ID, select **Develop** in the dbt Cloud navigation menu. You can use the URL to find the project ID. For example, in `https://cloud.getdbt.com/develop/26228/projects/123456`, the project ID is `123456`. + +### Set environment variables + +To set environment variables in the dbt Cloud CLI for your dbt project: + +1. Select the gear icon on the upper right of the page. +2. Then select **Profile Settings**, then **Credentials**. +3. Click on your project and scroll to the **Environment Variables** section. +4. Click **Edit** on the lower right and then set the user-level environment variables. + +## Use the dbt Cloud CLI + +- The dbt Cloud CLI shares the same set of [dbt commands](/reference/dbt-commands) as dbt Core and processes the commands you invoke. +- It allows you to use automatic deferral of build artifacts to your Cloud project's production environment. +- It also supports [project dependencies](/docs/collaborate/govern/project-dependencies), which allows you to depend on another project using the metadata service in dbt Cloud. + - Project dependencies instantly connect to and reference (or `ref`) public models defined in other projects. This means you don't need to execute or analyze these upstream models yourself. Instead, you treat them as an API that returns a dataset. + +:::tip Use the --help flag +As a tip, most command-line tools have a `--help` flag to show available commands and arguments. Use the `--help` flag with dbt in two ways: +- `dbt --help`: Lists the commands available for dbt
+- `dbt run --help`: Lists the flags available for the `run` command +::: diff --git a/website/docs/docs/cloud/connect-data-platform/about-connections.md b/website/docs/docs/cloud/connect-data-platform/about-connections.md index 65bfac3a90d..1fe89c7273c 100644 --- a/website/docs/docs/cloud/connect-data-platform/about-connections.md +++ b/website/docs/docs/cloud/connect-data-platform/about-connections.md @@ -3,6 +3,8 @@ title: "About data platform connections" id: about-connections description: "Information about data platform connections" sidebar_label: "About data platform connections" +pagination_next: "docs/cloud/connect-data-platform/connect-starburst-trino" +pagination_prev: null --- dbt Cloud can connect with a variety of data platform providers including: - [Amazon Redshift](/docs/cloud/connect-data-platform/connect-redshift-postgresql-alloydb) @@ -13,6 +15,10 @@ dbt Cloud can connect with a variety of data platform providers including: - [Snowflake](/docs/cloud/connect-data-platform/connect-snowflake) - [Starburst or Trino](/docs/cloud/connect-data-platform/connect-starburst-trino) +import MSCallout from '/snippets/_microsoft-adapters-soon.md'; + + + You can connect to your database in dbt Cloud by clicking the gear in the top right and selecting **Account Settings**. From the Account Settings page, click **+ New Project**. diff --git a/website/docs/docs/cloud/connect-data-platform/connect-apache-spark.md b/website/docs/docs/cloud/connect-data-platform/connect-apache-spark.md index 670b628547b..0186d821a54 100644 --- a/website/docs/docs/cloud/connect-data-platform/connect-apache-spark.md +++ b/website/docs/docs/cloud/connect-data-platform/connect-apache-spark.md @@ -3,6 +3,7 @@ title: "Connect Apache Spark" id: connect-apache-spark description: "Setup instructions for connecting Apache Spark to dbt Cloud" sidebar_label: "Connect Apache Spark" +pagination_next: null --- diff --git a/website/docs/docs/cloud/dbt-cloud-ide/dbt-cloud-ide.md b/website/docs/docs/cloud/dbt-cloud-ide/dbt-cloud-ide.md new file mode 100644 index 00000000000..3c41432bc62 --- /dev/null +++ b/website/docs/docs/cloud/dbt-cloud-ide/dbt-cloud-ide.md @@ -0,0 +1,37 @@ +--- +title: "dbt Cloud IDE" +description: "Learn how to configure Git in dbt Cloud" +pagination_next: "docs/cloud/dbt-cloud-ide/develop-in-the-cloud" +pagination_prev: null +--- + +
+ + + + + +
+
+
+ + + + +
\ No newline at end of file diff --git a/website/docs/docs/cloud/dbt-cloud-ide/dbt-cloud-tips.md b/website/docs/docs/cloud/dbt-cloud-ide/dbt-cloud-tips.md index cfae00b960e..39db7832d79 100644 --- a/website/docs/docs/cloud/dbt-cloud-ide/dbt-cloud-tips.md +++ b/website/docs/docs/cloud/dbt-cloud-ide/dbt-cloud-tips.md @@ -3,6 +3,7 @@ title: "Tips and tricks" id: dbt-cloud-tips description: "Check out any dbt Cloud and IDE-related tips." sidebar_label: "Tips and tricks" +pagination_next: null --- # dbt Cloud tips @@ -16,7 +17,7 @@ There are default keyboard shortcuts that can help make development more product - Press Fn-F1 to view a full list of the editor shortcuts - Command-O on macOS or Control-O on Windows to select a file to open - Command-P/Command-Shift-P on macOS or Control-P/Control-Shift-P on Windows to see the command palette -- Hold Option-click-on-area on macOS or Hold-Alt-click-on-area on Windows to select multiple lines and perform a multi-edit. You can also press Command-E to perform this operation on the command line. +- Hold Option-click-on-area or press Shift-Option-Command on macOS or Hold-Alt-click-on-area on Windows to select multiple lines and perform a multi-edit. You can also press Command-E to perform this operation on the command line. - Command-Enter on macOS or Control-Enter on Windows to Preview your code - Command-Shift-Enter on macOS or Control-Shift-Enter on Windows to Compile - Highlight a portion of code and use the above shortcuts to Preview or Compile code diff --git a/website/docs/docs/cloud/dbt-cloud-ide/develop-in-the-cloud.md b/website/docs/docs/cloud/dbt-cloud-ide/develop-in-the-cloud.md index c55e67cf93e..9fc382f0217 100644 --- a/website/docs/docs/cloud/dbt-cloud-ide/develop-in-the-cloud.md +++ b/website/docs/docs/cloud/dbt-cloud-ide/develop-in-the-cloud.md @@ -1,22 +1,31 @@ --- -title: "Develop in the IDE" +title: "About the dbt Cloud IDE" id: develop-in-the-cloud description: "Develop, test, run, and build in the Cloud IDE. With the Cloud IDE, you can compile dbt code into SQL and run it against your database directly" -sidebar_label: Develop in the IDE +sidebar_label: About the IDE tags: [IDE] +pagination_next: "docs/cloud/dbt-cloud-ide/ide-user-interface" +pagination_prev: null --- -The dbt Cloud integrated development environment (IDE) is a single interface for building, testing, running, and version-controlling dbt projects from your browser. With the Cloud IDE, you can compile dbt code into SQL and run it against your database directly. +The dbt Cloud integrated development environment (IDE) is a single web-based interface for building, testing, running, and version-controlling dbt projects. It compiles dbt code into SQL and executes it directly on your database. -## Prerequisites +The dbt Cloud IDE offers several [editing features](/docs/cloud/dbt-cloud-ide/ide-user-interface#editing-features) for faster and more efficient data platform development and governance: -To develop in the Cloud IDE, make sure you have the following: +- Syntax highlighting for SQL: Makes it easy to distinguish different parts of your code, reducing syntax errors and enhancing readability. +- Auto-completion: Suggests table names, arguments, and column names as you type, saving time and reducing typos. +- Code [formatting and linting](/docs/cloud/dbt-cloud-ide/lint-format): Help standardize and fix your SQL code effortlessly. +- Navigation tools: Easily move around your code, jump to specific lines, find and replace text, and navigate between project files. +- Version control: Manage code versions with a few clicks. -- A [dbt Cloud account](https://cloud.getdbt.com/) and [Developer seat license](/docs/cloud/manage-access/seats-and-users) -- A git repository set up and git provider must have `write` access enabled. See [Connecting your GitHub Account](/docs/cloud/git/connect-github) or [Importing a project by git URL](/docs/cloud/git/import-a-project-by-git-url) for detailed setup instructions -- A dbt project connected to a [data platform](/docs/cloud/connect-data-platform/about-connections) -- A [development environment and development credentials](#access-the-cloud-ide) set up -- The environment must be on dbt version 1.0 or higher +These [features](#dbt-cloud-ide-features) create a powerful editing environment for efficient SQL coding, suitable for both experienced and beginner developers. + + + + + + + :::tip Disable ad blockers @@ -24,21 +33,16 @@ To improve your experience using dbt Cloud, we suggest that you turn off ad bloc ::: -## Develop in the Cloud IDE - -The Cloud IDE is a powerful tool that can help streamline and govern your data platform development process. It offers a range of [editing features](/docs/cloud/dbt-cloud-ide/ide-user-interface#editing-features) that can help make your data platform development process faster and more efficient. Some of the editing features include: - -- The IDE has syntax highlighting for SQL. This makes it easy to visually distinguish between different parts of your code. This helps prevent syntax errors and improve readability. -- Use the IDE built-in auto-completion, which suggests table names, arguments, and column names as you type. This saves time and reduces the likelihood of typos or errors in your code. -- The code [formatting and linting](/docs/cloud/dbt-cloud-ide/lint-format) tools allow you to standardize and fix your SQL code with ease. -- The IDE has a range of navigation tools, making it easy to move around your code with ease. You can quickly jump to specific lines of code, find and replace text, and navigate between different files in your project. -- Use the version control menu and features to version-control your code with just a few clicks. +## Prerequisites -All of these [features](#cloud-ide-features) work together to create a powerful editing environment that can help you write and maintain high-quality SQL code in less time. Whether you're a seasoned developer or just starting out, the Cloud IDE has everything you need to be productive, collaborative, and efficient. +- A [dbt Cloud account](https://cloud.getdbt.com/) and [Developer seat license](/docs/cloud/manage-access/seats-and-users) +- A git repository set up and git provider must have `write` access enabled. See [Connecting your GitHub Account](/docs/cloud/git/connect-github) or [Importing a project by git URL](/docs/cloud/git/import-a-project-by-git-url) for detailed setup instructions +- A dbt project connected to a [data platform](/docs/cloud/connect-data-platform/about-connections) +- A [development environment and development credentials](#access-the-cloud-ide) set up +- The environment must be on dbt version 1.0 or higher - -## Cloud IDE features +## dbt Cloud IDE features The dbt Cloud IDE comes with [tips](/docs/cloud/dbt-cloud-ide/dbt-cloud-tips) and [features](/docs/cloud/dbt-cloud-ide/ide-user-interface) that make it easier for you to develop, build, compile, run, and test data models. @@ -89,9 +93,9 @@ The Cloud IDE needs explicit action to save your changes. There are three ways y ## Access the Cloud IDE -:::info📌 +:::tip Disable ad blockers -New to dbt? Check out our [quickstart guides](/quickstarts) to build your first dbt project in the Cloud IDE! +To improve your experience using dbt Cloud, we suggest that you turn off ad blockers. This is because some project file names, such as `google_adwords.sql`, might resemble ad traffic and trigger ad blockers. ::: @@ -155,13 +159,15 @@ The dbt Cloud IDE makes it possible to [build and view](/docs/collaborate/build-
- What is the difference between developing on the Cloud IDE and on the CLI? + What is the difference between developing on the dbt Cloud IDE, the dbt Cloud CLI, and dbt Core?
-
There are two main ways to develop with dbt: using the web-based IDE in dbt Cloud or using the command-line interface (CLI) in dbt Core:

- — - dbt Cloud IDE dbt Cloud is a web-based application that allows you to develop dbt projects with the IDE, includes a purpose-built scheduler, and provides an easier way to share your dbt documentation with your team. The IDE is a faster and more reliable way to deploy your dbt models and provides a real-time editing and execution environment for your dbt project.

- — - dbt Core CLI The command line interface (CLI) uses dbt Core, an open-source software that’s freely available. You can build your dbt project in a code editor, like Jetbrains or VSCode, and run dbt commands from the command line. +
You can develop dbt using the web-based IDE in dbt Cloud or on the command line interface using the dbt Cloud CLI or open-source dbt Core, all of which enable you to execute dbt commands. The key distinction between the dbt Cloud CLI and dbt Core is the dbt Cloud CLI is tailored for dbt Cloud's infrastructure and integrates with all its features.

+ — + dbt Cloud IDE: dbt Cloud is a web-based application that allows you to develop dbt projects with the IDE, includes a purpose-built scheduler, and provides an easier way to share your dbt documentation with your team. The IDE is a faster and more reliable way to deploy your dbt models and provides a real-time editing and execution environment for your dbt project.

+ — + dbt Cloud CLI: The dbt Cloud CLI allows you to run dbt commands against your dbt Cloud development environment from your local command line or code editor. It supports cross-project ref, speedier, lower-cost builds, automatic deferral of build artifacts, and more.

+ — + dbt Core: dbt Core is an open-sourced software that’s freely available. You can build your dbt project in a code editor, and run dbt commands from the command line.
diff --git a/website/docs/docs/cloud/dbt-cloud-ide/ide-user-interface.md b/website/docs/docs/cloud/dbt-cloud-ide/ide-user-interface.md index de643413a8a..05910b23e7f 100644 --- a/website/docs/docs/cloud/dbt-cloud-ide/ide-user-interface.md +++ b/website/docs/docs/cloud/dbt-cloud-ide/ide-user-interface.md @@ -36,11 +36,13 @@ The IDE streamlines your workflow, and features a popular user interface layout * Added (A) — The IDE detects added files * Deleted (D) — The IDE detects deleted files. - + 5. **Command bar —** The Command bar, located in the lower left of the IDE, is used to invoke [dbt commands](/reference/dbt-commands). When a command is invoked, the associated logs are shown in the Invocation History Drawer. -6. **IDE Status button —** The IDE Status button, located on the lower right of the IDE, displays the current IDE status. If there is an error in the status or in the dbt code that stops the project from parsing, the button will turn red and display "Error". If there aren't any errors, the button will display a green "Ready" status. To access the [IDE Status modal](#modals-and-menus), simply click on this button. +6. **Defer to production —** The **Defer to production** toggle allows developers to only build and run and test models they've edited without having to first run and build all the models that come before them (upstream parents). Refer to [Using defer in dbt Cloud](/docs/cloud/about-cloud-develop-defer#defer-in-the-dbt-cloud-ide) for more info. + +7. **Status button —** The IDE Status button, located on the lower right of the IDE, displays the current IDE status. If there is an error in the status or in the dbt code that stops the project from parsing, the button will turn red and display "Error". If there aren't any errors, the button will display a green "Ready" status. To access the [IDE Status modal](#modals-and-menus), simply click on this button. ## Editing features diff --git a/website/docs/docs/cloud/dbt-cloud-ide/lint-format.md b/website/docs/docs/cloud/dbt-cloud-ide/lint-format.md index 8ffd83ef00e..6a86f1aa14b 100644 --- a/website/docs/docs/cloud/dbt-cloud-ide/lint-format.md +++ b/website/docs/docs/cloud/dbt-cloud-ide/lint-format.md @@ -45,7 +45,11 @@ With the dbt Cloud IDE, you can seamlessly use [SQLFluff](https://sqlfluff.com/) - Works with Jinja and SQL, - Comes with built-in [linting rules](https://docs.sqlfluff.com/en/stable/rules.html). You can also [customize](#customize-linting) your own linting rules. - Empowers you to [enable linting](#enable-linting) with options like **Lint** (displays linting errors and recommends actions) or **Fix** (auto-fixes errors in the IDE). -- Displays a **Code Quality** tab to view code errors, and provides code quality visibility and management. +- Displays a **Code Quality** tab to view code errors, and provides code quality visibility and management. + +:::info Ephemeral models not supported +Linting doesn't support ephemeral models in dbt v1.5 and lower. Refer to the [FAQs](#faqs) for more info. +::: ### Enable linting @@ -223,6 +227,12 @@ Currently, running SQLFluff commands from the terminal isn't supported. Make sure you're on a development branch. Formatting or Linting isn't available on "main" or "read-only" branches. +
+Why is there inconsistent SQLFluff behavior when running outside the dbt Cloud IDE (such as a GitHub Action)? +— Double-check your SQLFluff version matches the one in dbt Cloud IDE (found in the Code Quality tab after a lint operation).

+— If your lint operation passes despite clear rule violations, confirm you're not linting models with ephemeral models. Linting doesn't support ephemeral models in dbt v1.5 and lower. +
+ ## Related docs - [User interface](/docs/cloud/dbt-cloud-ide/ide-user-interface) diff --git a/website/docs/docs/cloud/git/authenticate-azure.md b/website/docs/docs/cloud/git/authenticate-azure.md index 03020ccca73..42028bf993b 100644 --- a/website/docs/docs/cloud/git/authenticate-azure.md +++ b/website/docs/docs/cloud/git/authenticate-azure.md @@ -3,10 +3,11 @@ title: "Authenticate with Azure DevOps" id: "authenticate-azure" description: "dbt Cloud developers need to authenticate with Azure DevOps." sidebar_label: "Authenticate with Azure DevOps" +pagination_next: null --- -If you use the dbt Cloud IDE to collaborate on your team's Azure DevOps dbt repo, you need to [link your dbt Cloud profile to Azure DevOps](#link-your-dbt-cloud-profile-to-azure-devops), which provides an extra layer of authentication. +If you use the dbt Cloud IDE or dbt Cloud CLI to collaborate on your team's Azure DevOps dbt repo, you need to [link your dbt Cloud profile to Azure DevOps](#link-your-dbt-cloud-profile-to-azure-devops), which provides an extra layer of authentication. ## Link your dbt Cloud profile to Azure DevOps diff --git a/website/docs/docs/cloud/git/connect-azure-devops.md b/website/docs/docs/cloud/git/connect-azure-devops.md index bc5bb81dd24..c138e042abc 100644 --- a/website/docs/docs/cloud/git/connect-azure-devops.md +++ b/website/docs/docs/cloud/git/connect-azure-devops.md @@ -1,6 +1,7 @@ --- title: "Connect to Azure DevOps" id: "connect-azure-devops" +pagination_next: "docs/cloud/git/setup-azure" --- @@ -13,7 +14,7 @@ Connect your Azure DevOps cloud account in dbt Cloud to unlock new product exper - Import new Azure DevOps repos with a couple clicks during dbt Cloud project setup. - Clone repos using HTTPS rather than SSH - Enforce user authorization with OAuth 2.0. -- Carry Azure DevOps user repository permissions (read / write access) through to dbt Cloud IDE's git actions. +- Carry Azure DevOps user repository permissions (read / write access) through to dbt Cloud IDE or dbt Cloud CLI's git actions. - Trigger Continuous integration (CI) builds when pull requests are opened in Azure DevOps. diff --git a/website/docs/docs/cloud/git/connect-github.md b/website/docs/docs/cloud/git/connect-github.md index 771e4286ef6..ff0f2fff18f 100644 --- a/website/docs/docs/cloud/git/connect-github.md +++ b/website/docs/docs/cloud/git/connect-github.md @@ -74,7 +74,7 @@ To connect a personal GitHub account: 4. Once you approve authorization, you will be redirected to dbt Cloud, and you should now see your connected account. -The next time you log into dbt Cloud, you will be able to do so via OAuth through GitHub, and if you're on the Enterprise plan, you're ready to use the dbt Cloud IDE. +The next time you log into dbt Cloud, you will be able to do so via OAuth through GitHub, and if you're on the Enterprise plan, you're ready to use the dbt Cloud IDE or dbt Cloud CLI. ## FAQs diff --git a/website/docs/docs/cloud/git/connect-gitlab.md b/website/docs/docs/cloud/git/connect-gitlab.md index 53fde5f4878..e55552e2d86 100644 --- a/website/docs/docs/cloud/git/connect-gitlab.md +++ b/website/docs/docs/cloud/git/connect-gitlab.md @@ -8,7 +8,7 @@ id: "connect-gitlab" Connecting your GitLab account to dbt Cloud provides convenience and another layer of security to dbt Cloud: - Import new GitLab repos with a couple clicks during dbt Cloud project setup. - Clone repos using HTTPS rather than SSH. -- Carry GitLab user permissions through to dbt Cloud IDE's git actions. +- Carry GitLab user permissions through to dbt Cloud or dbt Cloud CLI's git actions. - Trigger [Continuous integration](/docs/deploy/continuous-integration) builds when merge requests are opened in GitLab. The steps to integrate GitLab in dbt Cloud depend on your plan. If you are on: @@ -35,7 +35,7 @@ Once you've accepted, you should be redirected back to dbt Cloud, and you'll see dbt Cloud enterprise customers have the added benefit of bringing their own GitLab OAuth application to dbt Cloud. This tier benefits from extra security, as dbt Cloud will: - Enforce user authorization with OAuth. -- Carry GitLab's user repository permissions (read / write access) through to dbt Cloud IDE's git actions. +- Carry GitLab's user repository permissions (read / write access) through to dbt Cloud or dbt Cloud CLI's git actions. In order to connect GitLab in dbt Cloud, a GitLab account admin must: 1. [Set up a GitLab OAuth application](#setting-up-a-gitlab-oauth-application). @@ -97,7 +97,7 @@ You will then be redirected to GitLab and prompted to sign into your account. Gi Once you've accepted, you should be redirected back to dbt Cloud, and your integration is ready for developers on your team to [personally authenticate with](#personally-authenticating-with-gitlab). ### Personally authenticating with GitLab -dbt Cloud developers on the Enterprise plan must each connect their GitLab profiles to dbt Cloud, as every developer's read / write access for the dbt repo is checked in the dbt Cloud IDE. +dbt Cloud developers on the Enterprise plan must each connect their GitLab profiles to dbt Cloud, as every developer's read / write access for the dbt repo is checked in the dbt Cloud IDE or dbt Cloud CLI. To connect a personal GitLab account, dbt Cloud developers should navigate to Your Profile settings by clicking the gear icon in the top right, then select **Linked Accounts** in the left menu. @@ -105,7 +105,7 @@ If your GitLab account is not connected, you’ll see "No connected account". Se -Once you approve authorization, you will be redirected to dbt Cloud, and you should see your connected account. You're now ready to start developing in the dbt Cloud IDE. +Once you approve authorization, you will be redirected to dbt Cloud, and you should see your connected account. You're now ready to start developing in the dbt Cloud IDE or dbt Cloud CLI. ## Troubleshooting diff --git a/website/docs/docs/cloud/git/git-configuration-in-dbt-cloud.md b/website/docs/docs/cloud/git/git-configuration-in-dbt-cloud.md new file mode 100644 index 00000000000..fb8c0186236 --- /dev/null +++ b/website/docs/docs/cloud/git/git-configuration-in-dbt-cloud.md @@ -0,0 +1,37 @@ +--- +title: "Git configuration in dbt Cloud" +description: "Learn about the Git providers supported in dbt Cloud" +pagination_next: "docs/cloud/git/import-a-project-by-git-url" +pagination_prev: null +--- + +
+ + + + + +
+
+
+ + + + +
\ No newline at end of file diff --git a/website/docs/docs/cloud/git/import-a-project-by-git-url.md b/website/docs/docs/cloud/git/import-a-project-by-git-url.md index ba53baa33ea..83846bb1f0b 100644 --- a/website/docs/docs/cloud/git/import-a-project-by-git-url.md +++ b/website/docs/docs/cloud/git/import-a-project-by-git-url.md @@ -1,6 +1,8 @@ --- title: "Import a project by git URL" id: "import-a-project-by-git-url" +pagination_next: "docs/cloud/git/connect-github" +pagination_prev: null --- In dbt Cloud, you can import a git repository from any valid git URL that points to a dbt project. There are some important considerations to keep in mind when doing this. diff --git a/website/docs/docs/cloud/git/setup-azure.md b/website/docs/docs/cloud/git/setup-azure.md index 9eca77d7014..843371be6ea 100644 --- a/website/docs/docs/cloud/git/setup-azure.md +++ b/website/docs/docs/cloud/git/setup-azure.md @@ -93,7 +93,7 @@ Once you connect your Azure AD app and Azure DevOps, you need to provide dbt Clo - **Directory(tenant) ID:** Found in the Azure AD App. -Your Azure AD app should now be added to your dbt Cloud Account. People on your team who want to develop in dbt Cloud's IDE can now personally [authorize Azure DevOps from their profiles](/docs/cloud/git/authenticate-azure). +Your Azure AD app should now be added to your dbt Cloud Account. People on your team who want to develop in the dbt Cloud IDE or dbt Cloud CLI can now personally [authorize Azure DevOps from their profiles](/docs/cloud/git/authenticate-azure). ## Connect a service user diff --git a/website/docs/docs/cloud/manage-access/about-access.md b/website/docs/docs/cloud/manage-access/about-access.md index f9f97bc555d..d394c79baa3 100644 --- a/website/docs/docs/cloud/manage-access/about-access.md +++ b/website/docs/docs/cloud/manage-access/about-access.md @@ -2,6 +2,8 @@ title: "About user access in dbt Cloud" description: "Learn how dbt Cloud administrators can use dbt Cloud's permissioning model to control user-level access in a dbt Cloud account." id: "about-user-access" +pagination_next: "docs/cloud/manage-access/seats-and-users" +pagination_prev: null --- :::info "User access" is not "Model access" diff --git a/website/docs/docs/cloud/manage-access/audit-log.md b/website/docs/docs/cloud/manage-access/audit-log.md index 98bf660b259..b90bceef570 100644 --- a/website/docs/docs/cloud/manage-access/audit-log.md +++ b/website/docs/docs/cloud/manage-access/audit-log.md @@ -3,6 +3,8 @@ title: "The audit log for dbt Cloud Enterprise" id: audit-log description: "You can troubleshoot possible issues and provide security audits by reviewing event activity in your organization." sidebar_label: "Audit log" +pagination_next: null +pagination_prev: "docs/cloud/manage-access/about-user-access" --- To review actions performed by people in your organization, dbt provides logs of audited user and system events in real time. The audit log appears as events happen and includes details such as who performed the action, what the action was, and when it was performed. You can use these details to troubleshoot access issues, perform security audits, or analyze specific events. diff --git a/website/docs/docs/cloud/manage-access/cloud-seats-and-users.md b/website/docs/docs/cloud/manage-access/cloud-seats-and-users.md index 04dfbe093c3..24c64a5abed 100644 --- a/website/docs/docs/cloud/manage-access/cloud-seats-and-users.md +++ b/website/docs/docs/cloud/manage-access/cloud-seats-and-users.md @@ -3,6 +3,8 @@ title: "Users and licenses" description: "Learn how dbt Cloud administrators can use licenses and seats to control access in a dbt Cloud account." id: "seats-and-users" sidebar: "Users and licenses" +pagination_next: "docs/cloud/manage-access/self-service-permissions" +pagination_prev: null --- In dbt Cloud, _licenses_ are used to allocate users to your account. There are three different types of licenses in dbt Cloud: @@ -16,6 +18,7 @@ The user's assigned license determines the specific capabilities they can access | Functionality | Developer User | Read-Only Users | IT Users* | | ------------- | -------------- | --------------- | -------- | | Use the dbt Cloud IDE | ✅ | ❌ | ❌ | +| Use the dbt Cloud CLI | ✅ | ❌ | ❌ | | Use Jobs | ✅ | ❌ | ❌ | | Manage Account | ✅ | ❌ | ✅ | | API Access | ✅ | ❌ | ❌ | diff --git a/website/docs/docs/cloud/manage-access/enterprise-permissions.md b/website/docs/docs/cloud/manage-access/enterprise-permissions.md index 5bf3623b105..dcacda20deb 100644 --- a/website/docs/docs/cloud/manage-access/enterprise-permissions.md +++ b/website/docs/docs/cloud/manage-access/enterprise-permissions.md @@ -3,6 +3,7 @@ title: "Enterprise permissions" id: "enterprise-permissions" description: "Permission sets for Enterprise plans." hide_table_of_contents: true #For the sake of the tables on this page +pagination_next: null --- import Permissions from '/snippets/_enterprise-permissions-table.md'; @@ -21,10 +22,6 @@ The following roles and permission sets are available for assignment in dbt Clou -## Diagram of the permission sets - - - ## How to set up RBAC Groups in dbt Cloud Role-Based Access Control (RBAC) is helpful for automatically assigning permissions to dbt admins based on their SSO provider group associations. diff --git a/website/docs/docs/cloud/manage-access/self-service-permissions.md b/website/docs/docs/cloud/manage-access/self-service-permissions.md index 21cc765b76d..d3c9cf8f5ea 100644 --- a/website/docs/docs/cloud/manage-access/self-service-permissions.md +++ b/website/docs/docs/cloud/manage-access/self-service-permissions.md @@ -12,7 +12,8 @@ The permissions afforded to each role are described below: | ------ | ------ | ----- | | View and edit resources | ✅ | ✅ | | Trigger runs | ✅ | ✅ | -| Access the IDE | ✅ | ✅ | +| Access the dbt Cloud IDE | ✅ | ✅ | +| Access the dbt Cloud CLI | ✅ | ✅ | | Invite Members to the account | ✅ | ✅ | | Manage billing | ❌ | ✅ | | Manage team permissions | ❌ | ✅ | diff --git a/website/docs/docs/cloud/manage-access/set-up-bigquery-oauth.md b/website/docs/docs/cloud/manage-access/set-up-bigquery-oauth.md index 516a340c951..87018b14d56 100644 --- a/website/docs/docs/cloud/manage-access/set-up-bigquery-oauth.md +++ b/website/docs/docs/cloud/manage-access/set-up-bigquery-oauth.md @@ -1,7 +1,8 @@ --- title: "Set up BigQuery OAuth" -description: "Learn how dbt Cloud administrators can use licenses and seats to control access in a dbt Cloud account." +description: "Learn how dbt Cloud administrators can use BigQuery OAuth to control access in a dbt Cloud account" id: "set-up-bigquery-oauth" +pagination_next: null --- :::info Enterprise Feature @@ -73,3 +74,7 @@ You will then be redirected to BigQuery and asked to approve the drive, cloud pl Select **Allow**. This redirects you back to dbt Cloud. You should now be an authenticated BigQuery user, ready to use the dbt Cloud IDE. + +## FAQs + + diff --git a/website/docs/docs/cloud/manage-access/set-up-databricks-oauth.md b/website/docs/docs/cloud/manage-access/set-up-databricks-oauth.md new file mode 100644 index 00000000000..679133b7844 --- /dev/null +++ b/website/docs/docs/cloud/manage-access/set-up-databricks-oauth.md @@ -0,0 +1,77 @@ +--- +title: "Set up Databricks OAuth" +description: "Learn how dbt Cloud administrators can use Databricks OAuth to control access in a dbt Cloud account." +id: "set-up-databricks-oauth" +--- + +:::info Enterprise Feature + +This guide describes a feature of the dbt Cloud Enterprise plan. If you’re interested in learning more about an Enterprise plan, contact us at sales@getdbt.com. + +::: + +dbt Cloud supports developer OAuth ([OAuth for partner solutions](https://docs.databricks.com/en/integrations/manage-oauth.html)) with Databricks, providing an additional layer of security for dbt enterprise users. When you enable Databricks OAuth for a dbt Cloud project, all dbt Cloud developers must authenticate with Databricks in order to use the dbt Cloud IDE. The project's deployment environments will still leverage the Databricks authentication method set at the environment level. + +:::tip Beta Feature + +Databricks OAuth support in dbt Cloud is a [beta feature](/docs/dbt-versions/product-lifecycles#dbt-cloud) and subject to change without notification. More updates to this feature coming soon. + +Current limitations: +- Databrick's OAuth applications are in public preview +- The current experience requires the IDE to be restarted every hour (access tokens expire after 1 hour - [workaround](https://docs.databricks.com/en/integrations/manage-oauth.html#override-the-default-token-lifetime-policy-for-dbt-core-power-bi-or-tableau-desktop)) + +::: + +### Configure Databricks OAuth (Databricks admin) + +To get started, you will need to [add dbt as an OAuth application](https://docs.databricks.com/en/integrations/configure-oauth-dbt.html) with Databricks, in 2 steps: + +1. From your terminal, [authenticate to the Databricks Account API](https://docs.databricks.com/en/integrations/configure-oauth-dbt.html#authenticate-to-the-account-api) with the Databricks CLI. You authenticate using: + - OAuth for users ([prerequisites](https://docs.databricks.com/en/dev-tools/auth.html#oauth-u2m-auth)) + - Oauth for service principals ([prerequisites](https://docs.databricks.com/en/dev-tools/auth.html#oauth-m2m-auth)) + - Username and password (must be account admin) +2. In the same terminal, **add dbt Cloud as an OAuth application** using `curl` and the [OAuth Custom App Integration API](https://docs.databricks.com/api/account/customappintegration/create) + +For the second step, you can use this example `curl` to authenticate with your username and password, replacing values as defined in the following table: + +```shell +curl -u USERNAME:PASSWORD https://accounts.cloud.databricks.com/api/2.0/accounts/ACCOUNT_ID/oauth2/custom-app-integrations -d '{"redirect_urls": ["https://YOUR_ACCESS_URL", "https://YOUR_ACCESS_URL/complete/databricks"], "confidential": true, "name": "NAME", "scopes": ["sql", "offline_access"]}' +``` + +These parameters and descriptions will help you authenticate with your username and password: + +| Parameter | Description | +| ------ | ----- | +| **USERNAME** | Your Databricks username (account admin level) | +| **PASSWORD** | Your Databricks password (account admin level) | +| **ACCOUNT_ID** | Your Databricks [account ID](https://docs.databricks.com/en/administration-guide/account-settings/index.html#locate-your-account-id) | +| **YOUR_ACCESS_URL** | The [appropriate Access URL](/docs/cloud/about-cloud/regions-ip-addresses) for your dbt Cloud account region and plan | +| **NAME** | The integration name (i.e 'databricks-dbt-cloud') + +After running the `curl`, you'll get an API response that includes the `client_id` and `client_secret` required in the following section. At this time, this is the only way to retrieve the secret. If you lose the secret, then the integration needs to be [deleted](https://docs.databricks.com/api/account/customappintegration/delete) and re-created. + + +### Configure the Connection in dbt Cloud (dbt Cloud project admin) + +Now that you have an OAuth app set up in Databricks, you'll need to add the client ID and secret to dbt Cloud. To do so: + - go to Settings by clicking the gear in the top right. + - on the left, select **Projects** under **Account Settings** + - choose your project from the list + - select **Connection** to edit the connection details + - add the `OAuth Client ID` and `OAuth Client Secret` from the Databricks OAuth app under the **Optional Settings** section + + + +### Authenticating to Databricks (dbt Cloud IDE developer) + +Once the Databricks connection via OAuth is set up for a dbt Cloud project, each dbt Cloud user will need to authenticate with Databricks in order to use the IDE. To do so: + +- Click the gear icon at the top right and select **Profile settings**. +- Select **Credentials**. +- Choose your project from the list +- Select `OAuth` as the authentication method, and click **Save** +- Finalize by clicking the **Connect Databricks Account** button + + + +You will then be redirected to Databricks and asked to approve the connection. This redirects you back to dbt Cloud. You should now be an authenticated Databricks user, ready to use the dbt Cloud IDE. diff --git a/website/docs/docs/cloud/manage-access/set-up-sso-okta.md b/website/docs/docs/cloud/manage-access/set-up-sso-okta.md index 41381b57eca..5ec70443d1f 100644 --- a/website/docs/docs/cloud/manage-access/set-up-sso-okta.md +++ b/website/docs/docs/cloud/manage-access/set-up-sso-okta.md @@ -16,8 +16,6 @@ dbt Cloud Enterprise supports single-sign on via Okta (using SAML). Currently su * Just-in-time provisioning This guide outlines the setup process for authenticating to dbt Cloud with Okta. -If you have any questions during the setup process, please contact support -(support@getdbt.com) for assistance. ## Configuration in Okta @@ -63,7 +61,7 @@ Click **Next** to continue. ### Configure SAML Settings -The SAML Settings page configures how Okta and dbt Cloud communicate. You will want to use an [appropriate Access URL](/docs/cloud/about-cloud/regions-ip-addresses) for your region and plan. If you aren't sure which values you should use, please contact support (support@getdbt.com). +The SAML Settings page configures how Okta and dbt Cloud communicate. You will want to use an [appropriate Access URL](/docs/cloud/about-cloud/regions-ip-addresses) for your region and plan. To complete this section, you will need a _login slug_. This slug controls the URL where users on your account can log into your application via Okta. Login diff --git a/website/docs/docs/cloud/manage-access/set-up-sso-saml-2.0.md b/website/docs/docs/cloud/manage-access/set-up-sso-saml-2.0.md index be46e965fe3..db3efdbeb74 100644 --- a/website/docs/docs/cloud/manage-access/set-up-sso-saml-2.0.md +++ b/website/docs/docs/cloud/manage-access/set-up-sso-saml-2.0.md @@ -75,7 +75,6 @@ provider to provide group membership information in user attribute called You may use a restricted group attribute statement to limit the groups set to dbt Cloud for each authenticated user. For example, if all of your dbt Cloud groups start with `DBT_CLOUD_...`, you may optionally apply a filter like `Starts With: DBT_CLOUD_`. -Please contact support if you have any questions. ::: ### Collect integration secrets diff --git a/website/docs/docs/cloud/manage-access/sso-overview.md b/website/docs/docs/cloud/manage-access/sso-overview.md index 7e44859c73a..f613df7907e 100644 --- a/website/docs/docs/cloud/manage-access/sso-overview.md +++ b/website/docs/docs/cloud/manage-access/sso-overview.md @@ -1,7 +1,8 @@ --- -title: "SSO Overview" +title: "Single sign-on (SSO) Overview" id: "sso-overview" - +pagination_next: "docs/cloud/manage-access/set-up-sso-saml-2.0" +pagination_prev: null --- This overview explains how users are provisioned in dbt Cloud via Single Sign-On (SSO). diff --git a/website/docs/docs/cloud/secure/about-privatelink.md b/website/docs/docs/cloud/secure/about-privatelink.md index 77ee8a6af7a..29003f65a21 100644 --- a/website/docs/docs/cloud/secure/about-privatelink.md +++ b/website/docs/docs/cloud/secure/about-privatelink.md @@ -15,6 +15,7 @@ dbt Labs has a worldwide network of regional VPCs. These VPCs are specifically u dbt Cloud supports the following data platforms for use with the PrivateLink feature. Instructions for enabling PrivateLink for the various data platform providers are unique. The following guides will walk you through the necessary steps, including working with [dbt Support](https://docs.getdbt.com/community/resources/getting-help#dbt-cloud-support) to complete the connection in the dbt private network and setting up the endpoint in dbt Cloud. -- [Redshift](/docs/cloud/secure/redshift-privatelink) - [Snowflake](/docs/cloud/secure/snowflake-privatelink) - [Databricks](/docs/cloud/secure/databricks-privatelink) +- [Redshift](/docs/cloud/secure/redshift-privatelink) +- [Postgres](/docs/cloud/secure/postgres-privatelink) diff --git a/website/docs/docs/cloud/secure/databricks-privatelink.md b/website/docs/docs/cloud/secure/databricks-privatelink.md index c136cd8a0f9..a2c9e208459 100644 --- a/website/docs/docs/cloud/secure/databricks-privatelink.md +++ b/website/docs/docs/cloud/secure/databricks-privatelink.md @@ -3,6 +3,7 @@ title: "Configuring Databricks PrivateLink" id: databricks-privatelink description: "Configuring PrivateLink for Databricks" sidebar_label: "PrivateLink for Databricks" +pagination_next: null --- The following steps will walk you through the setup of a Databricks AWS PrivateLink endpoint in the dbt Cloud multi-tenant environment. diff --git a/website/docs/docs/cloud/secure/ip-restrictions.md b/website/docs/docs/cloud/secure/ip-restrictions.md index 49a7fe4f267..093d2a1c876 100644 --- a/website/docs/docs/cloud/secure/ip-restrictions.md +++ b/website/docs/docs/cloud/secure/ip-restrictions.md @@ -3,6 +3,8 @@ title: "Configuring IP restrictions" id: ip-restrictions description: "Configuring IP restrictions to outside traffic from accessing your dbt Cloud environment" sidebar_label: "IP restrictions" +pagination_next: "docs/cloud/secure/about-privatelink" +pagination_prev: null --- import SetUpPages from '/snippets/_available-tiers-iprestrictions.md'; @@ -34,7 +36,7 @@ To add an IP to the allowlist, from the **IP Restrictions** page: 4. Select **Allow** 5. Add the ranges in the CIDR notation - For example, 1.1.1.1/8 - - You can add multiple ranges followed by commas + - You cannot add multiple ranges in the same rule. Instead, create a rule per CIDR range. 6. Click **Save** Note that simply adding the IP Ranges will not enforce IP restrictions. For more information, see the section “Enabling Restrictions.” diff --git a/website/docs/docs/cloud/secure/postgres-privatelink.md b/website/docs/docs/cloud/secure/postgres-privatelink.md new file mode 100644 index 00000000000..482aeb4040d --- /dev/null +++ b/website/docs/docs/cloud/secure/postgres-privatelink.md @@ -0,0 +1,76 @@ +--- +title: "Configure AWS PrivateLink for Postgres" +id: postgres-privatelink +description: "Configuring PrivateLink for Postgres" +sidebar_label: "PrivateLink for Postgres" +--- + +A Postgres database, hosted either in AWS or in a properly connected on-prem data center, can be accessed through a private network connection using AWS Interface-type PrivateLink. The type of Target Group connected to the Network Load Balancer (NLB) may vary based on the location and type of Postgres instance being connected, as explained in the following steps. + +## Configuring Postgres interface-type PrivateLink + +### 1. Provision AWS resources + +Creating an Interface VPC PrivateLink connection requires creating multiple AWS resources in the account containing, or connected to, the Postgres instance: + +- **Security Group (AWS hosted only)** — If you are connecting to an existing Postgres instance, this likely already exists, however, you may need to add or modify Security Group rules to accept traffic from the Network Load Balancer (NLB) created for this Endpoint Service. +- **Target Group** — The Target Group will be attached to the NLB to tell it where to route requests. There are various target types available for NLB Target Groups, so choose the one appropriate for your Postgres setup. + + - Target Type: + + - _[Amazon RDS for PostgreSQL](https://aws.amazon.com/rds/postgresql/)_ - **IP** + + - Find the IP address of your RDS instance using a command line tool such as `nslookup ` or `dig +short ` with your RDS DNS endpoint + + - _Note_: With RDS Multi-AZ failover capabilities the IP address of your RDS instance can change, at which point your Target Group would need to be updated. See [this AWS blog post](https://aws.amazon.com/blogs/database/access-amazon-rds-across-vpcs-using-aws-privatelink-and-network-load-balancer/) for more details and a possible solution. + + - _On-prem Postgres server_ - **IP** + + - Use the IP address of the on-prem Postgres server linked to AWS through AWS Direct Connect or a Site-to-Site VPN connection + + - _Postgres on EC2_ - **Instance/ASG** (or **IP**) + + - If your Postgres instance is hosted on EC2 the _instance_ Target Group type (or ideally [using the instance type to connect to an auto-scaling group](https://docs.aws.amazon.com/autoscaling/ec2/userguide/attach-load-balancer-asg.html)) can be used to attach the instance without needing a static IP address + + - The IP type can also be used, with the understanding that the IP of the EC2 instance can change if the instance is relaunched for any reason + + - Target Group protocol: **TCP** + +- **Network Load Balancer (NLB)** — Requires creating a Listener that attaches to the newly created Target Group for port `5432` +- **VPC Endpoint Service** — Attach to the newly created NLB. + - Acceptance required (optional) — Requires you to [accept our connection request](https://docs.aws.amazon.com/vpc/latest/privatelink/configure-endpoint-service.html#accept-reject-connection-requests) after dbt creates the endpoint. + +### 2. Grant dbt AWS account access to the VPC Endpoint Service + +On the provisioned VPC endpoint service, click the **Allow principals** tab. Click **Allow principals** to grant access. Enter the ARN of the root user in the appropriate production AWS account and save your changes. + + - Principal: `arn:aws:iam::346425330055:role/MTPL_Admin` + + + +### 3. Obtain VPC Endpoint Service Name + +Once the VPC Endpoint Service is provisioned, you can find the service name in the AWS console by navigating to **VPC** → **Endpoint Services** and selecting the appropriate endpoint service. You can copy the service name field value and include it in your communication to dbt Cloud support. + + + +### 4. Add the required information to the template below, and submit your request to [dbt Support](https://docs.getdbt.com/community/resources/getting-help#dbt-cloud-support): +``` +Subject: New Multi-Tenant PrivateLink Request +- Type: Postgres Interface-type +- VPC Endpoint Service Name: +- Postgres server AWS Region (e.g., us-east-1, eu-west-2): +- dbt Cloud multi-tenant environment (US, EMEA, AU): +``` + +dbt Labs will work on your behalf to complete the PrivateLink setup. Please allow 1-2 business days for this process to complete. Support will contact you when the endpoint is available. + +## Create Connection in dbt Cloud + +Once dbt Cloud support completes the configuration, you can start creating new connections using PrivateLink. + +1. Navigate to **settings** → **Create new project** → select **PostgreSQL** +2. You will see two radio buttons: **Public** and **Private.** Select **Private**. +3. Select the private endpoint from the dropdown (this will automatically populate the hostname/account field). +4. Configure the remaining data platform details. +5. Test your connection and save it. diff --git a/website/docs/docs/cloud/secure/redshift-privatelink.md b/website/docs/docs/cloud/secure/redshift-privatelink.md index fc0ceeea334..3ed49e7bb34 100644 --- a/website/docs/docs/cloud/secure/redshift-privatelink.md +++ b/website/docs/docs/cloud/secure/redshift-privatelink.md @@ -79,7 +79,7 @@ Creating an Interface VPC PrivateLink connection requires creating multiple AWS On the provisioned VPC endpoint service, click the **Allow principals** tab. Click **Allow principals** to grant access. Enter the ARN of the root user in the appropriate production AWS account and save your changes. - - Principal: `arn:aws:iam::346425330055:root` + - Principal: `arn:aws:iam::346425330055:role/MTPL_Admin` diff --git a/website/docs/docs/cloud/secure/secure-your-tenant.md b/website/docs/docs/cloud/secure/secure-your-tenant.md new file mode 100644 index 00000000000..95cb8adffba --- /dev/null +++ b/website/docs/docs/cloud/secure/secure-your-tenant.md @@ -0,0 +1,49 @@ +--- +title: "Secure your tenant" +description: "Learn how to secure your tenant for dbt Cloud" +pagination_next: "docs/cloud/secure/ip-restrictions" +pagination_prev: null +--- + +
+ + + + + + + +
+
+
+ + + + + + +
\ No newline at end of file diff --git a/website/docs/docs/collaborate/cloud-build-and-view-your-docs.md b/website/docs/docs/collaborate/cloud-build-and-view-your-docs.md index 36f4781bfde..a54b8fcdc2b 100644 --- a/website/docs/docs/collaborate/cloud-build-and-view-your-docs.md +++ b/website/docs/docs/collaborate/cloud-build-and-view-your-docs.md @@ -2,6 +2,7 @@ title: "Build and view your docs with dbt Cloud" id: "build-and-view-your-docs" description: "Automatically generate project documentation as you run jobs." +pagination_next: null --- dbt enables you to generate documentation for your project and data warehouse, and renders the documentation in a website. For more information, see [Documentation](/docs/collaborate/documentation). diff --git a/website/docs/docs/collaborate/collaborate-with-others.md b/website/docs/docs/collaborate/collaborate-with-others.md new file mode 100644 index 00000000000..7875a8044b6 --- /dev/null +++ b/website/docs/docs/collaborate/collaborate-with-others.md @@ -0,0 +1,38 @@ +--- +title: "Collaborate with others" +description: "Learn how dbt Cloud makes it easier to collaborate with others" +pagination_next: "docs/collaborate/explore-projects" +pagination_prev: null +--- + +
+ + + + + +
+
+
+ + + + + +
\ No newline at end of file diff --git a/website/docs/docs/collaborate/documentation.md b/website/docs/docs/collaborate/documentation.md index 429b5187152..0fa00c7cca2 100644 --- a/website/docs/docs/collaborate/documentation.md +++ b/website/docs/docs/collaborate/documentation.md @@ -2,6 +2,8 @@ title: "About documentation" description: "Learn how good documentation for your dbt models helps stakeholders discover and understand your datasets." id: "documentation" +pagination_next: "docs/collaborate/build-and-view-your-docs" +pagination_prev: null --- ## Related documentation diff --git a/website/docs/docs/collaborate/explore-projects.md b/website/docs/docs/collaborate/explore-projects.md index a4c914259ef..b041cd0c915 100644 --- a/website/docs/docs/collaborate/explore-projects.md +++ b/website/docs/docs/collaborate/explore-projects.md @@ -1,25 +1,16 @@ --- -title: "Explore your dbt projects (beta)" -sidebar_label: "Explore dbt projects (beta)" +title: "Explore your dbt projects" +sidebar_label: "Explore dbt projects" description: "Learn about dbt Explorer and how to interact with it to understand, improve, and leverage your data pipelines." +pagination_next: null +pagination_prev: null --- -With dbt Explorer, you can view your project's [resources](/docs/build/projects) (such as models, tests, and metrics) and their lineage to gain a better understanding of its latest production state. Navigate and manage your projects within dbt Cloud to help your data consumers discover and leverage your dbt resources. +With dbt Explorer, you can view your project's [resources](/docs/build/projects) (such as models, tests, and metrics) and their lineage to gain a better understanding of its latest production state. Navigate and manage your projects within dbt Cloud to help you and other data developers, analysts, and consumers discover and leverage your dbt resources. -To display the details about your [project state](/docs/dbt-cloud-apis/project-state), dbt Explorer utilizes the metadata provided through the [Discovery API](/docs/dbt-cloud-apis/discovery-api). The metadata that's available on your project depends on the [deployment environment](/docs/deploy/deploy-environments) you've designated as _production_ in your dbt Cloud project. dbt Explorer automatically retrieves the metadata updates after each job run in the production deployment environment so it will always have the latest state on your project. The metadata it displays depends on the [commands executed by the jobs](/docs/deploy/job-commands). For instance: +:::tip Public preview -- To update model details or results, you must run `dbt run` or `dbt build` on a given model within a job in the environment. -- To view catalog statistics and columns, you must run `dbt docs generate` within a job in the environment. -- To view test results, you must run `dbt test` or `dbt build` within a job in the environment. -- To view source freshness check results, you must run `dbt source freshness` within a job in the environment. - -The need to run these commands will diminish, and richer, more timely metadata will become available as the Discovery API and its underlying platform evolve. - -:::tip Join the beta - -dbt Explorer is a [beta feature](/docs/dbt-versions/product-lifecycles#dbt-cloud) and subject to change without notification. More updates to this feature coming soon. - -If you’re interested in joining the beta, please contact your account team. +Try dbt Explorer! It's available in [Public Preview](/docs/dbt-versions/product-lifecycles#dbt-cloud) as of October 17, 2023 for dbt Cloud customers. More updates coming soon. ::: @@ -28,115 +19,218 @@ If you’re interested in joining the beta, please contact your account team. - You have a [multi-tenant](/docs/cloud/about-cloud/tenancy#multi-tenant) or AWS single-tenant dbt Cloud account on the [Team or Enterprise plan](https://www.getdbt.com/pricing/). - You have set up a [production deployment environment](/docs/deploy/deploy-environments#set-as-production-environment-beta) for each project you want to explore. - There has been at least one successful job run in the production deployment environment. -- You are on the dbt Explorer page. This requires the feature to be enabled for your account. - - To go to the page, select **Explore (Beta)** from the top navigation bar in dbt Cloud. +- You are on the dbt Explorer page. To do this, select **Explore** from the top navigation bar in dbt Cloud. + + +## Generate metadata + +dbt Explorer uses the metadata provided by the [Discovery API](/docs/dbt-cloud-apis/discovery-api) to display the details about [the state of your project](/docs/dbt-cloud-apis/project-state). The metadata that's available depends on the [deployment environment](/docs/deploy/deploy-environments) you've designated as _production_ in your dbt Cloud project. dbt Explorer automatically retrieves the metadata updates after each job run in the production deployment environment so it always has the latest results for your project. + +To view a resource and its metadata, you must define the resource in your project and run a job in the production environment. The resulting metadata depends on the [commands executed by the jobs](/docs/deploy/job-commands). + +For a richer experience with dbt Explorer, you must: + +- Run [dbt run](/reference/commands/run) or [dbt build](/reference/commands/build) on a given model within a job in the environment to update model details or results. +- Run [dbt docs generate](/reference/commands/cmd-docs) within a job in the environment to view catalog statistics and columns for models, sources, and snapshots. +- Run [dbt test](/reference/commands/test) or [dbt build](/reference/commands/build) within a job in the environment to view test results. +- Run [dbt source freshness](/reference/commands/source#dbt-source-freshness) within a job in the environment to view source freshness data. +- Run [dbt snapshot](/reference/commands/snapshot) or [dbt build](/reference/commands/build) within a job in the environment to view snapshot details. + +Richer and more timely metadata will become available as dbt, the Discovery API, and the underlying dbt Cloud platform evolves. -## Explore the project’s lineage +## Explore your project's lineage graph {#project-lineage} -dbt Explorer provides a visualization of your project’s DAG that you can interact with. To start, select **Overview** in the left sidebar and click the **Explore Lineage** button on the main (center) section of the page. +dbt Explorer provides a visualization of your project’s DAG that you can interact with. To access the project's full lineage graph, select **Overview** in the left sidebar and click the **Explore Lineage** button on the main (center) section of the page. -If you don't see the lineage graph immediately, click **Render Lineage**. It can take some time for the graph to render depending on the size of your project and your computer’s available memory. The graph of very large projects might not render so, instead, you can select a subset of nodes by using selectors. +If you don't see the project lineage graph immediately, click **Render Lineage**. It can take some time for the graph to render depending on the size of your project and your computer’s available memory. The graph of very large projects might not render so you can select a subset of nodes by using selectors, instead. -The nodes in the lineage graph represent the project’s resources and the edges represent the relationships between the nodes. Resources like tests and macros display in the lineage within their [resource details pages](#view-resource-details) but not within the overall project lineage graph. Nodes are color-coded and include iconography according to their resource type. +The nodes in the lineage graph represent the project’s resources and the edges represent the relationships between the nodes. Nodes are color-coded and include iconography according to their resource type. -To interact with the lineage graph, you can: +To explore the lineage graphs of tests and macros, view [their resource details pages](#view-resource-details). By default, dbt Explorer excludes these resources from the full lineage graph unless a search query returns them as results. + +To interact with the full lineage graph, you can: - Hover over any item in the graph to display the resource’s name and type. - Zoom in and out on the graph by mouse-scrolling. -- Grab and move the graph. -- Click on a resource to highlight its relationship with other resources in your project. -- [Search and select specific resources](#search-resources) or a subset of the DAG using selectors and lineage (for example, `+[YOUR_RESOURCE_NAME]` displays all nodes upstream of a particular resource). -- [View resource details](#view-resource-details) by selecting a node in the graph (double-clicking). +- Grab and move the graph and the nodes. +- Select a resource to highlight its relationship with other resources in your project. A panel opens on the graph’s right-hand side that displays a high-level summary of the resource’s details. The side panel includes a **General** tab for information like description, materialized type, and other details. + - Click the Share icon in the side panel to copy the graph’s link to your clipboard. + - Click the View Resource icon in the side panel to [view the resource details](#view-resource-details). +- [Search and select specific resources](#search-resources) or a subset of the DAG using selectors and graph operators. For example: + - `+[RESOURCE_NAME]` — Displays all parent nodes of the resource + - `resource_type:model [RESOURCE_NAME]` — Displays all models matching the name search +- [View resource details](#view-resource-details) by selecting a node (double-clicking) in the graph. +- Click the List view icon in the graph's upper right corner to return to the main **Explore** page. - + ## Search for resources {#search-resources} -With the search bar (on the upper left of the page or in a lineage graph), you can search using keywords or selectors (also known as *selector methods*). The resources that match your search criteria will display as a table in the main section of the page. When you select a resource in the table, its [resource details page](#view-resource-details) will display. +With the search bar (on the upper left corner of the page or in a lineage graph), you can search with keywords or by using [node selection syntax](/reference/node-selection/syntax). The resources that match your search criteria will display as a lineage graph and a table in the main section of the page. + +Select a node (single-click) in the lineage graph to highlight its relationship with your other search results and to display which project contains the resource's definition. When you choose a node (double-click) in the lineage graph or when you select a resource in the table, dbt Explorer displays the [resource's details page](#view-resource-details). -When using keyword search, dbt Explorer will search through your resources using metadata such as resource type, resource name, column name, source name, tags, schema, database, version, alias/identifier, and package name. +### Search with keywords +When searching with keywords, dbt Explorer searches through your resource metadata (such as resource type, resource name, column name, source name, tags, schema, database, version, alias/identifier, and package name) and returns any matches. -When using selector search, you can utilize the dbt node selection syntax including set and graph operators (like `+`). To learn more about selectors, refer to [Syntax overview](/reference/node-selection/syntax), [Graph operators](/reference/node-selection/graph-operators), and [Set operators](/reference/node-selection/set-operators). +### Search with selector methods -Below are the selection methods currently available in dbt Explorer. For more information about each of them, refer to [Methods](/reference/node-selection/methods). +You can search with [selector methods](/reference/node-selection/methods). Below are the selectors currently available in dbt Explorer: -- **fqn:** — Find resources by [file or fully qualified name](/reference/node-selection/methods#the-file-or-fqn-method). -- **source:** — Find resources by a specified [source](/reference/node-selection/methods#the-source-method). -- **resource_type:** — Find resources by their [type](/reference/node-selection/methods#the-resource_type-method). -- **package:** — Find resources by the [dbt package](/reference/node-selection/methods#the-package-method) that defines them. -- **tag:** — Find resources by a specified [tag](/reference/node-selection/methods#the-tag-method). +- `fqn:` — Find resources by [file or fully qualified name](/reference/node-selection/methods#the-fqn-method). This selector is the search bar's default. If you want to use the default, it's unnecessary to add `fqn:` before the search term. +- `source:` — Find resources by a specified [source](/reference/node-selection/methods#the-source-method). +- `resource_type:` — Find resources by their [type](/reference/node-selection/methods#the-resource_type-method). +- `package:` — Find resources by the [dbt package](/reference/node-selection/methods#the-package-method) that defines them. +- `tag:` — Find resources by a specified [tag](/reference/node-selection/methods#the-tag-method). -- **group:** — Find models defined within a specified [group](/reference/node-selection/methods#the-group-method). -- **access:** — Find models based on their [access](/reference/node-selection/methods#the-access-method) property. +- `group:` — Find models defined within a specified [group](/reference/node-selection/methods#the-group-method). +- `access:` — Find models based on their [access](/reference/node-selection/methods#the-access-method) property. - +### Search with graph operators + +You can use [graph operators](/reference/node-selection/graph-operators) on keywords or selector methods. For example, `+orders` returns all the parents of `orders`. + +### Search with set operators + +You can use multiple selector methods in your search query with [set operators](/reference/node-selection/set-operators). A space implies a union set operator and a comma for an intersection. For example: +- `resource_type:metric,tag:nightly` — Returns metrics with the tag `nightly` +- `+snowplow_sessions +fct_orders` — Returns resources that are parent nodes of either `snowplow_sessions` or `fct_orders` -## Use the catalog sidebar +### Search with both keywords and selector methods -By default, the catalog sidebar lists all your project’s resources. Select any resource type in the list and all those resources in the project will display as a table in the main section of the page. For a description on the different resource types (like models, metrics, and so on), refer to [About dbt projects](https://docs.getdbt.com/docs/build/projects). +You can use keyword search to highlight results that are filtered by the selector search. For example, if you don't have a resource called `customers`, then `resource_type:metric customers` returns all the metrics in your project and highlights those that are related to the term `customers` in the name, in a column, tagged as customers, and so on. + +When searching in this way, the selectors behave as filters that you can use to narrow the search and keywords as a way to find matches within those filtered results. + + + +## Browse with the sidebar + +By default, the catalog sidebar lists all your project’s resources. Select any resource type in the list and all those resources in the project will display as a table in the main section of the page. For a description on the different resource types (like models, metrics, and so on), refer to [About dbt projects](/docs/build/projects). To browse using a different view, you can choose one of these options from the **View by** dropdown: - **Resources** (default) — All resources in the project organized by type. -- **Packages** — All resources in the project organized by the project in which they are defined. +- **Packages** — All resources in the project organized by the dbt package in which they are defined. - **File Tree** — All resources in the project organized by the file in which they are defined. This mirrors the file tree in your dbt project repository. -- **Database** — All resources in the project organized by the database and schema in which they are built. This mirrors your data platform structure. +- **Database** — All resources in the project organized by the database and schema in which they are built. This mirrors your data platform's structure that represents the [applied state](/docs/dbt-cloud-apis/project-state) of your project. + + - +## View model versions + +If models in the project are versioned, you can see which [version of the model](/docs/collaborate/govern/model-versions) is being applied — `prerelease`, `latest`, and `old` — in the title of the model’s details page and in the model list from the sidebar. ## View resource details {#view-resource-details} -You can view the definition and latest run results of any resource in your project. To find a resource and view its details, you can interact with the lineage graph, use search, or browse the catalog. The details (metadata) available to you depends on the resource’s type, its definition, and the [commands](/docs/deploy/job-commands) run within jobs in the production environment. +You can view the definition and latest run results of any resource in your project. To find a resource and view its details, you can interact with the lineage graph, use search, or browse the catalog. - +The details (metadata) available to you depends on the resource’s type, its definition, and the [commands](/docs/deploy/job-commands) that run within jobs in the production environment. + ### Example of model details An example of the details you might get for a model: -- **General** — The model’s lineage graph that you can interact with. -- **Code** — The source code and compiled code for the model. -- **Columns** — The available columns in the model. -- **Description** — A [description of the model](/docs/collaborate/documentation#adding-descriptions-to-your-project). -- **Recent** — Information on the last time the model ran, how long it ran for, whether the run was successful, the job ID, and the run ID. -- **Tests** — [Tests](/docs/build/tests) for the model. -- **Details** — Key properties like the model’s relation name (for example, how it’s represented and how you can query it in the data platform: `database.schema.identifier`); model governance attributes like access, group, and if contracted; and more. -- **Relationships** — The nodes the model **Depends On** and is **Referenced by.** +- Status bar (below the page title) — Information on the last time the model ran, whether the run was successful, how the data is materialized, number of rows, and the size of the model. +- **General** tab includes: + - **Lineage** graph — The model’s lineage graph that you can interact with. The graph includes one parent node and one child node from the model. Click the Expand icon in the graph's upper right corner to view the model in full lineage graph mode. + - **Description** section — A [description of the model](/docs/collaborate/documentation#adding-descriptions-to-your-project). + - **Recent** section — Information on the last time the model ran, how long it ran for, whether the run was successful, the job ID, and the run ID. + - **Tests** section — [Tests](/docs/build/tests) for the model. + - **Details** section — Key properties like the model’s relation name (for example, how it’s represented and how you can query it in the data platform: `database.schema.identifier`); model governance attributes like access, group, and if contracted; and more. + - **Relationships** section — The nodes the model **Depends On**, is **Referenced by**, and (if applicable) is **Used by** for projects that have declared the models' project as a dependency. +- **Code** tab — The source code and compiled code for the model. +- **Columns** tab — The available columns in the model. This tab also shows tests results (if any) that you can select to view the test's details page. A :white_check_mark: denotes a passing test. + ### Example of exposure details An example of the details you might get for an exposure: -- **Status** — The status on data freshness and data quality. -- **Lineage** — The exposure’s lineage graph. -- **Description** — A description of the exposure. -- **Details** — Details like exposure type, maturity, owner information, and more. -- **Relationships** — The nodes the exposure **Depends On**. +- Status bar (below the page title) — Information on the last time the exposure was updated. +- **General** tab includes: + - **Status** section — The status on data freshness and data quality. + - **Lineage** graph — The exposure’s lineage graph. Click the Expand icon in the graph's upper right corner to view the exposure in full lineage graph mode. + - **Description** section — A description of the exposure. + - **Details** section — Details like exposure type, maturity, owner information, and more. + - **Relationships** section — The nodes the exposure **Depends On**. ### Example of test details An example of the details you might get for a test: -- **General** — The test’s lineage graph that you can interact with. -- **Code** — The source code and compiled code for the test. -- **Description** — A description of the test. -- **Recent** — Information on the last time the test ran, how long it ran for, whether the test passed, the job ID, and the run ID. -- **Details** — Details like schema, severity, package, and more. -- **Relationships** — The nodes the test **Depends On**. +- Status bar (below the page title) — Information on the last time the test ran, whether the test passed, test name, test target, and column name. +- **General** tab includes: + - **Lineage** graph — The test’s lineage graph that you can interact with. The graph includes one parent node and one child node from the test resource. Click the Expand icon in the graph's upper right corner to view the test in full lineage graph mode. + - **Description** section — A description of the test. + - **Recent** section — Information on the last time the test ran, how long it ran for, whether the test passed, the job ID, and the run ID. + - **Details** section — Details like schema, severity, package, and more. + - **Relationships** section — The nodes the test **Depends On**. +- **Code** tab — The source code and compiled code for the test. + ### Example of source details An example of the details you might get for each source table within a source collection: -- **General** — The source’s lineage graph that you can interact with. -- **Columns** — The available columns in the source. -- **Description** — A description of the source. -- **Source freshness** — Information on whether refreshing the data was successful, the last time the source was loaded, the timestamp of when a run generated data, and the run ID. -- **Details** — Details like database, schema, and more. -- **Relationships** — A table that lists all the sources used with their freshness status, the timestamp of when freshness was last checked, and the timestamp of when the source was last loaded. \ No newline at end of file +- Status bar (below the page title) — Information on the last time the source was updated and the number of tables the source uses. +- **General** tab includes: + - **Lineage** graph — The source’s lineage graph that you can interact with. The graph includes one parent node and one child node from the source. Click the Expand icon in the graph's upper right corner to view the source in full lineage graph mode. + - **Description** section — A description of the source. + - **Source freshness** section — Information on whether refreshing the data was successful, the last time the source was loaded, the timestamp of when a run generated data, and the run ID. + - **Details** section — Details like database, schema, and more. + - **Relationships** section — A table that lists all the sources used with their freshness status, the timestamp of when freshness was last checked, and the timestamp of when the source was last loaded. +- **Columns** tab — The available columns in the source. This tab also shows tests results (if any) that you can select to view the test's details page. A :white_check_mark: denotes a passing test. + +## About project-level lineage +You can also view all the different projects and public models in the account, where the public models are defined, and how they are used to gain a better understanding about your cross-project resources. + +When viewing the resource-level lineage graph for a given project that uses cross-project references, you can see cross-project relationships represented in the DAG. The iconography is slightly different depending on whether you're viewing the lineage of an upstream producer project or a downstream consumer project. + +When viewing an upstream (parent) project that produces public models that are imported by downstream (child) projects, public models will have a counter icon in their upper right corner that indicates the number of projects that declare the current project as a dependency. Selecting that model reveals the lineage to show the specific projects that are dependent on this model. Projects show up in this counter if they declare the parent project as a dependency in its `dependencies.yml` regardless of whether or not there's a direct `{{ ref() }}` against the public model. Selecting a project node from a public model opens the resource-level lineage graph for that project, which is subject to your permissions. + + + +When viewing a downstream (child) project that imports and refs public models from upstream (parent) projects, public models will show up in the lineage graph and display an icon on the graph edge that indicates what the relationship is to a model from another project. Hovering over this icon indicates the specific dbt Cloud project that produces that model. Double-clicking on a model from another project opens the resource-level lineage graph of the parent project, which is subject to your permissions. + + + + +### Explore the project-level lineage graph + +For cross-project collaboration, you can interact with the DAG in all the same ways as described in [Explore your project's lineage](#project-lineage) but you can also interact with it at the project level and view the details. + +To get a list view of all the projects, select the account name at the top of the **Explore** page near the navigation bar. This view includes a public model list, project list, and a search bar for project searches. You can also view the project-level lineage graph by clicking the Lineage view icon in the page's upper right corner. + +If you have permissions for a project in the account, you can view all public models used across the entire account. However, you can only view full public model details and private models if you have permissions for a project where the models are defined. + +From the project-level lineage graph, you can: + +- Click the Lineage view icon (in the graph’s upper right corner) to view the cross-project lineage graph. +- Click the List view icon (in the graph’s upper right corner) to view the project list. + - Select a project from the **Projects** tab to switch to that project’s main **Explore** page. + - Select a model from the **Public Models** tab to view the [model’s details page](#view-resource-details). + - Perform searches on your projects with the search bar. +- Select a project node in the graph (double-clicking) to switch to that particular project’s lineage graph. + +When you select a project node in the graph, a project details panel opens on the graph’s right-hand side where you can: + +- View counts of the resources defined in the project. +- View a list of its public models, if any. +- View a list of other projects that uses the project, if any. +- Click **Open Project Lineage** to switch to the project’s lineage graph. +- Click the Share icon to copy the project panel link to your clipboard so you can share the graph with someone. + + + +## Related content +- [Enterprise permissions](/docs/cloud/manage-access/enterprise-permissions) +- [About model governance](/docs/collaborate/govern/about-model-governance) +- [What is data mesh?](https://www.getdbt.com/blog/what-is-data-mesh-the-definition-and-importance-of-data-mesh) blog diff --git a/website/docs/docs/collaborate/git-version-control.md b/website/docs/docs/collaborate/git-version-control.md index 4444f381bb5..392e2c3baa5 100644 --- a/website/docs/docs/collaborate/git-version-control.md +++ b/website/docs/docs/collaborate/git-version-control.md @@ -3,6 +3,8 @@ title: "About git" id: git-version-control description: "Git overview" sidebar_label: "About git" +pagination_next: "docs/collaborate/git/version-control-basics" +pagination_prev: null --- A [version control](https://en.wikipedia.org/wiki/Version_control) system allows you and your teammates to work collaboratively, safely, and simultaneously on a single project. Version control helps you track all the code changes made in your dbt project. @@ -22,3 +24,4 @@ When you develop in the command line interface (CLI) or Cloud integrated develo - [Merge conflicts](/docs/collaborate/git/merge-conflicts) - [Connect to GitHub](/docs/cloud/git/connect-github) - [Connect to GitLab](/docs/cloud/git/connect-gitlab) +- [Connect to Azure DevOps](/docs/cloud/git/connect-azure-devops) diff --git a/website/docs/docs/collaborate/git/managed-repository.md b/website/docs/docs/collaborate/git/managed-repository.md index d7beb38c4f5..db8e9840ccd 100644 --- a/website/docs/docs/collaborate/git/managed-repository.md +++ b/website/docs/docs/collaborate/git/managed-repository.md @@ -15,6 +15,6 @@ To set up a project with a managed repository: 6. Click **Create**. -dbt Cloud will host and manage this repository for you. If in the future you choose to host this repository yourself, you can contact support to have the contents of your repo transferred to you. +dbt Cloud will host and manage this repository for you. If in the future you choose to host this repository elsewhere, you can export the information from dbt Cloud at any time. ** We do not recommend using a managed repository in a production environment. You will not be able to use git features like pull requests which are part of our recommended version control best practices. diff --git a/website/docs/docs/collaborate/git/merge-conflicts.md b/website/docs/docs/collaborate/git/merge-conflicts.md index b109cacb511..c3c19b1e2a1 100644 --- a/website/docs/docs/collaborate/git/merge-conflicts.md +++ b/website/docs/docs/collaborate/git/merge-conflicts.md @@ -1,6 +1,7 @@ --- title: "Merge conflicts" id: "merge-conflicts" +pagination_next: null --- [Merge conflicts](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/addressing-merge-conflicts/about-merge-conflicts) in the [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud) often occur when multiple users are simultaneously making edits to the same section in the same file. This makes it difficult for Git to decide what changes to incorporate in the final merge. diff --git a/website/docs/docs/collaborate/govern/about-model-governance.md b/website/docs/docs/collaborate/govern/about-model-governance.md index efeb2836bc6..bbc430845d2 100644 --- a/website/docs/docs/collaborate/govern/about-model-governance.md +++ b/website/docs/docs/collaborate/govern/about-model-governance.md @@ -2,6 +2,8 @@ title: "About model governance" id: about-model-governance description: "Information about new features related to model governance" +pagination_next: "docs/collaborate/govern/model-access" +pagination_prev: null --- diff --git a/website/docs/docs/collaborate/govern/model-access.md b/website/docs/docs/collaborate/govern/model-access.md index 64b70416a2f..765e833ac0c 100644 --- a/website/docs/docs/collaborate/govern/model-access.md +++ b/website/docs/docs/collaborate/govern/model-access.md @@ -25,7 +25,7 @@ The two concepts will be closely related, as we develop multi-project collaborat ## Related documentation * [`groups`](/docs/build/groups) -* [`access`](/reference/resource-properties/access) +* [`access`](/reference/resource-configs/access) ## Groups diff --git a/website/docs/docs/collaborate/govern/project-dependencies.md b/website/docs/docs/collaborate/govern/project-dependencies.md index 1dbc967e74e..9a1d8b59b68 100644 --- a/website/docs/docs/collaborate/govern/project-dependencies.md +++ b/website/docs/docs/collaborate/govern/project-dependencies.md @@ -3,18 +3,17 @@ title: "Project dependencies" id: project-dependencies sidebar_label: "Project dependencies" description: "Reference public models across dbt projects" +pagination_next: null --- -:::caution Closed Beta - dbt Cloud Enterprise -"Project" dependencies and cross-project `ref` are features of dbt Cloud Enterprise, currently in Closed Beta. To access these features while they are in beta, please contact your account team at dbt Labs. +:::info Available in Public Preview for dbt Cloud Enterprise accounts -**Prerequisites:** In order to add project dependencies and resolve cross-project `ref`, you must: -- Have the feature enabled (speak to your account team) -- Use dbt v1.6 for **both** the upstream ("producer") project and the downstream ("consumer") project. -- Have a deployment environment in the upstream ("producer") project [that is set to be your production environment](/docs/deploy/deploy-environments#set-as-production-environment-beta) -- Have a successful run of the upstream ("producer") project +Project dependencies and cross-project `ref` are features available in [dbt Cloud Enterprise](https://www.getdbt.com/pricing), currently in [Public Preview](/docs/dbt-versions/product-lifecycles#dbt-cloud). + +Enterprise users can use these features by designating a [public model](/docs/collaborate/govern/model-access) and adding a [cross-project ref](#how-to-use-ref). ::: + For a long time, dbt has supported code reuse and extension by installing other projects as [packages](/docs/build/packages). When you install another project as a package, you are pulling in its full source code, and adding it to your own. This enables you to call macros and run models defined in that other project. While this is a great way to reuse code, share utility macros, and establish a starting point for common transformations, it's not a great way to enable collaboration across teams and at scale, especially at larger organizations. @@ -23,6 +22,33 @@ This year, dbt Labs is introducing an expanded notion of `dependencies` across m - **Packages** — Familiar and pre-existing type of dependency. You take this dependency by installing the package's full source code (like a software library). - **Projects** — A _new_ way to take a dependency on another project. Using a metadata service that runs behind the scenes, dbt Cloud resolves references on-the-fly to public models defined in other projects. You don't need to parse or run those upstream models yourself. Instead, you treat your dependency on those models as an API that returns a dataset. The maintainer of the public model is responsible for guaranteeing its quality and stability. + +Starting in dbt v1.6 or higher, `packages.yml` has been renamed to `dependencies.yml`. However, if you need use Jinja within your packages config, such as an environment variable for your private package, you need to keep using `packages.yml` for your packages for now. Refer to the [FAQs](#faqs) for more info. + +## Prerequisites + +In order to add project dependencies and resolve cross-project `ref`, you must: +- Use dbt v1.6 or higher for **both** the upstream ("producer") project and the downstream ("consumer") project. +- Have a deployment environment in the upstream ("producer") project [that is set to be your production environment](/docs/deploy/deploy-environments#set-as-production-environment-beta) +- Have a successful run of the upstream ("producer") project +- Have a multi-tenant or single-tenant [dbt Cloud Enterprise](https://www.getdbt.com/pricing) account (Azure ST is not supported but coming soon) + + ## Example As an example, let's say you work on the Marketing team at the Jaffle Shop. The name of your team's project is `jaffle_marketing`: @@ -36,7 +62,7 @@ name: jaffle_marketing As part of your modeling of marketing data, you need to take a dependency on two other projects: -- `dbt_utils` as a [package](#packages-use-case): An collection of utility macros that you can use while writing the SQL for your own models. This package is, open-source public, and maintained by dbt Labs. +- `dbt_utils` as a [package](#packages-use-case): A collection of utility macros that you can use while writing the SQL for your own models. This package is, open-source public, and maintained by dbt Labs. - `jaffle_finance` as a [project use-case](#projects-use-case): Data models about the Jaffle Shop's revenue. This project is private and maintained by your colleagues on the Finance team. You want to select from some of this project's final models, as a starting point for your own work. @@ -66,7 +92,7 @@ When you're building on top of another team's work, resolving the references in - You don't need to mirror any conditional configuration of the upstream project such as `vars`, environment variables, or `target.name`. You can reference them directly wherever the Finance team is building their models in production. Even if the Finance team makes changes like renaming the model, changing the name of its schema, or [bumping its version](/docs/collaborate/govern/model-versions), your `ref` would still resolve successfully. - You eliminate the risk of accidentally building those models with `dbt run` or `dbt build`. While you can select those models, you can't actually build them. This prevents unexpected warehouse costs and permissions issues. This also ensures proper ownership and cost allocation for each team's models. -### Usage +### How to use ref **Writing `ref`:** Models referenced from a `project`-type dependency must use [two-argument `ref`](/reference/dbt-jinja-functions/ref#two-argument-variant), including the project name: @@ -87,6 +113,8 @@ with monthly_revenue as ( **Cycle detection:** Currently, "project" dependencies can only go in one direction, meaning that the `jaffle_finance` project could not add a new model that depends, in turn, on `jaffle_marketing.roi_by_channel`. dbt will check for cycles across projects and raise errors if any are detected. We are considering support for this pattern in the future, whereby dbt would still check for node-level cycles while allowing cycles at the project level. +For more guidance on how to use dbt Mesh, refer to the dedicated [dbt Mesh guide](/guides/best-practices/how-we-mesh/mesh-1-intro). + ### Comparison If you were to instead install the `jaffle_finance` project as a `package` dependency, you would instead be pulling down its full source code and adding it to your runtime environment. This means: @@ -99,4 +127,16 @@ There are a few cases where installing another internal project as a package can - Unified deployments — In a production environment, if the central data platform team of Jaffle Shop wanted to schedule the deployment of models across both `jaffle_finance` and `jaffle_marketing`, they could use dbt's [selection syntax](/reference/node-selection/syntax) to create a new "passthrough" project that installed both projects as packages. - Coordinated changes — In development, if you wanted to test the effects of a change to a public model in an upstream project (`jaffle_finance.monthly_revenue`) on a downstream model (`jaffle_marketing.roi_by_channel`) _before_ introducing changes to a staging or production environment, you can install the `jaffle_finance` package as a package within `jaffle_marketing`. The installation can point to a specific git branch, however, if you find yourself frequently needing to perform end-to-end testing across both projects, we recommend you re-examine if this represents a stable interface boundary. -These are the exceptions, rather than the rule. Installing another team's project as a package adds complexity, latency, and risk of unnecessary costs. By defining clear interface boundaries across teams, by serving one team's public models as "APIs" to another, and by enabling practitioners to develop with a more narrowly-defined scope, we can enable more people to contribute, with more confidence, while requiring less context upfront. +These are the exceptions, rather than the rule. Installing another team's project as a package adds complexity, latency, and risk of unnecessary costs. By defining clear interface boundaries across teams, by serving one team's public models as "APIs" to another, and by enabling practitioners to develop with a more narrowly defined scope, we can enable more people to contribute, with more confidence, while requiring less context upfront. + +## FAQs + +
+Can I define private packages in the dependencies.yml file? + +If you're using private packages with the [git token method](/docs/build/packages#git-token-method), you must define them in the `packages.yml` file instead of the `dependencies.yml` file. This is because conditional rendering (like Jinja-in-yaml) is not supported. +
+ + +## Related docs +- Refer to the [dbt Mesh](/guides/best-practices/how-we-mesh/mesh-1-intro) guide for more guidance on how to use dbt Mesh. diff --git a/website/docs/docs/connect-adapters.md b/website/docs/docs/connect-adapters.md index f45da732abb..77ead34e51d 100644 --- a/website/docs/docs/connect-adapters.md +++ b/website/docs/docs/connect-adapters.md @@ -11,9 +11,9 @@ This section provides more details on different ways you can connect dbt to an a Explore the fastest and most reliable way to deploy dbt using dbt Cloud, a hosted architecture that runs dbt Core across your organization. dbt Cloud lets you seamlessly [connect](/docs/cloud/about-cloud-setup) with a variety of [verified](/docs/supported-data-platforms) data platform providers directly in the dbt Cloud UI. -### Install using the CLI +### Install with dbt Core -Install dbt Core, which is an open-source tool, locally using the CLI. dbt communicates with a number of different data platforms by using a dedicated adapter plugin for each. When you install dbt Core, you'll also need to install the specific adapter for your database, [connect to dbt Core](/docs/core/about-core-setup), and set up a `profiles.yml` file. +Install dbt Core, an open-source tool, locally using the command line. dbt communicates with a number of different data platforms by using a dedicated adapter plugin for each. When you install dbt Core, you'll also need to install the specific adapter for your database, [connect to dbt Core](/docs/core/about-core-setup), and set up a `profiles.yml` file. With a few exceptions [^1], you can install all [Verified adapters](/docs/supported-data-platforms) from PyPI using `pip install adapter-name`. For example to install Snowflake, use the command `pip install dbt-snowflake`. The installation will include `dbt-core` and any other required dependencies, which may include both other dependencies and even other adapter plugins. Read more about [installing dbt](/docs/core/installation). diff --git a/website/docs/docs/contribute-core-adapters.md b/website/docs/docs/contribute-core-adapters.md index 6e66a5d28ff..553361ee1a2 100644 --- a/website/docs/docs/contribute-core-adapters.md +++ b/website/docs/docs/contribute-core-adapters.md @@ -1,6 +1,7 @@ --- title: "Contribute to adapters" id: "contribute-core-adapters" +pagination_next: null --- The dbt Community exists to allow analytics practitioners share their knowledge, help others and collectively to drive forward the discipline of analytics engineering. There are opportunities here for everyone to contribute whether you're at the beginning your analytics engineering journey or you are a seasoned data professional. diff --git a/website/docs/docs/core/about-core-setup.md b/website/docs/docs/core/about-core-setup.md index 0408e529b2d..a4d5ff09ee3 100644 --- a/website/docs/docs/core/about-core-setup.md +++ b/website/docs/docs/core/about-core-setup.md @@ -3,13 +3,15 @@ title: About dbt Core setup id: about-core-setup description: "Configuration settings for dbt Core." sidebar_label: "About dbt Core setup" +pagination_next: "docs/core/about-dbt-core" +pagination_prev: null --- dbt Core is an [open-source](https://github.com/dbt-labs/dbt-core) tool that enables data teams to transform data using analytics engineering best practices. You can install dbt locally in your environment and use dbt Core on the command line. It can communicate with databases through adapters. This section of our docs will guide you through various settings to get started: -- [About the CLI](/docs/core/about-the-cli) +- [About dbt Core](/docs/core/about-dbt-core) - [Installing dbt](/docs/core/installation) - [Connecting to a data platform](/docs/core/connect-data-platform/profiles.yml) - [How to run your dbt projects](/docs/running-a-dbt-project/run-your-dbt-projects) diff --git a/website/docs/docs/core/about-dbt-core.md b/website/docs/docs/core/about-dbt-core.md new file mode 100644 index 00000000000..a35d92420f3 --- /dev/null +++ b/website/docs/docs/core/about-dbt-core.md @@ -0,0 +1,25 @@ +--- +title: "About dbt Core" +id: "about-dbt-core" +sidebar_label: "About dbt Core" +--- + +[dbt Core](https://github.com/dbt-labs/dbt-core) is an open sourced project where you can develop from the command line and run your dbt project. + +To use dbt Core, your workflow generally looks like: + +1. **Build your dbt project in a code editor —** popular choices include VSCode and Atom. + +2. **Run your project from the command line —** macOS ships with a default Terminal program, however you can also use iTerm or the command line prompt within a code editor to execute dbt commands. + +:::info How we set up our computers for working on dbt projects + +We've written a [guide](https://discourse.getdbt.com/t/how-we-set-up-our-computers-for-working-on-dbt-projects/243) for our recommended setup when running dbt projects using dbt Core. + +::: + +If you're using the command line, we recommend learning some basics of your terminal to help you work more effectively. In particular, it's important to understand `cd`, `ls` and `pwd` to be able to navigate through the directory structure of your computer easily. + +You can find more information on installing and setting up the dbt Core [here](/docs/core/installation). + +**Note** — dbt supports a dbt Cloud CLI and dbt Core, both command line interface tools that enable you to run dbt commands. The key distinction is the dbt Cloud CLI is tailored for dbt Cloud's infrastructure and integrates with all its [features](/docs/cloud/about-cloud/dbt-cloud-features). diff --git a/website/docs/docs/core/about-the-cli.md b/website/docs/docs/core/about-the-cli.md deleted file mode 100644 index d05fb514dfa..00000000000 --- a/website/docs/docs/core/about-the-cli.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: "About the CLI" -id: "about-the-cli" -sidebar_label: "About the CLI" ---- - -dbt ships with a command line interface (CLI) for running your dbt project. This way of running dbt and a dbt project is free and open source. - -To use the CLI, your workflow generally looks like: -1. **Build your dbt project in a code editor —** popular choices include VSCode and Atom. - -1. **Run your project from the command line —** macOS ships with a default Terminal program, however you can also use iTerm or the command line prompt within a code editor to execute dbt commands. - -:::info How we set up our computers for working on dbt projects - -We've written a [guide](https://discourse.getdbt.com/t/how-we-set-up-our-computers-for-working-on-dbt-projects/243) for our recommended setup when running dbt projects using the CLI. - -::: - -If you're using the CLI, we recommend learning some basics of your terminal to help you work more effectively. In particular, it's important to understand `cd`, `ls` and `pwd` to be able to navigate through the directory structure of your computer easily. - -You can find more information on installing and setting up the dbt CLI [here](/dbt-cli/cli-overview). diff --git a/website/docs/docs/core/connect-data-platform/about-core-connections.md b/website/docs/docs/core/connect-data-platform/about-core-connections.md index 802e197514c..a85a32cc031 100644 --- a/website/docs/docs/core/connect-data-platform/about-core-connections.md +++ b/website/docs/docs/core/connect-data-platform/about-core-connections.md @@ -4,6 +4,8 @@ id: "about-core-connections" description: "Information about data platform connections in dbt Core" sidebar_label: "About data platform connections in dbt Core" hide_table_of_contents: true +pagination_next: "docs/core/connect-data-platform/profiles.yml" +pagination_prev: null --- dbt Core can connect with a variety of data platform providers including: diff --git a/website/docs/docs/core/connect-data-platform/bigquery-setup.md b/website/docs/docs/core/connect-data-platform/bigquery-setup.md index 7a2a445be3f..4169b782594 100644 --- a/website/docs/docs/core/connect-data-platform/bigquery-setup.md +++ b/website/docs/docs/core/connect-data-platform/bigquery-setup.md @@ -74,10 +74,10 @@ my-bigquery-db: dev: type: bigquery method: oauth - project: [GCP project id] - dataset: [the name of your dbt dataset] # You can also use "schema" here - threads: [1 or more] - [](#optional-configurations): + project: GCP_PROJECT_ID + dataset: DBT_DATASET_NAME # You can also use "schema" here + threads: 4 # Must be a value of 1 or greater + [OPTIONAL_CONFIG](#optional-configurations): VALUE ``` @@ -90,14 +90,7 @@ If you do not specify a `project`/`database` and are using the `oauth` method, d See [docs](https://developers.google.com/identity/protocols/oauth2) on using OAuth 2.0 to access Google APIs. - - - +#### Refresh token Using the refresh token and client information, dbt will mint new access tokens as necessary. @@ -110,21 +103,19 @@ my-bigquery-db: dev: type: bigquery method: oauth-secrets - project: [GCP project id] - dataset: [the name of your dbt dataset] # You can also use "schema" here - threads: [1 or more] - refresh_token: [token] - client_id: [client id] - client_secret: [client secret] - token_uri: [redirect URI] - [](#optional-configurations): + project: GCP_PROJECT_ID + dataset: DBT_DATASET_NAME # You can also use "schema" here + threads: 4 # Must be a value of 1 or greater + refresh_token: TOKEN + client_id: CLIENT_ID + client_secret: CLIENT_SECRET + token_uri: REDIRECT_URI + [OPTIONAL_CONFIG](#optional-configurations): VALUE ``` - - - +#### Temporary token dbt will use the one-time access token, no questions asked. This approach makes sense if you have an external deployment process that can mint new access tokens and update the profile file accordingly. @@ -137,18 +128,15 @@ my-bigquery-db: dev: type: bigquery method: oauth-secrets - project: [GCP project id] - dataset: [the name of your dbt dataset] # You can also use "schema" here - threads: [1 or more] - token: [temporary access token] # refreshed + updated by external process - [](#optional-configurations): + project: GCP_PROJECT_ID + dataset: DBT_DATASET_NAME # You can also use "schema" here + threads: 4 # Must be a value of 1 or greater + token: TEMPORARY_ACCESS_TOKEN # refreshed + updated by external process + [OPTIONAL_CONFIG](#optional-configurations): VALUE ``` - - - ### Service Account File @@ -161,11 +149,11 @@ my-bigquery-db: dev: type: bigquery method: service-account - project: [GCP project id] - dataset: [the name of your dbt dataset] - threads: [1 or more] - keyfile: [/path/to/bigquery/keyfile.json] - [](#optional-configurations): + project: GCP_PROJECT_ID + dataset: DBT_DATASET_NAME + threads: 4 # Must be a value of 1 or greater + keyfile: /PATH/TO/BIGQUERY/keyfile.json + [OPTIONAL_CONFIG](#optional-configurations): VALUE ``` @@ -189,10 +177,10 @@ my-bigquery-db: dev: type: bigquery method: service-account-json - project: [GCP project id] - dataset: [the name of your dbt dataset] - threads: [1 or more] - [](#optional-configurations): + project: GCP_PROJECT_ID + dataset: DBT_DATASET_NAME + threads: 4 # Must be a value of 1 or greater + [OPTIONAL_CONFIG](#optional-configurations): VALUE # These fields come from the service account json keyfile keyfile_json: diff --git a/website/docs/docs/core/connect-data-platform/profiles.yml.md b/website/docs/docs/core/connect-data-platform/profiles.yml.md index 67b0eb15fbe..97254dda1c4 100644 --- a/website/docs/docs/core/connect-data-platform/profiles.yml.md +++ b/website/docs/docs/core/connect-data-platform/profiles.yml.md @@ -3,7 +3,7 @@ title: "About profiles.yml" id: profiles.yml --- -If you're using dbt from the [command line (CLI)](/docs/core/about-the-cli), you'll need a `profiles.yml` file that contains the connection details for your data platform. When you run dbt from the CLI, it reads your `dbt_project.yml` file to find the `profile` name, and then looks for a profile with the same name in your `profiles.yml` file. This profile contains all the information dbt needs to connect to your data platform. +If you're using [dbt Core](/docs/core/about-dbt-core), you'll need a `profiles.yml` file that contains the connection details for your data platform. When you run dbt Core from the command line, it reads your `dbt_project.yml` file to find the `profile` name, and then looks for a profile with the same name in your `profiles.yml` file. This profile contains all the information dbt needs to connect to your data platform. For detailed info, you can refer to the [Connection profiles](/docs/core/connect-data-platform/connection-profiles). diff --git a/website/docs/docs/core/connect-data-platform/trino-setup.md b/website/docs/docs/core/connect-data-platform/trino-setup.md index 396634dc6e6..39d8ed8ab3f 100644 --- a/website/docs/docs/core/connect-data-platform/trino-setup.md +++ b/website/docs/docs/core/connect-data-platform/trino-setup.md @@ -83,7 +83,7 @@ The following profile fields are optional to set up. They let you configure your | Profile field | Example | Description | | ----------------------------- | -------------------------------- | ----------------------------------------------------------------------------------------------------------- | | `threads` | `8` | How many threads dbt should use (default is `1`) | -| `roles` | `system: analyst` | Catalog roles | +| `roles` | `system: analyst` | Catalog roles can be set under the optional `roles` parameter using the following format: `catalog: role`. | | `session_properties` | `query_max_run_time: 4h` | Sets Trino session properties used in the connection. Execute `SHOW SESSION` to see available options | | `prepared_statements_enabled` | `true` or `false` | Enable usage of Trino prepared statements (used in `dbt seed` commands) (default: `true`) | | `retries` | `10` | Configure how many times all database operation is retried when connection issues arise (default: `3`) | diff --git a/website/docs/docs/core/connect-data-platform/upsolver-setup.md b/website/docs/docs/core/connect-data-platform/upsolver-setup.md index 68cfa3045cd..6b2f410fc07 100644 --- a/website/docs/docs/core/connect-data-platform/upsolver-setup.md +++ b/website/docs/docs/core/connect-data-platform/upsolver-setup.md @@ -14,6 +14,7 @@ meta: slack_channel_link: 'https://join.slack.com/t/upsolvercommunity/shared_invite/zt-1zo1dbyys-hj28WfaZvMh4Z4Id3OkkhA' platform_name: 'Upsolver' config_page: '/reference/resource-configs/upsolver-configs' +pagination_next: null ---

Overview of {frontMatter.meta.pypi_package}

diff --git a/website/docs/docs/core/dbt-core-environments.md b/website/docs/docs/core/dbt-core-environments.md index 5daf17bddf9..c7f340557fd 100644 --- a/website/docs/docs/core/dbt-core-environments.md +++ b/website/docs/docs/core/dbt-core-environments.md @@ -1,6 +1,7 @@ --- title: "dbt Core environments" id: "dbt-core-environments" +pagination_next: "docs/running-a-dbt-project/run-your-dbt-projects" --- dbt makes it easy to maintain separate production and development environments through the use of [targets](/reference/dbt-jinja-functions/target.md) within a [profile](/docs/core/connect-data-platform/profiles.yml). A typical profile, when using dbt locally (for example, running from your command line), will have a target named `dev` and have this set as the default. This means that while making changes, your objects will be built in your _development_ target without affecting production queries made by your end users. Once you are confident in your changes, you can deploy the code to _production_, by running your dbt project with a _prod_ target. diff --git a/website/docs/docs/core/installation-overview.md b/website/docs/docs/core/installation-overview.md index f1fdb800fdf..25628b76358 100644 --- a/website/docs/docs/core/installation-overview.md +++ b/website/docs/docs/core/installation-overview.md @@ -2,6 +2,8 @@ title: "About installing dbt" id: "installation" description: "You can install dbt Core using a few different tested methods." +pagination_next: "docs/core/homebrew-install" +pagination_prev: null --- You can install dbt Core on the command line by using one of these methods: @@ -11,6 +13,14 @@ You can install dbt Core on the command line by using one of these methods: - [Use a Docker image to install dbt](/docs/core/docker-install) - [Install dbt from source](/docs/core/source-install) +:::tip Pro tip: Using the --help flag + +Most command-line tools, including dbt, have a `--help` flag that you can use to show available commands and arguments. For example, you can use the `--help` flag with dbt in two ways:

+— `dbt --help`: Lists the commands available for dbt
+— `dbt run --help`: Lists the flags available for the `run` command + +::: + ## Upgrading dbt Core dbt provides a number of resources for understanding [general best practices](/blog/upgrade-dbt-without-fear) while upgrading your dbt project as well as detailed [migration guides](/guides/migration/versions/upgrading-to-v1.4) highlighting the changes required for each minor and major release, and [core versions](/docs/dbt-versions/core) diff --git a/website/docs/docs/core/pip-install.md b/website/docs/docs/core/pip-install.md index a35ad5f0d77..44fac00e493 100644 --- a/website/docs/docs/core/pip-install.md +++ b/website/docs/docs/core/pip-install.md @@ -5,7 +5,7 @@ description: "You can use pip to install dbt Core and adapter plugins from the c You need to use `pip` to install dbt Core on Windows or Linux operating systems. You can use `pip` or [Homebrew](/docs/core/homebrew-install) for installing dbt Core on a MacOS. -You can install dbt Core and plugins using `pip` because they are Python modules distributed on [PyPI](https://pypi.org/project/dbt/). +You can install dbt Core and plugins using `pip` because they are Python modules distributed on [PyPI](https://pypi.org/project/dbt-core/). diff --git a/website/docs/docs/core/source-install.md b/website/docs/docs/core/source-install.md index be9918223fe..42086159c03 100644 --- a/website/docs/docs/core/source-install.md +++ b/website/docs/docs/core/source-install.md @@ -1,6 +1,7 @@ --- title: "Install from source" description: "You can install dbt Core from its GitHub code source." +pagination_next: null --- dbt Core and almost all of its adapter plugins are open source software. As such, the codebases are freely available to download and build from source. You might install from source if you want the latest code or want to install dbt from a specific commit. This might be helpful when you are contributing changes, or if you want to debug a past change. diff --git a/website/docs/docs/dbt-cloud-apis/admin-cloud-api.md b/website/docs/docs/dbt-cloud-apis/admin-cloud-api.md index 8a5712f40df..168ec0c80f4 100644 --- a/website/docs/docs/dbt-cloud-apis/admin-cloud-api.md +++ b/website/docs/docs/dbt-cloud-apis/admin-cloud-api.md @@ -1,6 +1,7 @@ --- title: "dbt Cloud Administrative API" id: "admin-cloud-api" +pagination_next: "docs/dbt-cloud-apis/discovery-api" --- The dbt Cloud Administrative API is enabled by default for [Team and Enterprise plans](https://www.getdbt.com/pricing/). It can be used to: diff --git a/website/docs/docs/dbt-cloud-apis/apis-overview.md b/website/docs/docs/dbt-cloud-apis/apis-overview.md index b7d722747d8..eef64992af9 100644 --- a/website/docs/docs/dbt-cloud-apis/apis-overview.md +++ b/website/docs/docs/dbt-cloud-apis/apis-overview.md @@ -2,6 +2,8 @@ title: "APIs Overview" description: "Learn how dbt accounts on the Team and Enterprise plans can query the dbt Cloud APIs." id: "overview" +pagination_next: "docs/dbt-cloud-apis/user-tokens" +pagination_prev: null --- ## Overview diff --git a/website/docs/docs/dbt-cloud-apis/authentication.md b/website/docs/docs/dbt-cloud-apis/authentication.md new file mode 100644 index 00000000000..7deadd68f18 --- /dev/null +++ b/website/docs/docs/dbt-cloud-apis/authentication.md @@ -0,0 +1,22 @@ +--- +title: "Authentication" +description: "Learn how to authenticate with user tokens and service account tokens " +pagination_next: "docs/dbt-cloud-apis/user-tokens" +pagination_prev: null +--- + +
+ + + + + +
\ No newline at end of file diff --git a/website/docs/docs/dbt-cloud-apis/discovery-api.md b/website/docs/docs/dbt-cloud-apis/discovery-api.md index e4441aa55a2..747128cf7bc 100644 --- a/website/docs/docs/dbt-cloud-apis/discovery-api.md +++ b/website/docs/docs/dbt-cloud-apis/discovery-api.md @@ -1,5 +1,6 @@ --- title: "About the Discovery API" +pagination_next: "docs/dbt-cloud-apis/discovery-use-cases-and-examples" --- Every time dbt Cloud runs a project, it generates and stores information about the project. The metadata includes details about your project’s models, sources, and other nodes along with their execution results. With the dbt Cloud Discovery API, you can query this comprehensive information to gain a better understanding of your DAG and the data it produces. diff --git a/website/docs/docs/dbt-cloud-apis/discovery-querying.md b/website/docs/docs/dbt-cloud-apis/discovery-querying.md index ba1365e632b..35c092adb4b 100644 --- a/website/docs/docs/dbt-cloud-apis/discovery-querying.md +++ b/website/docs/docs/dbt-cloud-apis/discovery-querying.md @@ -2,6 +2,7 @@ title: "Query the Discovery API" id: "discovery-querying" sidebar_label: "Query the Discovery API" +pagination_next: "docs/dbt-cloud-apis/discovery-schema-environment" --- The Discovery API supports ad-hoc queries and integrations. If you are new to the API, refer to [About the Discovery API](/docs/dbt-cloud-apis/discovery-api) for an introduction. diff --git a/website/docs/docs/dbt-cloud-apis/schema-discovery-job.mdx b/website/docs/docs/dbt-cloud-apis/schema-discovery-job.mdx index bb30786e19d..8b02c5601ad 100644 --- a/website/docs/docs/dbt-cloud-apis/schema-discovery-job.mdx +++ b/website/docs/docs/dbt-cloud-apis/schema-discovery-job.mdx @@ -2,6 +2,8 @@ title: "Job object schema" sidebar_label: "Job" id: "discovery-schema-job" +pagination_next: "docs/dbt-cloud-apis/discovery-schema-job-model" +pagination_prev: null --- import { QueryArgsTable, SchemaTable } from "./schema"; diff --git a/website/docs/docs/dbt-cloud-apis/sl-api-overview.md b/website/docs/docs/dbt-cloud-apis/sl-api-overview.md index efe54cbd833..3ddbf76d152 100644 --- a/website/docs/docs/dbt-cloud-apis/sl-api-overview.md +++ b/website/docs/docs/dbt-cloud-apis/sl-api-overview.md @@ -1,9 +1,10 @@ --- -title: "Semantic Layer API" +title: "Semantic Layer APIs" id: sl-api-overview -description: "Integrate and query using the Semantic Layer API." +description: "Integrate and query metrics and dimensions in downstream tools using the Semantic Layer APIs" tags: [Semantic Layer, API] hide_table_of_contents: true +pagination_next: "docs/dbt-cloud-apis/sl-jdbc" --- @@ -31,14 +32,11 @@ You can use the dbt Semantic Layer for a variety of tools and applications of da import Features from '/snippets/_sl-plan-info.md' -

-
+
- "} ``` -Each GQL request also comes with a dbt Cloud environmentId. The API uses both the service token in the header and environmentId for authentication. +Each GQL request also requires a dbt Cloud `environmentId`. The API uses both the service token in the header and environmentId for authentication. + +### Metadata calls -### Metric metadata calls +**Fetch data platform dialect** -Use the following example calls to provide you with an idea of the types of commands you can use: +In some cases in your application, it may be useful to know the dialect or data platform that's internally used for the dbt Semantic Layer connection (such as if you are building `where` filters from a user interface rather than user-inputted SQL). + +The GraphQL API has an easy way to fetch this with the following query: + +```graphql +{ + environmentInfo(environmentId: BigInt!) { + dialect + } +} +``` **Fetch available metrics** ```graphql -metrics(environmentId: Int!): [Metric!]! +metrics(environmentId: BigInt!): [Metric!]! ``` **Fetch available dimensions for metrics** ```graphql dimensions( -environmentId: Int! -metrics: [String!]! + environmentId: BigInt! + metrics: [MetricInput!]! ): [Dimension!]! ``` -**Fetch available time granularities given metrics** +**Fetch available granularities given metrics** + +Note: This call for `queryableGranularities` returns only queryable granularities for metric time - the primary time dimension across all metrics selected. ```graphql queryableGranularities( -environmentId: Int! -metrics: [String!]! + environmentId: BigInt! + metrics: [MetricInput!]! ): [TimeGranularity!]! ``` -**Fetch available metrics given a set of a dimensions** +You can also get queryable granularities for all other dimensions using the `dimensions` call: + +```graphql +{ + dimensions(environmentId: BigInt!, metrics:[{name:"order_total"}]) { + name + queryableGranularities # --> ["DAY", "WEEK", "MONTH", "QUARTER", "YEAR"] + } +} +``` + +You can also optionally access it from the metrics endpoint: + +```graphql +{ + metrics(environmentId: BigInt!) { + name + dimensions { + name + queryableGranularities + } + } +} +``` + +**Fetch measures** + +```graphql +{ + measures(environmentId: BigInt!, metrics: [{name:"order_total"}]) { + name + aggTimeDimension + } +} +``` + +`aggTimeDimension` tells you the name of the dimension that maps to `metric_time` for a given measure. You can also query `measures` from the `metrics` endpoint, which allows you to see what dimensions map to `metric_time` for a given metric: + +```graphql +{ + metrics(environmentId: BigInt!) { + measures { + name + aggTimeDimension + } + } +} +``` + +**Fetch available metrics given a set of dimensions** ```graphql metricsForDimensions( -environmentId: Int! -dimensions: [String!]! + environmentId: BigInt! + dimensions: [GroupByInput!]! ): [Metric!]! ``` -**Fetch dimension values for metrics and a given dimension** +**Create Dimension Values query** ```graphql -dimensionValues( -environmentId: Int! -metrics: [String!]! -dimension: String! -``` -### Metric value query parameters +mutation createDimensionValuesQuery( + environmentId: BigInt! + metrics: [MetricInput!] + groupBy: [GroupByInput!]! +): CreateDimensionValuesQueryResult! + +``` -The mutation is `createQuery`. The parameters are as follows: +**Create Metric query** ```graphql createQuery( -environmentId: Int! -metrics: [String!]! -dimensions: [String!] = null -limit: Int = null -startTime: String = null -endTime: String = null -where: String = null -order: [String!] = null -): String + environmentId: BigInt! + metrics: [MetricInput!]! + groupBy: [GroupByInput!] = null + limit: Int = null + where: [WhereInput!] = null + order: [OrderByInput!] = null +): CreateQueryResult +``` + +```graphql +MetricInput { + name: String! +} + +GroupByInput { + name: String! + grain: TimeGranularity = null +} + +WhereInput { + sql: String! +} + +OrderByinput { # -- pass one and only one of metric or groupBy + metric: MetricInput = null + groupBy: GroupByInput = null + descending: Boolean! = false +} +``` + +**Fetch query result** + +```graphql +query( + environmentId: BigInt! + queryId: String! +): QueryResult! +``` + +**Metric Types** + +```graphql +Metric { + name: String! + description: String + type: MetricType! + typeParams: MetricTypeParams! + filter: WhereFilter + dimensions: [Dimension!]! + queryableGranularities: [TimeGranularity!]! +} +``` + +``` +MetricType = [SIMPLE, RATIO, CUMULATIVE, DERIVED] +``` + +**Metric Type parameters** + +```graphql +MetricTypeParams { + measure: MetricInputMeasure + inputMeasures: [MetricInputMeasure!]! + numerator: MetricInput + denominator: MetricInput + expr: String + window: MetricTimeWindow + grainToDate: TimeGranularity + metrics: [MetricInput!] +} ``` + +**Dimension Types** + +```graphql +Dimension { + name: String! + description: String + type: DimensionType! + typeParams: DimensionTypeParams + isPartition: Boolean! + expr: String + queryableGranularities: [TimeGranularity!]! +} +``` + +``` +DimensionType = [CATEGORICAL, TIME] +``` + +### Create Query examples + +The following section provides query examples for the GraphQL API, such as how to query metrics, dimensions, where filters, and more. + +**Query two metrics grouped by time** + +```graphql +mutation { + createQuery( + environmentId: BigInt! + metrics: [{name: "food_order_amount"}] + groupBy: [{name: "metric_time}, {name: "customer__customer_type"}] + ) { + queryId + } +} +``` + +**Query with a time grain** + +```graphql +mutation { + createQuery( + environmentId: BigInt! + metrics: [{name: "order_total"}] + groupBy: [{name: "metric_time", grain: "month"}] + ) { + queryId + } +} +``` + +Note that when using granularity in the query, the output of a time dimension with a time grain applied to it always takes the form of a dimension name appended with a double underscore and the granularity level - `{time_dimension_name}__{DAY|WEEK|MONTH|QUARTER|YEAR}`. Even if no granularity is specified, it will also always have a granularity appended to it and will default to the lowest available (usually daily for most data sources). It is encouraged to specify a granularity when using time dimensions so that there won't be any unexpected results with the output data. + +**Query two metrics with a categorical dimension** + +```graphql +mutation { + createQuery( + environmentId: BigInt! + metrics: [{name: "food_order_amount"}, {name: "order_gross_profit"}] + groupBy: [{name: "metric_time, grain: "month"}, {name: "customer__customer_type"}] + ) { + queryId + } +} +``` + +**Query with a where filter** + +The `where` filter takes a list argument (or a string for a single input). Depending on the object you are filtering, there are a couple of parameters: + + - `Dimension()` — Used for any categorical or time dimensions. If used for a time dimension, granularity is required. For example, `Dimension('metric_time').grain('week')` or `Dimension('customer__country')`. + +- `Entity()` — Used for entities like primary and foreign keys, such as `Entity('order_id')`. + +Note: If you prefer a more strongly typed `where` clause, you can optionally use `TimeDimension()` to separate out categorical dimensions from time ones. The `TimeDimension` input takes the time dimension name and also requires granularity. For example, `TimeDimension('metric_time', 'MONTH')`. + +```graphql +mutation { + createQuery( + environmentId: BigInt! + metrics:[{name: "order_total"}] + groupBy:[{name: "customer__customer_type"}, {name: "metric_time", grain: "month"}] + where:[{sql: "{{ Dimension('customer__customer_type') }} = 'new'"}, {sql:"{{ Dimension('metric_time').grain('month') }} > '2022-10-01'"}] + ) { + queryId + } +} +``` + +**Query with Order** + +```graphql +mutation { + createQuery( + environmentId: BigInt! + metrics: [{name: "order_total"}] + groupBy: [{name: "metric_time", grain: "month"}] + orderBy: [{metric: {name: "order_total"}}, {groupBy: {name: "metric_time", grain: "month"}, descending:true}] + ) { + queryId + } +} +``` + + +**Query with Limit** + +```graphql +mutation { + createQuery( + environmentId: BigInt! + metrics: [{name:"food_order_amount"}, {name: "order_gross_profit"}] + groupBy: [{name:"metric_time, grain: "month"}, {name: "customer__customer_type"}] + limit: 10 + ) { + queryId + } +} +``` + +**Query with Explain** + +This takes the same inputs as the `createQuery` mutation. + +```graphql +mutation { + compileSql( + environmentId: BigInt! + metrics: [{name:"food_order_amount"} {name:"order_gross_profit"}] + groupBy: [{name:"metric_time, grain:"month"}, {name:"customer__customer_type"}] + ) { + sql + } +} +``` + +### Output format and pagination + +**Output format** + +By default, the output is in Arrow format. You can switch to JSON format using the following parameter. However, due to performance limitations, we recommend using the JSON parameter for testing and validation. The JSON received is a base64 encoded string. To access it, you can decode it using a base64 decoder. The JSON is created from pandas, which means you can change it back to a dataframe using `pandas.read_json(json, orient="table")`. Or you can work with the data directly using `json["data"]`, and find the table schema using `json["schema"]["fields"]`. Alternatively, you can pass `encoded:false` to the jsonResult field to get a raw JSON string directly. + + +```graphql +{ + query(environmentId: BigInt!, queryId: Int!, pageNum: Int! = 1) { + sql + status + error + totalPages + arrowResult + jsonResult(orient: PandasJsonOrient! = TABLE, encoded: Boolean! = true) + } +} +``` + +The results default to the table but you can change it to any [pandas](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_json.html) supported value. + +**Pagination** + +By default, we return 1024 rows per page. If your result set exceeds this, you need to increase the page number using the `pageNum` option. + +### Run a Python query + +The `arrowResult` in the GraphQL query response is a byte dump, which isn't visually useful. You can convert this byte data into an Arrow table using any Arrow-supported language. Refer to the following Python example explaining how to query and decode the arrow result: + + +```python +import base64 +import pyarrow as pa + +headers = {"Authorization":"Bearer "} +query_result_request = """ +{ + query(environmentId: 70, queryId: "12345678") { + sql + status + error + arrowResult + } +} +""" + +gql_response = requests.post( + "https://semantic-layer.cloud.getdbt.com/api/graphql", + json={"query": query_result_request}, + headers=headers, +) + +""" +gql_response.json() => +{ + "data": { + "query": { + "sql": "SELECT\n ordered_at AS metric_time__day\n , SUM(order_total) AS order_total\nFROM semantic_layer.orders orders_src_1\nGROUP BY\n ordered_at", + "status": "SUCCESSFUL", + "error": null, + "arrowResult": "arrow-byte-data" + } + } +} +""" + +def to_arrow_table(byte_string: str) -> pa.Table: + """Get a raw base64 string and convert to an Arrow Table.""" + with pa.ipc.open_stream(base64.b64decode(res)) as reader: + return pa.Table.from_batches(reader, reader.schema) + + +arrow_table = to_arrow_table(gql_response.json()["data"]["query"]["arrowResult"]) + +# Perform whatever functionality is available, like convert to a pandas table. +print(arrow_table.to_pandas()) +""" +order_total ordered_at + 3 2023-08-07 + 112 2023-08-08 + 12 2023-08-09 + 5123 2023-08-10 +""" +``` diff --git a/website/docs/docs/dbt-cloud-apis/sl-jdbc.md b/website/docs/docs/dbt-cloud-apis/sl-jdbc.md index c238dcad680..4d0d4f6d1a2 100644 --- a/website/docs/docs/dbt-cloud-apis/sl-jdbc.md +++ b/website/docs/docs/dbt-cloud-apis/sl-jdbc.md @@ -5,7 +5,6 @@ description: "Integrate and use the JDBC API to query your metrics." tags: [Semantic Layer, API] --- - import LegacyInfo from '/snippets/_legacy-sl-callout.md'; @@ -30,6 +29,8 @@ dbt Labs partners can use the JDBC API to build integrations in their tools with If you are a dbt user or partner with access to dbt Cloud and the [dbt Semantic Layer](/docs/use-dbt-semantic-layer/dbt-sl), you can [setup](/docs/use-dbt-semantic-layer/setup-sl) and test this API with data from your own instance by configuring the Semantic Layer and obtaining the right JDBC connection parameters described in this document. +You *may* be able to use our JDBC API with tools that do not have an official integration with the dbt Semantic Layer. If the tool you use allows you to write SQL and either supports a generic JDBC driver option (such as DataGrip) or supports Dremio and uses ArrowFlightSQL driver version 12.0.0 or higher, you can access the Semantic Layer API. + Refer to [Get started with the dbt Semantic Layer](/docs/use-dbt-semantic-layer/quickstart-sl) for more info. ## Authentication @@ -57,11 +58,13 @@ jdbc:arrow-flight-sql://semantic-layer.cloud.getdbt.com:443?&environmentId=20233 ## Querying the API for metric metadata -The Semantic Layer JDBC API has built-in metadata calls which can provide a user with information about their metrics and dimensions. Here are some metadata commands and examples: +The Semantic Layer JDBC API has built-in metadata calls which can provide a user with information about their metrics and dimensions. + +Refer to the following tabs for metadata commands and examples: - + Use this query to fetch all defined metrics in your dbt project: @@ -72,7 +75,7 @@ select * from {{ ``` - + Use this query to fetch all dimensions for a metric. @@ -85,7 +88,7 @@ select * from {{ - + Use this query to fetch dimension values for one or multiple metrics and single dimension. @@ -98,7 +101,7 @@ semantic_layer.dimension_values(metrics=['food_order_amount'], group_by=['custom - + Use this query to fetch queryable granularities for a list of metrics. This API request allows you to only show the time granularities that make sense for the primary time dimension of the metrics (such as `metric_time`), but if you want queryable granularities for other time dimensions, you can use the `dimensions()` call, and find the column queryable_granularities. @@ -111,6 +114,9 @@ select * from {{ + + + @@ -142,9 +148,10 @@ select NAME, QUERYABLE_GRANULARITIES from {{ - + It may be useful in your application to expose the names of the time dimensions that represent `metric_time` or the common thread across all metrics. + You can first query the `metrics()` argument to fetch a list of measures, then use the `measures()` call which will return the name(s) of the time dimensions that make up metric time. ```bash @@ -165,12 +172,13 @@ To query metric values, here are the following parameters that are available: | `metrics` | The metric name as defined in your dbt metric configuration | `metrics=['revenue']` | Required | | `group_by` | Dimension names or entities to group by. We require a reference to the entity of the dimension (other than for the primary time dimension), which is pre-appended to the front of the dimension name with a double underscore. | `group_by=['user__country', 'metric_time']` | Optional | | `grain` | A parameter specific to any time dimension and changes the grain of the data from the default for the metric. | `group_by=[Dimension('metric_time')`
`grain('week\|day\|month\|quarter\|year')]` | Optional | -| `where` | A where clause that allows you to filter on dimensions and entities using parameters - comes with `TimeDimension`, `Dimension`, and `Entity` objects. Granularity is required with `TimeDimension` | `"{{ where=Dimension('customer__country') }} = 'US')"` | Optional | +| `where` | A where clause that allows you to filter on dimensions and entities using parameters. This takes a filter list OR string. Inputs come with `Dimension`, and `Entity` objects. Granularity is required if the `Dimension` is a time dimension | `"{{ where=Dimension('customer__country') }} = 'US')"` | Optional | | `limit` | Limit the data returned | `limit=10` | Optional | -|`order` | Order the data returned | `order_by=['-order_gross_profit']` (remove `-` for ascending order) | Optional | +|`order` | Order the data returned by a particular field | `order_by=['order_gross_profit']`, use `-` for descending, or full object notation if the object is operated on: `order_by=[Metric('order_gross_profit').descending(True)`] | Optional | | `compile` | If true, returns generated SQL for the data platform but does not execute | `compile=True` | Optional | + ## Note on time dimensions and `metric_time` You will notice that in the list of dimensions for all metrics, there is a dimension called `metric_time`. `Metric_time` is a reserved keyword for the measure-specific aggregation time dimensions. For any time-series metric, the `metric_time` keyword should always be available for use in queries. This is a common dimension across *all* metrics in a semantic graph. @@ -244,13 +252,13 @@ select * from {{ Where filters in API allow for a filter list or string. We recommend using the filter list for production applications as this format will realize all benefits from the where possible. -Where filters have the following components that you can use: +Where Filters have a few objects that you can use: - `Dimension()` - This is used for any categorical or time dimensions. If used for a time dimension, granularity is required - `Dimension('metric_time').grain('week')` or `Dimension('customer__country')` -- `TimeDimension()` - This is used for all time dimensions and requires a granularity argument - `TimeDimension('metric_time', 'MONTH)` +- `Entity()` - Used for entities like primary and foreign keys - `Entity('order_id')` -- `Entity()` - This is used for entities like primary and foreign keys - `Entity('order_id')` +Note: If you prefer a more explicit path to create the `where` clause, you can optionally use the `TimeDimension` feature. This helps separate out categorical dimensions from time-related ones. The `TimeDimesion` input takes the time dimension name and also requires granularity, like this: `TimeDimension('metric_time', 'MONTH')`. Use the following example to query using a `where` filter with the string format: @@ -259,7 +267,7 @@ Use the following example to query using a `where` filter with the string format select * from {{ semantic_layer.query(metrics=['food_order_amount', 'order_gross_profit'], group_by=[Dimension('metric_time').grain('month'),'customer__customer_type'], -where="{{ TimeDimension('metric_time', 'MONTH') }} >= '2017-03-09' AND {{ Dimension('customer__customer_type' }} in ('new') AND {{ Entity('order_id') }} = 10") +where="{{ Dimension('metric_time').grain('month') }} >= '2017-03-09' AND {{ Dimension('customer__customer_type' }} in ('new') AND {{ Entity('order_id') }} = 10") }} ``` @@ -269,11 +277,11 @@ Use the following example to query using a `where` filter with a filter list for select * from {{ semantic_layer.query(metrics=['food_order_amount', 'order_gross_profit'], group_by=[Dimension('metric_time').grain('month'),'customer__customer_type'], -where=[{{ TimeDimension('metric_time', 'MONTH')}} >= '2017-03-09', {{ Dimension('customer__customer_type' }} in ('new'), {{ Entity('order_id') }} = 10]) +where=[{{ Dimension('metric_time').grain('month') }} >= '2017-03-09', {{ Dimension('customer__customer_type' }} in ('new'), {{ Entity('order_id') }} = 10]) }} ``` -### Query with a limit and order by +### Query with a limit Use the following example to query using a `limit` or `order_by` clauses: @@ -281,10 +289,55 @@ Use the following example to query using a `limit` or `order_by` clauses: select * from {{ semantic_layer.query(metrics=['food_order_amount', 'order_gross_profit'], group_by=[Dimension('metric_time')], + limit=10) + }} +``` +### Query with Order By Examples + +Order By can take a basic string that's a Dimension, Metric, or Entity and this will default to ascending order + +```bash +select * from {{ +semantic_layer.query(metrics=['food_order_amount', 'order_gross_profit'], + group_by=[Dimension('metric_time')], + limit=10, + order_by=['order_gross_profit'] + }} +``` + +For descending order, you can add a `-` sign in front of the object. However, you can only use this short hand notation if you aren't operating on the object or using the full object notation. + +```bash +select * from {{ +semantic_layer.query(metrics=['food_order_amount', 'order_gross_profit'], + group_by=[Dimension('metric_time')], + limit=10, + order_by=[-'order_gross_profit'] + }} +``` +If you are ordering by an object that's been operated on (e.g., change granularity), or you are using the full object notation, descending order must look like: + +```bash +select * from {{ +semantic_layer.query(metrics=['food_order_amount', 'order_gross_profit'], + group_by=[Dimension('metric_time').grain('week')], limit=10, - order_by=['order_gross_profit']) + order_by=[Metric('order_gross_profit').descending(True), Dimension('metric_time').grain('week').descending(True) ] }} ``` + +Similarly, this will yield ascending order: + +```bash +select * from {{ +semantic_layer.query(metrics=['food_order_amount', 'order_gross_profit'], + group_by=[Dimension('metric_time').grain('week')], + limit=10, + order_by=[Metric('order_gross_profit'), Dimension('metric_time').grain('week')] + }} +``` + + ### Query with compile keyword Use the following example to query using a `compile` keyword: diff --git a/website/docs/docs/dbt-cloud-apis/sl-manifest.md b/website/docs/docs/dbt-cloud-apis/sl-manifest.md index 47304accea3..6ecac495869 100644 --- a/website/docs/docs/dbt-cloud-apis/sl-manifest.md +++ b/website/docs/docs/dbt-cloud-apis/sl-manifest.md @@ -4,6 +4,7 @@ id: sl-manifest description: "Learn about the semantic manifest.json file and how you can use artifacts to gain insights about your dbt Semantic Layer." tags: [Semantic Layer, APIs] sidebar_label: "Semantic manifest" +pagination_next: null --- diff --git a/website/docs/docs/dbt-cloud-apis/user-tokens.md b/website/docs/docs/dbt-cloud-apis/user-tokens.md index e56d8b2f974..77e536b12a5 100644 --- a/website/docs/docs/dbt-cloud-apis/user-tokens.md +++ b/website/docs/docs/dbt-cloud-apis/user-tokens.md @@ -1,6 +1,7 @@ --- title: "User tokens" id: "user-tokens" +pagination_next: "docs/dbt-cloud-apis/service-tokens" --- ## User API tokens @@ -13,7 +14,7 @@ permissions of the user the that they were created for. You can find your User API token in the Profile page under the `API Access` label. - + ## FAQs diff --git a/website/docs/docs/dbt-cloud-environments.md b/website/docs/docs/dbt-cloud-environments.md index f61ec5ef72b..8fa4522d47c 100644 --- a/website/docs/docs/dbt-cloud-environments.md +++ b/website/docs/docs/dbt-cloud-environments.md @@ -2,9 +2,10 @@ title: "dbt Cloud environments" id: "dbt-cloud-environments" description: "Learn about dbt Cloud's development environment to execute your project in the IDE" +pagination_next: null --- -An environment determines how dbt Cloud will execute your project in both the dbt Cloud IDE (for development) and scheduled jobs (for deployment). +An environment determines how dbt Cloud will execute your project in the [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud) or [dbt Cloud CLI](/docs/cloud/cloud-cli-installation) (for development) and scheduled jobs (for deployment). Critically, in order to execute dbt, environments define three variables: @@ -34,7 +35,7 @@ To create a new dbt Cloud development environment: ### Set developer credentials -To use the IDE, each developer will need to set up [personal development credentials](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud#access-the-cloud-ide) to your warehouse connection in their **Profile Settings**. This allows you to set separate target information and maintain individual credentials to connect to your warehouse via the dbt Cloud IDE. +To use the dbt Cloud IDE or dbt Cloud CLI, each developer will need to set up [personal development credentials](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud#access-the-cloud-ide) to your warehouse connection in their **Profile Settings**. This allows you to set separate target information and maintain individual credentials to connect to your warehouse. diff --git a/website/docs/docs/dbt-support.md b/website/docs/docs/dbt-support.md index f63e016b03e..513d5fff588 100644 --- a/website/docs/docs/dbt-support.md +++ b/website/docs/docs/dbt-support.md @@ -1,6 +1,8 @@ --- title: "dbt support" id: "dbt-support" +pagination_next: null +pagination_prev: null --- ## dbt Core support diff --git a/website/docs/docs/dbt-versions/core-versions.md b/website/docs/docs/dbt-versions/core-versions.md index 2a5ce6daeb7..5e8e437f0b1 100644 --- a/website/docs/docs/dbt-versions/core-versions.md +++ b/website/docs/docs/dbt-versions/core-versions.md @@ -2,6 +2,8 @@ title: "About dbt Core versions" id: "core" description: "Learn about semantic versioning for dbt Core, and how long those versions are supported." +pagination_next: "docs/dbt-versions/upgrade-core-in-cloud" +pagination_prev: null --- dbt Core releases follow [semantic versioning](https://semver.org/) guidelines. For more on how we use semantic versions, see [How dbt Core uses semantic versioning](#how-dbt-core-uses-semantic-versioning). diff --git a/website/docs/docs/dbt-versions/experimental-features.md b/website/docs/docs/dbt-versions/experimental-features.md index 5ed0cf037ca..a621bd4ac44 100644 --- a/website/docs/docs/dbt-versions/experimental-features.md +++ b/website/docs/docs/dbt-versions/experimental-features.md @@ -3,6 +3,7 @@ title: "Preview new and experimental features in dbt Cloud" id: "experimental-features" sidebar_label: "Preview new dbt Cloud features" description: "Gain early access to many new dbt Labs experimental features by enabling this in your profile." +pagination_next: null --- dbt Labs often tests experimental features before deciding to continue on the [Product lifecycle](https://docs.getdbt.com/docs/dbt-versions/product-lifecycles#dbt-cloud). diff --git a/website/docs/docs/dbt-versions/release-notes.md b/website/docs/docs/dbt-versions/release-notes.md index db25af163ae..6f7be90e60d 100644 --- a/website/docs/docs/dbt-versions/release-notes.md +++ b/website/docs/docs/dbt-versions/release-notes.md @@ -2,6 +2,8 @@ title: "About dbt Cloud Release Notes" id: "dbt-cloud-release-notes" description: "Release notes for dbt Cloud" +pagination_next: null +pagination_prev: null --- dbt provides release notes for dbt Cloud so you can see recent and historical changes. Generally, you'll see release notes for these changes: diff --git a/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/cloud-cli-pp.md b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/cloud-cli-pp.md new file mode 100644 index 00000000000..d96b82636f8 --- /dev/null +++ b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/cloud-cli-pp.md @@ -0,0 +1,31 @@ +--- +title: "New: dbt Cloud CLI in Public Preview" +description: "October 2023: Learn about the new dbt Cloud CLI development experience, now in public preview," +sidebar_position: 04 +sidebar_label: "New: dbt Cloud CLI in Public Preview" +tags: [Oct-2023, CLI, dbt Cloud] +date: 2023-10-17 +--- + +We are excited to announce the dbt Cloud CLI, **unified command line for dbt**, is available in public preview. It’s a local development experience, powered by dbt Cloud. It’s easy to get started: `pip3 install dbt` or `brew install dbt` and you’re ready to go. + +We will continue to invest in the dbt Cloud IDE as the easiest and most accessible way to get started using dbt, especially for data analysts who have never developed software using the command line before. We will keep improving the speed, stability, and feature richness of the IDE, as we have been [all year long](https://www.getdbt.com/blog/improvements-to-the-dbt-cloud-ide/). + +We also know that many people developing in dbt have a preference for local development, where they can use their favorite terminal, text editor, keybindings, color scheme, and so on. This includes people with data engineering backgrounds, as well as those analytics engineers who started writing code in the dbt Cloud IDE and have expanded their skills. + +The new dbt Cloud CLI offers the best of both worlds, including: + +- The power of developing against the dbt Cloud platform +- The flexibility of your own local setup + +Run whichever community-developed plugins, pre-commit hooks, or other arbitrary scripts you like. + +Some of the unique capabilities of this dbt Cloud CLI include: + +- Automatic deferral of build artifacts to your Cloud project's production environment +- Secure credential storage in the dbt Cloud platform +- Support for dbt Mesh ([cross-project `ref`](/docs/collaborate/govern/project-dependencies)) +- Development workflow for dbt Semantic Layer +- Speedier, lower cost builds + +Refer to [dbt Cloud CLI](/docs/cloud/cloud-cli-installation) to learn more. diff --git a/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/custom-branch-fix-rn.md b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/custom-branch-fix-rn.md new file mode 100644 index 00000000000..06550b7d863 --- /dev/null +++ b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/custom-branch-fix-rn.md @@ -0,0 +1,14 @@ +--- +title: "Fix: Default behavior for CI job runs without a custom branch" +description: "October 2023: CI job runs now default to the main branch of the Git repository when a custom branch isn't set" +sidebar_label: "Fix: Default behavior for CI job runs without a custom branch" +tags: [Oct-2023, CI] +date: 2023-10-06 +sidebar_position: 08 +--- + +If you don't set a [custom branch](/docs/dbt-cloud-environments#custom-branch-behavior) for your dbt Cloud environment, it now defaults to the default branch of your Git repository (for example, `main`). Previously, [CI jobs](/docs/deploy/ci-jobs) would run for pull requests (PRs) that were opened against _any branch_ or updated with new commits if the **Custom Branch** option wasn't set. + +## Azure DevOps + +Your Git pull requests (PRs) might not trigger against your default branch if you're using Azure DevOps and the default branch isn't `main` or `master`. To resolve this, [set up a custom branch](/faqs/Environments/custom-branch-settings) with the branch you want to target. diff --git a/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/dbt-deps-auto-install.md b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/dbt-deps-auto-install.md new file mode 100644 index 00000000000..80963a9d550 --- /dev/null +++ b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/dbt-deps-auto-install.md @@ -0,0 +1,21 @@ +--- +title: "Enhancement: dbt Cloud auto-installs 'dbt deps' on startup" +description: "October 2023 :The dbt Cloud IDE and dbt Cloud CLI auto-handles 'dbt deps' on startup; manual run needed for 'packages.yml' changes. Available for multi-tenant users (single-tenant support coming soon) and applies to all dbt versions." +sidebar_label: "Enhancement: dbt Cloud auto-installs 'dbt deps' on startup" +tags: [Oct-2023, IDE] +date: 2023-10-17 +sidebar_position: 06 +--- + +The dbt Cloud IDE and dbt Cloud CLI now automatically installs `dbt deps` when your environment starts or when necessary. Previously, it would prompt you to run `dbt deps` during initialization. + +This improved workflow is available to all multi-tenant dbt Cloud users (Single-tenant support coming next week) and applies to dbt versions. + +However, you should still run the `dbt deps` command in these situations: + +- When you make changes to the `packages.yml` or `dependencies.yml` file during a session +- When you update the package version in the `packages.yml` or `dependencies.yml` file. +- If you edit the `dependencies.yml` file and the number of packages remains the same, run `dbt deps`. (Note that this is a known bug dbt Labs will fix in the future.) + + + diff --git a/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/explorer-public-preview-rn.md b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/explorer-public-preview-rn.md new file mode 100644 index 00000000000..ebf5add8d03 --- /dev/null +++ b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/explorer-public-preview-rn.md @@ -0,0 +1,13 @@ +--- +title: "New: dbt Explorer Public Preview" +description: "October 2023: dbt Explorer is now available in Public Preview. You can use it to understand, improve, and leverage your dbt projects." +sidebar_label: "New: dbt Explorer Public Preview" +tags: [Oct-2023, Explorer] +date: 2023-10-13 +sidebar_position: 07 +--- + +On Oct 17, 2023, a Public Preview of dbt Explorer will become available to dbt Cloud customers. With dbt Explorer, you can view your project's resources (such as models, tests, and metrics) and their lineage — including interactive DAGs — to gain a better understanding of its latest production state. Navigate and manage your projects within dbt Cloud to help you and other data developers, analysts, and consumers discover and leverage your dbt resources. + +For details, refer to [Explore your dbt projects](/docs/collaborate/explore-projects). + diff --git a/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/native-retry-support-rn.md b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/native-retry-support-rn.md new file mode 100644 index 00000000000..20e56879940 --- /dev/null +++ b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/native-retry-support-rn.md @@ -0,0 +1,15 @@ +--- +title: "Enhancement: Native support for the dbt retry command" +description: "October 2023: Rerun errored jobs from start or from the failure point" +sidebar_label: "Enhancement: Support for dbt retry" +tags: [Oct-2023, Scheduler] +date: 2023-10-06 +sidebar_position: 10 +--- + +Previously in dbt Cloud, you could only rerun an errored job from start but now you can also rerun it from its point of failure. + +You can view which job failed to complete successully, which command failed in the run step, and choose how to rerun it. To learn more, refer to [Retry jobs](/docs/deploy/retry-jobs). + + + diff --git a/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/product-docs-sept-rn.md b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/product-docs-sept-rn.md new file mode 100644 index 00000000000..e669b037d17 --- /dev/null +++ b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/product-docs-sept-rn.md @@ -0,0 +1,38 @@ +--- +title: "September 2023 product docs updates" +id: "product-docs-sept" +description: "September 2023: The Product docs team merged 107 PRs, made various updates to dbt Cloud and Core, such as GAing continuous integration jobs, Semantic Layer GraphQL API doc, a new community plugin, and more" +sidebar_label: "Update: Product docs changes" +tags: [Sept-2023, product-docs] +date: 2023-10-10 +sidebar_position: 09 +--- + +Hello from the dbt Docs team: @mirnawong1, @matthewshaver, @nghi-ly, and @runleonarun! First, we’d like to thank the 15 new community contributors to docs.getdbt.com. We merged [107 PRs](https://github.com/dbt-labs/docs.getdbt.com/pulls?q=is%3Apr+merged%3A2023-09-01..2023-09-31) in September. + +Here's what's new to [docs.getdbt.com](http://docs.getdbt.com/): + +* Migrated docs.getdbt.com from Netlify to Vercel. + +## ☁ Cloud projects +- Continuous integration jobs are now generally available and no longer in beta! +- Added [Postgres PrivateLink set up page](/docs/cloud/secure/postgres-privatelink) +- Published beta docs for [dbt Explorer](/docs/collaborate/explore-projects). +- Added a new Semantic Layer [GraphQL API doc](/docs/dbt-cloud-apis/sl-graphql) and updated the [integration docs](/docs/use-dbt-semantic-layer/avail-sl-integrations) to include Hex. Responded to dbt community feedback and clarified Metricflow use cases for dbt Core and dbt Cloud. +- Added an [FAQ](/faqs/Git/git-migration) describing how to migrate from one git provider to another in dbt Cloud. +- Clarified an example and added a [troubleshooting section](/docs/cloud/connect-data-platform/connect-snowflake#troubleshooting) to Snowflake connection docs to address common errors and provide solutions. + + +## 🎯 Core projects + +- Deprecated dbt Core v1.0 and v1.1 from the docs. +- Added configuration instructions for the [AWS Glue](/docs/core/connect-data-platform/glue-setup) community plugin. +- Revised the dbt Core quickstart, making it easier to follow. Divided this guide into steps that align with the [other guides](/quickstarts/manual-install?step=1). + +## New 📚 Guides, ✏️ blog posts, and FAQs + +Added a [style guide template](/guides/best-practices/how-we-style/6-how-we-style-conclusion#style-guide-template) that you can copy & paste to make sure you adhere to best practices when styling dbt projects! + +## Upcoming changes + +Stay tuned for a flurry of releases in October and a filterable guides section that will make guides easier to find! diff --git a/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/sl-ga.md b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/sl-ga.md new file mode 100644 index 00000000000..5e53363f62a --- /dev/null +++ b/website/docs/docs/dbt-versions/release-notes/03-Oct-2023/sl-ga.md @@ -0,0 +1,29 @@ +--- +title: "Update: dbt Cloud Semantic Layer is Generally Available" +description: "October 2023: dbt Cloud Semantic Layer is Generally Available for all users" +sidebar_label: "Update: dbt Cloud Semantic Layer is GA" +sidebar_position: 05 +date: 2023-10-17 +tags: [Oct-2023] +--- + +:::important +If you're using the legacy Semantic Layer, we **highly** recommend you [upgrade your dbt version](/docs/dbt-versions/upgrade-core-in-cloud) to dbt v1.6 or higher and [migrate](/guides/migration/sl-migration) to the latest Semantic Layer. +::: + +dbt Labs is thrilled to announce that the [dbt Semantic Layer](/docs/use-dbt-semantic-layer/dbt-sl) is now generally available. It offers consistent data organization, improved governance, reduced costs, enhanced efficiency, and accessible data for better decision-making and collaboration across organizations. + +It aims to bring the best of modeling and semantics to downstream applications by introducing: + +- Brand new [integrations](/docs/use-dbt-semantic-layer/avail-sl-integrations) such as Tableau, Google Sheets, Hex, Mode, and Lightdash. +- New [Semantic Layer APIs](/docs/dbt-cloud-apis/sl-api-overview) using GraphQL and JDBC to query metrics and build integrations. +- dbt Cloud [multi-tenant regional](/docs/cloud/about-cloud/regions-ip-addresses) support for North America, EMEA, and APAC. Single-tenant support coming soon. +- Use the APIs to call an export (a way to build tables in your data platform), then access them in your preferred BI tool. Starting from dbt v1.7 or higher, you will be able to schedule exports as part of your dbt job. + + + +The dbt Semantic Layer is available to [dbt Cloud Team or Enterprise](https://www.getdbt.com/) multi-tenant plans on dbt v1.6 or higher. +- Team and Enterprise customers can use 1,000 Queried Units per month for no additional cost on a limited trial basis, subject to reasonable use limitations. Refer to [Billing](/docs/cloud/billing#what-counts-as-a-query-unit) for more information. +- dbt Cloud Developer plans and dbt Core users can define metrics but won't be able to query them with integrated tools. + + diff --git a/website/docs/docs/dbt-versions/release-notes/04-Sept-2023/ci-updates-phase2-rn.md b/website/docs/docs/dbt-versions/release-notes/04-Sept-2023/ci-updates-phase2-rn.md index fefa07e6d6c..fd2d163b748 100644 --- a/website/docs/docs/dbt-versions/release-notes/04-Sept-2023/ci-updates-phase2-rn.md +++ b/website/docs/docs/dbt-versions/release-notes/04-Sept-2023/ci-updates-phase2-rn.md @@ -1,19 +1,19 @@ --- title: "Update: Improvements to dbt Cloud continuous integration" -description: "September 2023: dbt Cloud now has two types of jobs — deploy jobs and CI jobs — with streamlined setup and improved efficiency. " +description: "September 2023: dbt Cloud now has two types of jobs -- deploy jobs and CI jobs -- with streamlined setup and improved efficiency. " sidebar_label: "Update: Improvements to dbt jobs" tags: [Sept-2023, CI] date: 2023-09-11 sidebar_position: 10 --- -dbt Cloud now has two distinct job types: [deploy jobs](/docs/deploy/deploy-jobs) for building production data assets, and [CI jobs](/docs/deploy/ci-jobs) for checking code changes. These jobs perform fundamentally different tasks so dbt Labs improved the setup experience with better defaults for each. +dbt Cloud now has two distinct job types: [deploy jobs](/docs/deploy/deploy-jobs) for building production data assets, and [continuous integration (CI) jobs](/docs/deploy/ci-jobs) for checking code changes. These jobs perform fundamentally different tasks so dbt Labs improved the setup experience with better defaults for each. With two types of jobs, instead of one generic type, we can better guide you through the setup flow. Best practices are built into the default settings so you can go from curious to being set up in seconds. - + -And, we now have more efficient state comparisons on CI checks: never waste a build or test on code that hasn’t been changed. We now diff between the Git PR code and what’s running in production more efficiently with the introduction of deferral to an environment versus a job. To learn more, refer to [Continuous integration in dbt Cloud](/docs/deploy/continuous-integration) and [Get started with continuous integration tests](/guides/orchestration/set-up-ci/overview). +And, we now have more efficient state comparisons on CI checks: never waste a build or test on code that hasn’t been changed. We now diff between the Git pull request (PR) code and what’s running in production more efficiently with the introduction of deferral to an environment versus a job. To learn more, refer to [Continuous integration in dbt Cloud](/docs/deploy/continuous-integration). Below is a comparison table that describes how deploy jobs and CI jobs behave differently: @@ -29,4 +29,14 @@ Below is a comparison table that describes how deploy jobs and CI jobs behave di ## What you need to update -If you previously set up a job using the [Create Job](/dbt-cloud/api-v2#/operations/Create%20Job) API endpoint before September 11, 2023, you must re-create the job as described in [Trigger a CI job with the API](/docs/deploy/ci-jobs#trigger-a-ci-job-with-the-api). This is because you must set the `job_type` to be `ci`. \ No newline at end of file +- If you want to set up a CI environment for your jobs, dbt Labs recommends that you create your CI job in a dedicated [deployment environment](/docs/deploy/deploy-environments#create-a-deployment-environment) that's connected to a staging database. To learn more about these environment best practices, refer to the guide [Get started with continuous integration tests](/guides/orchestration/set-up-ci/overview). + +- If you had set up a CI job before October 2, 2023, the job might've been misclassified as a deploy job with this update. Below describes how to fix the job type: + + If you used the [Create Job](/dbt-cloud/api-v2#/operations/Create%20Job) API endpoint but didn't set `"triggers":triggers.git_provider_webhook`, the job was misclassified as a deploy job and you must re-create it as described in [Trigger a CI job with the API](/docs/deploy/ci-jobs#trigger-a-ci-job-with-the-api). + + If you used the dbt Cloud UI but didn't enable the **Run on Pull Requests** option that was in the **Continuous Integration** (CI) tab, the job was misclassified as a deploy job and you must re-create it as described in [Set up CI jobs](/docs/deploy/ci-jobs#set-up-ci-jobs). + + To check for the job type, review your CI jobs in dbt Cloud's [Run History](/docs/deploy/run-visibility#run-history) and check for the **CI Job** tag below the job name. If it doesn't have this tag, it was misclassified and you need to re-create the job. + + diff --git a/website/docs/docs/deploy/ci-jobs.md b/website/docs/docs/deploy/ci-jobs.md index fb603e2864e..d10bc780fc2 100644 --- a/website/docs/docs/deploy/ci-jobs.md +++ b/website/docs/docs/deploy/ci-jobs.md @@ -27,6 +27,7 @@ To make CI job creation easier, many options on the **CI job** page are set to d - **Job Name** — Specify the name for this CI job. - **Environment** — By default, it’s set to the environment you created the CI job from. - **Triggered by pull requests** — By default, it’s enabled. Every time a developer opens up a pull request or pushes a commit to an existing pull request, this job will get triggered to run. + - **Run on Draft Pull Request** — Enable this option if you want to also trigger the job to run every time a developer opens up a draft pull request or pushes a commit to that draft pull request. 3. Options in the **Execution Settings** section: - **Commands** — By default, it includes the `dbt build --select state:modified+` command. This informs dbt Cloud to build only new or changed models and their downstream dependents. Importantly, state comparison can only happen when there is a deferred environment selected to compare state to. Click **Add command** to add more [commands](/docs/deploy/job-commands) that you want to be invoked when this job runs. @@ -62,13 +63,13 @@ If you're not using dbt Cloud’s native Git integration with [GitHub](/docs/cl 1. Set up a CI job with the [Create Job](/dbt-cloud/api-v2#/operations/Create%20Job) API endpoint using `"job_type": ci` or from the [dbt Cloud UI](#set-up-ci-jobs). -1. Call the [Trigger Job Run](/dbt-cloud/api-v2#/operations/Trigger%20Job%20Run) API endpoint to trigger the CI job. Provide the pull request (PR) ID to the payload using one of these fields, even if you're using a different Git provider (like Bitbucket): +1. Call the [Trigger Job Run](/dbt-cloud/api-v2#/operations/Trigger%20Job%20Run) API endpoint to trigger the CI job. You must include these fields to the payload: + - Provide the pull request (PR) ID with one of these fields, even if you're using a different Git provider (like Bitbucket). This can make your code less human-readable but it will _not_ affect dbt functionality. - - `github_pull_request_id` - - `gitlab_merge_request_id` - - `azure_devops_pull_request_id`  - - This can make your code less human-readable but it will _not_ affect dbt functionality. + - `github_pull_request_id` + - `gitlab_merge_request_id` + - `azure_devops_pull_request_id`  + - Provide the `git_sha` or `git_branch` to target the correct commit or branch to run the job against. ## Example pull requests @@ -94,10 +95,18 @@ If you're experiencing any issues, review some of the common questions and answe
Temporary schemas aren't dropping
-
If your temporary schemas aren't dropping after a PR merges or closes, this typically indicates you have overridden the generate_schema_name macro and it isn't using dbt_cloud_pr_ as the prefix.



To resolve this, change your macro so that the temporary PR schema name contains the required prefix. For example: +
If your temporary schemas aren't dropping after a PR merges or closes, this typically indicates one of these issues: +
    +
  • You have overridden the generate_schema_name macro and it isn't using dbt_cloud_pr_ as the prefix.



    To resolve this, change your macro so that the temporary PR schema name contains the required prefix. For example:



    - • ✅ Temporary PR schema name contains the prefix dbt_cloud_pr_ (like dbt_cloud_pr_123_456_marketing)

    - • ❌ Temporary PR schema name doesn't contain the prefix dbt_cloud_pr_ (like marketing).

    + ✅ Temporary PR schema name contains the prefix dbt_cloud_pr_ (like dbt_cloud_pr_123_456_marketing).

    + ❌ Temporary PR schema name doesn't contain the prefix dbt_cloud_pr_ (like marketing).

    +
    • +
    +
  • + A macro is creating a schema but there are no dbt models writing to that schema. dbt Cloud doesn't drop temporary schemas that weren't written to as a result of running a dbt model. +
    • +
@@ -153,6 +162,3 @@ If you're experiencing any issues, review some of the common questions and answe If you're on a Virtual Private dbt Enterprise plan using security features like ingress PrivateLink or IP Allowlisting, registering CI hooks may not be available and can cause the job to fail silently.
- - - diff --git a/website/docs/docs/deploy/continuous-integration.md b/website/docs/docs/deploy/continuous-integration.md index 23ed37afaa0..0f87965aada 100644 --- a/website/docs/docs/deploy/continuous-integration.md +++ b/website/docs/docs/deploy/continuous-integration.md @@ -16,7 +16,7 @@ Using CI helps: ## How CI works -When you [set up CI jobs](/docs/deploy/ci-jobs#set-up-ci-jobs), dbt Cloud listens for webhooks from your Git provider indicating that a new PR has been opened or updated with new commits. When dbt Cloud receives one of these webhooks, it enqueues a new run of the CI job. If you want CI checks to run on each new commit, you need to mark your PR as **Ready for review** in your Git provider — draft PRs _don't_ trigger CI jobs. +When you [set up CI jobs](/docs/deploy/ci-jobs#set-up-ci-jobs), dbt Cloud listens for webhooks from your Git provider indicating that a new PR has been opened or updated with new commits. When dbt Cloud receives one of these webhooks, it enqueues a new run of the CI job. dbt Cloud builds and tests the models affected by the code change in a temporary schema, unique to the PR. This process ensures that the code builds without error and that it matches the expectations as defined by the project's dbt tests. The unique schema name follows the naming convention `dbt_cloud_pr__` (for example, `dbt_cloud_pr_1862_1704`) and can be found in the run details for the given run, as shown in the following image: @@ -50,8 +50,3 @@ When you push a new commit to a PR, dbt Cloud enqueues a new CI run for the late -### Run slot treatment - -Your CI runs don't consume run slots so a CI check will never block a production run. - - diff --git a/website/docs/docs/deploy/deployment-overview.md b/website/docs/docs/deploy/deployment-overview.md index 5883ecaa3f1..29934663544 100644 --- a/website/docs/docs/deploy/deployment-overview.md +++ b/website/docs/docs/deploy/deployment-overview.md @@ -4,6 +4,8 @@ id: "deployments" sidebar: "Use dbt Cloud's capabilities to seamlessly run a dbt job in production." hide_table_of_contents: true tags: ["scheduler"] +pagination_next: "docs/deploy/job-scheduler" +pagination_prev: null --- Use dbt Cloud's capabilities to seamlessly run a dbt job in production or staging environments. Rather than run dbt commands manually from the command line, you can leverage the [dbt Cloud's in-app scheduling](/docs/deploy/job-scheduler) to automate how and when you execute dbt. @@ -58,6 +60,12 @@ Learn how to use dbt Cloud's features to help your team ship timely and quality link="/docs/deploy/run-visibility" icon="dbt-bit"/> + + + +## Related content +- [Retry a failed run for a job](/dbt-cloud/api-v2#/operations/Retry%20a%20failed%20run%20for%20a%20job) API endpoint +- [Run visibility](/docs/deploy/run-visibility) +- [Jobs](/docs/deploy/jobs) +- [Job commands](/docs/deploy/job-commands) \ No newline at end of file diff --git a/website/docs/docs/environments-in-dbt.md b/website/docs/docs/environments-in-dbt.md index 54eaa68f667..70bc096cf4f 100644 --- a/website/docs/docs/environments-in-dbt.md +++ b/website/docs/docs/environments-in-dbt.md @@ -2,6 +2,7 @@ title: "About environments" id: "environments-in-dbt" hide_table_of_contents: true +pagination_next: null --- In software engineering, environments are used to enable engineers to develop and test code without impacting the users of their software. Typically, there are two types of environments in dbt: @@ -18,7 +19,7 @@ Configure environments to tell dbt Cloud or dbt Core how to build and execute yo diff --git a/website/docs/docs/introduction.md b/website/docs/docs/introduction.md index c4cfd6e45ac..0aeef0201cb 100644 --- a/website/docs/docs/introduction.md +++ b/website/docs/docs/introduction.md @@ -1,6 +1,8 @@ --- title: "What is dbt?" id: "introduction" +pagination_next: null +pagination_prev: null --- @@ -28,6 +30,7 @@ Read more about why we want to enable analysts to work more like software engine You can access dbt using dbt Core or dbt Cloud. dbt Cloud is built around dbt Core, but it also provides: - Web-based UI so it’s more accessible +- dbt Cloud-powered command line (CLI) to develop, test, version control dbt projects, and run dbt commands - Hosted environment so it’s faster to get up and running - Differentiated features, such as metadata, in-app job scheduler, observability, integrations with other tools, integrated development environment (IDE), and more. @@ -35,7 +38,8 @@ You can learn about plans and pricing on [www.getdbt.com](https://www.getdbt.com ### dbt Cloud -dbt Cloud is the fastest and most reliable way to deploy dbt. Develop, test, schedule, and investigate data models all in one web-based UI. Learn more about [dbt Cloud features](/docs/cloud/about-cloud/dbt-cloud-features) and try one of the [dbt Cloud quickstarts](/quickstarts). +dbt Cloud is the fastest and most reliable way to deploy dbt. Develop, test, schedule, and investigate data models all in one web-based UI. It also natively supports developing using a command line with the [dbt Cloud CLI](/docs/cloud/cloud-cli-installation). +Learn more about [dbt Cloud features](/docs/cloud/about-cloud/dbt-cloud-features) and try one of the [dbt Cloud quickstarts](/quickstarts). ### dbt Core diff --git a/website/docs/docs/running-a-dbt-project/run-your-dbt-projects.md b/website/docs/docs/running-a-dbt-project/run-your-dbt-projects.md index 9bd57e0b280..b3b6ffb3e45 100644 --- a/website/docs/docs/running-a-dbt-project/run-your-dbt-projects.md +++ b/website/docs/docs/running-a-dbt-project/run-your-dbt-projects.md @@ -1,14 +1,25 @@ --- title: "Run your dbt projects" id: "run-your-dbt-projects" +pagination_prev: null --- -You can run your dbt projects with [dbt Cloud](/docs/cloud/about-cloud/dbt-cloud-features) and [dbt Core](https://github.com/dbt-labs/dbt-core). dbt Cloud is a hosted application where you can develop directly from a web browser. dbt Core is an open source project where you can develop from the command line. +You can run your dbt projects with [dbt Cloud](/docs/cloud/about-cloud/dbt-cloud-features) or [dbt Core](https://github.com/dbt-labs/dbt-core): -Among other features, dbt Cloud provides a development environment to help you build, test, run, and [version control](/docs/collaborate/git-version-control) your project faster. It also includes an easier way to share your [dbt project's documentation](/docs/collaborate/build-and-view-your-docs) with your team. These development tasks are directly built into dbt Cloud for an _integrated development environment_ (IDE). Refer to [Develop in the Cloud](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud) for more details. +- **dbt Cloud**: A hosted application where you can develop directly from a web browser using the [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud). It also natively supports developing using a command line interface, [dbt Cloud CLI](/docs/cloud/cloud-cli-installation). Among other features, dbt Cloud provides: -With dbt Core, you can run your dbt projects from the command line. The command line interface (CLI) is available from your computer's terminal application such as Terminal and iTerm. When using the command line, you can run commands and do other work from the current working directory on your computer. Before running the dbt project from the command line, make sure you are working in your dbt project directory. Learning terminal commands such as `cd` (change directory), `ls` (list directory contents), and `pwd` (present working directory) can help you navigate the directory structure on your system. + - Development environment to help you build, test, run, and [version control](/docs/collaborate/git-version-control) your project faster. + - Share your [dbt project's documentation](/docs/collaborate/build-and-view-your-docs) with your team. + - Integrates with the dbt Cloud IDE, allowing you to run development tasks and environment in the dbt Cloud UI for a seamless experience. + - The dbt Cloud CLI to develop and run dbt commands against your dbt Cloud development environment from your local command line. + - For more details, refer to [Develop in the Cloud](/docs/cloud/about-cloud-develop). -When running your project from dbt Core or dbt Cloud, the commands you commonly use are: +- **dbt Core**: An open source project where you can develop from the [command line](/docs/core/about-dbt-core). + +The dbt Cloud CLI and dbt Core are both command line tools that enable you to run dbt commands. The key distinction is the dbt Cloud CLI is tailored for dbt Cloud's infrastructure and integrates with all its [features](/docs/cloud/about-cloud/dbt-cloud-features). + +The command line is available from your computer's terminal application such as Terminal and iTerm. With the command line, you can run commands and do other work from the current working directory on your computer. Before running the dbt project from the command line, make sure you are working in your dbt project directory. Learning terminal commands such as `cd` (change directory), `ls` (list directory contents), and `pwd` (present working directory) can help you navigate the directory structure on your system. + +In dbt Cloud or dbt Core, the commands you commonly use are: - [dbt run](/reference/commands/run) — Runs the models you defined in your project - [dbt build](/reference/commands/build) — Builds and tests your selected resources such as models, seeds, snapshots, and tests @@ -20,6 +31,7 @@ For information on all dbt commands and their arguments (flags), see the [dbt co - [How we set up our computers for working on dbt projects](https://discourse.getdbt.com/t/how-we-set-up-our-computers-for-working-on-dbt-projects/243) - [Model selection syntax](/reference/node-selection/syntax) +- [dbt Cloud CLI](/docs/cloud/cloud-cli-installation) - [Cloud IDE features](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud#ide-features) - [Does dbt offer extract and load functionality?](/faqs/Project/transformation-tool) - [Why does dbt compile need a data platform connection](/faqs/Warehouse/db-connection-dbt-compile) diff --git a/website/docs/docs/running-a-dbt-project/using-threads.md b/website/docs/docs/running-a-dbt-project/using-threads.md index 519ce8aab81..5eede7abc27 100644 --- a/website/docs/docs/running-a-dbt-project/using-threads.md +++ b/website/docs/docs/running-a-dbt-project/using-threads.md @@ -3,7 +3,7 @@ title: "Using threads" id: "using-threads" sidebar_label: "Use threads" description: "Understand what threads mean and how to use them." - +pagination_next: null --- When dbt runs, it creates a directed acyclic graph (DAG) of links between models. The number of threads represents the maximum number of paths through the graph dbt may work on at once – increasing the number of threads can minimize the run time of your project. @@ -18,7 +18,7 @@ Generally the optimal number of threads depends on your data warehouse and its c You can use a different number of threads than the value defined in your target by using the `--threads` option when executing a dbt command. -You will define the number of threads in your `profiles.yml` file (for CLI-users only), dbt Cloud job definition, and dbt Cloud development credentials under your profile. +You will define the number of threads in your `profiles.yml` file (for dbt Core users only), dbt Cloud job definition, and dbt Cloud development credentials under your profile. ## Related docs diff --git a/website/docs/docs/supported-data-platforms.md b/website/docs/docs/supported-data-platforms.md index 8ac782991c8..a8e146f49d0 100644 --- a/website/docs/docs/supported-data-platforms.md +++ b/website/docs/docs/supported-data-platforms.md @@ -4,14 +4,20 @@ id: "supported-data-platforms" sidebar_label: "Supported data platforms" description: "Connect dbt to any data platform in dbt Cloud or dbt Core, using a dedicated adapter plugin" hide_table_of_contents: true +pagination_next: "docs/connect-adapters" +pagination_prev: null --- dbt connects to and runs SQL against your database, warehouse, lake, or query engine. These SQL-speaking platforms are collectively referred to as _data platforms_. dbt connects with data platforms by using a dedicated adapter plugin for each. Plugins are built as Python modules that dbt Core discovers if they are installed on your system. Read [What are Adapters](/guides/dbt-ecosystem/adapter-development/1-what-are-adapters) for more info. -You can [connect](/docs/connect-adapters) to adapters and data platforms either directly in the dbt Cloud user interface (UI) or install them manually using the command line (CLI). +You can [connect](/docs/connect-adapters) to adapters and data platforms natively in dbt Cloud or install them manually using dbt Core. You can also further customize how dbt works with your specific data platform via configuration: see [Configuring Postgres](/reference/resource-configs/postgres-configs) for an example. +import MSCallout from '/snippets/_microsoft-adapters-soon.md'; + + + ## Types of Adapters There are three types of adapters available today: @@ -36,5 +42,5 @@ import AdaptersTrusted from '/snippets/_adapters-trusted.md'; -
* Install these adapters using the CLI as they're not currently supported in dbt Cloud.
+
* Install these adapters using dbt Core as they're not currently supported in dbt Cloud.
diff --git a/website/docs/docs/trusted-adapters.md b/website/docs/docs/trusted-adapters.md index e19bb40785f..08191e8ea42 100644 --- a/website/docs/docs/trusted-adapters.md +++ b/website/docs/docs/trusted-adapters.md @@ -6,7 +6,7 @@ hide_table_of_contents: true Trusted adapters are adapters not maintained by dbt Labs, that we feel comfortable recommending to users for use in production. -Free and open-source tools for the data professional are increasingly abundant. This is by-and-large a *good thing*, however it requires due dilligence that wasn't required in a paid-license, closed-source software world. As a user, there are questions to answer important before taking a dependency on an open-source project. The trusted adapter designation is meant to streamline this process for end users. +Free and open-source tools for the data professional are increasingly abundant. This is by-and-large a *good thing*, however it requires due diligence that wasn't required in a paid-license, closed-source software world. As a user, there are questions to answer important before taking a dependency on an open-source project. The trusted adapter designation is meant to streamline this process for end users.
Considerations for depending on an open-source project diff --git a/website/docs/docs/use-dbt-semantic-layer/avail-sl-integrations.md b/website/docs/docs/use-dbt-semantic-layer/avail-sl-integrations.md index 27cb83977de..a398ace164e 100644 --- a/website/docs/docs/use-dbt-semantic-layer/avail-sl-integrations.md +++ b/website/docs/docs/use-dbt-semantic-layer/avail-sl-integrations.md @@ -4,31 +4,42 @@ id: avail-sl-integrations description: "Discover the diverse range of partners that seamlessly integrate with the powerful dbt Semantic Layer, allowing you to query and unlock valuable insights from your data ecosystem." tags: [Semantic Layer] sidebar_label: "Available integrations" +hide_table_of_contents: true meta: - api_name: dbt Semantic Layer API + api_name: dbt Semantic Layer APIs --- -import NewSLChanges from '/snippets/_new-sl-changes.md'; - - - There are a number of data applications that seamlessly integrate with the dbt Semantic Layer, powered by MetricFlow, from business intelligence tools to notebooks, spreadsheets, data catalogs, and more. These integrations allow you to query and unlock valuable insights from your data ecosystem. -Query dbt metrics with external integrations using the sophisticated {frontMatter.meta.api_name}. The API enables you to query metrics, avoid duplicative coding, optimize your development workflow, ensure data governance for company metrics, and guarantee consistency for data consumers.

- - +Use the [dbt Semantic Layer APIs](/docs/dbt-cloud-apis/sl-api-overview) to simplify metric queries, optimize your development workflow, and reduce coding. This approach also ensures data governance and consistency for data consumers. import AvailIntegrations from '/snippets/_sl-partner-links.md'; +## Other integrations + +You can also integrate the following tools with the dbt Semantic Layer: +- [Push.ai](https://docs.push.ai/semantic-layer-integrations/dbt-semantic-layer) +- [Delphi](delphihq.com) +- KlipFolio Power Metrics - Documentation coming soon + +### Custom integration + +- You can create custom integrations using different languages and tools. We support connecting with JDBC, ADBC, and GraphQL APIs. For more info, check out [our examples on GitHub](https://github.com/dbt-labs/example-semantic-layer-clients/). +- You can also connect to tools that allow you to write SQL. These tools must meet one of the two criteria: + + - Supports a generic JDBC driver option (such as DataGrip) or + - Uses Arrow Flight SQL JDBC driver version 12.0.0 or higher. + ## Related docs -- {frontMatter.meta.api_name} to learn how to integrate with the JDBC to query your metrics in downstream tools. +- {frontMatter.meta.api_name} to learn how to integrate and query your metrics in downstream tools. - [dbt Semantic Layer API query syntax](/docs/dbt-cloud-apis/sl-jdbc#querying-the-api-for-metric-metadata) +- [Hex dbt Semantic Layer cells](https://learn.hex.tech/docs/logic-cell-types/transform-cells/dbt-metrics-cells) to set up SQL cells in Hex. diff --git a/website/docs/docs/use-dbt-semantic-layer/dbt-sl.md b/website/docs/docs/use-dbt-semantic-layer/dbt-sl.md index 8d073297f48..8868c68ed20 100644 --- a/website/docs/docs/use-dbt-semantic-layer/dbt-sl.md +++ b/website/docs/docs/use-dbt-semantic-layer/dbt-sl.md @@ -5,13 +5,12 @@ description: "Learn how the dbt Semantic Layer enables data teams to centrally d sidebar_label: "About the dbt Semantic Layer" tags: [Semantic Layer] hide_table_of_contents: true +pagination_next: "docs/use-dbt-semantic-layer/quickstart-sl" +pagination_prev: null --- -import NewSLChanges from '/snippets/_new-sl-changes.md'; - - The dbt Semantic Layer, powered by [MetricFlow](/docs/build/about-metricflow), simplifies the process of defining and using critical business metrics, like `revenue` in the modeling layer (your dbt project). By centralizing metric definitions, data teams can ensure consistent self-service access to these metrics in downstream data tools and applications. The dbt Semantic Layer eliminates duplicate coding by allowing data teams to define metrics on top of existing models and automatically handles data joins. @@ -26,10 +25,8 @@ Refer to the [Why we need a universal semantic layer](https://www.getdbt.com/blo import Features from '/snippets/_sl-plan-info.md'
@@ -59,8 +56,8 @@ instance="hosted in North America" icon="dbt-bit"/> diff --git a/website/docs/docs/use-dbt-semantic-layer/gsheets.md b/website/docs/docs/use-dbt-semantic-layer/gsheets.md new file mode 100644 index 00000000000..ee391c91b70 --- /dev/null +++ b/website/docs/docs/use-dbt-semantic-layer/gsheets.md @@ -0,0 +1,63 @@ +--- +title: "Google Sheets (beta)" +description: "Integrate with Google Sheets to query your metrics in a spreadsheet." +tags: [Semantic Layer] +sidebar_label: "Google Sheets (beta)" +--- + +:::info Beta functionality +Google Sheets integration with the dbt Semantic Layer is a [beta](/docs/dbt-versions/product-lifecycles#dbt-cloud) feature. +::: + +The dbt Semantic Layer offers a seamless integration with Google Sheets through a custom menu. This add-on allows you to build dbt Semantic Layer queries and return data on your metrics directly within Google Sheet. + +## Prerequisites + +1. You have a Google account with access to Google Sheets. +2. You can install Google add-ons. +3. You have [set up the dbt Semantic Layer](/docs/use-dbt-semantic-layer/setup-sl). +4. You have a dbt Cloud Environment ID and a [service token](/docs/dbt-cloud-apis/service-tokens) to authenticate with from a dbt Cloud account. + +## Installing the add-on + +1. Navigate to the [dbt Semantic Layer for Sheets App](https://gsuite.google.com/marketplace/app/foo/392263010968) to install the add-on. + + - You can also find it in Google Sheets by going to [**Extensions -> Add-on -> Get add-ons**](https://support.google.com/docs/answer/2942256?hl=en&co=GENIE.Platform%3DDesktop&oco=0#zippy=%2Cinstall-add-ons%2Cinstall-an-add-on) and searching for it there. +2. After installing, open the Add-On menu and select the "dbt Semantic Layer for Sheets". This will open a custom menu to the right-hand side of your screen. +3. Authenticate with your Host, dbt Cloud Environment ID, and Service Token. +4. Start querying your metrics using the **Query Builder**. For more info on the menu functions, refer to [Custom menu functions](#custom-menu-functions). + +When querying your data with Google Sheets: + +- It returns the data to the cell you have clicked on. +- The custom menu operation has a timeout limit of six (6) minutes. +- If you're using this extension, make sure you're signed into Chrome with the same Google profile you used to set up the Add-On. Log in with one Google profile at a time as using multiple Google profiles at once might cause issues. + + +## Custom menu functions + +The custom menu provides the following capabilities: + +| Menu items | Description | +|---------------|-------------------------------------------------------| +| Metrics | Search and select metrics. | +| Group By | Search and select dimensions to group by. Dimensions are grouped by the entity of the semantic model they come from. | +| Granularity | Modify the granularity of the primary time dimension. | +| Where | Filter your data. This includes categorical and time filters. | +| Order By | Return your data ordered. | +| Limit | Set a limit for the rows of your output. | + + +## Filtering data + +To use the filter functionality, choose the [dimension](docs/build/dimensions) you want to filter by and select the operation you want to filter on. + - For categorical dimensiosn, type in the dimension value you want to filter by (no quotes needed) and press enter. + - Continue adding additional filters as needed with AND and OR. If it's a time dimension, choose the operator and select from the calendar. + + + +**Limited Use Policy Disclosure** + +The dbt Semantic Layer for Sheet's use and transfer to any other app of information received from Google APIs will adhere to [Google API Services User Data Policy](https://developers.google.com/terms/api-services-user-data-policy), including the Limited Use requirements. + + diff --git a/website/docs/docs/use-dbt-semantic-layer/quickstart-sl.md b/website/docs/docs/use-dbt-semantic-layer/quickstart-sl.md index 542ab4896bb..d0e5df18d94 100644 --- a/website/docs/docs/use-dbt-semantic-layer/quickstart-sl.md +++ b/website/docs/docs/use-dbt-semantic-layer/quickstart-sl.md @@ -5,22 +5,20 @@ description: "Use this guide to build and define metrics, set up the dbt Semanti sidebar_label: "Get started with the dbt Semantic Layer" tags: [Semantic Layer] meta: - api_name: dbt Semantic Layer API + api_name: dbt Semantic Layer APIs --- -import NewSLChanges from '/snippets/_new-sl-changes.md'; -import InstallMetricFlow from '/snippets/_sl-install-metricflow.md'; + import CreateModel from '/snippets/_sl-create-semanticmodel.md'; import DefineMetrics from '/snippets/_sl-define-metrics.md'; import ConfigMetric from '/snippets/_sl-configure-metricflow.md'; import TestQuery from '/snippets/_sl-test-and-query-metrics.md'; +import ConnectQueryAPI from '/snippets/_sl-connect-and-query-api.md'; +import RunProdJob from '/snippets/_sl-run-prod-job.md'; - - - The dbt Semantic Layer, powered by [MetricFlow](/docs/build/about-metricflow), simplifies defining and using critical business metrics. It centralizes metric definitions, eliminates duplicate coding, and ensures consistent self-service access to metrics in downstream tools. MetricFlow, a powerful component of the dbt Semantic Layer, simplifies the creation and management of company metrics. It offers flexible abstractions, SQL query generation, and enables fast retrieval of metric datasets from a data platform. @@ -29,15 +27,15 @@ Use this guide to fully experience the power of the universal dbt Semantic Layer - [Create a semantic model](#create-a-semantic-model) in dbt Cloud using MetricFlow - [Define metrics](#define-metrics) in dbt Cloud using MetricFlow -- [Test and query metrics locally](#test-and-query-metrics) using MetricFlow +- [Test and query metrics](#test-and-query-metrics) with MetricFlow - [Run a production job](#run-a-production-job) in dbt Cloud - [Set up dbt Semantic Layer](#setup) in dbt Cloud - [Connect and query API](#connect-and-query-api) with dbt Cloud - -MetricFlow allows users to define metrics in their dbt project whether in dbt Cloud or in dbt Core. dbt Core users can use the [MetricFlow CLI](/docs/build/metricflow-cli) to define metrics in their local dbt Core project. +MetricFlow allows you to define metrics in your dbt project and query them whether in dbt Cloud or dbt Core with [MetricFlow commands](/docs/build/metricflow-commands). However, to experience the power of the universal [dbt Semantic Layer](/docs/use-dbt-semantic-layer/dbt-sl) and query those metrics in downstream tools, you'll need a dbt Cloud [Team or Enterprise](https://www.getdbt.com/pricing/) account. + ## Prerequisites import SetUp from '/snippets/_v2-sl-prerequisites.md'; @@ -62,13 +60,8 @@ New to dbt or metrics? Try our [Jaffle shop example project](https://github.com/ ## Run a production job -Once you’ve defined metrics in your dbt project, you can perform a job run in your deployment environment in dbt Cloud to materialize your metrics. The deployment environment is only supported for the dbt Semantic Layer at this moment. -1. Go to **Deploy** in the navigation header -2. Select **Jobs** to re-run the job with the most recent code in the deployment environment. -3. Your metric should appear as a red node in the dbt Cloud IDE and dbt directed acyclic graphs (DAG). - - +
@@ -88,16 +81,7 @@ import SlSetUp from '/snippets/_new-sl-setup.md'; ## Connect and query API -You can query your metrics in a JDBC-enabled tool or use existing first-class integrations with the dbt Semantic Layer. - -You must have a dbt Cloud Team or Enterprise [multi-tenant](/docs/cloud/about-cloud/regions-ip-addresses) deployment, hosted in North America (Additional region support coming soon). - -- To learn how to use the JDBC API and what tools you can query it with, refer to the {frontMatter.meta.api_name}.
- - * To authenticate, you need to [generate a service token](/docs/dbt-cloud-apis/service-tokens) with Semantic Layer Only and Metadata Only permissions. - * Refer to the [SQL query syntax](/docs/dbt-cloud-apis/sl-jdbc#querying-the-api-for-metric-metadata) to query metrics using the API. - -- To learn more about the sophisticated integrations that connect to the dbt Semantic Layer, refer to [Available integrations](/docs/use-dbt-semantic-layer/avail-sl-integrations) for more info. + ## FAQs diff --git a/website/docs/docs/use-dbt-semantic-layer/setup-sl.md b/website/docs/docs/use-dbt-semantic-layer/setup-sl.md index a2395d367e7..4c88ee50b25 100644 --- a/website/docs/docs/use-dbt-semantic-layer/setup-sl.md +++ b/website/docs/docs/use-dbt-semantic-layer/setup-sl.md @@ -8,9 +8,6 @@ tags: [Semantic Layer] -import NewSLChanges from '/snippets/_new-sl-changes.md'; - - With the dbt Semantic Layer, you can centrally define business metrics, reduce code duplication and inconsistency, create self-service in downstream tools, and more. Configure the dbt Semantic Layer in dbt Cloud to connect with your integrated partner tool. diff --git a/website/docs/docs/use-dbt-semantic-layer/sl-architecture.md b/website/docs/docs/use-dbt-semantic-layer/sl-architecture.md index 89cd9bc6ddc..dc75016eb91 100644 --- a/website/docs/docs/use-dbt-semantic-layer/sl-architecture.md +++ b/website/docs/docs/use-dbt-semantic-layer/sl-architecture.md @@ -4,12 +4,9 @@ id: sl-architecture description: "dbt Semantic Layer product architecture and related questions." sidebar_label: "Architecture" tags: [Semantic Layer] +pagination_next: null --- -import NewSLChanges from '/snippets/_new-sl-changes.md'; - - - @@ -27,7 +24,7 @@ The dbt Semantic Layer includes the following components: | **[MetricFlow](/docs/build/about-metricflow)** | MetricFlow in dbt allows users to centrally define their semantic models and metrics with YAML specifications. | ✅ | ✅ | ✅ | BSL package (code is source available) | | **MetricFlow Server**| A proprietary server that takes metric requests and generates optimized SQL for the specific data platform. | ❌ | ✅ | ✅ | Proprietary, Cloud (Team & Enterprise)| | **Semantic Layer Gateway** | A service that passes queries to MetricFlow server and executes the SQL generated by MetricFlow against the data platform|

❌| ✅ | ✅ | Proprietary, Cloud (Team & Enterprise) | -| **Semantic Layer API** | The interfaces that allow users to submit metric queries include the MetricFlow CLI and JDBC API. They also serve as the foundation for building first-class integrations with various tools. | ❌ | ✅ | ✅ | Proprietary, Cloud (Team & Enterprise)| +| **Semantic Layer APIs** | The interfaces that allow users to submit metric queries using the GraphQL and JDBC APIs. They also serve as the foundation for building first-class integrations with various tools. | ❌ | ✅ | ✅ | Proprietary, Cloud (Team & Enterprise)| ## Related questions diff --git a/website/docs/docs/use-dbt-semantic-layer/tableau.md b/website/docs/docs/use-dbt-semantic-layer/tableau.md new file mode 100644 index 00000000000..ac3e6b64ab0 --- /dev/null +++ b/website/docs/docs/use-dbt-semantic-layer/tableau.md @@ -0,0 +1,67 @@ +--- +title: "Tableau (beta)" +description: "Use Tableau worksheets to query the dbt Semantic Layer and produce dashboards with trusted date." +tags: [Semantic Layer] +sidebar_label: "Tableau (beta)" +--- + +:::info Beta functionality +The Tableau integration with the dbt Semantic Layer is a [beta feature](/docs/dbt-versions/product-lifecycles#dbt-cloud). +::: + + +The Tableau integration allows you to use worksheets to query the Semantic Layer directly and produce your dashboards with trusted data. This integration provides a live connection to the dbt Semantic Layer through Tableau Desktop. + +## Prerequisites + +1. You must have [Tableau Desktop](https://www.tableau.com/en-gb/products/desktop) installed +2. Authenticate with either Tableau Server or Tableau Cloud +3. You need your dbt Cloud host, [Environment ID](/docs/use-dbt-semantic-layer/setup-sl#set-up-dbt-semantic-layer) and [service token](/docs/dbt-cloud-apis/service-tokens) to log in. This account should be set up with the dbt Semantic Layer. +4. You must a dbt Cloud Team or Enterprise [account](https://www.getdbt.com/pricing) and multi-tenant [deployment](/docs/cloud/about-cloud/regions-ip-addresses). (Single-Tenant coming soon) + + +## Installing + +1. Download our [connector file](https://github.com/dbt-labs/semantic-layer-tableau-connector/releases/download/v1.0.0/dbt_semantic_layer.taco) locally and add it to your default folder: + - Windows: `C:\Users\\[Windows User]\Documents\My Tableau Repository\Connectors` + - Mac: `/Users/[user]/Documents/My Tableau Repository/Connectors` + - Linux: `/opt/tableau/connectors` +2. Install the [JDBC driver](/docs/dbt-cloud-apis/sl-jdbc) to the folder based on your operating system: + - Windows: `C:\Program Files\Tableau\Drivers` + - Mac: `~/Library/Tableau/Drivers` + - Linux: ` /opt/tableau/tableau_driver/jdbc` +3. Open Tableau Desktop and find the **dbt Semantic Layer by dbt Labs** connector on the left-hand side. +4. Connect with your Host, Environment ID, and service token information that's provided to you in your dbt Cloud Semantic Layer configuration. + + +## Using the integration + +Once you authenticate, the system will direct you to the data source page with all the metrics and dimensions configured in your Semantic Layer. + +- From there, go directly to a worksheet in the bottom left hand corner. +- Then, you'll find all the metrics and dimensions that are available to query on the left-hand side of your window. + +Visit the [Tableau documentation](https://help.tableau.com/current/pro/desktop/en-us/gettingstarted_overview.htm) to learn more about how to use Tableau worksheets and dashboards. + +## Things to note + +- All metrics use the "SUM" aggregation type, and this can't be altered. The dbt Semantic Layer controls the aggregation type and it is intentionally fixed. Keep in mind that the underlying aggregation in the dbt Semantic Layer might not be "SUM" (even though "SUM" is Tableau's default). +- Tableau surfaces all metrics and dimensions from the dbt Semantic Layer on the left-hand side. Note, that not all metrics and dimensions can be combined with one another. You will receive an error message if a particular dimension cannot be sliced with a metric (or vice versa). + - To display available metrics and dimensions, dbt Semantic Layer returns metadata for a fake table with the dimensions and metrics as 'columns' on this table. Because of this, you can't actually query this table for previews or extracts. + - Since this is treated as a table, dbt Semantic Layer can't dynamically change what is available. This means we display _all_ available metrics and dimensions even if a particular metric and dimension combination isn't available. + +- Certain Table calculations like "Totals" and "Percent Of" may not be accurate when using metrics aggregated in a non-additive way (such as count distinct) + +## Unsupported functionality + +The following Tableau features aren't supported at this time, however, the dbt Semantic Layer may support some of this functionality in a future release: + +- Updating the data source page +- Using "Extract" mode to view your data +- Unioning Tables +- Writing Custom SQL +- Table Extensions +- Cross Database Joins +- All functions in Analysis --> Create Calculated Field +- Filtering on a Date Part time dimension for a Cumulative metric type +- Changing your date dimension to use "Week Number" diff --git a/website/docs/docs/verified-adapters.md b/website/docs/docs/verified-adapters.md index a2d28a612d6..170bc8f885b 100644 --- a/website/docs/docs/verified-adapters.md +++ b/website/docs/docs/verified-adapters.md @@ -13,6 +13,10 @@ The verification process serves as the on-ramp to integration with dbt Cloud. As To learn more, see [Verifying a new adapter](/guides/dbt-ecosystem/adapter-development/7-verifying-a-new-adapter). +import MSCallout from '/snippets/_microsoft-adapters-soon.md'; + + + Here are the verified data platforms that connect to dbt and its latest version. import AdaptersVerified from '/snippets/_adapters-verified.md'; diff --git a/website/docs/faqs/Models/reference-models-in-another-project.md b/website/docs/faqs/Models/reference-models-in-another-project.md deleted file mode 100644 index 19f3f52da31..00000000000 --- a/website/docs/faqs/Models/reference-models-in-another-project.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -title: How can I reference models or macros in another project? -description: "Use packages to add another project to your dbt project" -sidebar_label: 'Reference models or macros in another project' -id: reference-models-in-another-project - ---- - -You can use [packages](/docs/build/packages) to add another project to your dbt -project, including other projects you've created. Check out the [docs](/docs/build/packages) -for more information! diff --git a/website/docs/faqs/Warehouse/bq-oauth-drive-scope.md b/website/docs/faqs/Warehouse/bq-oauth-drive-scope.md new file mode 100644 index 00000000000..ae6da82c47a --- /dev/null +++ b/website/docs/faqs/Warehouse/bq-oauth-drive-scope.md @@ -0,0 +1,8 @@ +--- +title: Why does the BigQuery OAuth application require scopes to Google Drive? +description: "Learn more about Google Drive scopes in the BigQuery OAuth application" +sidebar_label: "BigQuery OAuth Drive Scopes" +id: bq-oauth-drive-scope +--- + +BigQuery supports external tables over both personal Google Drive files and shared files. For more information, refer to [Create Google Drive external tables](https://cloud.google.com/bigquery/docs/external-data-drive). diff --git a/website/docs/guides/best-practices/how-we-build-our-metrics/semantic-layer-2-setup.md b/website/docs/guides/best-practices/how-we-build-our-metrics/semantic-layer-2-setup.md index 34c0e813725..801227924dd 100644 --- a/website/docs/guides/best-practices/how-we-build-our-metrics/semantic-layer-2-setup.md +++ b/website/docs/guides/best-practices/how-we-build-our-metrics/semantic-layer-2-setup.md @@ -13,7 +13,7 @@ git clone git@github.com:dbt-labs/jaffle-sl-template.git cd path/to/project ``` -Next before we start writing code, we'll need to install the MetricFlow CLI as an extension of a dbt adapter from PyPI. The MetricFlow CLI is compatible with Python versions 3.8 through 3.11. +Next, before you start writing code, you need to install MetricFlow as an extension of a dbt adapter from PyPI (dbt Core users only). The MetricFlow is compatible with Python versions 3.8 through 3.11. We'll use pip to install MetricFlow and our dbt adapter: @@ -33,7 +33,7 @@ Lastly, to get to the pre-Semantic Layer starting state, checkout the `start-her git checkout start-here ``` -For more information you can [look at the docs](/docs/build/metricflow-cli) or checkout a [Quickstart](https://docs.getdbt.com/quickstarts) to get more familiar with setting up a dbt project. +For more information, refer to the [MetricFlow commands](/docs/build/metricflow-commands) or a [quickstart](/quickstarts) to get more familiar with setting up a dbt project. ## Basic commands diff --git a/website/docs/guides/best-practices/how-we-mesh/mesh-1-intro.md b/website/docs/guides/best-practices/how-we-mesh/mesh-1-intro.md new file mode 100644 index 00000000000..ba1660a8d82 --- /dev/null +++ b/website/docs/guides/best-practices/how-we-mesh/mesh-1-intro.md @@ -0,0 +1,39 @@ +--- +title: "Intro to dbt Mesh" +description: Getting started with dbt Mesh patterns +hoverSnippet: Learn how to get started with dbt Mesh +--- + +## What is dbt Mesh? + +Organizations of all sizes rely upon dbt to manage their data transformations, from small startups to large enterprises. At scale, it can be challenging to coordinate all the organizational and technical requirements demanded by your stakeholders within the scope of a single dbt project. To date, there also hasn't been a first-class way to effectively manage the dependencies, governance, and workflows between multiple dbt projects. + +Regardless of your organization's size and complexity, dbt should empower data teams to work independently and collaboratively; sharing data, code, and best practices without sacrificing security or autonomy. dbt Mesh provides the tooling for teams to finally achieve this. + +dbt Mesh is not a single product: it is a pattern enabled by a convergence of several features in dbt: + +- **[Cross-project references](/docs/collaborate/govern/project-dependencies#how-to-use-ref)** - this is the foundational feature that enables the multi-project deployments. `{{ ref() }}`s now work across dbt Cloud projects on Enterprise plans. +- **[dbt Explorer](/docs/collaborate/explore-projects)** - dbt Cloud's metadata-powered documentation platform, complete with full, cross-project lineage. +- **Governance** - dbt's new governance features allow you to manage access to your dbt models both within and across projects. + - **[Groups](/docs/collaborate/govern/model-access#groups)** - groups allow you to assign models to subsets within a project. + - **[Access](/docs/collaborate/govern/model-access#access-modifiers)** - access configs allow you to control who can reference models. +- **[Model Versions](/docs/collaborate/govern/model-versions)** - when coordinating across projects and teams, we recommend treating your data models as stable APIs. Model versioning is the mechanism to allow graceful adoption and deprecation of models as they evolve. +- **[Model Contracts](/docs/collaborate/govern/model-contracts)** - data contracts set explicit expectations on the shape of the data to ensure data changes upstream of dbt or within a project's logic don't break downstream consumers' data products. + +## Who is dbt Mesh for? + +The multi-project architecture helps organizations with mature, complex transformation workflows in dbt increase the flexibility and performance of their dbt projects. If you're already using dbt and your project has started to experience any of the following, you're likely ready to start exploring this paradigm: + +- The **number of models** in your project is degrading performance and slowing down development. +- Teams have developed **separate workflows** and need to decouple development from each other. +- **Security and governance** requirements are increasing and would benefit from increased isolation. + +dbt Cloud is designed to coordinate the features above and simplify the complexity to solve for these problems. + +If you're just starting your dbt journey, don't worry about building a multi-project architecture right away. You can _incrementally_ adopt the features in this guide as you scale. The collection of features work effectively as independent tools. Familiarizing yourself with the tooling and features that make up a multi-project architecture, and how they can apply to your organization will help you make better decisions as you grow. + +## Learning goals + +- Understand the **purpose and tradeoffs** of building a multi-project architecture. +- Develop an intuition for various **dbt Mesh patterns** and how to design a multi-project architecture for your organization. +- Establish recommended steps to **incrementally adopt** these patterns in your dbt implementation. diff --git a/website/docs/guides/best-practices/how-we-mesh/mesh-2-structures.md b/website/docs/guides/best-practices/how-we-mesh/mesh-2-structures.md new file mode 100644 index 00000000000..937515954af --- /dev/null +++ b/website/docs/guides/best-practices/how-we-mesh/mesh-2-structures.md @@ -0,0 +1,52 @@ +--- +title: Deciding how to structure your dbt Mesh +description: Getting started with dbt Mesh patterns +hoverSnippet: Learn how to get started with dbt Mesh +--- +## Exploring mesh patterns + +When adopting a multi-project architecture, where do you draw the lines between projects? + +How should you organize data workflows in a world where instead of having a single dbt DAG, you have multiple projects speaking to each other, each comprised of their own DAG? + +Adopting the dbt Mesh pattern is not a one-size-fits-all process. In fact, it's the opposite! It's about customizing your project structure to fit _your_ team and _your_ data. Now you can mold your organizational knowledge graph to your organizational people graph, bringing people and data closer together rather than compromising one for the other. + +While there is not a single best way to implement this pattern, there are some common decision points that will be helpful for you to consider. + +At a high level, you’ll need to decide: + +- Where to draw the lines between your dbt Projects -- i.e. how do you determine where to split your DAG and which models go in which project? +- How to manage your code -- do you want multiple dbt Projects living in the same repository (mono-repo) or do you want to have multiple repos with one repo per project? + +## Define your project interfaces by splitting your DAG + +The first (and perhaps most difficult!) decision when migrating to a multi-project architecture is deciding where to draw the line in your DAG to define the interfaces between your projects. Let's explore some language for discussing the design of these patterns. + +### Vertical splits + +Vertical splits separate out layers of transformation in DAG order. Let's look at some examples. + +- **Splitting up staging and mart layers** to create a more tightly-controlled, shared set of components that other projects build on but can't edit. +- **Isolating earlier models for security and governance requirements** to separate out and mask PII data so that downstream consumers can't access it is a common use case for a vertical split. +- **Protecting complex or expensive data** to isolate large or complex models that are expensive to run so that they are safe from accidental selection, independently deployable, and easier to debug when they have issues. + +### Horizontal splits + +Horizontal splits separate your DAG based on source or domain. These splits are often based around the shape and size of the data and how it's used. Let's consider some possibilities for horizontal splitting. + +- **Team consumption patterns.** For example, splitting out the marketing team's data flow into a separate project. +- **Data from different sources.** For example, clickstream event data and transactional ecommerce data may need to be modeled independently of each other. +- **Team workflows.** For example, if two embedded groups operate at different paces, you may want to split the projects up so they can move independently. + +### Combining these strategies + +- **These are not either/or techniques**. You should consider both types of splits, and combine them in any way that makes sense for your organization. +- **Pick one type of split and focus on that first**. If you have a hub-and-spoke team topology for example, handle breaking out the central platform project before you split the remainder into domains. Then if you need to break those domains up horizontally you can focus on that after the fact. +- **DRY applies to underlying data, not just code.** Regardless of your strategy, you should not be sourcing the same rows and columns into multiple nodes. When working within a mesh pattern it becomes increasingly important that we don't duplicate logic or data. + +## Determine your git strategy + +A multi-project architecture can exist in a single repo (monorepo) or as multiple projects, with each one being in their own repository (multi-repo). + +- If you're a **smaller team** looking primarily to speed up and simplify development, a **monorepo** is likely the right choice, but can become unwieldy as the number of projects, models and contributors grow. +- If you’re a **larger team with multiple groups**, and need to decouple projects for security and enablement of different development styles and rhythms, a **multi-repo setup** is your best bet. diff --git a/website/docs/guides/best-practices/how-we-mesh/mesh-3-implementation.md b/website/docs/guides/best-practices/how-we-mesh/mesh-3-implementation.md new file mode 100644 index 00000000000..cfbbc7a1f28 --- /dev/null +++ b/website/docs/guides/best-practices/how-we-mesh/mesh-3-implementation.md @@ -0,0 +1,130 @@ +--- +title: "Implementing your mesh plan" +description: Getting started with dbt Mesh patterns +hoverSnippet: Learn how to get started with dbt Mesh +--- + +As mentioned before, the key decision in migrating to a multi-project architecture is understanding how your project is already being grouped, built, and deployed. We can use this information to inform our decision to split our project apart. + +- **Examine your jobs** - which sets of models are most often built together? +- **Look at your lineage graph** - how are models connected? +- **Look at your selectors** defined in `selectors.yml` - how do people already define resource groups? +- **Talk to teams** about what sort of separation naturally exists right now. + - Are there various domains people are focused on? + - Are there various sizes, shapes, and sources of data that get handled separately (such as click event data)? + - Are there people focused on separate levels of transformation, such as landing and staging data or building marts? + +## Add groups and access + +Once you have a sense of some initial groupings, you can first implement **group and access permissions** within a single project. + +- First you can create a [group](/docs/build/groups) to define the owner of a set of models. + +```yml +# in models/__groups.yml + +groups: + - name: marketing + owner: + - name: Ben Jaffleck + email: ben.jaffleck@jaffleshop.com +``` + +- Then, we can add models to that group using the `group:` key in the model's YAML entry. + +```yml +# in models/marketing/__models.yml + +models: + - name: fct_marketing_model + group: marketing + - name: stg_marketing_model + group: marketing +``` + +- Once you've added models to the group, you can **add [access](/docs/collaborate/govern/model-access) settings to the models** based on their connections between groups, *opting for the most private access that will maintain current functionality*. This means that any model that has *only* relationships to other models in the same group should be `private` , and any model that has cross-group relationships, or is a terminal node in the group DAG should be `protected` so that other parts of the DAG can continue to reference it. + +```yml +# in models/marketing/__models.yml + +models: + - name: fct_marketing_model + group: marketing + access: protected + - name: stg_marketing_model + group: marketing + access: private +``` + +- **Validate these groups by incrementally migrating your jobs** to execute these groups specifically via selection syntax. We would recommend doing this in parallel to your production jobs until you’re sure about them. This will help you feel out if you’ve drawn the lines in the right place. +- If you find yourself **consistently making changes across multiple groups** when you update logic, that’s a sign that **you may want to rethink your groups**. + +## Split your projects + +1. **Move your grouped models into a subfolder**. This will include any model in the selected group, it's associated YAML entry, as well as its parent or child resources as appropriate depending on where this group sits in your DAG. + 1. Note that just like in your dbt project, circular refereneces are not allowed! Project B cannot have parents and children in Project A, for example. +2. **Create a new `dbt_project.yml` file** in the subdirectory. +3. **Copy any macros** used by the resources you moved. +4. **Create a new `packages.yml` file** in your subdirectory with the packages that are used by the resources you moved. +5. **Update `{{ ref }}` functions** — For any model that has a cross-project dependency (this may be in the files you moved, or in the files that remain in your project): + 1. Update the `{{ ref() }}` function to have two arguments, where the first is the name of the source project and the second is the name of the model: e.g. `{{ ref('jaffle_shop', 'my_upstream_model') }}` + 2. Update the upstream, cross-project parents’ `access` configs to `public` , ensuring any project can safely `{{ ref() }}` those models. + 3. We *highly* recommend adding a [model contract](/docs/collaborate/govern/model-contracts) to the upstream models to ensure the data shape is consistent and reliable for your downstream consumers. +6. **Create a `dependencies.yml` file** ([docs](/docs/collaborate/govern/project-dependencies)) for the downstream project, declaring the upstream project as a dependency. + +```yml + +# in dependencies.yml +projects: + - name: jaffle_shop +``` + +### Best practices + +- When you’ve **confirmed the right groups**, it's time to split your projects. + - **Do *one* group at a time**! + - **Do *not* refactor as you migrate**, however tempting that may be. Focus on getting 1-to-1 parity and log any issues you find in doing the migration for later. Once you’ve fully migrated the project then you can start optimizing it for its new life as part of your mesh. +- Start by splitting your project within the same repository for full git tracking and easy reversion if you need to start from scratch. + + +## Connecting existing projects + +Some organizations may already be coordinating across multiple dbt projects. Most often this is via: + +1. Installing parent projects as dbt packages +2. Using `{{ source() }}` functions to read the outputs of a parent project as inputs to a child project. + +This has a few drawbacks: + +1. If using packages, each project has to include *all* resources from *all* projects in its manifest, slowing down dbt and the development cycle. +2. If using sources, there are breakages in the lineage, as there's no real connection between the parent and child projects. + +The migration steps here are much simpler than splitting up a monolith! + +1. If using the `package` method: + 1. In the parent project: + 1. mark all models being referenced downstream as `public` and add a model contract. + 2. In the child project: + 1. Remove the package entry from `packages.yml` + 2. Add the upstream project to your `dependencies.yml` + 3. Update the `{{ ref() }}` functions to models from the upstream project to include the project name argument. +1. If using `source` method: + 1. In the parent project: + 1. mark all models being imported downstream as `public` and add a model contract. + 2. In the child project: + 1. Add the upstream project to your `dependencies.yml` + 2. Replace the `{{ source() }}` functions with cross project `{{ ref() }}` functions. + 3. Remove the unnecessary `source` definitions. + +## Additional Resources +### Our example projects + +We've provided a set of example projects you can use to explore the topics covered here. We've split our [Jaffle Shop](https://github.com/dbt-labs/jaffle-shop) project into 3 separate projects in a multi-repo dbt Mesh. Note that you'll need to leverage dbt Cloud to use multi-project architecture, as cross-project references are powered via dbt Cloud's APIs. + +- **[Platform](https://github.com/dbt-labs/jaffle-shop-mesh-platform)** - containing our centralized staging models. +- **[Marketing](https://github.com/dbt-labs/jaffle-shop-mesh-marketing)** - containing our marketing marts. +- **[Finance](https://github.com/dbt-labs/jaffle-shop-mesh-finance)** - containing our finance marts. + +### dbt-meshify + +We recommend using the `dbt-meshify` [command line tool]() to help you do this. This comes with CLI operations to automate most of the above steps. diff --git a/website/docs/guides/dbt-ecosystem/databricks-guides/productionizing-your-dbt-databricks-project.md b/website/docs/guides/dbt-ecosystem/databricks-guides/productionizing-your-dbt-databricks-project.md index 2f052ae47d2..a3b4be5a051 100644 --- a/website/docs/guides/dbt-ecosystem/databricks-guides/productionizing-your-dbt-databricks-project.md +++ b/website/docs/guides/dbt-ecosystem/databricks-guides/productionizing-your-dbt-databricks-project.md @@ -121,7 +121,6 @@ The five key steps for troubleshooting dbt Cloud issues are: 2. Inspect the problematic file and look for an immediate fix. 3. Isolate the problem by running one model at a time in the IDE or undoing the code that caused the issue. 4. Check for problems in compiled files and logs. -5. Seek help from the [dbt Cloud support team](/docs/dbt-support) if needed. Consult the [Debugging errors documentation](/guides/best-practices/debugging-errors) for a comprehensive list of error types and diagnostic methods. diff --git a/website/docs/guides/dbt-ecosystem/sl-partner-integration-guide.md b/website/docs/guides/dbt-ecosystem/sl-partner-integration-guide.md index 39e93987b20..522bcf70c65 100644 --- a/website/docs/guides/dbt-ecosystem/sl-partner-integration-guide.md +++ b/website/docs/guides/dbt-ecosystem/sl-partner-integration-guide.md @@ -4,11 +4,6 @@ id: "sl-partner-integration-guide" description: Learn about partner integration guidelines, roadmap, and connectivity. --- - -import NewChanges from '/snippets/_new-sl-changes.md'; - - - To fit your tool within the world of the Semantic Layer, dbt Labs offers some best practice recommendations for how to expose metrics and allow users to interact with them seamlessly. :::note @@ -20,7 +15,7 @@ This is an evolving guide that is meant to provide recommendations based on our To build a dbt Semantic Layer integration: -- We offer a [JDBC](/docs/dbt-cloud-apis/sl-jdbc) API (and will soon offer a GraphQL API). Refer to the dedicated [dbt Semantic Layer API](/docs/dbt-cloud-apis/sl-api-overview) for more technical integration details. +- We offer a [JDBC](/docs/dbt-cloud-apis/sl-jdbc) API and [GraphQL API](/docs/dbt-cloud-apis/sl-graphql). Refer to the dedicated [dbt Semantic Layer API](/docs/dbt-cloud-apis/sl-api-overview) for more technical integration details. - Familiarize yourself with the [dbt Semantic Layer](/docs/use-dbt-semantic-layer/dbt-sl) and [MetricFlow](/docs/build/about-metricflow)'s key concepts. There are two main objects: @@ -114,7 +109,7 @@ For better analysis, it's best to have the context of the metrics close to where These are recommendations on how to evolve a Semantic Layer integration and not a strict runbook. **Stage 1 - The basic** -* Supporting and using the new [JDBC](/docs/dbt-cloud-apis/sl-jdbc) is the first step. Refer to the [dbt Semantic Layer API](/docs/dbt-cloud-apis/sl-api-overview) for more technical details. +* Supporting and using [JDBC](/docs/dbt-cloud-apis/sl-jdbc) or [GraphQL](/docs/dbt-cloud-apis/sl-graphql) is the first step. Refer to the [dbt Semantic Layer APIs](/docs/dbt-cloud-apis/sl-api-overview) for more technical details. **Stage 2 - More discoverability and basic querying** * Support listing metrics defined in the project diff --git a/website/docs/guides/migration/sl-migration.md b/website/docs/guides/migration/sl-migration.md index baa7ae4a567..56cd6dc9d80 100644 --- a/website/docs/guides/migration/sl-migration.md +++ b/website/docs/guides/migration/sl-migration.md @@ -12,7 +12,7 @@ The legacy Semantic Layer will be deprecated in H2 2023. Additionally, the `dbt_ The metrics specification in dbt Core is changed in v1.6 to support the integration of MetricFlow. It's strongly recommended that you refer to [Build your metrics](/docs/build/build-metrics-intro) and before getting started so you understand the core concepts of the Semantic Layer. -dbt Labs recommends completing these steps in a local dev environment instead of the IDE: +dbt Labs recommends completing these steps in a local dev environment (such as the [dbt Cloud CLI](/docs/cloud/cloud-cli-installation)) instead of the dbt Cloud IDE: 1. Create new Semantic Model configs as YAML files in your dbt project.* 1. Upgrade the metrics configs in your project to the new spec.* @@ -63,11 +63,19 @@ You might need to audit metric values during the migration to ensure that the hi This step is only relevant to users who want the legacy and new semantic layer to run in parallel for a short time. This will let you recreate content in downstream tools like Hex and Mode with minimal downtime. If you do not need to recreate assets in these tools skip to step 5. 1. Create a new deployment environment in dbt Cloud and set the dbt version to 1.6 or higher. -2. Choose `Only run on a custom branch` and point to the branch that has the updated metric definition + +2. Select **Only run on a custom branch** and point to the branch that has the updated metric definition. + 3. Set the deployment schema to a temporary migration schema, such as `tmp_sl_migration`. Optional, you can create a new database for the migration. -4. Create a job to parse your project, such as `dbt parse`, and run it. Make sure this job succeeds, There needs to be a successful job in your environment in order to set up the semantic layer -5. In Account Settings > Projects > Project details click `Configure the Semantic Layer`. Under **Environment**select the deployment environment you created in the previous step. Save your configuration. -6. In the Project details page, click `Generate service token` and grant it `Semantic Layer Only` and `Metadata Only` permissions. Save this token securely - you will need it to connect to the semantic layer. + +4. Create a job to parse your project, such as `dbt parse`, and run it. Make sure this job succeeds. There needs to be a successful job in your environment in order to set up the semantic layer. + +5. Select **Account Settings** -> **Projects** -> **Project details** and choose **Configure the Semantic Layer**. + +6. Under **Environment**, select the deployment environment you created in the previous step. Save your configuration. + +7. In the **Project details** page, click **Generate service token** and grant it **Semantic Layer Only** and **Metadata Only** permissions. Save this token securely. You will need it to connect to the semantic layer. + At this point, both the new semantic layer and the old semantic layer will be running. The new semantic layer will be pointing at your migration branch with the updated metrics definitions. @@ -77,10 +85,7 @@ Now that your Semantic Layer is set up, you will need to update any downstream i ### Migration guide for Hex -:::important Hex integration coming soon - -Hex’s Semantic Layer integration will be available for use in the coming weeks. This section will have updated instructions once the integration is available. -::: +To learn more about integrating with Hex, check out their [documentation](https://learn.hex.tech/docs/connect-to-data/data-connections/dbt-integration#dbt-semantic-layer-integration) for more info. Additionally, refer to [dbt Semantic Layer cells](https://learn.hex.tech/docs/logic-cell-types/transform-cells/dbt-metrics-cells) to set up SQL cells in Hex. 1. Set up a new connection for the Semantic Layer for your account. Something to note is that your old connection will still work. The following Loom video guides you in setting up your Semantic Layer with Hex: @@ -109,7 +114,7 @@ Hex’s Semantic Layer integration will be available for use in the coming weeks If you created a new environment in [Step 3](#step-3-setup-the-semantic-layer-in-a-new-environment): -3. Update your Environment in Account Settings > Project Details > Edit Semantic Layer Configuration to point to your production environment +3. Update your Environment in **Account Settings** -> **Project Details** -> **Edit Semantic Layer Configuration** to point to your production environment 4. Delete your migration environment. Be sure to update your connection details in any downstream tools to account for the environment change. diff --git a/website/docs/guides/migration/versions/00-upgrading-to-v1.7.md b/website/docs/guides/migration/versions/00-upgrading-to-v1.7.md index 036c734dfb1..f350e8955f7 100644 --- a/website/docs/guides/migration/versions/00-upgrading-to-v1.7.md +++ b/website/docs/guides/migration/versions/00-upgrading-to-v1.7.md @@ -8,7 +8,7 @@ description: New features and changes in dbt Core v1.7 - [Changelog](https://github.com/dbt-labs/dbt-core/blob/8aaed0e29f9560bc53d9d3e88325a9597318e375/CHANGELOG.md) - [CLI Installation guide](/docs/core/installation) - [Cloud upgrade guide](/docs/dbt-versions/upgrade-core-in-cloud) -- [Release schedule](https://github.com/dbt-labs/dbt-core/issues/7481) +- [Release schedule](https://github.com/dbt-labs/dbt-core/issues/8260) ## What to know before upgrading @@ -16,9 +16,49 @@ dbt Labs is committed to providing backward compatibility for all versions 1.x, ### Behavior changes -**COMING SOON** +dbt Core v1.7 expands the amount of sources you can configure freshness for. Previously, freshness was limited to sources with a `loaded_at_field`; now, freshness can be generated from warehouse metadata tables when available. -### Quick hits +As part of this change, the `loaded_at_field` is no longer required to generate source freshness. If a source has a `freshness:` block, dbt will attempt to calculate freshness for that source: +- If a `loaded_at_field` is provided, dbt will calculate freshness via a select query (previous behavior). +- If a `loaded_at_field` is _not_ provided, dbt will calculate freshness via warehouse metadata tables when possible (new behavior). + +This is a relatively small behavior change, but worth calling out in case you notice that dbt is calculating freshness for _more_ sources than before. To exclude a source from freshness calculations, you have two options: +- Don't add a `freshness:` block. +- Explicitly set `freshness: null` + +## New and changed features and functionality + +- [`dbt docs generate`](/reference/commands/cmd-docs) now supports `--select` to generate [catalog metadata](/reference/artifacts/catalog-json) for a subset of your project. Currently available for Snowflake and Postgres only, but other adapters are coming soon. +- [Source freshness](/docs/deploy/source-freshness) can now be generated from warehouse metadata tables, currently Snowflake only, but other adapters that have metadata tables are coming soon. + +### MetricFlow enhancements + +- Automatically create metrics on measures with [`create_metric: true`](/docs/build/semantic-models). +- Optional [`label`](/docs/build/semantic-models) in semantic_models, measures, dimensions and entities. +- New configurations for semantic models - [enable/disable](/reference/resource-configs/enabled), [group](/reference/resource-configs/group), and [meta](/reference/resource-configs/meta). +- Support `fill_nulls_with` and `join_to_timespine` for metric nodes. +- `saved_queries` extends governance beyond the semantic objects to their consumption. -**COMING SOON** +### For consumers of dbt artifacts (metadata) + +- The [manifest](/reference/artifacts/manifest-json) schema version has been updated to v11. +- The [run_results](/reference/artifacts/run-results-json) schema version has been updated to v5. +- There are a few specific changes to the [catalog.json](/reference/artifacts/catalog-json): + - Added [node attributes](/reference/artifacts/run-results-json) related to compilation (`compiled`, `compiled_code`, `relation_name`) to the `catalog.json`. + - The nodes dictionary in the `catalog.json` can now be "partial" if `dbt docs generate` is run with a selector. + +### Model governance + +dbt Core v1.5 introduced model governance which we're continuing to refine. v1.7 includes these additional features and functionality: + +- **[Breaking change detection](/reference/resource-properties/versions#detecting-breaking-changes) for models with contracts enforced:** When dbt detects a breaking change to a model with an enforced contract during state comparison, it will now raise an error for versioned models and a warning for models that are not versioned. +- **[Set `access` as a config](/reference/resource-configs/access):** You can now set a model's `access` within config blocks in the model's file or in the `dbt_project.yml` for an entire subfolder at once. +- **[Type aliasing for model contracts](/reference/resource-configs/contract):** dbt will use each adapter's built-in type aliasing for user-provided data types—meaning you can now write `string` always, and dbt will translate to `text` on Postgres/Redshift. This is "on" by default, but you can opt-out. +- **[Raise warning for numeric types](/reference/resource-configs/contract):** Because of issues when putting `numeric` in model contracts without considering that default values such as `numeric(38,0)` might round decimals accordingly. dbt will now warn you if it finds a numeric type without specified precision/scale. + +### Quick hits +With these quick hits, you can now: +- Configure a [`delimiter`](/reference/resource-configs/delimiter) for a seed file. +- Use packages with the same git repo and unique subdirectory. +- Access the `date_spine` macro directly from dbt-core (moved over from dbt-utils). diff --git a/website/docs/guides/migration/versions/01-upgrading-to-v1.6.md b/website/docs/guides/migration/versions/01-upgrading-to-v1.6.md index bdb47bbf2ea..50b0ca8bc58 100644 --- a/website/docs/guides/migration/versions/01-upgrading-to-v1.6.md +++ b/website/docs/guides/migration/versions/01-upgrading-to-v1.6.md @@ -59,7 +59,7 @@ Supported on: - [Postgres](/reference/resource-configs/postgres-configs#materialized-view) - [Redshift](/reference/resource-configs/redshift-configs#materialized-view) - [Snowflake](/reference/resource-configs/snowflake-configs#dynamic-tables) -- Databricks (docs forthcoming) +- [Databricks](/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables) Support for BigQuery coming soon. @@ -90,4 +90,5 @@ More consistency and flexibility around packages. Resources defined in a package - [`dbt debug --connection`](/reference/commands/debug) to test just the data platform connection specified in a profile - [`dbt docs generate --empty-catalog`](/reference/commands/cmd-docs) to skip catalog population while generating docs - [`--defer-state`](/reference/node-selection/defer) enables more-granular control +- [`dbt ls`](/reference/commands/list) adds the Semantic model selection method to allow for `dbt ls -s "semantic_model:*"` and the ability to execute `dbt ls --resource-type semantic_model`. diff --git a/website/docs/guides/orchestration/airflow-and-dbt-cloud/1-airflow-and-dbt-cloud.md b/website/docs/guides/orchestration/airflow-and-dbt-cloud/1-airflow-and-dbt-cloud.md index a377554c317..d6760771b79 100644 --- a/website/docs/guides/orchestration/airflow-and-dbt-cloud/1-airflow-and-dbt-cloud.md +++ b/website/docs/guides/orchestration/airflow-and-dbt-cloud/1-airflow-and-dbt-cloud.md @@ -15,17 +15,17 @@ In some cases, [Airflow](https://airflow.apache.org/) may be the preferred orche ### Airflow + dbt Core -There are so many great examples from Gitlab through their open source data engineering work. Example: [here](https://gitlab.com/gitlab-data/analytics/-/blob/master/dags/transformation/dbt_snowplow_backfill.py). This is especially appropriate if you are well-versed in Kubernetes, CI/CD, and docker task management when building your airflow pipelines. If this is you and your team, you’re in good hands reading through more details: [here](https://about.gitlab.com/handbook/business-technology/data-team/platform/infrastructure/#airflow) and [here](https://about.gitlab.com/handbook/business-technology/data-team/platform/dbt-guide/) +There are [so many great examples](https://gitlab.com/gitlab-data/analytics/-/blob/master/dags/transformation/dbt_snowplow_backfill.py) from GitLab through their open source data engineering work. This is especially appropriate if you are well-versed in Kubernetes, CI/CD, and docker task management when building your airflow pipelines. If this is you and your team, you’re in good hands reading through more details [here](https://about.gitlab.com/handbook/business-technology/data-team/platform/infrastructure/#airflow) and [here](https://about.gitlab.com/handbook/business-technology/data-team/platform/dbt-guide/). ### Airflow + dbt Cloud API w/Custom Scripts -This has served as a bridge until the fabled Astronomer + dbt Labs-built dbt Cloud provider became generally available: [here](https://registry.astronomer.io/providers/dbt-cloud?type=Sensors&utm_campaign=Monthly%20Product%20Updates&utm_medium=email&_hsmi=208603877&utm_content=208603877&utm_source=hs_email) +This has served as a bridge until the fabled Astronomer + dbt Labs-built dbt Cloud provider became generally available [here](https://registry.astronomer.io/providers/dbt%20Cloud/versions/latest). There are many different permutations of this over time: -- [Custom Python Scripts](https://github.com/sungchun12/airflow-dbt-cloud/blob/main/archive/dbt_cloud_example.py): This is an airflow DAG based on custom python API utilities [here](https://github.com/sungchun12/airflow-dbt-cloud/blob/main/archive/dbt_cloud_utils.py) +- [Custom Python Scripts](https://github.com/sungchun12/airflow-dbt-cloud/blob/main/archive/dbt_cloud_example.py): This is an airflow DAG based on [custom python API utilities](https://github.com/sungchun12/airflow-dbt-cloud/blob/main/archive/dbt_cloud_utils.py) - [Make API requests directly through the BashOperator based on the docs](https://docs.getdbt.com/dbt-cloud/api-v2-legacy#operation/triggerRun): You can make cURL requests to invoke dbt Cloud to do what you want -- [Other ways to run dbt in airflow](/docs/deploy/deployments#airflow): Official dbt Docs on how teams are running dbt in airflow +- For more options, check out the [official dbt Docs](/docs/deploy/deployments#airflow) on the various ways teams are running dbt in airflow ## This guide's process diff --git a/website/docs/guides/orchestration/custom-cicd-pipelines/4-dbt-cloud-job-on-pr.md b/website/docs/guides/orchestration/custom-cicd-pipelines/4-dbt-cloud-job-on-pr.md index b58bab175b3..1a75fdc17ac 100644 --- a/website/docs/guides/orchestration/custom-cicd-pipelines/4-dbt-cloud-job-on-pr.md +++ b/website/docs/guides/orchestration/custom-cicd-pipelines/4-dbt-cloud-job-on-pr.md @@ -94,7 +94,7 @@ Add this as a macro to your project. It takes 2 arguments that lets you control ```sql {# This macro finds PR schemas older than a set date and drops them - The maco defaults to 10 days old, but can be configued with the input argument age_in_days + The macro defaults to 10 days old, but can be configured with the input argument age_in_days Sample usage with different date: dbt run-operation pr_schema_cleanup --args "{'database_to_clean': 'analytics','age_in_days':'15'}" #} diff --git a/website/docs/quickstarts/bigquery-qs.md b/website/docs/quickstarts/bigquery-qs.md index ee88dc93402..5f0b641875f 100644 --- a/website/docs/quickstarts/bigquery-qs.md +++ b/website/docs/quickstarts/bigquery-qs.md @@ -76,7 +76,6 @@ In order to let dbt connect to your warehouse, you'll need to generate a keyfile 1. Start the [GCP credentials wizard](https://console.cloud.google.com/apis/credentials/wizard). Make sure your new project is selected in the header. If you do not see your account or project, click your profile picture to the right and verify you are using the correct email account. For **Credential Type**: - From the **Select an API** dropdown, choose **BigQuery API** - Select **Application data** for the type of data you will be accessing - - Select **No, I’m not using them** and click **Next**. - Click **Next** to create a new service account. 2. Create a service account for your new project from the [Service accounts page](https://console.cloud.google.com/projectselector2/iam-admin/serviceaccounts?supportedpurview=project). For more information, refer to [Create a service account](https://developers.google.com/workspace/guides/create-credentials#create_a_service_account) in the Google Cloud docs. As an example for this guide, you can: - Type `dbt-user` as the **Service account name** diff --git a/website/docs/quickstarts/manual-install-qs.md b/website/docs/quickstarts/manual-install-qs.md index 05336178ff6..2444cf29d7e 100644 --- a/website/docs/quickstarts/manual-install-qs.md +++ b/website/docs/quickstarts/manual-install-qs.md @@ -9,11 +9,11 @@ hide_table_of_contents: true --- ## Introduction -When you use dbt Core to work with dbt, you will be editing files locally using a code editor, and running projects using the dbt command line interface (dbt CLI). If you'd rather edit files and run projects using the web-based Integrated Development Environment (IDE), you should refer to the [dbt Cloud quickstarts](/quickstarts). +When you use dbt Core to work with dbt, you will be editing files locally using a code editor, and running projects using a command line interface (CLI). If you'd rather edit files and run projects using the web-based Integrated Development Environment (IDE), you should refer to the [dbt Cloud quickstarts](/quickstarts). You can also develop and run dbt commands using the [dbt Cloud CLI](/docs/cloud/cloud-cli-installation) — a dbt Cloud powered command line. ### Prerequisites -* To use the dbt CLI, it's important that you know some basics of the Terminal. In particular, you should understand `cd`, `ls` and `pwd` to navigate through the directory structure of your computer easily. +* To use dbt Core, it's important that you know some basics of the Terminal. In particular, you should understand `cd`, `ls` and `pwd` to navigate through the directory structure of your computer easily. * Install dbt Core using the [installation instructions](/docs/core/installation) for your operating system. * Complete [Setting up (in BigQuery)](/quickstarts/bigquery?step=2) and [Loading data (BigQuery)](/quickstarts/bigquery?step=3). * [Create a GitHub account](https://github.com/join) if you don't already have one. @@ -103,16 +103,16 @@ When developing locally, dbt connects to your using jaffle_shop: # this needs to match the profile in your dbt_project.yml file target: dev outputs: - dev: - type: bigquery - method: service-account - keyfile: /Users/BBaggins/.dbt/dbt-tutorial-project-331118.json # replace this with the full path to your keyfile - project: grand-highway-265418 # Replace this with your project id - dataset: dbt_bbagins # Replace this with dbt_your_name, e.g. dbt_bilbo - threads: 1 - timeout_seconds: 300 - location: US - priority: interactive + dev: + type: bigquery + method: service-account + keyfile: /Users/BBaggins/.dbt/dbt-tutorial-project-331118.json # replace this with the full path to your keyfile + project: grand-highway-265418 # Replace this with your project id + dataset: dbt_bbagins # Replace this with dbt_your_name, e.g. dbt_bilbo + threads: 1 + timeout_seconds: 300 + location: US + priority: interactive ``` diff --git a/website/docs/reference/artifacts/manifest-json.md b/website/docs/reference/artifacts/manifest-json.md index 5e8dcedd2d5..47a9849eda5 100644 --- a/website/docs/reference/artifacts/manifest-json.md +++ b/website/docs/reference/artifacts/manifest-json.md @@ -3,15 +3,9 @@ title: "Manifest JSON file" sidebar_label: "Manifest" --- -| dbt Core version | Manifest version | -|------------------|---------------------------------------------------------------| -| v1.6 | [v10](https://schemas.getdbt.com/dbt/manifest/v10/index.html) | -| v1.5 | [v9](https://schemas.getdbt.com/dbt/manifest/v9/index.html) | -| v1.4 | [v8](https://schemas.getdbt.com/dbt/manifest/v8/index.html) | -| v1.3 | [v7](https://schemas.getdbt.com/dbt/manifest/v7/index.html) | -| v1.2 | [v6](https://schemas.getdbt.com/dbt/manifest/v6/index.html) | -| v1.1 | [v5](https://schemas.getdbt.com/dbt/manifest/v5/index.html) | -| v1.0 | [v4](https://schemas.getdbt.com/dbt/manifest/v4/index.html) | +import ManifestVersions from '/snippets/_manifest-versions.md'; + + **Produced by:** Any command that parses your project. This includes all commands **except** [`deps`](/reference/commands/deps), [`clean`](/reference/commands/clean), [`debug`](/reference/commands/debug), [`init`](/reference/commands/init) diff --git a/website/docs/reference/commands/clone.md b/website/docs/reference/commands/clone.md index a3c8bb236c7..ea3e570447d 100644 --- a/website/docs/reference/commands/clone.md +++ b/website/docs/reference/commands/clone.md @@ -21,7 +21,7 @@ The `clone` command is useful for: dbt clone --state path/to/artifacts # clone one_specific_model of my models from specified state to my target schema(s) -dbt clone --select one_specific_model --state path/to/artifacts +dbt clone --select "one_specific_model" --state path/to/artifacts # clone all of my models from specified state to my target schema(s) and recreate all pre-existing relations in the current target dbt clone --state path/to/artifacts --full-refresh diff --git a/website/docs/reference/commands/cmd-docs.md b/website/docs/reference/commands/cmd-docs.md index 754c5e93baf..bc4840464b8 100644 --- a/website/docs/reference/commands/cmd-docs.md +++ b/website/docs/reference/commands/cmd-docs.md @@ -19,6 +19,18 @@ The command is responsible for generating your project's documentation website b dbt docs generate ``` + + +Use the `--select` argument to limit the nodes included within `catalog.json`. When this flag is provided, step (3) will be restricted to the selected nodes. All other nodes will be excluded. Step (2) is unaffected. + +**Example**: +```shell +dbt docs generate --select +orders +``` + + + + Use the `--no-compile` argument to skip re-compilation. When this flag is provided, `dbt docs generate` will skip step (2) described above. **Example**: diff --git a/website/docs/reference/commands/compile.md b/website/docs/reference/commands/compile.md index ed403d2af32..cde65b7c6b6 100644 --- a/website/docs/reference/commands/compile.md +++ b/website/docs/reference/commands/compile.md @@ -29,7 +29,7 @@ This will log the compiled SQL to the terminal, in addition to writing to the `t For example: ```bash -dbt compile --select stg_payments +dbt compile --select "stg_payments" dbt compile --inline "select * from {{ ref('raw_orders') }}" ``` @@ -37,7 +37,7 @@ returns the following: ```bash -dbt compile --select stg_orders +dbt compile --select "stg_orders" 21:17:09 Running with dbt=1.5.0-b5 21:17:09 Found 5 models, 20 tests, 0 snapshots, 0 analyses, 425 macros, 0 operations, 3 seed files, 0 sources, 0 exposures, 0 metrics, 0 groups 21:17:09 diff --git a/website/docs/reference/commands/deps.md b/website/docs/reference/commands/deps.md index 4c7a36606e2..f4f8153c115 100644 --- a/website/docs/reference/commands/deps.md +++ b/website/docs/reference/commands/deps.md @@ -57,3 +57,31 @@ Installing calogica/dbt_date@0.4.0 Updates available for packages: ['tailsdotcom/dbt_artifacts', 'dbt-labs/snowplow'] Update your versions in packages.yml, then run dbt deps ``` + + + +dbt generates the `package-lock.yml` file in the _project_root_ where `packages.yml` is recorded, which contains all the resolved packages, the first time you run `dbt deps`. Each subsequent run records the packages installed in this file. If the subsequent `dbt deps` runs contain no updated packages in `depenedencies.yml` or `packages.yml`, dbt-core installs from `package-lock.yml`. + +When you update the package spec and run `dbt deps` again, the package-lock and package files update accordingly. You can run `dbt deps --lock` to update the `package-lock.yml` with the most recent dependencies from `packages`. + +The `--add` flag allows you to add a package to the `packages.yml` with configurable `--version` and `--source` information. The `--dry-run` flag, when set to `False`(default), recompiles the `package-lock.yml` file after a new package is added to the `packages.yml` file. Set the flag to `True` for the changes to not persist. + +Examples of the `--add` flag: +```shell +# add package from hub (--source arg defaults to "hub") +dbt deps add --package dbt-labs/dbt_utils --version 1.0.0 + +# add package from hub with semantic version +dbt deps add --package dbt-labs/snowplow --version ">=0.7.0,<0.8.0" + +# add package from git +dbt deps add --package https://github.com/fivetran/dbt_amplitude --version v0.3.0 --source git + +# add package from local (--version not required for local) +dbt deps add --package /opt/dbt/redshift --source local + +# add package to packages.yml WITHOUT updating package-lock.yml +dbt deps add --package dbt-labs/dbt_utils --version 1.0.0 --dry-run True + +``` + \ No newline at end of file diff --git a/website/docs/reference/commands/init.md b/website/docs/reference/commands/init.md index 873647814ec..ac55717c0ec 100644 --- a/website/docs/reference/commands/init.md +++ b/website/docs/reference/commands/init.md @@ -17,10 +17,21 @@ Then, it will: - Create a new folder with your project name and sample files, enough to get you started with dbt - Create a connection profile on your local machine. The default location is `~/.dbt/profiles.yml`. Read more in [configuring your profile](/docs/core/connect-data-platform/connection-profiles). + + +When using `dbt init` to initialize your project, include the `--profile` flag to specify an existing `profiles.yml` as the `profile:` key to use instead of creating a new one. For example, `dbt init --profile`. + + + +If the profile does not exist in `profiles.yml` or the command is run inside an existing project, the command raises an error. + + + ## Existing project If you've just cloned or downloaded an existing dbt project, `dbt init` can still help you set up your connection profile so that you can start working quickly. It will prompt you for connection information, as above, and add a profile (using the `profile` name from the project) to your local `profiles.yml`, or create the file if it doesn't already exist. + ## profile_template.yml `dbt init` knows how to prompt for connection information by looking for a file named `profile_template.yml`. It will look for this file in two places: diff --git a/website/docs/reference/commands/list.md b/website/docs/reference/commands/list.md index 6084b3dec70..5caabdc2b2e 100644 --- a/website/docs/reference/commands/list.md +++ b/website/docs/reference/commands/list.md @@ -8,9 +8,10 @@ id: "list" The `dbt ls` command lists resources in your dbt project. It accepts selector arguments that are similar to those provided in [dbt run](/reference/commands/run). `dbt list` is an alias for `dbt ls`. While `dbt ls` will read your [connection profile](/docs/core/connect-data-platform/connection-profiles) to resolve [`target`](/reference/dbt-jinja-functions/target)-specific logic, this command will not connect to your database or run any queries. ### Usage + ``` dbt ls - [--resource-type {model,source,seed,snapshot,metric,test,exposure,analysis,default,all}] + [--resource-type {model,semantic_model,source,seed,snapshot,metric,test,exposure,analysis,default,all}] [--select SELECTION_ARG [SELECTION_ARG ...]] [--models SELECTOR [SELECTOR ...]] [--exclude SELECTOR [SELECTOR ...]] @@ -85,7 +86,7 @@ $ dbt ls --select snowplow.* --output json --output-keys "name resource_type des ``` -$ dbt ls --select snowplow.* --output json --output-keys name resource_type description +$ dbt ls --select snowplow.* --output json --output-keys "name resource_type description" {"name": "snowplow_events", "description": "This is a pretty cool model", ...} {"name": "snowplow_page_views", "description": "This model is even cooler", ...} ... @@ -93,6 +94,16 @@ $ dbt ls --select snowplow.* --output json --output-keys name resource_type desc + + +**Listing Semantic models** + +List all resources upstream of your orders semantic model: +``` +dbt ls -s +semantic_model:orders +``` + + **Listing file paths** ``` diff --git a/website/docs/reference/commands/retry.md b/website/docs/reference/commands/retry.md index d494a46cf1f..8da5d5a77a6 100644 --- a/website/docs/reference/commands/retry.md +++ b/website/docs/reference/commands/retry.md @@ -4,14 +4,6 @@ sidebar_label: "retry" id: "retry" --- -:::info Support in dbt Cloud - -`dbt retry` is supported in the dbt Cloud IDE. - -Native support for restarting scheduled runs from point of failure is currently in development & coming soon. - -::: - `dbt retry` re-executes the last `dbt` command from the node point of failure. If the previously executed `dbt` command was successful, `retry` will finish as `no operation`. Retry works with the following commands: diff --git a/website/docs/reference/commands/seed.md b/website/docs/reference/commands/seed.md index 8a410706842..d0cd199ea12 100644 --- a/website/docs/reference/commands/seed.md +++ b/website/docs/reference/commands/seed.md @@ -12,7 +12,7 @@ The `dbt seed` command will load `csv` files located in the `seed-paths` directo Specific seeds can be run using the `--select` flag to `dbt seed`. Example: ``` -$ dbt seed --select country_codes +$ dbt seed --select "country_codes" Found 2 models, 3 tests, 0 archives, 0 analyses, 53 macros, 0 operations, 2 seed files 14:46:15 | Concurrency: 1 threads (target='dev') diff --git a/website/docs/reference/commands/show.md b/website/docs/reference/commands/show.md index 5bdcfacc1e8..a0e5d68c83f 100644 --- a/website/docs/reference/commands/show.md +++ b/website/docs/reference/commands/show.md @@ -16,7 +16,7 @@ The results of the preview query are not materialized in the data warehouse, or Example: ``` -dbt show --select model_name.sql +dbt show --select "model_name.sql" ``` or ``` @@ -26,7 +26,7 @@ dbt show --inline "select * from {{ ref('model_name') }}" The following is an example of `dbt show` output for a model named `stg_orders`: ```bash -dbt show --select stg_orders +dbt show --select "stg_orders" 21:17:38 Running with dbt=1.5.0-b5 21:17:38 Found 5 models, 20 tests, 0 snapshots, 0 analyses, 425 macros, 0 operations, 3 seed files, 0 sources, 0 exposures, 0 metrics, 0 groups 21:17:38 @@ -46,7 +46,7 @@ dbt show --select stg_orders For example, if you've just built a model that has a failing test, you can quickly preview the test failures right in the terminal, to find values of `id` that are duplicated: ```bash -$ dbt build -s my_model_with_duplicates +$ dbt build -s "my_model_with_duplicates" 13:22:47 Running with dbt=1.5.0 ... 13:22:48 Completed with 1 error and 0 warnings: @@ -58,7 +58,7 @@ $ dbt build -s my_model_with_duplicates 13:22:48 13:22:48 Done. PASS=1 WARN=0 ERROR=1 SKIP=0 TOTAL=2 -$ dbt show -s unique_my_model_with_duplicates_id +$ dbt show -s "unique_my_model_with_duplicates_id" 13:22:53 Running with dbt=1.5.0 13:22:53 Found 4 models, 2 tests, 0 snapshots, 0 analyses, 309 macros, 0 operations, 0 seed files, 0 sources, 0 exposures, 0 metrics, 0 groups 13:22:53 diff --git a/website/docs/reference/commands/source.md b/website/docs/reference/commands/source.md index b29bf7dadc6..697ae2b5fcc 100644 --- a/website/docs/reference/commands/source.md +++ b/website/docs/reference/commands/source.md @@ -20,10 +20,10 @@ By default, `dbt source freshness` will calculate freshness information for all ```bash # Snapshot freshness for all Snowplow tables: -$ dbt source freshness --select source:snowplow +$ dbt source freshness --select "source:snowplow" # Snapshot freshness for a particular source table: -$ dbt source freshness --select source:snowplow.event +$ dbt source freshness --select "source:snowplow.event" ``` ### Configuring source freshness output diff --git a/website/docs/reference/commands/test.md b/website/docs/reference/commands/test.md index a1a63729568..c050d82a0ab 100644 --- a/website/docs/reference/commands/test.md +++ b/website/docs/reference/commands/test.md @@ -10,22 +10,22 @@ The tests to run can be selected using the `--select` flag discussed [here](/ref ```bash # run tests for one_specific_model -dbt test --select one_specific_model +dbt test --select "one_specific_model" # run tests for all models in package -dbt test --select some_package.* +dbt test --select "some_package.*" # run only tests defined singularly -dbt test --select test_type:singular +dbt test --select "test_type:singular" # run only tests defined generically -dbt test --select test_type:generic +dbt test --select "test_type:generic" # run singular tests limited to one_specific_model -dbt test --select one_specific_model,test_type:singular +dbt test --select "one_specific_model,test_type:singular" # run generic tests limited to one_specific_model -dbt test --select one_specific_model,test_type:generic +dbt test --select "one_specific_model,test_type:generic" ``` For more information on writing tests, see the [Testing Documentation](/docs/build/tests). diff --git a/website/docs/reference/dbt-commands.md b/website/docs/reference/dbt-commands.md index 862829ef809..1448d9849d3 100644 --- a/website/docs/reference/dbt-commands.md +++ b/website/docs/reference/dbt-commands.md @@ -2,54 +2,58 @@ title: "dbt Command reference" --- -dbt is typically run one of two ways: +You can run dbt using the following tools: -* In [dbt Cloud](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud) -* On the [command line interface](/docs/core/about-the-cli) (CLI) +- In your browser with the [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud) +- On the command line interface using the [dbt Cloud CLI](/docs/cloud/cloud-cli-installation) or open-source [dbt Core](/docs/core/about-dbt-core), both of which enable you to execute dbt commands. The key distinction is the dbt Cloud CLI is tailored for dbt Cloud's infrastructure and integrates with all its [features](/docs/cloud/about-cloud/dbt-cloud-features). The following sections outline the commands supported by dbt and their relevant flags. For information about selecting models on the command line, consult the docs on [Model selection syntax](/reference/node-selection/syntax). ### Available commands - - -Use the following dbt commands in the [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud) or [CLI](/docs/core/about-the-cli). Use the `dbt` prefix. For example, to run the `test` command, type `dbt test`. - -| Command | Description | Version | -| ------- | ----------- | ------- | -| [build](/reference/commands/build) | Build and test all selected resources (models, seeds, snapshots, tests) | All [supported versions](/docs/dbt-versions/core) | -| [clean](/reference/commands/clean) | Deletes artifacts present in the dbt project | All [supported versions](/docs/dbt-versions/core) | -| [clone](/reference/commands/clone) | Clone selected models from the specified state | Requires [dbt v1.6 or higher](/docs/dbt-versions/core) | -| [compile](/reference/commands/compile) | Compiles (but does not run) the models in a project | All [supported versions](/docs/dbt-versions/core) | -| [debug](/reference/commands/debug) | Debugs dbt connections and projects | All [supported versions](/docs/dbt-versions/core) | -| [deps](/reference/commands/deps) | Downloads dependencies for a project | All [supported versions](/docs/dbt-versions/core) | -| [docs](/reference/commands/cmd-docs) | Generates documentation for a project | All [supported versions](/docs/dbt-versions/core) | -| [list](/reference/commands/list) | Lists resources defined in a dbt project | All [supported versions](/docs/dbt-versions/core) | -| [parse](/reference/commands/parse) | Parses a project and writes detailed timing info | All [supported versions](/docs/dbt-versions/core) | -| [retry](/reference/commands/retry) | Retry the last run `dbt` command from the point of failure | Requires [dbt v1.6 or higher](/docs/dbt-versions/core) | -| [run](/reference/commands/run) | Runs the models in a project | All [supported versions](/docs/dbt-versions/core) | -| [run-operation](/reference/commands/run-operation) | Invoke a macro, including running arbitrary maintenance SQL against
the database | All [supported versions](/docs/dbt-versions/core) | -| [seed](/reference/commands/seed) | Loads CSV files into the database | All [supported versions](/docs/dbt-versions/core) | -| [show](/reference/commands/show) | Preview table rows post-transformation | All [supported versions](/docs/dbt-versions/core) | -| [snapshot](/reference/commands/snapshot) | Executes "snapshot" jobs defined in a project | All [supported versions](/docs/dbt-versions/core) | -| [source](/reference/commands/source) | Provides tools for working with source data (including validating that
sources are "fresh") | All [supported versions](/docs/dbt-versions/core) | -| [test](/reference/commands/test) | Executes tests defined in a project | All [supported versions](/docs/dbt-versions/core) | -| [init](/reference/commands/init) | Initializes a new dbt project (CLI only) | All [supported versions](/docs/dbt-versions/core) | - + + +All commands in the table are compatible with either the dbt Cloud IDE, dbt Cloud CLI, or dbt Core. + +You can run dbt commands in your specific tool by prefixing them with `dbt`. For example, to run the `test` command, type `dbt test`. + +| Command | Description | Compatible tools | Version | +| ------- | ----------- | ---------------- | ------- | +| [build](/reference/commands/build) | Build and test all selected resources (models, seeds, snapshots, tests) | All | All [supported versions](/docs/dbt-versions/core) | +| cancel | Cancels the most recent invocation.| dbt Cloud CLI | Requires [dbt v1.6 or higher](/docs/dbt-versions/core) | +| [clean](/reference/commands/clean) | Deletes artifacts present in the dbt project | All | All [supported versions](/docs/dbt-versions/core) | +| [clone](/reference/commands/clone) | Clone selected models from the specified state | dbt Cloud CLI
dbt Core | Requires [dbt v1.6 or higher](/docs/dbt-versions/core) | +| [compile](/reference/commands/compile) | Compiles (but does not run) the models in a project | All | All [supported versions](/docs/dbt-versions/core) | +| [debug](/reference/commands/debug) | Debugs dbt connections and projects | dbt Core | All [supported versions](/docs/dbt-versions/core) | +| [deps](/reference/commands/deps) | Downloads dependencies for a project | All | All [supported versions](/docs/dbt-versions/core) | +| [docs](/reference/commands/cmd-docs) | Generates documentation for a project | All | All [supported versions](/docs/dbt-versions/core) | +| help | Displays help information for any command | dbt Core
dbt Cloud CLI | All [supported versions](/docs/dbt-versions/core) | +| [list](/reference/commands/list) | Lists resources defined in a dbt project | All | All [supported versions](/docs/dbt-versions/core) | +| [parse](/reference/commands/parse) | Parses a project and writes detailed timing info | All | All [supported versions](/docs/dbt-versions/core) | +| reattach | Reattaches to the most recent invocation to retrieve logs and artifacts. | dbt Cloud CLI | Requires [dbt v1.6 or higher](/docs/dbt-versions/core) | +| [retry](/reference/commands/retry) | Retry the last run `dbt` command from the point of failure | All | Requires [dbt v1.6 or higher](/docs/dbt-versions/core) | +| [run](/reference/commands/run) | Runs the models in a project | All | All [supported versions](/docs/dbt-versions/core) | +| [run-operation](/reference/commands/run-operation) | Invoke a macro, including running arbitrary maintenance SQL against the database | All | All [supported versions](/docs/dbt-versions/core) | +| [seed](/reference/commands/seed) | Loads CSV files into the database | All | All [supported versions](/docs/dbt-versions/core) | +| [show](/reference/commands/show) | Preview table rows post-transformation | All | All [supported versions](/docs/dbt-versions/core) | +| [snapshot](/reference/commands/snapshot) | Executes "snapshot" jobs defined in a project | All | All [supported versions](/docs/dbt-versions/core) | +| [source](/reference/commands/source) | Provides tools for working with source data (including validating that sources are "fresh") | All | All [supported versions](/docs/dbt-versions/core) | +| [test](/reference/commands/test) | Executes tests defined in a project | All | All [supported versions](/docs/dbt-versions/core) | +| [init](/reference/commands/init) | Initializes a new dbt project | dbt Core | All [supported versions](/docs/dbt-versions/core) | - + Select the tabs that are relevant to your development workflow. For example, if you develop in the dbt Cloud IDE, select **dbt Cloud**. - + Use the following dbt commands in the [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud) and use the `dbt` prefix. For example, to run the `test` command, type `dbt test`. - [build](/reference/commands/build): build and test all selected resources (models, seeds, snapshots, tests) -- [clone](/reference/commands/clone): clone selected nodes from specified state (requires dbt 1.6 or higher) +- [clone](/reference/commands/clone): clone selected nodes from the specified state (requires dbt 1.6 or higher) - [compile](/reference/commands/compile): compiles (but does not run) the models in a project - [deps](/reference/commands/deps): downloads dependencies for a project - [docs](/reference/commands/cmd-docs) : generates documentation for a project @@ -64,13 +68,13 @@ Use the following dbt commands in the [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/ - + -Use the following dbt commands in the [CLI](/docs/core/about-the-cli) and use the `dbt` prefix. For example, to run the `test` command, type `dbt test`. +Use the following dbt commands in [dbt Core](/docs/core/about-dbt-core) and use the `dbt` prefix. For example, to run the `test` command, type `dbt test`. - [build](/reference/commands/build): build and test all selected resources (models, seeds, snapshots, tests) - [clean](/reference/commands/clean): deletes artifacts present in the dbt project -- [clone](/reference/commands/clone): clone selected models from specified state (requires dbt 1.6 or higher) +- [clone](/reference/commands/clone): clone selected models from the specified state (requires dbt 1.6 or higher) - [compile](/reference/commands/compile): compiles (but does not run) the models in a project - [debug](/reference/commands/debug): debugs dbt connections and projects - [deps](/reference/commands/deps): downloads dependencies for a project diff --git a/website/docs/reference/dbt-jinja-functions/model.md b/website/docs/reference/dbt-jinja-functions/model.md index e967debd01f..9ccf0759470 100644 --- a/website/docs/reference/dbt-jinja-functions/model.md +++ b/website/docs/reference/dbt-jinja-functions/model.md @@ -52,15 +52,9 @@ To view the structure of `models` and their definitions: Use the following table to understand how the versioning pattern works and match the Manifest version with the dbt version: -| dbt version | Manifest version | -| ----------- | ---------------- | -| `v1.5` | [Manifest v9](https://schemas.getdbt.com/dbt/manifest/v9/index.html) -| `v1.4` | [Manifest v8](https://schemas.getdbt.com/dbt/manifest/v8/index.html) -| `v1.3` | [Manifest v7](https://schemas.getdbt.com/dbt/manifest/v7/index.html) -| `v1.2` | [Manifest v6](https://schemas.getdbt.com/dbt/manifest/v6/index.html) -| `v1.1` | [Manifest v5](https://schemas.getdbt.com/dbt/manifest/v5/index.html) - +import ManifestVersions from '/snippets/_manifest-versions.md'; + ## Related docs diff --git a/website/docs/reference/dbt_project.yml.md b/website/docs/reference/dbt_project.yml.md index c706b57a73b..571e930d7da 100644 --- a/website/docs/reference/dbt_project.yml.md +++ b/website/docs/reference/dbt_project.yml.md @@ -11,6 +11,8 @@ By default, dbt will look for `dbt_project.yml` in your current working director By default, dbt will look for `dbt_project.yml` in your current working directory and its parents, but you can set a different directory using the `--project-dir` flag or the `DBT_PROJECT_DIR` environment variable. +Starting from dbt v1.5 and higher, you can specify your dbt Cloud project ID in the `dbt_project.yml` file using the `dbt-cloud` config, which doesn't require validation or storage in the project config class. To find your project ID, check your dbt Cloud project URL, such as `https://cloud.getdbt.com/11/projects/123456`, where the project ID is `123456`. + The following is a list of all available configurations in the `dbt_project.yml` file. @@ -19,6 +21,81 @@ The following is a list of all available configurations in the `dbt_project.yml` dbt uses YAML in a few different places. If you're new to YAML, it would be worth taking the time to learn how arrays, dictionaries and strings are represented. ::: + + + + + +```yml +[name](/reference/project-configs/name): string + +[config-version](/reference/project-configs/config-version): 2 +[version](/reference/project-configs/version): version + +[profile](/reference/project-configs/profile): profilename + +[model-paths](/reference/project-configs/model-paths): [directorypath] +[seed-paths](/reference/project-configs/seed-paths): [directorypath] +[test-paths](/reference/project-configs/test-paths): [directorypath] +[analysis-paths](/reference/project-configs/analysis-paths): [directorypath] +[macro-paths](/reference/project-configs/macro-paths): [directorypath] +[snapshot-paths](/reference/project-configs/snapshot-paths): [directorypath] +[docs-paths](/reference/project-configs/docs-paths): [directorypath] +[asset-paths](/reference/project-configs/asset-paths): [directorypath] + +[target-path](/reference/project-configs/target-path): directorypath +[log-path](/reference/project-configs/log-path): directorypath +[packages-install-path](/reference/project-configs/packages-install-path): directorypath + +[clean-targets](/reference/project-configs/clean-targets): [directorypath] + +[query-comment](/reference/project-configs/query-comment): string + +[require-dbt-version](/reference/project-configs/require-dbt-version): version-range | [version-range] + +[dbt-cloud](/docs/cloud/cloud-cli-installation): + project-id: project_id #Required + defer-env-id: 5678 #Optional + +[quoting](/reference/project-configs/quoting): + database: true | false + schema: true | false + identifier: true | false + +models: + [](/reference/model-configs) + +seeds: + [](/reference/seed-configs) + +snapshots: + [](/reference/snapshot-configs) + +sources: + [](source-configs) + +tests: + [](/reference/test-configs) + +vars: + [](/docs/build/project-variables) + +[on-run-start](/reference/project-configs/on-run-start-on-run-end): sql-statement | [sql-statement] +[on-run-end](/reference/project-configs/on-run-start-on-run-end): sql-statement | [sql-statement] + +[dispatch](/reference/project-configs/dispatch-config): + - macro_namespace: packagename + search_order: [packagename] + +[restrict-access](/docs/collaborate/govern/model-access): true | false + +``` + + + + + + ```yml @@ -79,6 +156,9 @@ vars: search_order: [packagename] [restrict-access](/docs/collaborate/govern/model-access): true | false + ``` + + diff --git a/website/docs/reference/global-configs/command-line-flags.md b/website/docs/reference/global-configs/command-line-flags.md index 6496c92da6d..fbe89ce28f1 100644 --- a/website/docs/reference/global-configs/command-line-flags.md +++ b/website/docs/reference/global-configs/command-line-flags.md @@ -4,60 +4,95 @@ id: "command-line-flags" sidebar: "Command line flags" --- -Command line (CLI) flags immediately follow `dbt` and precede your subcommand. When set, CLI flags override environment variables and profile configs. +For consistency, command-line interface (CLI) flags should come right after the `dbt` prefix and its subcommands. This includes "global" flags (supported for all commands). When set, CLI flags override environment variables and profile configs. -Use this non-boolean config structure, replacing `` with the config you are enabling or disabling, `` with the new setting for the config, and `` with the command this config applies to: +For example, instead of using: + +```bash +dbt --no-populate-cache run +``` + +You should use: + +```bash +dbt run --no-populate-cache +``` + +Historically, passing flags (such as "global flags") _before_ the subcommand is a legacy functionality that dbt Labs can remove at any time. We do not support using the same flag before and after the subcommand. + +## Using boolean and non-boolean flags + +You can construct your commands with boolean flags to enable or disable or with non-boolean flags that use specific values, such as strings. + + + + + +Use this non-boolean config structure: +- Replacing `` with the command this config applies to. +- `` with the config you are enabling or disabling, and +- `` with the new setting for the config. ```text -$ --= + --= ``` -Non-boolean config examples: +### Example ```text -dbt --printer-width=80 run -dbt --indirect-selection=eager test +dbt run --printer-width=80 +dbt test --indirect-selection=eager ``` -To turn on boolean configs, you would use the `--` CLI flag, and a `--no-` CLI flag to turn off boolean configs, replacing `` with the config you are enabling or disabling and `` with the command this config applies to. + + + + +To enable or disable boolean configs: +- Use `` this config applies to. +- Followed by `--` to turn it on, or `--no-` to turn it off. +- Replace `` with the config you are enabling or disabling -Boolean config structure: ```text -dbt -- -dbt --no- +dbt -- +dbt --no- ``` -Boolean config example: +### Example ```text -dbt --version-check run -dbt --no-version-check run +dbt run --version-check +dbt run --no-version-check ``` - \ No newline at end of file + + + + + diff --git a/website/docs/reference/global-configs/logs.md b/website/docs/reference/global-configs/logs.md index f5f1b3f814b..8c819193fc6 100644 --- a/website/docs/reference/global-configs/logs.md +++ b/website/docs/reference/global-configs/logs.md @@ -14,6 +14,9 @@ The `LOG_FORMAT` config specifies how dbt's logs should be formatted. If the val dbt --log-format json run {"code": "A001", "data": {"v": "=1.0.0"}, "invocation_id": "1193e449-4b7a-4eb1-8e8e-047a8b3b7973", "level": "info", "log_version": 1, "msg": "Running with dbt=1.0.0", "node_info": {}, "pid": 35098, "thread_name": "MainThread", "ts": "2021-12-03T10:46:59.928217Z", "type": "log_line"} ``` + + + To set the `LOG_FORMAT_FILE` type output for the file without impacting the console log format, use the `log-format-file` flag. @@ -37,8 +40,6 @@ See [structured logging](/reference/events-logging#structured-logging) for more ::: - - ### Log Level @@ -124,7 +125,16 @@ dbt --quiet run ### Color -You can set the color preferences for the file logs only using the `--use-colors-file / --no-use-colors-file` flags. +You can set the color preferences for the file logs only within `profiles.yml` or using the `--use-colors-file / --no-use-colors-file` flags. + + + +```yaml +config: + use_colors_file: False +``` + + ```text dbt --use-colors-file run diff --git a/website/docs/reference/global-configs/print-output.md b/website/docs/reference/global-configs/print-output.md index 83280677229..112b92b546f 100644 --- a/website/docs/reference/global-configs/print-output.md +++ b/website/docs/reference/global-configs/print-output.md @@ -74,13 +74,24 @@ config: use_colors: False ``` + + ```text dbt --use-colors run dbt --no-use-colors run ``` -You can set the color preferences for the file logs only using the `--use-colors-file / --no-use-colors-file` flags. +You can set the color preferences for the file logs only within `profiles.yml` or using the `--use-colors-file / --no-use-colors-file` flags. + + + +```yaml +config: + use_colors_file: False +``` + + ```text dbt --use-colors-file run @@ -88,5 +99,3 @@ dbt --no-use-colors-file run ``` - - \ No newline at end of file diff --git a/website/docs/reference/model-properties.md b/website/docs/reference/model-properties.md index 730432c88af..63adc1f0d63 100644 --- a/website/docs/reference/model-properties.md +++ b/website/docs/reference/model-properties.md @@ -18,7 +18,7 @@ models: show: true | false [latest_version](/reference/resource-properties/latest_version): [deprecation_date](/reference/resource-properties/deprecation_date): - [access](/reference/resource-properties/access): private | protected | public + [access](/reference/resource-configs/access): private | protected | public [config](/reference/resource-properties/config): [](/reference/model-configs): [constraints](/reference/resource-properties/constraints): @@ -46,7 +46,7 @@ models: [description](/reference/resource-properties/description): [docs](/reference/resource-configs/docs): show: true | false - [access](/reference/resource-properties/access): private | protected | public + [access](/reference/resource-configs/access): private | protected | public [constraints](/reference/resource-properties/constraints): - [config](/reference/resource-properties/config): diff --git a/website/docs/reference/node-selection/defer.md b/website/docs/reference/node-selection/defer.md index e13a4f6648a..03c3b2aac12 100644 --- a/website/docs/reference/node-selection/defer.md +++ b/website/docs/reference/node-selection/defer.md @@ -17,16 +17,16 @@ It is possible to use separate state for `state:modified` and `--defer`, by pass ### Usage ```shell -$ dbt run --select [...] --defer --state path/to/artifacts -$ dbt test --select [...] --defer --state path/to/artifacts +dbt run --select [...] --defer --state path/to/artifacts +dbt test --select [...] --defer --state path/to/artifacts ``` ```shell -$ dbt run --models [...] --defer --state path/to/artifacts -$ dbt test --models [...] --defer --state path/to/artifacts +dbt run --models [...] --defer --state path/to/artifacts +dbt test --models [...] --defer --state path/to/artifacts ``` @@ -101,7 +101,7 @@ I want to test my changes. Nothing exists in my development schema, `dev_alice`. ```shell -$ dbt run --select model_b +dbt run --select "model_b" ``` @@ -128,7 +128,7 @@ Unless I had previously run `model_a` into this development environment, `dev_al ```shell -$ dbt run --select model_b --defer --state prod-run-artifacts +dbt run --select "model_b" --defer --state prod-run-artifacts ``` @@ -186,7 +186,7 @@ models: ```shell -dbt test --select model_b +dbt test --select "model_b" ``` @@ -211,7 +211,7 @@ The `relationships` test requires both `model_a` and `model_b`. Because I did no ```shell -dbt test --select model_b --defer --state prod-run-artifacts +dbt test --select "model_b" --defer --state prod-run-artifacts ``` diff --git a/website/docs/reference/node-selection/exclude.md b/website/docs/reference/node-selection/exclude.md index 9ad4bd1cc0e..d2c140d1bb5 100644 --- a/website/docs/reference/node-selection/exclude.md +++ b/website/docs/reference/node-selection/exclude.md @@ -7,19 +7,19 @@ sidebar_label: "Exclude" dbt provides an `--exclude` flag with the same semantics as `--select`. Models specified with the `--exclude` flag will be removed from the set of models selected with `--select`. ```bash -$ dbt run --select my_package.*+ --exclude my_package.a_big_model+ # select all models in my_package and their children except a_big_model and its children +dbt run --select "my_package".*+ --exclude "my_package.a_big_model+" # select all models in my_package and their children except a_big_model and its children ``` Exclude a specific resource by its name or lineage: ```bash # test -$ dbt test --exclude not_null_orders_order_id # test all models except the not_null_orders_order_id test -$ dbt test --exclude orders # test all models except tests associated with the orders model +dbt test --exclude "not_null_orders_order_id" # test all models except the not_null_orders_order_id test +dbt test --exclude "orders" # test all models except tests associated with the orders model # seed -$ dbt seed --exclude account_parent_mappings # load all seeds except account_parent_mappings +dbt seed --exclude "account_parent_mappings" # load all seeds except account_parent_mappings # snapshot -$ dbt snapshot --exclude snap_order_statuses # execute all snapshots except snap_order_statuses +dbt snapshot --exclude "snap_order_statuses" # execute all snapshots except snap_order_statuses ``` diff --git a/website/docs/reference/node-selection/graph-operators.md b/website/docs/reference/node-selection/graph-operators.md index 4fdc2f10628..8cba43e1b52 100644 --- a/website/docs/reference/node-selection/graph-operators.md +++ b/website/docs/reference/node-selection/graph-operators.md @@ -7,9 +7,9 @@ If placed at the front of the model selector, `+` will select all parents of the ```bash - $ dbt run --select my_model+ # select my_model and all children - $ dbt run --select +my_model # select my_model and all parents - $ dbt run --select +my_model+ # select my_model, and all of its parents and children +dbt run --select "my_model+" # select my_model and all children +dbt run --select "+my_model" # select my_model and all parents +dbt run --select "+my_model+" # select my_model, and all of its parents and children ``` @@ -20,9 +20,9 @@ to step through. ```bash - $ dbt run --select my_model+1 # select my_model and its first-degree children - $ dbt run --select 2+my_model # select my_model, its first-degree parents, and its second-degree parents ("grandparents") - $ dbt run --select 3+my_model+4 # select my_model, its parents up to the 3rd degree, and its children down to the 4th degree +dbt run --select "my_model+1" # select my_model and its first-degree children +dbt run --select "2+my_model" # select my_model, its first-degree parents, and its second-degree parents ("grandparents") +dbt run --select "3+my_model+4" # select my_model, its parents up to the 3rd degree, and its children down to the 4th degree ``` @@ -32,5 +32,5 @@ The `@` operator is similar to `+`, but will also include _the parents of the ch ```bash -$ dbt run --models @my_model # select my_model, its children, and the parents of its children +dbt run --models @my_model # select my_model, its children, and the parents of its children ``` diff --git a/website/docs/reference/node-selection/methods.md b/website/docs/reference/node-selection/methods.md index 2647f3416a3..e29612e3401 100644 --- a/website/docs/reference/node-selection/methods.md +++ b/website/docs/reference/node-selection/methods.md @@ -34,8 +34,8 @@ The `tag:` method is used to select models that match a specified [tag](/referen ```bash - $ dbt run --select tag:nightly # run all models with the `nightly` tag - ``` +dbt run --select "tag:nightly" # run all models with the `nightly` tag +``` ### The "source" method @@ -43,22 +43,22 @@ The `source` method is used to select models that select from a specified [sourc ```bash - $ dbt run --select source:snowplow+ # run all models that select from Snowplow sources - ``` +dbt run --select "source:snowplow+" # run all models that select from Snowplow sources +``` ### The "resource_type" method Use the `resource_type` method to select nodes of a particular type (`model`, `test`, `exposure`, and so on). This is similar to the `--resource-type` flag used by the [`dbt ls` command](/reference/commands/list). ```bash - $ dbt build --select resource_type:exposure # build all resources upstream of exposures - $ dbt list --select resource_type:test # list all tests in your project - ``` +dbt build --select "resource_type:exposure" # build all resources upstream of exposures +dbt list --select "resource_type:test" # list all tests in your project +``` Note: This method doesn't work for sources, so use the [`--resource-type`](/reference/commands/list) option of the list command instead: ```bash - $ dbt list --resource-type source - ``` +dbt list --resource-type source +``` ### The "path" method The `path` method is used to select models/sources defined at or under a specific path. @@ -69,12 +69,12 @@ selectors unambiguous. ```bash # These two selectors are equivalent - dbt run --select path:models/staging/github - dbt run --select models/staging/github + dbt run --select "path:models/staging/github" + dbt run --select "models/staging/github" # These two selectors are equivalent - dbt run --select path:models/staging/github/stg_issues.sql - dbt run --select models/staging/github/stg_issues.sql + dbt run --select "path:models/staging/github/stg_issues.sql" + dbt run --select "models/staging/github/stg_issues.sql" ``` @@ -85,9 +85,9 @@ The `file` method can be used to select a model by its filename, including the f ```bash # These are equivalent -dbt run --select file:some_model.sql -dbt run --select some_model.sql -dbt run --select some_model +dbt run --select "file:some_model.sql" +dbt run --select "some_model.sql" +dbt run --select "some_model" ``` @@ -96,10 +96,10 @@ dbt run --select some_model The `fqn` method is used to select nodes based off their "fully qualified names" (FQN) within the dbt graph. The default output of [`dbt list`](/reference/commands/list) is a listing of FQN. -``` -dbt run --select fqn:some_model -dbt run --select fqn:your_project.some_model -dbt run --select fqn:some_package.some_other_model +```bash +dbt run --select "fqn:some_model" +dbt run --select "fqn:your_project.some_model" +dbt run --select "fqn:some_package.some_other_model" ``` ### The "package" method @@ -111,10 +111,10 @@ selectors unambiguous. ```bash # These three selectors are equivalent - dbt run --select package:snowplow - dbt run --select snowplow - dbt run --select snowplow.* - ``` + dbt run --select "package:snowplow" + dbt run --select "snowplow" + dbt run --select "snowplow.*" +``` ### The "config" method @@ -124,10 +124,10 @@ The `config` method is used to select models that match a specified [node config ```bash - $ dbt run --select config.materialized:incremental # run all models that are materialized incrementally - $ dbt run --select config.schema:audit # run all models that are created in the `audit` schema - $ dbt run --select config.cluster_by:geo_country # run all models clustered by `geo_country` - ``` +dbt run --select "config.materialized:incremental" # run all models that are materialized incrementally +dbt run --select "config.schema:audit" # run all models that are created in the `audit` schema +dbt run --select "config.cluster_by:geo_country" # run all models clustered by `geo_country` +``` @@ -135,7 +135,8 @@ The `config` method is used to select models that match a specified [node config While most config values are strings, you can also use the `config` method to match boolean configs, dictionary keys, and values in lists. For example, given a model with the following configurations: -``` + +```bash {{ config( materialized = 'incremental', unique_key = ['column_a', 'column_b'], @@ -148,10 +149,10 @@ select ... You can select using any of the following: ```bash -$ dbt ls -s config.materialized:incremental -$ dbt ls -s config.unique_key:column_a -$ dbt ls -s config.grants.select:reporter -$ dbt ls -s config.transient:true +dbt ls -s config.materialized:incremental +dbt ls -s config.unique_key:column_a +dbt ls -s config.grants.select:reporter +dbt ls -s config.transient:true ``` @@ -162,10 +163,10 @@ The `test_type` method is used to select tests based on their type, `singular` o - ```bash - $ dbt test --select test_type:generic # run all generic tests - $ dbt test --select test_type:singular # run all singular tests - ``` +```bash +dbt test --select "test_type:generic" # run all generic tests +dbt test --select "test_type:singular" # run all singular tests +``` ### The "test_name" method @@ -176,10 +177,10 @@ that defines it. For more information about how generic tests are defined, read ```bash - $ dbt test --select test_name:unique # run all instances of the `unique` test - $ dbt test --select test_name:equality # run all instances of the `dbt_utils.equality` test - $ dbt test --select test_name:range_min_max # run all instances of a custom schema test defined in the local project, `range_min_max` - ``` +dbt test --select "test_name:unique" # run all instances of the `unique` test +dbt test --select "test_name:equality" # run all instances of the `dbt_utils.equality` test +dbt test --select "test_name:range_min_max" # run all instances of a custom schema test defined in the local project, `range_min_max` +``` ### The "state" method @@ -204,9 +205,9 @@ The `state` method is used to select nodes by comparing them against a previous ```bash - $ dbt test --select state:new # run all tests on new models + and new tests on old models - $ dbt run --select state:modified # run all models that have been modified - $ dbt ls --select state:modified # list all modified nodes (not just models) +dbt test --select "state:new " # run all tests on new models + and new tests on old models +dbt run --select "state:modified" # run all models that have been modified +dbt ls --select "state:modified" # list all modified nodes (not just models) ``` @@ -236,18 +237,18 @@ The `exposure` method is used to select parent resources of a specified [exposur ```bash - $ dbt run --select +exposure:weekly_kpis # run all models that feed into the weekly_kpis exposure - $ dbt test --select +exposure:* # test all resources upstream of all exposures - $ dbt ls --select +exposure:* --resource-type source # list all sources upstream of all exposures - ``` +dbt run --select "+exposure:weekly_kpis" # run all models that feed into the weekly_kpis exposure +dbt test --select "+exposure:*" # test all resources upstream of all exposures +dbt ls --select "+exposure:*" --resource-type source # list all sources upstream of all exposures +``` ### The "metric" method The `metric` method is used to select parent resources of a specified [metric](/docs/build/metrics). Use in conjunction with the `+` operator. ```bash -$ dbt build --select +metric:weekly_active_users # build all resources upstream of weekly_active_users metric -$ dbt ls --select +metric:* --resource-type source # list all source tables upstream of all metrics +dbt build --select "+metric:weekly_active_users" # build all resources upstream of weekly_active_users metric +dbt ls --select "+metric:*" --resource-type source # list all source tables upstream of all metrics ``` ### The "result" method @@ -255,10 +256,10 @@ $ dbt ls --select +metric:* --resource-type source # list all source tables The `result` method is related to the `state` method described above and can be used to select resources based on their result status from a prior run. Note that one of the dbt commands [`run`, `test`, `build`, `seed`] must have been performed in order to create the result on which a result selector operates. You can use `result` selectors in conjunction with the `+` operator. ```bash -$ dbt run --select result:error --state path/to/artifacts # run all models that generated errors on the prior invocation of dbt run -$ dbt test --select result:fail --state path/to/artifacts # run all tests that failed on the prior invocation of dbt test -$ dbt build --select 1+result:fail --state path/to/artifacts # run all the models associated with failed tests from the prior invocation of dbt build -$ dbt seed --select result:error --state path/to/artifacts # run all seeds that generated errors on the prior invocation of dbt seed. +dbt run --select "result:error" --state path/to/artifacts # run all models that generated errors on the prior invocation of dbt run +dbt test --select "result:fail" --state path/to/artifacts # run all tests that failed on the prior invocation of dbt test +dbt build --select "1+result:fail" --state path/to/artifacts # run all the models associated with failed tests from the prior invocation of dbt build +dbt seed --select "result:error" --state path/to/artifacts # run all seeds that generated errors on the prior invocation of dbt seed. ``` ### The "source_status" method @@ -276,8 +277,8 @@ After issuing one of the above commands, you can reference the source freshness ```bash # You can also set the DBT_ARTIFACT_STATE_PATH environment variable instead of the --state flag. -$ dbt source freshness # must be run again to compare current to previous state -$ dbt build --select source_status:fresher+ --state path/to/prod/artifacts +dbt source freshness # must be run again to compare current to previous state +dbt build --select "source_status:fresher+" --state path/to/prod/artifacts ``` @@ -286,8 +287,8 @@ $ dbt build --select source_status:fresher+ --state path/to/prod/artifacts ```bash # You can also set the DBT_STATE environment variable instead of the --state flag. -$ dbt source freshness # must be run again to compare current to previous state -$ dbt build --select source_status:fresher+ --state path/to/prod/artifacts +dbt source freshness # must be run again to compare current to previous state +dbt build --select "source_status:fresher+" --state path/to/prod/artifacts ``` @@ -305,9 +306,9 @@ Supported in v1.5 or newer. The `group` method is used to select models defined within a [group](/reference/resource-configs/group). - ```bash - dbt run --select group:finance # run all models that belong to the finance group. - ``` +```bash +dbt run --select "group:finance" # run all models that belong to the finance group. +``` @@ -321,12 +322,12 @@ Supported in v1.5 or newer. -The `access` method selects models based on their [access](/reference/resource-properties/access) property. +The `access` method selects models based on their [access](/reference/resource-configs/access) property. ```bash -dbt list --select access:public # list all public models -dbt list --select access:private # list all private models -dbt list --select access:protected # list all protected models +dbt list --select "access:public" # list all public models +dbt list --select "access:private" # list all private models +dbt list --select "access:protected" # list all protected models ``` @@ -344,11 +345,26 @@ Supported in v1.5 or newer. The `version` method selects [versioned models](/docs/collaborate/govern/model-versions) based on their [version identifier](/reference/resource-properties/versions) and [latest version](/reference/resource-properties/latest_version). ```bash -dbt list --select version:latest # only 'latest' versions -dbt list --select version:prerelease # versions newer than the 'latest' version +dbt list --select "version:latest" # only 'latest' versions +dbt list --select "version:prerelease" # versions newer than the 'latest' version dbt list --select version:old # versions older than the 'latest' version -dbt list --select version:none # models that are *not* versioned +dbt list --select "version:none" # models that are *not* versioned ``` + +### The "semantic_model" method + +Supported in v1.6 or newer. + + + +The `semantic_model` method selects [semantic models](/docs/build/semantic-models). + +```bash +dbt list --select semantic_model:* # list all semantic models +dbt list --select +semantic_model:orders # list your semantic model named "orders" and all upstream resources +``` + + \ No newline at end of file diff --git a/website/docs/reference/node-selection/putting-it-together.md b/website/docs/reference/node-selection/putting-it-together.md index 8faf02e6cc9..48fc5188b32 100644 --- a/website/docs/reference/node-selection/putting-it-together.md +++ b/website/docs/reference/node-selection/putting-it-together.md @@ -4,16 +4,16 @@ title: "Putting it together" ```bash - $ dbt run --select my_package.*+ # select all models in my_package and their children - $ dbt run --select +some_model+ # select some_model and all parents and children +dbt run --select "my_package.*+" # select all models in my_package and their children +dbt run --select "+some_model+" # select some_model and all parents and children - $ dbt run --select tag:nightly+ # select "nightly" models and all children - $ dbt run --select +tag:nightly+ # select "nightly" models and all parents and children +dbt run --select "tag:nightly+" # select "nightly" models and all children +dbt run --select "+tag:nightly+" # select "nightly" models and all parents and children - $ dbt run --select @source:snowplow # build all models that select from snowplow sources, plus their parents +dbt run --select "@source:snowplow" # build all models that select from snowplow sources, plus their parents - $ dbt test --select config.incremental_strategy:insert_overwrite,test_name:unique # execute all `unique` tests that select from models using the `insert_overwrite` incremental strategy - ``` +dbt test --select "config.incremental_strategy:insert_overwrite,test_name:unique" # execute all `unique` tests that select from models using the `insert_overwrite` incremental strategy +``` @@ -22,8 +22,8 @@ and feed exports, while _excluding_ the biggest incremental models (and one othe ```bash - $ dbt run --select @source:snowplow,tag:nightly models/export --exclude package:snowplow,config.materialized:incremental export_performance_timing - ``` +dbt run --select "@source:snowplow,tag:nightly models/export" --exclude "package:snowplow,config.materialized:incremental export_performance_timing" +``` This command selects all models that: diff --git a/website/docs/reference/node-selection/set-operators.md b/website/docs/reference/node-selection/set-operators.md index 7d6b6c2411c..af399b9cad5 100644 --- a/website/docs/reference/node-selection/set-operators.md +++ b/website/docs/reference/node-selection/set-operators.md @@ -11,7 +11,7 @@ Run snowplow_sessions, all ancestors of snowplow_sessions, fct_orders, and all a ```bash - $ dbt run --select +snowplow_sessions +fct_orders +dbt run --select "+snowplow_sessions +fct_orders" ``` ### Intersections @@ -22,15 +22,15 @@ Run all the common ancestors of snowplow_sessions and fct_orders: ```bash - $ dbt run --select +snowplow_sessions,+fct_orders - ``` +dbt run --select "+snowplow_sessions,+fct_orders" +``` Run all the common descendents of stg_invoices and stg_accounts: ```bash - $ dbt run --select stg_invoices+,stg_accounts+ +dbt run --select "stg_invoices+,stg_accounts+" ``` @@ -38,5 +38,5 @@ Run models that are in the marts/finance subdirectory *and* tagged nightly: ```bash - $ dbt run --select marts.finance,tag:nightly - ``` +dbt run --select "marts.finance,tag:nightly" +``` diff --git a/website/docs/reference/node-selection/state-comparison-caveats.md b/website/docs/reference/node-selection/state-comparison-caveats.md index baeeb7e4c75..73947c80a66 100644 --- a/website/docs/reference/node-selection/state-comparison-caveats.md +++ b/website/docs/reference/node-selection/state-comparison-caveats.md @@ -27,8 +27,8 @@ The command `dbt test -s state:modified` will include both: As long as you're adding or changing tests at the same time that you're adding or changing the resources (models, seeds, snapshots) they select from, all should work the way you expect with "simple" state selection: ```shell -$ dbt run -s state:modified -$ dbt test -s state:modified +dbt run -s "state:modified" +dbt test -s "state:modified" ``` This can get complicated, however. If you add a new test without modifying its underlying model, or add a test that selects from a new model and an old unmodified one, you may need to test a model without having first run it. @@ -36,8 +36,8 @@ This can get complicated, however. If you add a new test without modifying its u In v0.18.0, you needed to handle this by building the unmodified models needed for modified tests: ```shell -$ dbt run -s state:modified @state:modified,1+test_type:data -$ dbt test -s state:modified +dbt run -s "state:modified @state:modified,1+test_type:data" +dbt test -s "state:modified" ``` In v0.19.0, dbt added support for deferring upstream references when testing. If a test selects from a model that doesn't exist as a database object in your current environment, dbt will look to the other environment instead—the one defined in your state manifest. This enables you to use "simple" state selection without risk of query failure, but it may have some surprising consequences for tests with multiple parents. For instance, if you have a `relationships` test that depends on one modified model and one unmodified model, the test query will select from data "across" two different environments. If you limit or sample your data in development and CI, it may not make much sense to test for referential integrity, knowing there's a good chance of mismatch. @@ -45,8 +45,8 @@ In v0.19.0, dbt added support for deferring upstream references when testing. If If you're a frequent user of `relationships` tests or data tests, or frequently find yourself adding tests without modifying their underlying models, consider tweaking the selection criteria of your CI job. For instance: ```shell -$ dbt run -s state:modified -$ dbt test -s state:modified --exclude test_name:relationships +dbt run -s "state:modified" +dbt test -s "state:modified" --exclude "test_name:relationships" ``` ### False positives @@ -58,7 +58,7 @@ State comparison works by identifying discrepancies between two manifests. Thos dbt will do its best to capture *only* changes that are the result of modifications made in development. In projects with intricate env-aware logic, dbt will err on the side of running too many models (i.e. false positives). Over the next several versions of dbt, we're working on: - iterative improvements to dbt's built-in detective abilities -- better options for more complex projects, in the form of more-specific subselectors (see [this issue](https://github.com/dbt-labs/dbt-core/issues/2704)) +- better options for more complex projects, in the form of more-specific sub-selectors (see [this issue](https://github.com/dbt-labs/dbt-core/issues/2704)) State comparison is now able to detect env-aware config in `dbt_project.yml`. For instance, this target-based config would register as a modification in v0.18.0, but in v0.19.0 it no longer will: diff --git a/website/docs/reference/node-selection/syntax.md b/website/docs/reference/node-selection/syntax.md index 7c165b0f4ff..bb2aeefd742 100644 --- a/website/docs/reference/node-selection/syntax.md +++ b/website/docs/reference/node-selection/syntax.md @@ -14,6 +14,7 @@ dbt's node selection syntax makes it possible to run only specific resources in | [compile](/reference/commands/compile) | `--select`, `--exclude`, `--selector`, `--inline` | | [freshness](/reference/commands/source) | `--select`, `--exclude`, `--selector` | | [build](/reference/commands/build) | `--select`, `--exclude`, `--selector`, `--resource-type`, `--defer` | +| [docs generate](/reference/commands/cmd-docs) | `--select`, `--exclude`, `--selector` | :::info Nodes and resources @@ -24,6 +25,8 @@ We use the terms " By default, `dbt run` executes _all_ of the models in the dependency graph; `dbt seed` creates all seeds, `dbt snapshot` performs every snapshot. The `--select` flag is used to specify a subset of nodes to execute. +To follow [POSIX standards](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap12.html) and make things easier to understand, we recommend CLI users use quotes when passing arguments to the `--select` or `--exclude` option (including single or multiple space-delimited, or comma-delimited arguments). Not using quotes might not work reliably on all operating systems, terminals, and user interfaces. For example, `dbt run --select "my_dbt_project_name"` runs all models in your project. + ### How does selection work? 1. dbt gathers all the resources that are matched by one or more of the `--select` criteria, in the order of selection methods (e.g. `tag:`), then graph operators (e.g. `+`), then finally set operators ([unions](/reference/node-selection/set-operators#unions), [intersections](/reference/node-selection/set-operators#intersections), [exclusions](/reference/node-selection/exclude)). @@ -51,28 +54,28 @@ Examples: ```bash - $ dbt run --select my_dbt_project_name # runs all models in your project - $ dbt run --select my_dbt_model # runs a specific model - $ dbt run --select path.to.my.models # runs all models in a specific directory - $ dbt run --select my_package.some_model # run a specific model in a specific package - $ dbt run --select tag:nightly # run models with the "nightly" tag - $ dbt run --select path/to/models # run models contained in path/to/models - $ dbt run --select path/to/my_model.sql # run a specific model by its path +dbt run --select "my_dbt_project_name" # runs all models in your project +dbt run --select "my_dbt_model" # runs a specific model +dbt run --select "path.to.my.models" # runs all models in a specific directory +dbt run --select "my_package.some_model" # run a specific model in a specific package +dbt run --select "tag:nightly" # run models with the "nightly" tag +dbt run --select "path/to/models" # run models contained in path/to/models +dbt run --select "path/to/my_model.sql" # run a specific model by its path ``` dbt supports a shorthand language for defining subsets of nodes. This language uses the characters `+`, `@`, `*`, and `,`. ```bash - # multiple arguments can be provided to --select - $ dbt run --select my_first_model my_second_model +# multiple arguments can be provided to --select + dbt run --select "my_first_model my_second_model" - # these arguments can be projects, models, directory paths, tags, or sources - $ dbt run --select tag:nightly my_model finance.base.* +# these arguments can be projects, models, directory paths, tags, or sources +dbt run --select "tag:nightly my_model finance.base.*" - # use methods and intersections for more complex selectors - $ dbt run --select path:marts/finance,tag:nightly,config.materialized:table - ``` +# use methods and intersections for more complex selectors +dbt run --select "path:marts/finance,tag:nightly,config.materialized:table" +``` As your selection logic gets more complex, and becomes unwieldly to type out as command-line arguments, consider using a [yaml selector](/reference/node-selection/yaml-selectors). You can use a predefined definition with the `--selector` flag. @@ -150,7 +153,7 @@ After issuing one of the above commands, you can reference the results by adding ```bash # You can also set the DBT_ARTIFACT_STATE_PATH environment variable instead of the --state flag. -$ dbt run --select result: --defer --state path/to/prod/artifacts +dbt run --select "result: --defer --state path/to/prod/artifacts" ``` The available options depend on the resource (node) type: @@ -169,7 +172,7 @@ The available options depend on the resource (node) type: The state and result selectors can also be combined in a single invocation of dbt to capture errors from a previous run OR any new or modified models. ```bash -$ dbt run --select result:+ state:modified+ --defer --state ./ +dbt run --select "result:+ state:modified+ --defer --state ./" ``` ### Fresh rebuilds @@ -183,7 +186,7 @@ As example: ```bash # Command step order dbt source freshness -dbt build --select source_status:fresher+ +dbt build --select "source_status:fresher+" ``` @@ -202,6 +205,6 @@ After issuing one of the above commands, you can reference the source freshness ```bash # You can also set the DBT_ARTIFACT_STATE_PATH environment variable instead of the --state flag. -$ dbt source freshness # must be run again to compare current to previous state -$ dbt build --select source_status:fresher+ --state path/to/prod/artifacts +dbt source freshness # must be run again to compare current to previous state +dbt build --select "source_status:fresher+" --state path/to/prod/artifacts ``` diff --git a/website/docs/reference/node-selection/test-selection-examples.md b/website/docs/reference/node-selection/test-selection-examples.md index 52439d95d97..feb3898c230 100644 --- a/website/docs/reference/node-selection/test-selection-examples.md +++ b/website/docs/reference/node-selection/test-selection-examples.md @@ -19,14 +19,14 @@ Run generic tests only: ```bash - $ dbt test --select test_type:generic + dbt test --select "test_type:generic" ``` Run singular tests only: ```bash - $ dbt test --select test_type:singular + dbt test --select "test_type:singular" ``` In both cases, `test_type` checks a property of the test itself. These are forms of "direct" test selection. @@ -87,8 +87,8 @@ By default, a test will run when ANY parent is selected; we call this "eager" in In this mode, any test that depends on unbuilt resources will raise an error. ```shell -$ dbt test --select orders -$ dbt build --select orders +dbt test --select "orders" +dbt build --select "orders" ``` @@ -102,8 +102,10 @@ It will only include tests whose references are each within the selected nodes. Put another way, it will prevent tests from running if one or more of its parents is unselected. ```shell -$ dbt test --select orders --indirect-selection=cautious -$ dbt build --select orders --indirect-selection=cautious + +dbt test --select "orders" --indirect-selection=cautious +dbt build --select "orders" --indirect-selection=cautious + ``` @@ -122,8 +124,8 @@ By default, a test will run when ANY parent is selected; we call this "eager" in In this mode, any test that depends on unbuilt resources will raise an error. ```shell -$ dbt test --select orders -$ dbt build --select orders +dbt test --select "orders" +dbt build --select "orders" ``` @@ -137,8 +139,10 @@ It will only include tests whose references are each within the selected nodes. Put another way, it will prevent tests from running if one or more of its parents is unselected. ```shell -$ dbt test --select orders --indirect-selection=cautious -$ dbt build --select orders --indirect-selection=cautious + +dbt test --select "orders" --indirect-selection=cautious +dbt build --select "orders" --indirect-selection=cautious + ``` @@ -152,8 +156,9 @@ It will only include tests whose references are each within the selected nodes ( This is useful in the same scenarios as "cautious", but also includes when a test depends on a model **and** a direct ancestor of that model (like confirming an aggregation has the same totals as its input). ```shell -$ dbt test --select orders --indirect-selection=buildable -$ dbt build --select orders --indirect-selection=buildable +dbt test --select "orders" --indirect-selection=buildable +dbt build --select "orders" --indirect-selection=buildable + ``` @@ -172,8 +177,8 @@ By default, a test will run when ANY parent is selected; we call this "eager" in In this mode, any test that depends on unbuilt resources will raise an error. ```shell -$ dbt test --select orders -$ dbt build --select orders +dbt test --select "orders" +dbt build --select "orders" ``` @@ -187,8 +192,9 @@ It will only include tests whose references are each within the selected nodes. Put another way, it will prevent tests from running if one or more of its parents is unselected. ```shell -$ dbt test --select orders --indirect-selection=cautious -$ dbt build --select orders --indirect-selection=cautious +dbt test --select "orders" --indirect-selection=cautious +dbt build --select "orders" --indirect-selection=cautious + ``` @@ -202,8 +208,8 @@ It will only include tests whose references are each within the selected nodes ( This is useful in the same scenarios as "cautious", but also includes when a test depends on a model **and** a direct ancestor of that model (like confirming an aggregation has the same totals as its input). ```shell -$ dbt test --select orders --indirect-selection=buildable -$ dbt build --select orders --indirect-selection=buildable +dbt test --select "orders" --indirect-selection=buildable +dbt build --select "orders" --indirect-selection=buildable ``` @@ -213,8 +219,10 @@ $ dbt build --select orders --indirect-selection=buildable This mode will only include tests whose references are each within the selected nodes and will ignore all tests from attached nodes. ```shell -$ dbt test --select orders --indirect-selection=empty -$ dbt build --select orders --indirect-selection=empty + +dbt test --select "orders" --indirect-selection=empty +dbt build --select "orders" --indirect-selection=empty + ``` @@ -234,22 +242,25 @@ The following examples should feel somewhat familiar if you're used to executing ```bash # Run tests on a model (indirect selection) - $ dbt test --select customers + dbt test --select "customers" + + # Run tests on two or more specific models (indirect selection) + dbt test --select "customers orders" # Run tests on all models in the models/staging/jaffle_shop directory (indirect selection) - $ dbt test --select staging.jaffle_shop + dbt test --select "staging.jaffle_shop" # Run tests downstream of a model (note this will select those tests directly!) - $ dbt test --select stg_customers+ + dbt test --select "stg_customers+" # Run tests upstream of a model (indirect selection) - $ dbt test --select +stg_customers + dbt test --select "+stg_customers" # Run tests on all models with a particular tag (direct + indirect) - $ dbt test --select tag:my_model_tag + dbt test --select "tag:my_model_tag" # Run tests on all models with a particular materialization (indirect selection) - $ dbt test --select config.materialized:table + dbt test --select "config.materialized:table" ``` @@ -258,16 +269,20 @@ The following examples should feel somewhat familiar if you're used to executing ```bash # tests on all sources - $ dbt test --select source:* + + dbt test --select "source:*" # tests on one source - $ dbt test --select source:jaffle_shop + dbt test --select "source:jaffle_shop" + + # tests on two or more specific sources + dbt test --select "source:jaffle_shop source:raffle_bakery" # tests on one source table - $ dbt test --select source:jaffle_shop.customers + dbt test --select "source:jaffle_shop.customers" # tests on everything _except_ sources - $ dbt test --exclude source:* + dbt test --exclude "source:*" ``` ### More complex selection @@ -276,10 +291,12 @@ Through the combination of direct and indirect selection, there are many ways to ```bash - $ dbt test --select assert_total_payment_amount_is_positive # directly select the test by name - $ dbt test --select payments,test_type:singular # indirect selection, v1.2 - $ dbt test --select payments,test_type:data # indirect selection, v0.18.0 - $ dbt test --select payments --data # indirect selection, earlier versions + + dbt test --select "assert_total_payment_amount_is_positive" # directly select the test by name + dbt test --select "payments,test_type:singular" # indirect selection, v1.2 + dbt test --select "payments,test_type:data" # indirect selection, v0.18.0 + dbt test --select "payments" --data # indirect selection, earlier versions + ``` @@ -288,13 +305,14 @@ Through the combination of direct and indirect selection, there are many ways to ```bash # Run tests on all models with a particular materialization - $ dbt test --select config.materialized:table + dbt test --select "config.materialized:table" # Run tests on all seeds, which use the 'seed' materialization - $ dbt test --select config.materialized:seed + dbt test --select "config.materialized:seed" # Run tests on all snapshots, which use the 'snapshot' materialization - $ dbt test --select config.materialized:snapshot + dbt test --select "config.materialized:snapshot" + ``` Note that this functionality may change in future versions of dbt. @@ -312,8 +330,8 @@ models: - name: orders columns: - name: order_id - tests: tags: [my_column_tag] + tests: - unique ``` @@ -322,7 +340,8 @@ models: ```bash - $ dbt test --select tag:my_column_tag + dbt test --select "tag:my_column_tag" + ``` Currently, tests "inherit" tags applied to columns, sources, and source tables. They do _not_ inherit tags applied to models, seeds, or snapshots. In all likelihood, those tests would still be selected indirectly, because the tag selects its parent. This is a subtle distinction, and it may change in future versions of dbt. @@ -350,5 +369,6 @@ models: ```bash - $ dbt test --select tag:my_test_tag + dbt test --select "tag:my_test_tag" + ``` diff --git a/website/docs/reference/node-selection/yaml-selectors.md b/website/docs/reference/node-selection/yaml-selectors.md index 78342e32779..1e3f8d8d1e2 100644 --- a/website/docs/reference/node-selection/yaml-selectors.md +++ b/website/docs/reference/node-selection/yaml-selectors.md @@ -34,6 +34,7 @@ Each `definition` is comprised of one or more arguments, which can be one of the Use the `union` and `intersection` operator-equivalent keywords to organize multiple arguments. ### CLI-style + ```yml definition: 'tag:nightly' @@ -42,6 +43,7 @@ definition: This simple syntax supports use of the `+`, `@`, and `*` [graph](/reference/node-selection/graph-operators) operators, but it does not support [set](/reference/node-selection/set-operators) operators or `exclude`. ### Key-value + ```yml definition: tag: nightly @@ -317,7 +319,7 @@ selectors: Then in our job definition: ```bash -$ dbt run --selector nightly_diet_snowplow +dbt run --selector nightly_diet_snowplow ``` ## Default @@ -325,6 +327,7 @@ $ dbt run --selector nightly_diet_snowplow Selectors may define a boolean `default` property. If a selector has `default: true`, dbt will use this selector's criteria when tasks do not define their own selection criteria. Let's say we define a default selector that only selects resources defined in our root project: + ```yml selectors: - name: root_project_only @@ -338,16 +341,18 @@ selectors: ``` If I run an "unqualified" command, dbt will use the selection criteria defined in `root_project_only`—that is, dbt will only build / freshness check / generate compiled SQL for resources defined in my root project. + ``` -$ dbt build -$ dbt source freshness -$ dbt docs generate +dbt build +dbt source freshness +dbt docs generate ``` If I run a command that defines its own selection criteria (via `--select`, `--exclude`, or `--selector`), dbt will ignore the default selector and use the flag criteria instead. It will not try to combine the two. -``` -$ dbt run --select model_a -$ dbt run --exclude model_a + +```bash +dbt run --select "model_a" +dbt run --exclude model_a ``` Only one selector may set `default: true` for a given invocation; otherwise, dbt will return an error. You may use a Jinja expression to adjust the value of `default` depending on the environment, however: diff --git a/website/docs/reference/project-configs/on-run-start-on-run-end.md b/website/docs/reference/project-configs/on-run-start-on-run-end.md index 1ed8c570dd0..e1a3d7b761a 100644 --- a/website/docs/reference/project-configs/on-run-start-on-run-end.md +++ b/website/docs/reference/project-configs/on-run-start-on-run-end.md @@ -4,6 +4,8 @@ description: "Read this guide to understand the on-run-start and on-run-end conf datatype: sql-statement | [sql-statement] --- +import OnRunCommands from '/snippets/_onrunstart-onrunend-commands.md'; + ```yml @@ -15,14 +17,8 @@ on-run-end: sql-statement | [sql-statement] ## Definition -A SQL statement (or list of SQL statements) to be run at the start, or end, of the following commands: -- `dbt run` -- `dbt test` -- `dbt seed` -- `dbt snapshot` -- `dbt build` -- `dbt compile` -- `dbt docs generate` + +A SQL statement (or list of SQL statements) to be run at the start or end of the following commands: `on-run-start` and `on-run-end` hooks can also call macros that return SQL statements diff --git a/website/docs/reference/references-overview.md b/website/docs/reference/references-overview.md index 16afd01607c..85a374c5aa3 100644 --- a/website/docs/reference/references-overview.md +++ b/website/docs/reference/references-overview.md @@ -4,6 +4,8 @@ id: "references-overview" sidebar_label: "About References" description: "Connect dbt to any data platform in dbt Cloud or dbt Core, using a dedicated adapter plugin" hide_table_of_contents: true +pagination_next: null +pagination_prev: null --- The References section contains reference materials for developing with dbt, which includes dbt Cloud and dbt Core. diff --git a/website/docs/reference/resource-configs/access.md b/website/docs/reference/resource-configs/access.md new file mode 100644 index 00000000000..da50e48d2f0 --- /dev/null +++ b/website/docs/reference/resource-configs/access.md @@ -0,0 +1,97 @@ +--- +resource_types: [models] +datatype: access +--- + + + +```yml +version: 2 + +models: + - name: model_name + access: private | protected | public +``` + + + + + +Access modifiers may be applied to models one-by-one in YAML properties. In v1.5 and v1.6, you are unable to configure `access` for multiple models at once. Upgrade to v1.7 for additional configuration options. A group or subfolder contains models with varying access levels, so when you designate a model with `access: public`, make sure you intend for this behavior. + + + + + +You can apply access modifiers in config files, including `the dbt_project.yml`, or to models one-by-one in YAML properties. Applying access configs to a subfolder modifies the default for all models in that subfolder, so make sure you intend for this behavior. When setting individual model access, a group or subfolder might contain a variety of access levels, so when you designate a model with `access: public` make sure you intend for this behavior. + +There are multiple approaches to configuring access: + +In the model configs of `dbt_project.yml``: + +```yaml +models: + - name: my_public_model + access: public # Older method, still supported + +``` +Or (but not both) + +```yaml +models: + - name: my_public_model + config: + access: public # newly supported in v1.7 + +``` + +In a subfolder: +```yaml +models: + my_project_name: + subfolder_name: + +group: + +access: private # sets default for all models in this subfolder +``` + +In the model.sql file: + +```sql +-- models/my_public_model.sql + +{{ config(access = "public") }} + +select ... +``` + + + +## Definition +The access level of the model you are declaring properties for. + +Some models (not all) are designed to be referenced through the [ref](/reference/dbt-jinja-functions/ref) function across [groups](/docs/build/groups). + +| Access | Referenceable by | +|-----------|-------------------------------| +| private | same group | +| protected | same project/package | +| public | any group, package or project | + +If you try to reference a model outside of its supported access, you will see an error: + +```shell +dbt run -s marketing_model +... +dbt.exceptions.DbtReferenceError: Parsing Error + Node model.jaffle_shop.marketing_model attempted to reference node model.jaffle_shop.finance_model, + which is not allowed because the referenced node is private to the finance group. +``` + +## Default + +By default, all models are "protected." This means that other models in the same project can reference them. + +## Related docs + +* [Model Access](/docs/collaborate/govern/model-access#groups) +* [Group configuration](/reference/resource-configs/group) diff --git a/website/docs/reference/resource-configs/bigquery-configs.md b/website/docs/reference/resource-configs/bigquery-configs.md index 89a750f47bd..ffbaa37c059 100644 --- a/website/docs/reference/resource-configs/bigquery-configs.md +++ b/website/docs/reference/resource-configs/bigquery-configs.md @@ -414,7 +414,7 @@ models: columns: - name: field policy_tags: - - 'projects//locations//taxonomies//policyTags/' + - 'projects//locations//taxonomies//policyTags/' ``` diff --git a/website/docs/reference/resource-configs/contract.md b/website/docs/reference/resource-configs/contract.md index e8ea6d82287..59cc511890b 100644 --- a/website/docs/reference/resource-configs/contract.md +++ b/website/docs/reference/resource-configs/contract.md @@ -23,11 +23,34 @@ When the `contract` configuration is enforced, dbt will ensure that your model's This is to ensure that the people querying your model downstream—both inside and outside dbt—have a predictable and consistent set of columns to use in their analyses. Even a subtle change in data type, such as from `boolean` (`true`/`false`) to `integer` (`0`/`1`), could cause queries to fail in surprising ways. + + The `data_type` defined in your YAML file must match a data type your data platform recognizes. dbt does not do any type aliasing itself. If your data platform recognizes both `int` and `integer` as corresponding to the same type, then they will return a match. -When dbt is comparing data types, it will not compare granular details such as size, precision, or scale. We don't think you should sweat the difference between `varchar(256)` and `varchar(257)`, because it doesn't really affect the experience of downstream queriers. If you need a more-precise assertion, it's always possible to accomplish by [writing or using a custom test](/guides/best-practices/writing-custom-generic-tests). + + + + +dbt uses built-in type aliasing for the `data_type` defined in your YAML. For example, you can specify `string` in your contract, and on Postgres/Redshift, dbt will convert it to `text`. If dbt doesn't recognize the `data_type` name among its known aliases, it will pass it through as-is. This is enabled by default, but you can opt-out by setting `alias_types` to `false`. + +Example for disabling: + +```yml + +models: + - name: my_model + config: + contract: + enforced: true + alias_types: false # true by default + +``` + + + +When dbt compares data types, it will not compare granular details such as size, precision, or scale. We don't think you should sweat the difference between `varchar(256)` and `varchar(257)`, because it doesn't really affect the experience of downstream queriers. You can accomplish a more-precise assertion by [writing or using a custom test](/guides/best-practices/writing-custom-generic-tests). -That said, on certain data platforms, you will need to specify a varchar size or numeric scale if you do not want it to revert to the default. This is most relevant for the `numeric` type on Snowflake, which defaults to a precision of 38 and a scale of 0 (zero digits after the decimal, such as rounded to an integer). To avoid this implicit coercion, specify your `data_type` with a nonzero scale, like `numeric(38, 6)`. +Note that you need to specify a varchar size or numeric scale, otherwise dbt relies on default values. For example, if a `numeric` type defaults to a precision of 38 and a scale of 0, then the numeric column stores 0 digits to the right of the decimal (it only stores whole numbers), which might cause it to fail contract enforcement. To avoid this implicit coercion, specify your `data_type` with a nonzero scale, like `numeric(38, 6)`. dbt Core 1.7 and higher provides a warning if you don't specify precision and scale when providing a numeric data type. ## Example @@ -47,6 +70,8 @@ models: - type: not_null - name: customer_name data_type: string + - name: non_integer + data_type: numeric(38,3) ``` diff --git a/website/docs/reference/resource-configs/delimiter.md b/website/docs/reference/resource-configs/delimiter.md new file mode 100644 index 00000000000..58d6ba8344a --- /dev/null +++ b/website/docs/reference/resource-configs/delimiter.md @@ -0,0 +1,126 @@ +--- +resource_types: [seeds] +datatype: +default_value: "," +--- + +## Definition + +You can use this optional seed configuration to customize how you separate values in a [seed](/docs/build/seeds) by providing the one-character string. + +* The delimiter defaults to a comma when not specified. +* Explicitly set the `delimiter` configuration value if you want seed files to use a different delimiter, such as "|" or ";". + +:::info New in 1.7! + +Delimiter is new functionality available beginning with dbt Core v1.7. + +::: + + +## Usage + +Specify a delimiter in your `dbt_project.yml` file to customize the global separator for all seed values: + + + +```yml +seeds: + : + +delimiter: "|" # default project delimiter for seeds will be "|" + : + +delimiter: "," # delimiter for seeds in seed_subdirectory will be "," +``` + + + + +Or use a custom delimiter to override the values for a specific seed: + + + +```yml +version: 2 + +seeds: + - name: + config: + delimiter: "|" +``` + + + +## Examples +For a project with: + +* `name: jaffle_shop` in the `dbt_project.yml` file +* `seed-paths: ["seeds"]` in the `dbt_project.yml` file + +### Use a custom delimiter to override global values + +You can set a default behavior for all seeds with an exception for one seed, `seed_a`, which uses a comma: + + + +```yml +seeds: + jaffle_shop: + +delimiter: "|" # default delimiter for seeds in jaffle_shop project will be "|" + seed_a: + +delimiter: "," # delimiter for seed_a will be "," +``` + + + +Your corresponding seed files would be formatted like this: + + + +```text +col_a|col_b|col_c +1|2|3 +4|5|6 +... +``` + + + + + +```text +name,id +luna,1 +doug,2 +... +``` + + + +Or you can configure custom behavior for one seed. The `country_codes` uses the ";" delimiter: + + + +```yml +version: 2 + +seeds: + - name: country_codes + config: + delimiter: ";" +``` + + + +The `country_codes` seed file would be formatted like this: + + + +```text +country_code;country_name +US;United States +CA;Canada +GB;United Kingdom +... +``` + + diff --git a/website/docs/reference/resource-configs/enabled.md b/website/docs/reference/resource-configs/enabled.md index b6d0961ee60..52045503088 100644 --- a/website/docs/reference/resource-configs/enabled.md +++ b/website/docs/reference/resource-configs/enabled.md @@ -15,6 +15,7 @@ default_value: true { label: 'Sources', value: 'sources', }, { label: 'Metrics', value: 'metrics', }, { label: 'Exposures', value: 'exposures', }, + { label: 'Semantic models', value: 'semantic models', }, ] }> @@ -250,10 +251,39 @@ exposures: + + + + +Support for disabling semantic models has been added in dbt Core v1.7 + + + + + + + +```yml +semantic_models: + - name: semantic_people + model: ref('people') + config: + enabled: false + +``` + + + +The `enabled` configuration can be nested under the `config` key. + + + + + ## Definition -An optional configuration for disabling models, seeds, snapshots, and tests. +An optional configuration for disabling models, seeds, snapshots, tests, and semantic models. * Default: true diff --git a/website/docs/reference/resource-configs/group.md b/website/docs/reference/resource-configs/group.md index dd73d99edff..7515d8c5789 100644 --- a/website/docs/reference/resource-configs/group.md +++ b/website/docs/reference/resource-configs/group.md @@ -16,6 +16,7 @@ This functionality is new in v1.5. { label: 'Tests', value: 'tests', }, { label: 'Analyses', value: 'analyses', }, { label: 'Metrics', value: 'metrics', }, + { label: 'Semantic models', value: 'semantic models', }, ] }> @@ -265,6 +266,43 @@ metrics: + + + + +Support for grouping semantic models has been added in dbt Core v1.7. + + + + + + + +```yml +semantic_models: + - name: model_name + group: finance + +``` + + + + + +```yml +semantic_models: + [](resource-path): + +group: finance +``` + + + +The `group` configuration can be nested under the `config` key. + + + + + ## Definition diff --git a/website/docs/reference/resource-configs/meta.md b/website/docs/reference/resource-configs/meta.md index d24c5fbaee1..65c8b5f908e 100644 --- a/website/docs/reference/resource-configs/meta.md +++ b/website/docs/reference/resource-configs/meta.md @@ -14,6 +14,8 @@ default_value: {} { label: 'Tests', value: 'tests', }, { label: 'Analyses', value: 'analyses', }, { label: 'Macros', value: 'macros', }, + { label: 'Exposures', value: 'exposures', }, + { label: 'Semantic Models', value: 'semantic models', }, ] }> @@ -172,6 +174,34 @@ exposures: + + + + +Support for grouping semantic models was added in dbt Core v1.7 + + + + + + + +```yml +semantic_models: + - name: semantic_people + model: ref('people') + config: + meta: {} + +``` +The `meta` configuration can be nusted under the `config` key. + + + + + + + ## Definition @@ -248,3 +278,19 @@ select 1 as id ``` + +### Assign owner in the dbt_project.yml as a config property + + + +```yml +models: + jaffle_shop: + materialized: table + config: + meta: + owner: "@alice" +``` + + + diff --git a/website/docs/reference/resource-configs/store_failures.md b/website/docs/reference/resource-configs/store_failures.md index 3c965179211..6c71cdb9296 100644 --- a/website/docs/reference/resource-configs/store_failures.md +++ b/website/docs/reference/resource-configs/store_failures.md @@ -3,7 +3,7 @@ resource_types: [tests] datatype: boolean --- -The configured test(s) will store their failures when `dbt test --store-failures` is invoked. +The configured test(s) will store their failures when `dbt test --store-failures` is invoked. If you set this configuration as `false` but [`store_failures_as`](/reference/resource-configs/store_failures_as) is configured, it will be overriden. ## Description Optionally set a test to always or never store its failures in the database. diff --git a/website/docs/reference/resource-configs/store_failures_as.md b/website/docs/reference/resource-configs/store_failures_as.md new file mode 100644 index 00000000000..a9149360089 --- /dev/null +++ b/website/docs/reference/resource-configs/store_failures_as.md @@ -0,0 +1,76 @@ +--- +resource_types: [tests] +id: "store_failures_as" +--- + +For the `test` resource type, `store_failures_as` is an optional config that specifies how test failures should be stored in the database. If [`store_failures`](/reference/resource-configs/store_failures) is also configured, `store_failures_as` takes precedence. + +The three supported values are: + +- `ephemeral` — nothing stored in the database (default) +- `table` — test failures stored as a database table +- `view` — test failures stored as a database view + +You can configure it in all the same places as `store_failures`, including singular tests (.sql files), generic tests (.yml files), and dbt_project.yml. + +### Examples + +#### Singular test + +[Singular test](https://docs.getdbt.com/docs/build/tests#singular-tests) in `tests/singular/check_something.sql` file + +```sql +{{ config(store_failures_as="table") }} + +-- custom singular test +select 1 as id +where 1=0 +``` + +#### Generic test + +[Generic tests](https://docs.getdbt.com/docs/build/tests#generic-tests) in `models/_models.yml` file + +```yaml +models: + - name: my_model + columns: + - name: id + tests: + - not_null: + config: + store_failures_as: view + - unique: + config: + store_failures_as: ephemeral +``` + +#### Project level + +Config in `dbt_project.yml` + +```yaml +name: "my_project" +version: "1.0.0" +config-version: 2 +profile: "sandcastle" + +tests: + my_project: + +store_failures_as: table + my_subfolder_1: + +store_failures_as: view + my_subfolder_2: + +store_failures_as: ephemeral +``` + +### "Clobbering" configs + +As with most other configurations, `store_failures_as` is "clobbered" when applied hierarchically. Whenever a more specific value is available, it will completely replace the less specific value. + +Additional resources: + +- [Test configurations](/reference/test-configs#related-documentation) +- [Test-specific configurations](/reference/test-configs#test-specific-configurations) +- [Configuring directories of models in dbt_project.yml](/reference/model-configs#configuring-directories-of-models-in-dbt_projectyml) +- [Config inheritance](/reference/configs-and-properties#config-inheritance) \ No newline at end of file diff --git a/website/docs/reference/resource-properties/access.md b/website/docs/reference/resource-properties/access.md deleted file mode 100644 index 42b9893ed7f..00000000000 --- a/website/docs/reference/resource-properties/access.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -resource_types: [models] -datatype: access -required: no ---- - -:::info New functionality -This functionality is new in v1.5. -::: - - - -```yml -version: 2 - -models: - - name: model_name - access: private | protected | public -``` - - - -Access modifiers may be applied to models one-by-one in YAML properties. It is not currently possible to configure `access` for multiple models at once. A group or subfolder contains models with a variety of access levels, and designating a model with `access: public` should always be a conscious and intentional choice. - -## Definition -The access level of the model you are declaring properties for. - -Some models (not all) are designed to be referenced through the [ref](/reference/dbt-jinja-functions/ref) function across [groups](/docs/build/groups). - -| Access | Referenceable by | -|-----------|-------------------------------| -| private | same group | -| protected | same project/package | -| public | any group, package or project | - -If you try to reference a model outside of its supported access, you will see an error: - -```shell -dbt run -s marketing_model -... -dbt.exceptions.DbtReferenceError: Parsing Error - Node model.jaffle_shop.marketing_model attempted to reference node model.jaffle_shop.finance_model, - which is not allowed because the referenced node is private to the finance group. -``` - -## Default - -By default, all models are "protected." This means that other models in the same project can reference them. - -## Related docs - -* [Model Access](/docs/collaborate/govern/model-access#groups) -* [Group configuration](/reference/resource-configs/group) diff --git a/website/docs/reference/seed-configs.md b/website/docs/reference/seed-configs.md index d74f414cbfe..429aa9444ae 100644 --- a/website/docs/reference/seed-configs.md +++ b/website/docs/reference/seed-configs.md @@ -23,6 +23,7 @@ seeds: [](/reference/resource-configs/resource-path): [+](/reference/resource-configs/plus-prefix)[quote_columns](/reference/resource-configs/quote_columns): true | false [+](/reference/resource-configs/plus-prefix)[column_types](/reference/resource-configs/column_types): {column_name: datatype} + [+](/reference/resource-configs/plus-prefix)[delimiter](/reference/resource-configs/delimiter): ``` @@ -43,6 +44,7 @@ seeds: config: [quote_columns](/reference/resource-configs/quote_columns): true | false [column_types](/reference/resource-configs/column_types): {column_name: datatype} + [delimiter](/reference/resource-configs/grants): ``` diff --git a/website/docs/terms/materialization.md b/website/docs/terms/materialization.md index fdeaaebfcc8..328076f1483 100644 --- a/website/docs/terms/materialization.md +++ b/website/docs/terms/materialization.md @@ -11,7 +11,7 @@ hoverSnippet: The exact Data Definition Language (DDL) that dbt will use when cr :::important This page could use some love -This term would benefit from additional depth and examples. Have knowledge to contribute? [Create a discussion in the docs.getdbt.com GitHub repository](https://github.com/dbt-labs/docs.getdbt.com/discussions) to begin the process of becoming a glossary contributor! +This term would benefit from additional depth and examples. Have knowledge to contribute? [Create an issue in the docs.getdbt.com repository](https://github.com/dbt-labs/docs.getdbt.com/issues/new/choose) to begin the process of becoming a glossary contributor! ::: The exact Data Definition Language (DDL) that dbt will use when creating the model’s equivalent in a . It's the manner in which the data is represented, and each of those options is defined either canonically (tables, views, incremental), or bespoke. diff --git a/website/docs/terms/model.md b/website/docs/terms/model.md new file mode 100644 index 00000000000..c589cc196a7 --- /dev/null +++ b/website/docs/terms/model.md @@ -0,0 +1,9 @@ +--- +id: model +title: Model +description: A model is an essential building block of the DAG +displayText: model +hoverSnippet: A model is an essential building block of the DAG +--- + +A model is an essential building block of the DAG that lives in a single file and contains logic that transforms data. This logic can be expressed as a SQL `select` statement or a Python dataframe operation. Models can be materialized in the warehouse in different ways — most of these materializations require models to be built in the warehouse. \ No newline at end of file diff --git a/website/docs/terms/table.md b/website/docs/terms/table.md index 69fc2b3e6b6..cbe36ec1315 100644 --- a/website/docs/terms/table.md +++ b/website/docs/terms/table.md @@ -6,7 +6,7 @@ displayText: table hoverSnippet: In simplest terms, a table is the direct storage of data in rows and columns. Think excel sheet with raw values in each of the cells. --- :::important This page could use some love -This term would benefit from additional depth and examples. Have knowledge to contribute? [Create a discussion in the docs.getdbt.com GitHub repository](https://github.com/dbt-labs/docs.getdbt.com/discussions) to begin the process of becoming a glossary contributor! +This term would benefit from additional depth and examples. Have knowledge to contribute? [Create an issue in the docs.getdbt.com repository](https://github.com/dbt-labs/docs.getdbt.com/issues/new/choose) to begin the process of becoming a glossary contributor! ::: In simplest terms, a table is the direct storage of data in rows and columns. Think excel sheet with raw values in each of the cells. diff --git a/website/docs/terms/view.md b/website/docs/terms/view.md index 5d9238256e0..90cd5d1f36f 100644 --- a/website/docs/terms/view.md +++ b/website/docs/terms/view.md @@ -6,7 +6,7 @@ displayText: view hoverSnippet: A view (as opposed to a table) is a defined passthrough SQL query that can be run against a database (or data warehouse). --- :::important This page could use some love -This term would benefit from additional depth and examples. Have knowledge to contribute? [Create a discussion in the docs.getdbt.com GitHub repository](https://github.com/dbt-labs/docs.getdbt.com/discussions) to begin the process of becoming a glossary contributor! +This term would benefit from additional depth and examples. Have knowledge to contribute? [Create an issue in the docs.getdbt.com repository](https://github.com/dbt-labs/docs.getdbt.com/issues/new/choose) to begin the process of becoming a glossary contributor! ::: A view (as opposed to a ) is a defined passthrough SQL query that can be run against a database (or ). A view doesn’t store data, like a table does, but it defines the logic that you need to fetch the underlying data. diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 0cc6299ed39..ce81d614c65 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -71,13 +71,13 @@ var siteSettings = { announcementBar: { id: "biweekly-demos", content: - "Join our weekly demos and see dbt Cloud in action!", + "Join our weekly demos and dbt Cloud in action!", backgroundColor: "#047377", textColor: "#fff", isCloseable: true, }, announcementBarActive: true, - announcementBarLink: "https://www.getdbt.com/resources/dbt-cloud-demos-with-experts/?utm_medium=event&utm_source=docs&utm_campaign=q1-2024_cloud-demos-with-experts_awareness", + announcementBarLink: "https://www.getdbt.com/resources/dbt-cloud-demos-with-experts?utm_source=docs&utm_medium=event&utm_campaign=q1-2024_cloud-demos-with-experts_awareness", // Set community spotlight member on homepage // This is the ID for a specific file under docs/community/spotlight communitySpotlightMember: "faith-lierheimer", diff --git a/website/sidebars.js b/website/sidebars.js index af9482a8ddf..fc87c1142c7 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -7,6 +7,7 @@ const sidebarSettings = { collapsed: true, link: { type: "doc", id: "docs/supported-data-platforms" }, items: [ + "docs/supported-data-platforms", "docs/connect-adapters", "docs/verified-adapters", "docs/trusted-adapters", @@ -17,12 +18,12 @@ const sidebarSettings = { { type: "category", label: "About dbt Cloud", + link: { type: "doc", id: "docs/cloud/about-cloud/dbt-cloud-features" }, items: [ "docs/cloud/about-cloud/dbt-cloud-features", "docs/cloud/about-cloud/architecture", "docs/cloud/about-cloud/tenancy", "docs/cloud/about-cloud/regions-ip-addresses", - "docs/cloud/about-cloud/about-cloud-ide", "docs/cloud/about-cloud/browsers", ], }, // About dbt Cloud directory @@ -35,6 +36,7 @@ const sidebarSettings = { type: "category", label: "Set up dbt", collapsed: true, + link: { type: "doc", id: "docs/about-setup" }, items: [ "docs/about-setup", "docs/environments-in-dbt", @@ -42,12 +44,14 @@ const sidebarSettings = { type: "category", label: "dbt Cloud", collapsed: true, + link: { type: "doc", id: "docs/cloud/about-cloud-setup" }, items: [ "docs/cloud/about-cloud-setup", "docs/dbt-cloud-environments", { type: "category", label: "Connect data platform", + link: { type: "doc", id: "docs/cloud/connect-data-platform/about-connections" }, items: [ "docs/cloud/connect-data-platform/about-connections", "docs/cloud/connect-data-platform/connect-starburst-trino", @@ -61,13 +65,15 @@ const sidebarSettings = { { type: "category", label: "Manage access", + link: { type: "doc", id: "docs/cloud/manage-access/about-user-access" }, items: [ "docs/cloud/manage-access/about-user-access", - "docs/cloud/manage-access/seats-and-users", { type: "category", - label: "Permissions", + label: "User permissions and licenses", + link: { type: "doc", id: "docs/cloud/manage-access/seats-and-users" }, items: [ + "docs/cloud/manage-access/seats-and-users", "docs/cloud/manage-access/self-service-permissions", "docs/cloud/manage-access/enterprise-permissions", ], @@ -75,7 +81,8 @@ const sidebarSettings = { { type: "category", - label: "Single sign-on", + label: "Single sign-on and Oauth", + link: { type: "doc", id: "docs/cloud/manage-access/sso-overview" }, items: [ "docs/cloud/manage-access/sso-overview", "docs/cloud/manage-access/auth0-migration", @@ -83,16 +90,11 @@ const sidebarSettings = { "docs/cloud/manage-access/set-up-sso-okta", "docs/cloud/manage-access/set-up-sso-google-workspace", "docs/cloud/manage-access/set-up-sso-azure-active-directory", - ], - }, // SSO - { - type: "category", - label: "OAuth with data platforms", - items: [ "docs/cloud/manage-access/set-up-snowflake-oauth", + "docs/cloud/manage-access/set-up-databricks-oauth", "docs/cloud/manage-access/set-up-bigquery-oauth", ], - }, // oauth + }, // SSO "docs/cloud/manage-access/audit-log", ], }, // Manage access @@ -100,42 +102,65 @@ const sidebarSettings = { { type: "category", label: "Configure Git", + link: { type: "doc", id: "docs/cloud/git/git-configuration-in-dbt-cloud" }, items: [ + "docs/cloud/git/git-configuration-in-dbt-cloud", + "docs/cloud/git/import-a-project-by-git-url", "docs/cloud/git/connect-github", "docs/cloud/git/connect-gitlab", { type: "category", label: "Azure DevOps", + link: { type: "doc", id: "docs/cloud/git/connect-azure-devops" }, items: [ "docs/cloud/git/connect-azure-devops", "docs/cloud/git/setup-azure", "docs/cloud/git/authenticate-azure", ], }, - "docs/cloud/git/import-a-project-by-git-url", ], }, // Supported Git providers { type: "category", - label: "Develop in the IDE", - link: { - type: "doc", - id: "docs/cloud/dbt-cloud-ide/develop-in-the-cloud", - }, + label: "Develop in dbt Cloud", + link: { type: "doc", id: "docs/cloud/about-cloud-develop" }, items: [ - "docs/cloud/dbt-cloud-ide/ide-user-interface", - "docs/cloud/dbt-cloud-ide/lint-format", - "docs/cloud/dbt-cloud-ide/dbt-cloud-tips", + "docs/cloud/about-cloud-develop", + "docs/cloud/about-cloud-develop-defer", + { + type: "category", + label: "dbt Cloud CLI", + link: { type: "doc", id: "docs/cloud/cloud-cli-installation" }, + items: [ + "docs/cloud/cloud-cli-installation", + "docs/cloud/configure-cloud-cli", + ], + }, + { + type: "category", + label: "dbt Cloud IDE", + link: { type: "doc", id: "docs/cloud/dbt-cloud-ide/develop-in-the-cloud" }, + items: [ + "docs/cloud/dbt-cloud-ide/develop-in-the-cloud", + "docs/cloud/dbt-cloud-ide/ide-user-interface", + "docs/cloud/dbt-cloud-ide/lint-format", + "docs/cloud/dbt-cloud-ide/dbt-cloud-tips", + ], + }, ], - }, // dbt Cloud IDE directory + }, // dbt Cloud develop directory { type: "category", label: "Secure your tenant", + link: { type: "doc", id: "docs/cloud/secure/secure-your-tenant" }, items: [ + "docs/cloud/secure/secure-your-tenant", + "docs/cloud/secure/ip-restrictions", "docs/cloud/secure/about-privatelink", "docs/cloud/secure/snowflake-privatelink", - "docs/cloud/secure/redshift-privatelink", "docs/cloud/secure/databricks-privatelink", + "docs/cloud/secure/redshift-privatelink", + "docs/cloud/secure/postgres-privatelink", "docs/cloud/secure/ip-restrictions", ], }, // PrivateLink @@ -148,13 +173,15 @@ const sidebarSettings = { collapsed: true, link: { type: "doc", id: "docs/core/about-core-setup" }, items: [ - "docs/core/about-the-cli", + "docs/core/about-core-setup", + "docs/core/about-dbt-core", "docs/core/dbt-core-environments", { type: "category", label: "Install dbt", link: { type: "doc", id: "docs/core/installation" }, items: [ + "docs/core/installation", "docs/core/homebrew-install", "docs/core/pip-install", "docs/core/docker-install", @@ -169,6 +196,7 @@ const sidebarSettings = { id: "docs/core/connect-data-platform/about-core-connections", }, items: [ + "docs/core/connect-data-platform/about-core-connections", "docs/core/connect-data-platform/profiles.yml", "docs/core/connect-data-platform/connection-profiles", "docs/core/connect-data-platform/bigquery-setup", @@ -223,16 +251,19 @@ const sidebarSettings = { type: "category", label: "Build dbt projects", collapsed: true, + link: { type: "doc", id: "docs/build/projects" }, items: [ "docs/build/projects", { type: "category", label: "Build your DAG", collapsed: true, + link: { type: "doc", id: "docs/build/models" }, items: [ { type: "category", label: "Models", + link: { type: "doc", id: "docs/build/models" }, items: [ "docs/build/models", "docs/build/sql-models", @@ -256,38 +287,43 @@ const sidebarSettings = { link: { type: "doc", id: "docs/build/build-metrics-intro" }, collapsed: true, items: [ + "docs/build/build-metrics-intro", "docs/build/sl-getting-started", { type: "category", label: "About MetricFlow", link: { type: "doc", id: "docs/build/about-metricflow" }, items: [ + "docs/build/about-metricflow", "docs/build/join-logic", "docs/build/validation", + "docs/build/saved-queries", "docs/build/metricflow-time-spine", - "docs/build/metricflow-cli", - ] + "docs/build/metricflow-commands", + ], }, { type: "category", label: "Semantic models", link: { type: "doc", id: "docs/build/semantic-models" }, items: [ + "docs/build/semantic-models", "docs/build/dimensions", "docs/build/entities", - "docs/build/measures" - ] + "docs/build/measures", + ], }, { type: "category", label: "Metrics", link: { type: "doc", id: "docs/build/metrics-overview" }, items: [ + "docs/build/metrics-overview", "docs/build/cumulative", "docs/build/derived", "docs/build/ratio", "docs/build/simple", - ] + ], }, ], }, @@ -295,7 +331,9 @@ const sidebarSettings = { type: "category", label: "Enhance your models", collapsed: true, + link: { type: "doc", id: "docs/build/enhance-your-models" }, items: [ + "docs/build/enhance-your-models", "docs/build/materializations", "docs/build/incremental-models", ], @@ -304,7 +342,9 @@ const sidebarSettings = { type: "category", label: "Enhance your code", collapsed: true, + link: { type: "doc", id: "docs/build/enhance-your-code" }, items: [ + "docs/build/enhance-your-code", "docs/build/project-variables", "docs/build/environment-variables", "docs/build/packages", @@ -315,7 +355,9 @@ const sidebarSettings = { type: "category", label: "Organize your outputs", collapsed: true, + link: { type: "doc", id: "docs/build/organize-your-outputs" }, items: [ + "docs/build/organize-your-outputs", "docs/build/custom-schemas", "docs/build/custom-databases", "docs/build/custom-aliases", @@ -332,6 +374,7 @@ const sidebarSettings = { collapsed: true, link: { type: "doc", id: "docs/deploy/deployments" }, items: [ + "docs/deploy/deployments", "docs/deploy/job-scheduler", "docs/deploy/deploy-environments", "docs/deploy/continuous-integration", @@ -340,6 +383,7 @@ const sidebarSettings = { label: "Jobs", link: { type: "doc", id: "docs/deploy/jobs" }, items: [ + "docs/deploy/jobs", "docs/deploy/deploy-jobs", "docs/deploy/ci-jobs", "docs/deploy/job-commands", @@ -350,7 +394,9 @@ const sidebarSettings = { label: "Monitor jobs and alerts", link: { type: "doc", id: "docs/deploy/monitor-jobs" }, items: [ + "docs/deploy/monitor-jobs", "docs/deploy/run-visibility", + "docs/deploy/retry-jobs", "docs/deploy/job-notifications", "docs/deploy/webhooks", "docs/deploy/artifacts", @@ -364,11 +410,14 @@ const sidebarSettings = { { type: "category", label: "Collaborate with others", + link: { type: "doc", id: "docs/collaborate/collaborate-with-others" }, items: [ + "docs/collaborate/collaborate-with-others", "docs/collaborate/explore-projects", { type: "category", label: "Git version control", + link: { type: "doc", id: "docs/collaborate/git-version-control" }, items: [ "docs/collaborate/git-version-control", "docs/collaborate/git/version-control-basics", @@ -380,6 +429,7 @@ const sidebarSettings = { { type: "category", label: "Document your dbt projects", + link: { type: "doc", id: "docs/collaborate/documentation" }, items: [ "docs/collaborate/documentation", "docs/collaborate/build-and-view-your-docs", @@ -394,6 +444,7 @@ const sidebarSettings = { id: "docs/collaborate/govern/about-model-governance", }, items: [ + "docs/collaborate/govern/about-model-governance", "docs/collaborate/govern/model-access", "docs/collaborate/govern/model-contracts", "docs/collaborate/govern/model-versions", @@ -405,24 +456,38 @@ const sidebarSettings = { { type: "category", label: "Use the dbt Semantic Layer", + collapsed: true, link: { type: "doc", id: "docs/use-dbt-semantic-layer/dbt-sl" }, items: [ + "docs/use-dbt-semantic-layer/dbt-sl", "docs/use-dbt-semantic-layer/quickstart-sl", "docs/use-dbt-semantic-layer/setup-sl", - "docs/use-dbt-semantic-layer/avail-sl-integrations", "docs/use-dbt-semantic-layer/sl-architecture", + { + type: "category", + label: "Integrations", + link: { type: "doc", id: "docs/use-dbt-semantic-layer/avail-sl-integrations" }, + items: [ + "docs/use-dbt-semantic-layer/avail-sl-integrations", + "docs/use-dbt-semantic-layer/gsheets", + "docs/use-dbt-semantic-layer/tableau", + ], + }, ], }, { type: "category", label: "dbt Cloud APIs", collapsed: true, + link: { type: "doc", id: "docs/dbt-cloud-apis/overview" }, items: [ "docs/dbt-cloud-apis/overview", { type: "category", label: "Authentication", + link: { type: "doc", id: "docs/dbt-cloud-apis/authentication" }, items: [ + "docs/dbt-cloud-apis/authentication", "docs/dbt-cloud-apis/user-tokens", "docs/dbt-cloud-apis/service-tokens", ], @@ -432,6 +497,7 @@ const sidebarSettings = { label: "Administrative API", link: { type: "doc", id: "docs/dbt-cloud-apis/admin-cloud-api" }, items: [ + "docs/dbt-cloud-apis/admin-cloud-api", { type: "link", label: "API v2 (legacy docs)", @@ -454,18 +520,25 @@ const sidebarSettings = { label: "Discovery API", link: { type: "doc", id: "docs/dbt-cloud-apis/discovery-api" }, items: [ + "docs/dbt-cloud-apis/discovery-api", "docs/dbt-cloud-apis/discovery-use-cases-and-examples", "docs/dbt-cloud-apis/project-state", "docs/dbt-cloud-apis/discovery-querying", { type: "category", label: "Schema", + link: { type: "doc", id: "docs/dbt-cloud-apis/discovery-schema-environment" }, items: [ + "docs/dbt-cloud-apis/discovery-schema-environment", { type: "category", label: "Job", - link: { type: "doc", id: "docs/dbt-cloud-apis/discovery-schema-job" }, + link: { + type: "doc", + id: "docs/dbt-cloud-apis/discovery-schema-job", + }, items: [ + "docs/dbt-cloud-apis/discovery-schema-job", "docs/dbt-cloud-apis/discovery-schema-job-model", "docs/dbt-cloud-apis/discovery-schema-job-models", "docs/dbt-cloud-apis/discovery-schema-job-metric", @@ -485,11 +558,6 @@ const sidebarSettings = { ], }, { - type: "category", - label: "Environment", - link: { type: "doc", id: "docs/dbt-cloud-apis/discovery-schema-environment" }, - items: [ - { type: "category", label: "Applied", items: [ @@ -503,9 +571,7 @@ const sidebarSettings = { // items: [ // // insert pages here // ], - // }, - ], - }, + // }, ], }, ], @@ -515,7 +581,9 @@ const sidebarSettings = { label: "Semantic Layer APIs", link: { type: "doc", id: "docs/dbt-cloud-apis/sl-api-overview" }, items: [ + "docs/dbt-cloud-apis/sl-api-overview", "docs/dbt-cloud-apis/sl-jdbc", + "docs/dbt-cloud-apis/sl-graphql", "docs/dbt-cloud-apis/sl-manifest", ], }, @@ -524,6 +592,7 @@ const sidebarSettings = { { type: "category", label: "Available dbt versions", + link: { type: "doc", id: "docs/dbt-versions/core" }, items: [ "docs/dbt-versions/core", "docs/dbt-versions/upgrade-core-in-cloud", @@ -532,6 +601,7 @@ const sidebarSettings = { { type: "category", label: "dbt Cloud Release Notes", + link: { type: "doc", id: "docs/dbt-versions/dbt-cloud-release-notes" }, items: [ "docs/dbt-versions/dbt-cloud-release-notes", { @@ -631,7 +701,6 @@ const sidebarSettings = { type: "category", label: "General properties", items: [ - "reference/resource-properties/access", "reference/resource-properties/columns", "reference/resource-properties/config", "reference/resource-properties/constraints", @@ -648,6 +717,7 @@ const sidebarSettings = { type: "category", label: "General configs", items: [ + "reference/resource-configs/access", "reference/resource-configs/alias", "reference/resource-configs/database", "reference/resource-configs/enabled", @@ -682,6 +752,7 @@ const sidebarSettings = { "reference/seed-properties", "reference/seed-configs", "reference/resource-configs/column_types", + "reference/resource-configs/delimiter", "reference/resource-configs/quote_columns", ], }, @@ -709,6 +780,7 @@ const sidebarSettings = { "reference/resource-configs/limit", "reference/resource-configs/severity", "reference/resource-configs/store_failures", + "reference/resource-configs/store_failures_as", "reference/resource-configs/where", ], }, @@ -926,7 +998,19 @@ const sidebarSettings = { }, { type: "category", - label: "Materializations best practices", + label: "How we build our dbt Mesh projects", + link: { + type: "doc", + id: "guides/best-practices/how-we-mesh/mesh-1-intro", + }, + items: [ + "guides/best-practices/how-we-mesh/mesh-2-structures", + "guides/best-practices/how-we-mesh/mesh-3-implementation", + ], + }, + { + type: "category", + label: "Materialization best practices", link: { type: "doc", id: "guides/best-practices/materializations/materializations-guide-1-guide-overview", diff --git a/website/snippets/_adapters-trusted.md b/website/snippets/_adapters-trusted.md index 10af0218e22..7747ce16dec 100644 --- a/website/snippets/_adapters-trusted.md +++ b/website/snippets/_adapters-trusted.md @@ -2,7 +2,19 @@ + + + +
diff --git a/website/snippets/_adapters-verified.md b/website/snippets/_adapters-verified.md index 7caf099b7d1..3cc1e800448 100644 --- a/website/snippets/_adapters-verified.md +++ b/website/snippets/_adapters-verified.md @@ -2,61 +2,60 @@ -* Install these adapters using the CLI as they're not currently supported in dbt Cloud.
diff --git a/website/snippets/_cloud-cli-flag.md b/website/snippets/_cloud-cli-flag.md new file mode 100644 index 00000000000..523591a438c --- /dev/null +++ b/website/snippets/_cloud-cli-flag.md @@ -0,0 +1,5 @@ +:::info Public preview functionality + +The dbt Cloud CLI is currently in [public preview](/docs/dbt-versions/product-lifecycles#dbt-cloud). Share feedback or request features you'd like to see on the [dbt community Slack](https://getdbt.slack.com/archives/C05M77P54FL). + +::: diff --git a/website/snippets/_cloud-environments-info.md b/website/snippets/_cloud-environments-info.md index 5388379dc34..b0c8e86a2b5 100644 --- a/website/snippets/_cloud-environments-info.md +++ b/website/snippets/_cloud-environments-info.md @@ -3,17 +3,17 @@ In dbt Cloud, there are two types of environments: - Deployment environment — Determines the settings used when jobs created within that environment are executed. -- Development environment — Determines the settings used in the dbt Cloud IDE for that particular dbt Cloud project. +- Development environment — Determines the settings used in the dbt Cloud IDE or dbt Cloud CLI, for that particular project. Each dbt Cloud project can only have a single development environment but can have any number of deployment environments. | | Development Environments | Deployment Environments | | --- | --- | --- | -| Determines settings for | dbt Cloud IDE | dbt Cloud Job runs | +| Determines settings for | dbt Cloud IDE or dbt Cloud CLI | dbt Cloud Job runs | | How many can I have in my project? | 1 | Any number | :::note -For users familiar with development on the CLI, each environment is roughly analogous to an entry in your `profiles.yml` file, with some additional information about your repository to ensure the proper version of code is executed. More info on dbt core environments [here](/docs/core/dbt-core-environments). +For users familiar with development on dbt Core, each environment is roughly analogous to an entry in your `profiles.yml` file, with some additional information about your repository to ensure the proper version of code is executed. More info on dbt core environments [here](/docs/core/dbt-core-environments). ::: ## Common environment settings @@ -38,7 +38,7 @@ Both development and deployment environments have a section called **General Set By default, all environments will use the default branch in your repository (usually the `main` branch) when accessing your dbt code. This is overridable within each dbt Cloud Environment using the **Default to a custom branch** option. This setting have will have slightly different behavior depending on the environment type: -- **Development**: determines which branch in the dbt Cloud IDE developers create branches from and open PRs against +- **Development**: determines which branch in the dbt Cloud IDE or dbt Cloud CLI developers create branches from and open PRs against. - **Deployment:** determines the branch is cloned during job executions for each environment. For more info, check out this [FAQ page on this topic](/faqs/Environments/custom-branch-settings)! @@ -59,7 +59,7 @@ Something to note, Extended Attributes doesn't mask secret values. We recommend
-If you're developing in the [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud) or [orchestrating job runs](/docs/deploy/deployments), Extended Attributes parses through the provided YAML and extracts the `profiles.yml` attributes. For each individual attribute: +If you're developing in the [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud), [dbt Cloud CLI](/docs/cloud/cloud-cli-installation), or [orchestrating job runs](/docs/deploy/deployments), Extended Attributes parses through the provided YAML and extracts the `profiles.yml` attributes. For each individual attribute: - If the attribute exists in another source (such as your project settings), it will replace its value (like environment-level values) in the profile. It also overrides any custom environment variables. diff --git a/website/snippets/_enterprise-permissions-table.md b/website/snippets/_enterprise-permissions-table.md index 779c5bcb479..3eb313e0f5b 100644 --- a/website/snippets/_enterprise-permissions-table.md +++ b/website/snippets/_enterprise-permissions-table.md @@ -7,7 +7,7 @@ Key: Permissions: * Account-level permissions — Permissions related to management of the dbt Cloud account. For example, billing and account settings. -* Project-level permissions — Permissions related to the projects in dbt Cloud. For example, repos and access to the IDE. +* Project-level permissions — Permissions related to the projects in dbt Cloud. For example, repos and access to the IDE or dbt Cloud CLI. ### Account roles Account roles enable you to manage the dbt Cloud account and manage the account settings (for example, generating service tokens, inviting users, configuring SSO). They also provide project-level permissions. The **Account Admin** role is the highest level of access you can assign. @@ -20,8 +20,10 @@ Account roles enable you to manage the dbt Cloud account and manage the account | Audit logs | R | | | R | | | Auth provider | W | | | W | R | | Billing | W | W | | | R | +| Groups | W | | R | W | R | | Invitations | W | | W | W | R | | IP restrictions | W | | | W | R | +| Licenses | W | | W | W | R | | Members | W | | W | W | R | | Project (create) | W | | W | | | | Public models | R | R | R | R | R | @@ -34,25 +36,23 @@ Account roles enable you to manage the dbt Cloud account and manage the account |:-------------------------|:-------------:|:-------------:|:---------------:|:--------------:|:------:| | Connections | W | | W | | R | | Credentials | W | | W | | R | -| Custom env. variables | W | | W | | R | +| Custom env. variables | W | | W | | R | | dbt adapters | W | | W | | R | -| Develop (IDE) | W | | W | | | +| Develop (IDE or dbt Cloud CLI) | W | | W | | | | Environments | W | | W | | R | -| Groups | W | | R | W | R | | Jobs | W | | W | | R | -| Licenses | W | | W | W | R | | Metadata | R | | R | | R | | Permissions | W | | W | W | R | | Profile | W | | W | | R | | Projects | W | | W | R | R | | Repositories | W | | W | | R | | Runs | W | | W | | R | -| Semantic Layer Config | W | | W | | R | +| Semantic Layer Config | W | | W | | R | ### Project role permissions -The project roles enable you to work within the projects in various capacities. They primarily provide access to project-level permissions such as repos and the IDE, but may also provide some account-level permissions. +The project roles enable you to work within the projects in various capacities. They primarily provide access to project-level permissions such as repos and the IDE or dbt Cloud CLI, but may also provide some account-level permissions. #### Account permissions for project roles @@ -61,12 +61,14 @@ The project roles enable you to work within the projects in various capacities. | Account settings | R | | R | | R | | | | | | R | | | Auth provider | | | | | | | | | | | | | | Billing | | | | | | | | | | | | | -| Invitations | W | R | R | R | R | R | R | | | R | R | | -| Members | W | | R | R | R | | | | | R | R | | +| Groups | R | | R | R | R | | | | | R | R | | +| Invitations | W | R | R | R | R | R | R | | | R | R | | +| Licenses | W | R | R | R | R | R | R | | | | R | | +| Members | W | | R | R | R | | | | | R | R | | | Project (create) | | | | | | | | | | | | | -| Public models | R | R | R | R | R | R | R | R | R | R | R | R | +| Public models | R | R | R | R | R | R | R | R | R | R | R | R | | Service tokens | | | | | | | | | | | | | -| Webhooks | W | | | W | | | | | | | | W | +| Webhooks | W | | | W | | | | | | | | W | #### Project permissions for project roles @@ -74,13 +76,11 @@ The project roles enable you to work within the projects in various capacities. |--------------------------|:-----:|:-------:|:--------------:|:---------:|:---------:|:---------:|:-----------:|:--------:|:--------------:|:-----------:|:----------:|:------:| | Connections | W | R | W | R | R | R | | | | R | R | | | Credentials | W | W | W | W | R | W | | | | R | R | | -| Custom env. variables | W | W | W | W | W | W | R | | | R | W | | +| Custom env. variables | W | W | W | W | W | W | R | | | R | W | | | dbt adapters | W | W | W | W | R | W | | | | R | R | | -| Develop (IDE) | W | W | | W | | | | | | | | | +| Develop (IDE or dbt Cloud CLI) | W | W | | W | | | | | | | | | | Environments | W | R | R | R | R | W | R | | | R | R | | -| Groups | R | | R | R | R | | | | | R | R | | | Jobs | W | R | R | W | R | W | R | | | R | R | | -| Licenses | W | R | R | R | R | R | R | | | | R | | | Metadata | R | R | R | R | R | R | R | R | | R | R | | | Permissions | W | | R | R | R | | | | | | W | | | Profile | W | R | W | R | R | R | | | | R | R | | diff --git a/website/snippets/_manifest-versions.md b/website/snippets/_manifest-versions.md new file mode 100644 index 00000000000..c9b3e7af6ec --- /dev/null +++ b/website/snippets/_manifest-versions.md @@ -0,0 +1,11 @@ + +| dbt Core version | Manifest version | +|------------------|---------------------------------------------------------------| +| v1.7 | [v11](https://schemas.getdbt.com/dbt/manifest/v11/index.html) | +| v1.6 | [v10](https://schemas.getdbt.com/dbt/manifest/v10/index.html) | +| v1.5 | [v9](https://schemas.getdbt.com/dbt/manifest/v9/index.html) | +| v1.4 | [v8](https://schemas.getdbt.com/dbt/manifest/v8/index.html) | +| v1.3 | [v7](https://schemas.getdbt.com/dbt/manifest/v7/index.html) | +| v1.2 | [v6](https://schemas.getdbt.com/dbt/manifest/v6/index.html) | +| v1.1 | [v5](https://schemas.getdbt.com/dbt/manifest/v5/index.html) | +| v1.0 | [v4](https://schemas.getdbt.com/dbt/manifest/v4/index.html) | \ No newline at end of file diff --git a/website/snippets/_microsoft-adapters-soon.md b/website/snippets/_microsoft-adapters-soon.md new file mode 100644 index 00000000000..c3f30ef0939 --- /dev/null +++ b/website/snippets/_microsoft-adapters-soon.md @@ -0,0 +1,3 @@ +:::tip Coming soon +dbt Cloud support for the Microsoft Fabric and Azure Synapse Analytics adapters is coming soon! +::: \ No newline at end of file diff --git a/website/snippets/_new-sl-changes.md b/website/snippets/_new-sl-changes.md index fa7c7abf743..6eca327001a 100644 --- a/website/snippets/_new-sl-changes.md +++ b/website/snippets/_new-sl-changes.md @@ -3,6 +3,6 @@ The dbt Semantic Layer has been re-released with [significant improvements](https://www.getdbt.com/blog/dbt-semantic-layer-whats-next/), making it more efficient to define and query metrics. -The new version is available in [public beta](/docs/dbt-versions/release-notes/Aug-2023/sl-revamp-beta#public-beta) and introduces [MetricFlow](/docs/build/about-metricflow), an essential component. It also includes new semantic elements, better governance, improved efficiency, easier data access, and new Semantic Layer API. +The new version is available in [public beta](/docs/dbt-versions/release-notes/Aug-2023/sl-revamp-beta#public-beta) and introduces [MetricFlow](/docs/build/about-metricflow), an essential component. It also includes new semantic elements, better governance, improved efficiency, easier data access, and new dbt Semantic Layer APIs. ::: diff --git a/website/snippets/_new-sl-setup.md b/website/snippets/_new-sl-setup.md index 9f1fcef0fb6..ad248bc3ca9 100644 --- a/website/snippets/_new-sl-setup.md +++ b/website/snippets/_new-sl-setup.md @@ -1,13 +1,13 @@ You can set up the dbt Semantic Layer in dbt Cloud at the environment and project level. Before you begin: -- You must have a dbt Cloud Team or Enterprise [multi-tenant](/docs/cloud/about-cloud/regions-ip-addresses) deployment, hosted in North America. +- You must have a dbt Cloud Team or Enterprise [multi-tenant](/docs/cloud/about-cloud/regions-ip-addresses) deployment. Single-tenant coming soon. - You must be part of the Owner group, and have the correct [license](/docs/cloud/manage-access/seats-and-users) and [permissions](/docs/cloud/manage-access/self-service-permissions) to configure the Semantic Layer: * Enterprise plan — Developer license with Account Admin permissions. Or Owner with a Developer license, assigned Project Creator, Database Admin, or Admin permissions. * Team plan — Owner with a Developer license. - You must have a successful run in your new environment. :::tip -If you're using the legacy Semantic Layer, we **highly** recommend you [upgrade your dbt version](/docs/dbt-versions/upgrade-core-in-cloud) to dbt v1.6 or higher to use the new dbt Semantic Layer. Refer to the dedicated [migration guide](/guides/migration/sl-migration) for more info. +If you're using the legacy Semantic Layer, dbt Labs strongly recommends that you [upgrade your dbt version](/docs/dbt-versions/upgrade-core-in-cloud) to dbt version 1.6 or newer to use the latest dbt Semantic Layer. Refer to the dedicated [migration guide](/guides/migration/sl-migration) for details. ::: 1. In dbt Cloud, create a new [deployment environment](/docs/deploy/deploy-environments#create-a-deployment-environment) or use an existing environment on dbt 1.6 or higher. @@ -25,7 +25,7 @@ If you're using the legacy Semantic Layer, we **highly** recommend you [upgrade 5. Select the deployment environment you want for the Semantic Layer and click **Save**. -6. After saving it, you'll be provided with the connection information that allows you to connect to downstream tools. If your tool supports JDBC, save the JDBC URL or individual components (like environment id and host). +6. After saving it, you'll be provided with the connection information that allows you to connect to downstream tools. If your tool supports JDBC, save the JDBC URL or individual components (like environment id and host). If it uses the GraphQL API, save the GraphQL API host information instead. diff --git a/website/snippets/_onrunstart-onrunend-commands.md b/website/snippets/_onrunstart-onrunend-commands.md new file mode 100644 index 00000000000..68d693ce426 --- /dev/null +++ b/website/snippets/_onrunstart-onrunend-commands.md @@ -0,0 +1 @@ +dbt build, dbt compile, dbt docs generate, dbt run, dbt seed, dbt snapshot, or dbt test. diff --git a/website/snippets/_sl-connect-and-query-api.md b/website/snippets/_sl-connect-and-query-api.md new file mode 100644 index 00000000000..429f41c3bf6 --- /dev/null +++ b/website/snippets/_sl-connect-and-query-api.md @@ -0,0 +1,10 @@ +You can query your metrics in a JDBC-enabled tool or use existing first-class integrations with the dbt Semantic Layer. + +You must have a dbt Cloud Team or Enterprise [multi-tenant](/docs/cloud/about-cloud/regions-ip-addresses) deployment. Single-tenant coming soon. + +- To learn how to use the JDBC or GraphQL API and what tools you can query it with, refer to [dbt Semantic Layer APIs](/docs/dbt-cloud-apis/sl-api-overview). + + * To authenticate, you need to [generate a service token](/docs/dbt-cloud-apis/service-tokens) with Semantic Layer Only and Metadata Only permissions. + * Refer to the [SQL query syntax](/docs/dbt-cloud-apis/sl-jdbc#querying-the-api-for-metric-metadata) to query metrics using the API. + +- To learn more about the sophisticated integrations that connect to the dbt Semantic Layer, refer to [Available integrations](/docs/use-dbt-semantic-layer/avail-sl-integrations) for more info. diff --git a/website/snippets/_sl-create-semanticmodel.md b/website/snippets/_sl-create-semanticmodel.md index bc4276efcb6..6e0376ab10b 100644 --- a/website/snippets/_sl-create-semanticmodel.md +++ b/website/snippets/_sl-create-semanticmodel.md @@ -1,4 +1,4 @@ -The following steps will walk you through setting up semantic models, which you can do with the dbt Cloud IDE or the CLI. Semantic models consist of [entities](/docs/build/entities), [dimensions](/docs/build/dimensions), and [measures](/docs/build/measures). +The following steps describe how to set up semantic models. Semantic models consist of [entities](/docs/build/entities), [dimensions](/docs/build/dimensions), and [measures](/docs/build/measures). We highly recommend you read the overview of what a [semantic model](/docs/build/semantic-models) is before getting started. If you're working in the [Jaffle shop example](https://github.com/dbt-labs/jaffle-sl-template), delete the `orders.yml` config or delete the .yml extension so it's ignored during parsing. **We'll be rebuilding it step by step in this example.** diff --git a/website/snippets/_sl-define-metrics.md b/website/snippets/_sl-define-metrics.md index 29af3f5b7c3..3734e819c1b 100644 --- a/website/snippets/_sl-define-metrics.md +++ b/website/snippets/_sl-define-metrics.md @@ -1,4 +1,4 @@ -Now that you've created your first semantic model, it's time to define your first metric! You can define metrics with the dbt Cloud IDE or CLI. +Now that you've created your first semantic model, it's time to define your first metric! You can define metrics with the dbt Cloud IDE or command line. MetricFlow supports different metric types like [simple](/docs/build/simple), [ratio](/docs/build/ratio), [cumulative](/docs/build/cumulative), and [derived](/docs/build/derived). It's recommended that you read the [metrics overview docs](/docs/build/metrics-overview) before getting started. diff --git a/website/snippets/_sl-install-metricflow.md b/website/snippets/_sl-install-metricflow.md deleted file mode 100644 index 73e60d34e85..00000000000 --- a/website/snippets/_sl-install-metricflow.md +++ /dev/null @@ -1,8 +0,0 @@ -Install the [MetricFlow CLI](/docs/build/metricflow-cli) as an extension of a dbt adapter from PyPI. The MetricFlow CLI is compatible with Python versions 3.8, 3.9, 3.10 and 3.11 - -Use pip install `metricflow` and your [dbt adapter](/docs/supported-data-platforms): - -- Create or activate your virtual environment. `python -m venv venv` or `source your-venv/bin/activate` -- Run `pip install "dbt-metricflow[your_adapter_name]"` - - You must specify `[your_adapter_name]`. - - For example, run `pip install "dbt-metricflow[snowflake]"` if you use a Snowflake adapter. diff --git a/website/snippets/_sl-partner-links.md b/website/snippets/_sl-partner-links.md index 3e4173a6ae3..59e5b54b7c0 100644 --- a/website/snippets/_sl-partner-links.md +++ b/website/snippets/_sl-partner-links.md @@ -1,11 +1,62 @@ - -The dbt Semantic Layer integrations are capable of querying dbt metrics, importing definitions, surfacing the underlying data in partner tools, and more. These are the following tools that integrate with the dbt Semantic Layer: +The following tools integrate with the dbt Semantic Layer: -1. **Mode** — Refer to the [Mode docs](https://mode.com/help/articles/supported-databases/#dbt-semantic-layer) for info on how to integrate with Mode. -1. **Hex** — Hex’s Semantic Layer integration will be available in the coming weeks. Until then, refer to [this Loom video](https://www.loom.com/share/752e85aabfbf4fa585008a5598f3517a) for more info. -1. **Google Sheets** — Google Sheets integration coming soon. -1. **Tools that allows you to write SQL** — They must meet one of the two criteria: - * Supports a generic JDBC driver option (such as DataGrip) or - * Supports Dremio and uses ArrowFlightSQL driver version 12.0.0 or higher. +
+ + +Before you connect to these tools, you'll need to first [set up the dbt Semantic Layer](/docs/use-dbt-semantic-layer/setup-sl) and [generate a service token](/docs/dbt-cloud-apis/service-tokens) to create **Semantic Layer Only** and **Metadata Only** permissions. diff --git a/website/snippets/_sl-plan-info.md b/website/snippets/_sl-plan-info.md index db4aa0bfc25..083ab2209bc 100644 --- a/website/snippets/_sl-plan-info.md +++ b/website/snippets/_sl-plan-info.md @@ -1 +1,2 @@ -To define and query metrics with the {props.product}, you must be on a {props.plan} multi-tenant plan, {props.instance} (Additional region support coming soon).

The re-released dbt Semantic Layer is available on dbt v1.6 or higher. dbt Core users can use the MetricFlow CLI to define metrics in their local project, but won't be able to dynamically query them with integrated tools.

+To define and query metrics with the {props.product}, you must be on a {props.plan} multi-tenant plan .


+ diff --git a/website/snippets/_sl-run-prod-job.md b/website/snippets/_sl-run-prod-job.md new file mode 100644 index 00000000000..a637b0b431e --- /dev/null +++ b/website/snippets/_sl-run-prod-job.md @@ -0,0 +1,7 @@ +Once you’ve defined metrics in your dbt project, you can perform a job run in your deployment environment in dbt Cloud to materialize your metrics. The deployment environment is only supported for the dbt Semantic Layer currently. + +1. Select **Deploy** from the top navigation bar. +2. Select **Jobs** to rerun the job with the most recent code in the deployment environment. +3. Your metric should appear as a red node in the dbt Cloud IDE and dbt directed acyclic graphs (DAG). + + diff --git a/website/snippets/_sl-test-and-query-metrics.md b/website/snippets/_sl-test-and-query-metrics.md index 323ba2d83ad..43ebd929cb3 100644 --- a/website/snippets/_sl-test-and-query-metrics.md +++ b/website/snippets/_sl-test-and-query-metrics.md @@ -1,31 +1,68 @@ -:::important Testing and querying metrics in the dbt Cloud IDE not yet supported +This section explains how you can test and run MetricFlow commands with dbt Cloud or dbt Core (dbt Cloud IDE support coming soon). dbt Cloud IDE users can skip to [Run a production job](#run-a-production-job) to run a model. -Support for testing or querying metrics in the dbt Cloud IDE is not available in the current beta but is coming soon. +:::important Testing and querying metrics in the dbt Cloud IDE is currently not supported -You can use the **Preview** or **Compile** buttons in the IDE to run semantic validations and make sure your metrics are defined. You can [dynamically query metrics](#connect-and-query-api) with integrated tools on a dbt Cloud [Team or Enterprise](https://www.getdbt.com/pricing/) plan using the [Semantic Layer API](/docs/dbt-cloud-apis/sl-api-overview). +Support for running [MetricFlow commands](/docs/build/metricflow-commands) in the dbt Cloud IDE is not available but is coming soon. -Currently, you can define and test metrics using the MetricFlow CLI. dbt Cloud IDE support is coming soon. Alternatively, you can test using SQL client tools like DataGrip, DBeaver, or RazorSQL. +You can use the **Preview** or **Compile** buttons in the IDE to run semantic validations and make sure your metrics are defined. Alternatively, you can run commands with the [dbt Cloud CLI](/docs/cloud/cloud-cli-installation) or with SQL client tools like DataGrip, DBeaver, or RazorSQL. ::: -This section will explain how you can test and query metrics using the MetricFlow CLI (dbt Cloud IDE support coming soon). + -Before you begin, you'll need to install the [MetricFlow CLI](/docs/build/metricflow-cli) package and make sure you run at least one model. -### Install MetricFlow -import InstallMetricFlow from '/snippets/_sl-install-metricflow.md'; + - -### Query and commit your metrics using the CLI +This section is for people using the dbt Cloud CLI (support for dbt Cloud IDE is coming soon). With dbt Cloud: -MetricFlow needs a `semantic_manifest.json` in order to build a semantic graph. To generate a semantic_manifest.json artifact run `dbt parse`. This will create the file in your `/target` directory. If you're working from the Jaffle shop example, run `dbt seed && dbt run` before preceding to ensure the data exists in your warehouse. +- You can run MetricFlow commands after installing the dbt Cloud CLI. They're integrated with dbt Cloud so you can use them immediately. +- Your account will automatically manage version control for you. -1. Make sure you have the MetricFlow CLI installed and up to date. -2. Run `mf --help` to confirm you have MetricFlow installed and view the available commands. -3. Run `mf query --metrics --group-by ` to query the metrics and dimensions. For example, `mf query --metrics order_total --group-by metric_time` -4. Verify that the metric values are what you expect. To further understand how the metric is being generated, you can view the generated SQL if you type `--explain` in the CLI. -5. Run `mf validate-configs` to run validation on your semantic models and metrics. -6. Commit and merge the code changes that contain the metric definitions. +To get started: + +1. Make sure you've installed the [dbt Cloud CLI](/docs/cloud/cloud-cli-installation). +2. Navigate to your dbt project directory. +3. Run a dbt command, such as `dbt parse`, `dbt run`, `dbt compile`, or `dbt build`. If you don't, you'll receive an error message that begins with: "ensure that you've ran an artifacts...." + - MetricFlow builds a semantic graph and generates a `semantic_manifest.json` file in dbt Cloud, which is stored in the `/target` directory. If using the Jaffle shop example, run `dbt seed && dbt run` to ensure the required data is in your data platform before proceeding. + +4. Run `dbt sl --help` to confirm you have MetricFlow installed and that you can view the available commands. +5. Run `dbt sl query --metrics --group-by ` to query the metrics and dimensions. For example, `dbt sl query --metrics order_total --group-by metric_time` +6. Verify that the metric values are what you expect. To further understand how the metric is being generated, you can view the generated SQL if you type `--compile` in the command line. +7. Commit and merge the code changes that contain the metric definitions. + +To streamline your metric querying process, you can connect to the [dbt Semantic Layer APIs](/docs/dbt-cloud-apis/sl-api-overview) to access your metrics programmatically. For SQL syntax, refer to [Querying the API for metric metadata](/docs/dbt-cloud-apis/sl-jdbc#querying-the-api-for-metric-metadata) to query metrics using the API. + + + + + + + +This step is for dbt Core users only. MetricFlow is compatible with Python versions 3.8, 3.9, 3.10 and 3.11. You need to use `pip` to install MetricFlow on Windows or Linux operating systems: + +:::note +The dbt Cloud CLI is strongly recommended to define and query metrics for your dbt project in dbt Cloud or dbt Core with MetricFlow. If you're using dbt Core, you'll need to manage versioning between dbt Core, your adapter, and MetricFlow. +::: + + +1. Install [MetricFlow](/docs/build/metricflow-commands) as an extension of a dbt adapter from PyPI. +2. Create or activate your virtual environment with `python -m venv venv` or `source your-venv/bin/activate`. +3. Run `pip install dbt-metricflow`. + - You can install MetricFlow using PyPI as an extension of your dbt adapter in the command line. To install the adapter, run `pip install "dbt-metricflow[your_adapter_name]"` and add the adapter name at the end of the command. As an example for a Snowflake adapter, run `pip install "dbt-metricflow[snowflake]"`. + - You'll need to manage versioning between dbt Core, your adapter, and MetricFlow. +4. Run `dbt parse`. This allows MetricFlow to build a semantic graph and generate a `semantic_manifest.json`. + - This creates the file in your `/target` directory. If you're working from the Jaffle shop example, run `dbt seed && dbt run` before proceeding to ensure the data exists in your warehouse. +5. Run `mf --help` to confirm you have MetricFlow installed and that you can view the available commands. +6. Run `mf query --metrics --group-by ` to query the metrics and dimensions. For example, `mf query --metrics order_total --group-by metric_time`. +7. Verify that the metric values are what you expect. To further understand how the metric is being generated, you can view the generated SQL if you type `--explain` in the command line. +8. Run `mf validate-configs` to run validation on your semantic models and metrics. +9. Commit and merge the code changes that contain the metric definitions. + +To streamline your metric querying process, you can connect to the [dbt Semantic Layer APIs](/docs/dbt-cloud-apis/sl-api-overview) to access your metrics programmatically. For SQL syntax, refer to [Querying the API for metric metadata](/docs/dbt-cloud-apis/sl-jdbc#querying-the-api-for-metric-metadata) to query metrics using the API. + + + + + -To streamline your metric querying process, you can connect to the [dbt Semantic Layer API](/docs/dbt-cloud-apis/sl-api-overview) to access your metrics programmatically. For SQL syntax, refer to [Querying the API for metric metadata](/docs/dbt-cloud-apis/sl-jdbc#querying-the-api-for-metric-metadata) to query metrics using the API. diff --git a/website/snippets/_v2-sl-prerequisites.md b/website/snippets/_v2-sl-prerequisites.md index 9fdc3b53143..852730604d0 100644 --- a/website/snippets/_v2-sl-prerequisites.md +++ b/website/snippets/_v2-sl-prerequisites.md @@ -1,17 +1,14 @@ -To use the Semantic Layer, you must: - -- Have a dbt Cloud Team or Enterprise [multi-tenant](/docs/cloud/about-cloud/regions-ip-addresses) deployment, hosted in North America. +- Have a dbt Cloud Team or Enterprise [multi-tenant](/docs/cloud/about-cloud/regions-ip-addresses) deployment. Single-tenant coming soon. - Have both your production and development environments running dbt version 1.6 or higher. Refer to [upgrade in dbt Cloud](/docs/dbt-versions/upgrade-core-in-cloud) for more info. -- Use Snowflake, BigQuery, Databricks, or Redshift (dbt Cloud Postgres support coming soon). +- Use Snowflake, BigQuery, Databricks, or Redshift. - Create a successful run in the environment where you configure the Semantic Layer. - **Note:** Semantic Layer currently supports the Deployment environment for querying. (_development querying experience coming soon_) - Set up the [Semantic Layer API](/docs/dbt-cloud-apis/sl-api-overview) in the integrated tool to import metric definitions. - - **Note:** To access the API and query metrics in downstream tools, you must have a dbt Cloud [Team or Enterprise](https://www.getdbt.com/pricing/) account. dbt Core or Developer accounts can define metrics with the [MetricFlow CLI](/docs/build/metricflow-cli) or [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud) but won't be able to dynamically query them.
-- Understand [MetricFlow's](/docs/build/about-metricflow) key concepts, which powers the revamped dbt Semantic Layer. - + - To access the API and query metrics in downstream tools, you must have a dbt Cloud [Team or Enterprise](https://www.getdbt.com/pricing/) account. dbt Core or Developer accounts can define metrics but won't be able to dynamically query them.
+- Understand [MetricFlow's](/docs/build/about-metricflow) key concepts, which powers the latest dbt Semantic Layer. diff --git a/website/snippets/connect-starburst-trino/roles-starburst-enterprise.md b/website/snippets/connect-starburst-trino/roles-starburst-enterprise.md index ba11508f1b4..f832d52be20 100644 --- a/website/snippets/connect-starburst-trino/roles-starburst-enterprise.md +++ b/website/snippets/connect-starburst-trino/roles-starburst-enterprise.md @@ -1,3 +1,6 @@ -[comment: For context, the section title used for this snippet is "Roles in Starburst Enterprise" ]: # +[comment: For context, the section title used for this snippet is "Roles in Starburst Enterprise" ]: # -If connecting to a Starburst Enterprise cluster with built-in access controls enabled, you can't add the role as a suffix to the username, so the default role for the provided username is used instead. +If connecting to a Starburst Enterprise cluster with built-in access controls +enabled, you must specify a role using the format detailed in [Additional +parameters](#additional-parameters). If a role is not specified, the default +role for the provided username is used. \ No newline at end of file diff --git a/website/snippets/core-versions-table.md b/website/snippets/core-versions-table.md index 431e1f08b4c..b08c23c84c5 100644 --- a/website/snippets/core-versions-table.md +++ b/website/snippets/core-versions-table.md @@ -6,7 +6,7 @@ | [**v1.6**](/guides/migration/versions/upgrading-to-v1.6) | Jul 31, 2023 | Active | Jul 30, 2024 | | [**v1.5**](/guides/migration/versions/upgrading-to-v1.5) | Apr 27, 2023 | Critical | Apr 27, 2024 | | [**v1.4**](/guides/migration/versions/upgrading-to-v1.4) | Jan 25, 2023 | Critical | Jan 25, 2024 | -| [**v1.3**](/guides/migration/versions/upgrading-to-v1.3) | Oct 12, 2022 | Critical | Oct 12, 2023 | +| [**v1.3**](/guides/migration/versions/upgrading-to-v1.3) | Oct 12, 2022 | End of Life* ⚠️ | Oct 12, 2023 | | [**v1.2**](/guides/migration/versions/upgrading-to-v1.2) | Jul 26, 2022 | End of Life* ⚠️ | Jul 26, 2023 | | [**v1.1**](/guides/migration/versions/upgrading-to-v1.1) ⚠️ | Apr 28, 2022 | Deprecated ⛔️ | Deprecated ⛔️ | | [**v1.0**](/guides/migration/versions/upgrading-to-v1.0) ⚠️ | Dec 3, 2021 | Deprecated ⛔️ | Deprecated ⛔️ | diff --git a/website/snippets/quickstarts/schedule-a-job.md b/website/snippets/quickstarts/schedule-a-job.md index 59d428bdfaa..ab8f4350dbf 100644 --- a/website/snippets/quickstarts/schedule-a-job.md +++ b/website/snippets/quickstarts/schedule-a-job.md @@ -24,15 +24,16 @@ Jobs are a set of dbt commands that you want to run on a schedule. For example, As the `jaffle_shop` business gains more customers, and those customers create more orders, you will see more records added to your source data. Because you materialized the `customers` model as a table, you'll need to periodically rebuild your table to ensure that the data stays up-to-date. This update will happen when you run a job. -1. After creating your deployment environment, you should be directed to the page for new environment. If not, select **Deploy** in the upper left, then click **Jobs**. -2. Click **Create one** and provide a name, for example "Production run", and link to the Environment you just created. -3. Scroll down to "Execution Settings" and select **Generate docs on run**. -4. Under "Commands," add this command as part of your job if you don't see them: - * `dbt build` -5. For this exercise, do _not_ set a schedule for your project to run — while your organization's project should run regularly, there's no need to run this example project on a schedule. Scheduling a job is sometimes referred to as _deploying a project_. -6. Select **Save**, then click **Run now** to run your job. -7. Click the run and watch its progress under "Run history." -8. Once the run is complete, click **View Documentation** to see the docs for your project. +1. After creating your deployment environment, you should be directed to the page for a new environment. If not, select **Deploy** in the upper left, then click **Jobs**. +2. Click **Create one** and provide a name, for example, "Production run", and link to the Environment you just created. +3. Scroll down to the **Execution Settings** section. +4. Under **Commands**, add this command as part of your job if you don't see it: + * `dbt build` +5. Select the **Generate docs on run** checkbox to automatically [generate updated project docs](/docs/collaborate/build-and-view-your-docs) each time your job runs. +6. For this exercise, do _not_ set a schedule for your project to run — while your organization's project should run regularly, there's no need to run this example project on a schedule. Scheduling a job is sometimes referred to as _deploying a project_. +7. Select **Save**, then click **Run now** to run your job. +8. Click the run and watch its progress under "Run history." +9. Once the run is complete, click **View Documentation** to see the docs for your project. :::tip Congratulations 🎉! You've just deployed your first dbt project! diff --git a/website/src/css/custom.css b/website/src/css/custom.css index 3181738406d..fc51ef8a8ef 100644 --- a/website/src/css/custom.css +++ b/website/src/css/custom.css @@ -58,7 +58,7 @@ --pagination-icon-prev: "\2190"; --filter-brightness-low: 1.1; --filter-brightness-high: 1.5; - + --darkmode-link-color: #1FA4A3; --light-dark-toggle: "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTYiIGhlaWdodD0iMTYiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+PHBhdGggZD0iTTQuMzA4IDMuMzg1YzAtMS4xNzguMTczLTIuMzcuNjE1LTMuMzg1QzEuOTgzIDEuMjggMCA0LjI4MiAwIDcuNjkyQTguMzA4IDguMzA4IDAgMCAwIDguMzA4IDE2YzMuNDEgMCA2LjQxMi0xLjk4MyA3LjY5Mi00LjkyMy0xLjAxNS40NDItMi4yMDcuNjE1LTMuMzg1LjYxNWE4LjMwOCA4LjMwOCAwIDAgMS04LjMwNy04LjMwN1oiIGZpbGw9IiM5MkEwQjMiLz48L3N2Zz4="; /* search overrides */ @@ -104,10 +104,10 @@ html[data-theme="dark"] { /* Linked `code` tags visibility adjustment */ html[data-theme=dark] a code { - color: var(--ifm-link-color); + color: var(--darkmode-link-color); } html[data-theme=dark] a code:hover { - color: var(--ifm-link-hover-color);; + color: var(--darkmode-link-color); } /* For /dbt-cloud/api REDOC Page */ @@ -122,11 +122,11 @@ html[data-theme="dark"] .api-content h1 { html[data-theme="dark"] .api-content button, html[data-theme="dark"] .api-content a { - filter: brightness(1.25); + filter: brightness(var(--filter-brightness-low)); } html[data-theme="dark"] .api-content a:hover { - filter: brightness(1.25); + filter: brightness(var(--filter-brightness-low)); } .redoc-wrap .api-content a, @@ -165,8 +165,19 @@ table td { vertical-align: top; } +html[data-theme=dark] main .row .col:first-of-type a:not(.button) { + color: var(--darkmode-link-color); +} + +html[data-theme="dark"] main .row .col:first-of-type a:hover { + filter: brightness(var(--filter-brightness-low)); +} + +html[data-theme="dark"] main .row .col:first-of-type a article * { + color: white; +} + html[data-theme="dark"] table td { - filter: brightness(1.5); color: white; } @@ -668,6 +679,14 @@ i.theme-doc-sidebar-item-category.theme-doc-sidebar-item-category-level-2.menu__ color: var(--ifm-color-gray-900); } +.alert--secondary, +.alert--secondary a, +.alert--secondary svg { + --ifm-alert-background-color: #474748; + color: white !important; + fill: white !important; +} + html[data-theme="dark"] .alert * { --ifm-alert-foreground-color: var(--ifm-color-gray-900); } @@ -683,7 +702,7 @@ html[data-theme="dark"] .alert table { .alert--success a, .alert--danger a, .alert--warning a { - color: var(--ifm-color-gray-900); + color: var(--ifm-color-gray-900) !important; } .linkout { @@ -842,6 +861,14 @@ div .toggle_src-components-faqs-styles-module { gap: 1em; } +html[data-theme="dark"] .pagination-nav a { + color: var(--darkmode-link-color); +} + +html[data-theme="dark"] .pagination-nav a:hover { + filter: brightness(var(--filter-brightness-low)); +} + .pagination-nav__link { padding: 1rem 0; transition: 100ms all ease-in-out; @@ -948,8 +975,13 @@ html[data-theme="dark"] .blog-breadcrumbs a[href="#"] { filter: brightness(var(--filter-brightness-low)); } -html[data-theme="dark"] .blog-breadcrumbs a:not(:last-of-type):after { - color: var(--ifm-link-color); +html[data-theme="dark"] .blog-breadcrumbs a:hover { + filter: brightness(var(--filter-brightness-low)); +} + +html[data-theme="dark"] .blog-breadcrumbs a:not(:last-of-type):after, +html[data-theme="dark"] .blog-breadcrumbs a { + color: var(--darkmode-link-color); } html[data-theme="dark"] .breadcrumbs__item--active .breadcrumbs__link { @@ -993,6 +1025,21 @@ article[itemprop="blogPost"] h2 { font-size: 2rem; } +html[data-theme="dark"] article[itemprop="blogPost"] a { + color: var(--darkmode-link-color); +} + +html[data-theme="dark"] article[itemprop="blogPost"] a:hover { + filter: brightness(var(--filter-brightness-low)); +} + +/* Sidebar Nav */ +html[data-theme="dark"] .main-wrapper nav a:hover, +html[data-theme="dark"] .main-wrapper nav a:active { + color: var(--darkmode-link-color) !important; + filter: brightness(var(--filter-brightness-low)); +} + /* footer styles */ .footer { font-weight: var(--ifm-font-weight-narrow); @@ -1053,7 +1100,7 @@ article[itemprop="blogPost"] h2 { /* copyright */ .footer__bottom { text-align: left; - color: var(--color-footer-accent); + color: white; font-size: 0.875rem; } @@ -1975,6 +2022,18 @@ html[data-theme="dark"] .theme-doc-sidebar-container>div>button.button:hover { display: none; } +.card-container { + position: relative; +} + +.external-link { + position: absolute; + top: 0; + right: 0; + margin: 10px; + color: #818589; /* You can adjust the color as needed */ +} + @media (max-width: 996px) { .quickstart-container { flex-direction: column; diff --git a/website/static/img/Filtering.png b/website/static/img/Filtering.png index 5a15a59f23e..b05394bd459 100644 Binary files a/website/static/img/Filtering.png and b/website/static/img/Filtering.png differ diff --git a/website/static/img/Paginate.png b/website/static/img/Paginate.png index 84a15732c12..21e2fd138b8 100644 Binary files a/website/static/img/Paginate.png and b/website/static/img/Paginate.png differ diff --git a/website/static/img/api-access-profile.jpg b/website/static/img/api-access-profile.jpg new file mode 100644 index 00000000000..36ffd4beda8 Binary files /dev/null and b/website/static/img/api-access-profile.jpg differ diff --git a/website/static/img/api-access-profile.png b/website/static/img/api-access-profile.png deleted file mode 100644 index deade9f2135..00000000000 Binary files a/website/static/img/api-access-profile.png and /dev/null differ diff --git a/website/static/img/dbt-cloud-project-setup-flow-next.png b/website/static/img/dbt-cloud-project-setup-flow-next.png index 660e8ae446a..92f46bccd0a 100644 Binary files a/website/static/img/dbt-cloud-project-setup-flow-next.png and b/website/static/img/dbt-cloud-project-setup-flow-next.png differ diff --git a/website/static/img/delete_projects_from_dbt_cloud_20221023.gif b/website/static/img/delete_projects_from_dbt_cloud_20221023.gif index 246c912c55b..b579556d457 100644 Binary files a/website/static/img/delete_projects_from_dbt_cloud_20221023.gif and b/website/static/img/delete_projects_from_dbt_cloud_20221023.gif differ diff --git a/website/static/img/docs/collaborate/dbt-explorer/cross-project-lineage-child.png b/website/static/img/docs/collaborate/dbt-explorer/cross-project-lineage-child.png new file mode 100644 index 00000000000..666db3384fa Binary files /dev/null and b/website/static/img/docs/collaborate/dbt-explorer/cross-project-lineage-child.png differ diff --git a/website/static/img/docs/collaborate/dbt-explorer/cross-project-lineage-parent.png b/website/static/img/docs/collaborate/dbt-explorer/cross-project-lineage-parent.png new file mode 100644 index 00000000000..ee5d19de369 Binary files /dev/null and b/website/static/img/docs/collaborate/dbt-explorer/cross-project-lineage-parent.png differ diff --git a/website/static/img/docs/dbt-cloud/cloud-ide/ide-command-bar.jpg b/website/static/img/docs/dbt-cloud/cloud-ide/ide-command-bar.jpg new file mode 100644 index 00000000000..fe60ddd7e03 Binary files /dev/null and b/website/static/img/docs/dbt-cloud/cloud-ide/ide-command-bar.jpg differ diff --git a/website/static/img/docs/dbt-cloud/dbt-cloud-enterprise/azure/azure-redirect-uri.png b/website/static/img/docs/dbt-cloud/dbt-cloud-enterprise/azure/azure-redirect-uri.png index 7daaab4504d..3bb04467abd 100644 Binary files a/website/static/img/docs/dbt-cloud/dbt-cloud-enterprise/azure/azure-redirect-uri.png and b/website/static/img/docs/dbt-cloud/dbt-cloud-enterprise/azure/azure-redirect-uri.png differ diff --git a/website/static/img/docs/dbt-cloud/dbt-cloud-enterprise/enterprise-permission-sets-diagram.png b/website/static/img/docs/dbt-cloud/dbt-cloud-enterprise/enterprise-permission-sets-diagram.png deleted file mode 100644 index e8a80f29266..00000000000 Binary files a/website/static/img/docs/dbt-cloud/dbt-cloud-enterprise/enterprise-permission-sets-diagram.png and /dev/null differ diff --git a/website/static/img/docs/dbt-cloud/defer-toggle.jpg b/website/static/img/docs/dbt-cloud/defer-toggle.jpg new file mode 100644 index 00000000000..7bd5a1c1283 Binary files /dev/null and b/website/static/img/docs/dbt-cloud/defer-toggle.jpg differ diff --git a/website/static/img/docs/dbt-cloud/using-dbt-cloud/dbt-cloud-enterprise/DBX-auth/dbt-databricks-oauth-user.png b/website/static/img/docs/dbt-cloud/using-dbt-cloud/dbt-cloud-enterprise/DBX-auth/dbt-databricks-oauth-user.png new file mode 100644 index 00000000000..aecf99d726a Binary files /dev/null and b/website/static/img/docs/dbt-cloud/using-dbt-cloud/dbt-cloud-enterprise/DBX-auth/dbt-databricks-oauth-user.png differ diff --git a/website/static/img/docs/dbt-cloud/using-dbt-cloud/dbt-cloud-enterprise/DBX-auth/dbt-databricks-oauth.png b/website/static/img/docs/dbt-cloud/using-dbt-cloud/dbt-cloud-enterprise/DBX-auth/dbt-databricks-oauth.png new file mode 100644 index 00000000000..bb32fab2afb Binary files /dev/null and b/website/static/img/docs/dbt-cloud/using-dbt-cloud/dbt-cloud-enterprise/DBX-auth/dbt-databricks-oauth.png differ diff --git a/website/static/img/docs/deploy/native-retry.gif b/website/static/img/docs/deploy/native-retry.gif new file mode 100644 index 00000000000..020a9958fc5 Binary files /dev/null and b/website/static/img/docs/deploy/native-retry.gif differ diff --git a/website/static/img/docs/release-notes/ci-job-tag.png b/website/static/img/docs/release-notes/ci-job-tag.png new file mode 100644 index 00000000000..02f2cdc895c Binary files /dev/null and b/website/static/img/docs/release-notes/ci-job-tag.png differ diff --git a/website/static/img/icons/google-sheets-logo-icon.svg b/website/static/img/icons/google-sheets-logo-icon.svg new file mode 100644 index 00000000000..d080c1dd53d --- /dev/null +++ b/website/static/img/icons/google-sheets-logo-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/website/static/img/icons/hex.svg b/website/static/img/icons/hex.svg new file mode 100755 index 00000000000..00431ffe299 --- /dev/null +++ b/website/static/img/icons/hex.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/website/static/img/icons/lightdash.svg b/website/static/img/icons/lightdash.svg new file mode 100644 index 00000000000..96f4676e7ee --- /dev/null +++ b/website/static/img/icons/lightdash.svg @@ -0,0 +1,96 @@ + + + + + + + + \ No newline at end of file diff --git a/website/static/img/icons/materialize.svg b/website/static/img/icons/materialize.svg new file mode 100644 index 00000000000..92f693cd94f --- /dev/null +++ b/website/static/img/icons/materialize.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + diff --git a/website/static/img/icons/mode.svg b/website/static/img/icons/mode.svg new file mode 100644 index 00000000000..269c182cd8b --- /dev/null +++ b/website/static/img/icons/mode.svg @@ -0,0 +1,165 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/website/static/img/icons/oracle.svg b/website/static/img/icons/oracle.svg new file mode 100644 index 00000000000..6868dea2eb3 --- /dev/null +++ b/website/static/img/icons/oracle.svg @@ -0,0 +1,47 @@ + + + + + \ No newline at end of file diff --git a/website/static/img/icons/tableau-software.svg b/website/static/img/icons/tableau-software.svg new file mode 100644 index 00000000000..28996f1dadd --- /dev/null +++ b/website/static/img/icons/tableau-software.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/website/static/img/icons/white/google-sheets-logo-icon.svg b/website/static/img/icons/white/google-sheets-logo-icon.svg new file mode 100644 index 00000000000..d080c1dd53d --- /dev/null +++ b/website/static/img/icons/white/google-sheets-logo-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/website/static/img/icons/white/hex.svg b/website/static/img/icons/white/hex.svg new file mode 100644 index 00000000000..00431ffe299 --- /dev/null +++ b/website/static/img/icons/white/hex.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/website/static/img/icons/white/lightdash.svg b/website/static/img/icons/white/lightdash.svg new file mode 100644 index 00000000000..96f4676e7ee --- /dev/null +++ b/website/static/img/icons/white/lightdash.svg @@ -0,0 +1,96 @@ + + + + + + + + \ No newline at end of file diff --git a/website/static/img/icons/white/materialize.svg b/website/static/img/icons/white/materialize.svg new file mode 100644 index 00000000000..92f693cd94f --- /dev/null +++ b/website/static/img/icons/white/materialize.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + diff --git a/website/static/img/icons/white/mode.svg b/website/static/img/icons/white/mode.svg new file mode 100644 index 00000000000..269c182cd8b --- /dev/null +++ b/website/static/img/icons/white/mode.svg @@ -0,0 +1,165 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/website/static/img/icons/white/oracle.svg b/website/static/img/icons/white/oracle.svg new file mode 100644 index 00000000000..6868dea2eb3 --- /dev/null +++ b/website/static/img/icons/white/oracle.svg @@ -0,0 +1,47 @@ + + + + + \ No newline at end of file diff --git a/website/static/img/icons/white/tableau-software.svg b/website/static/img/icons/white/tableau-software.svg new file mode 100644 index 00000000000..28996f1dadd --- /dev/null +++ b/website/static/img/icons/white/tableau-software.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/website/static/img/node_color_example.png b/website/static/img/node_color_example.png index 83b26f5735a..a1a62742ca0 100644 Binary files a/website/static/img/node_color_example.png and b/website/static/img/node_color_example.png differ diff --git a/website/static/img/sample_email_data.png b/website/static/img/sample_email_data.png deleted file mode 100644 index 7224d42e60b..00000000000 Binary files a/website/static/img/sample_email_data.png and /dev/null differ diff --git a/website/vercel.json b/website/vercel.json index c5fb0638fba..14b3a0a6af0 100644 --- a/website/vercel.json +++ b/website/vercel.json @@ -2,6 +2,31 @@ "cleanUrls": true, "trailingSlash": false, "redirects": [ + { + "source": "/docs/build/metricflow-cli", + "destination": "/docs/build/metricflow-commands", + "permanent": true + }, + { + "source": "/docs/core/about-the-cli", + "destination": "/docs/core/about-dbt-core", + "permanent": true + }, + { + "source": "/docs/cloud/about-cloud/about-cloud-ide", + "destination": "/docs/cloud/about-cloud-develop", + "permanent": true + }, + { + "source": "/faqs/models/reference-models-in-another-project", + "destination": "/docs/collaborate/govern/project-dependencies", + "permanent": true + }, + { + "source": "/faqs/Models/reference-models-in-another-project", + "destination": "/docs/collaborate/govern/project-dependencies", + "permanent": true + }, { "source": "/docs/deploy/job-triggers", "destination": "/docs/deploy/deploy-jobs", @@ -3991,6 +4016,11 @@ "source": "/docs/dbt-cloud/on-premises/upgrading-kots", "destination": "/docs/deploy/single-tenant", "permanent": true + }, + { + "source": "/reference/resource-properties/access", + "destination": "/reference/resource-configs/access", + "permanent": true } ] }