From d3ea46405184c146cf9c5ca688a7dcb5de4f60b1 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 29 Jan 2024 00:56:44 +0100 Subject: [PATCH] First draft: global configs --> flags --- .github/pull_request_template.md | 2 +- website/docs/reference/commands/version.md | 32 ++++++++++ website/docs/reference/dbt-commands.md | 1 + website/docs/reference/dbt_project.yml.md | 3 + website/docs/reference/global-cli-flags.md | 41 ------------- .../global-configs/about-global-configs.md | 59 +++++++++++++++---- ...-line-flags.md => command-line-options.md} | 6 +- .../global-configs/legacy-behaviors.md | 26 ++++++++ ...aml-configurations.md => project-flags.md} | 17 ++++-- .../global-configs/record-timing-info.md | 17 ++++++ .../global-configs/version-compatibility.md | 4 +- .../project-configs/require-dbt-version.md | 5 ++ website/sidebars.js | 40 ++++++++----- website/vercel.json | 15 +++++ 14 files changed, 193 insertions(+), 75 deletions(-) create mode 100644 website/docs/reference/commands/version.md delete mode 100644 website/docs/reference/global-cli-flags.md rename website/docs/reference/global-configs/{command-line-flags.md => command-line-options.md} (95%) create mode 100644 website/docs/reference/global-configs/legacy-behaviors.md rename website/docs/reference/global-configs/{yaml-configurations.md => project-flags.md} (74%) create mode 100644 website/docs/reference/global-configs/record-timing-info.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 0534dd916cb..193e989b875 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -19,5 +19,5 @@ Uncomment when publishing docs for a prerelease version of dbt: Adding or removing pages (delete if not applicable): - [ ] Add/remove page in `website/sidebars.js` - [ ] Provide a unique filename for new pages -- [ ] Add an entry for deleted pages in `website/static/_redirects` +- [ ] Add an entry for deleted pages in `website/vercel.json` - [ ] Run link testing locally with `npm run build` to update the links that point to deleted pages diff --git a/website/docs/reference/commands/version.md b/website/docs/reference/commands/version.md new file mode 100644 index 00000000000..142fc148a1b --- /dev/null +++ b/website/docs/reference/commands/version.md @@ -0,0 +1,32 @@ +--- +title: "About dbt --version" +sidebar_label: "version" +id: "version" +--- + +The `--version` CLI option returns information about the currently installed version of dbt Core or the dbt Cloud CLI. + +This option is not supported when invoking dbt in other dbt Cloud runtimes (IDE or scheduled runs). + + + +```text +$ dbt --version +Core: + - installed: 1.7.6 + - latest: 1.7.6 - Up to date! + +Plugins: + - snowflake: 1.7.1 - Up to date! +``` + + + + + +```text +$ dbt --version +dbt Cloud CLI - 0.35.7 (fae78a6f5f6f2d7dff3cab3305fe7f99bd2a36f3 2024-01-18T22:34:52Z) +``` + + diff --git a/website/docs/reference/dbt-commands.md b/website/docs/reference/dbt-commands.md index 4cb20051ea2..fc6b4ad2c6e 100644 --- a/website/docs/reference/dbt-commands.md +++ b/website/docs/reference/dbt-commands.md @@ -40,6 +40,7 @@ You can run dbt commands in your specific tool by prefixing them with `dbt`. Fo | [snapshot](/reference/commands/snapshot) | Executes "snapshot" jobs defined in a project | All | All [supported versions](/docs/dbt-versions/core) | | [source](/reference/commands/source) | Provides tools for working with source data (including validating that sources are "fresh") | All | All [supported versions](/docs/dbt-versions/core) | | [test](/reference/commands/test) | Executes tests defined in a project | All | All [supported versions](/docs/dbt-versions/core) | +| [--version](/reference/commands/version) | Displays the currently installed version of dbt CLI | dbt Core
dbt Cloud CLI | All [supported versions](/docs/dbt-versions/core) | diff --git a/website/docs/reference/dbt_project.yml.md b/website/docs/reference/dbt_project.yml.md index ae911200b40..9772bf5f082 100644 --- a/website/docs/reference/dbt_project.yml.md +++ b/website/docs/reference/dbt_project.yml.md @@ -52,6 +52,9 @@ The following example is a list of all available configurations in the `dbt_proj [require-dbt-version](/reference/project-configs/require-dbt-version): version-range | [version-range] +[flags](/reference/global-configs/project-flags): + + [dbt-cloud](/docs/cloud/cloud-cli-installation): [project-id](/docs/cloud/configure-cloud-cli#configure-the-dbt-cloud-cli): project_id # Required [defer-env-id](/docs/cloud/about-cloud-develop-defer#defer-in-dbt-cloud-cli): environment_id # Optional diff --git a/website/docs/reference/global-cli-flags.md b/website/docs/reference/global-cli-flags.md deleted file mode 100644 index 1604bc545b5..00000000000 --- a/website/docs/reference/global-cli-flags.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: "Global CLI flags" -id: "global-cli-flags" ---- - -dbt commands, such as `run` or `test`, support their own command-specific CLI flags. In addition, the dbt command itself supports "global" flags applicable to *all* subcommands. - -As of v1.0, the vast majority of global CLI flags map to [global configs](/reference/global-configs/about-global-configs), which can also be configured via environment variables or in the `profiles.yml`. - -The `--version` and `--record-timing-info` flags remain available to the CLI only. - -## Version - -The `--version` flag returns information about the currently installed version of dbt. - - - -```text -$ dbt --version -installed version: 0.15.1 - latest version: 0.15.1 - -Up to date! -``` - - - -## Record timing info - -The `-r` or `--record-timing-info` flag saves performance profiling information to a file. This file can be visualized with `snakeviz` to understand the performance characteristics of a dbt invocation - - - -```text -$ dbt -r timing.txt run -... - -$ snakeviz timing.txt -``` - - diff --git a/website/docs/reference/global-configs/about-global-configs.md b/website/docs/reference/global-configs/about-global-configs.md index 9d1691812b5..637e7546d99 100644 --- a/website/docs/reference/global-configs/about-global-configs.md +++ b/website/docs/reference/global-configs/about-global-configs.md @@ -1,18 +1,57 @@ --- -title: "About global configs" +title: "About flags (global configs)" id: "about-global-configs" -sidebar: "About global configs" +sidebar: "About flags (global configs)" --- -Global configs enable you to fine-tune _how_ dbt runs projects on your machine—whether your personal laptop, an orchestration tool running remotely, or (in some cases) dbt Cloud. In general, they differ from most [project configs](/reference/dbt_project.yml) and [resource configs](/reference/configs-and-properties), which tell dbt _what_ to run. +In dbt, "flags" (also called "global configs") are configurations for fine-tuning _how_ dbt runs your project. They differ from [resource-specific configs](/reference/configs-and-properties) that tell dbt about _what_ to run. -Global configs control things like the visual output of logs, the manner in which dbt parses your project, and what to do when dbt finds a version mismatch or a failing model. These configs are "global" because they are available for all dbt commands, and because they can be set for all projects running on the same machine or in the same environment. +Flags control things like the visual output of logs, whether to treat specific warning messages as errors, or whether to "fail fast" after encountering the first error. These flags are "global" because they are available for all dbt commands, and because they can be set in multiple places. -### Global config precedence +There is significant overlap between dbt's flags and dbt's command line options, but they are not the same: +- Certain flags can only be set in `dbt_project.yml`, and cannot be overridden for specific invocations via CLI option. +- If a CLI option is supported by specific commands, rather than supported by all commands ("global"), it is generally not considered to be a "flag". -Starting in v1.0, you can set global configs in three places. dbt will evaluate the configs in the following order: -1. [user config](https://docs.getdbt.com/reference/global-configs/yaml-configurations) -1. [environment variable](https://docs.getdbt.com/reference/global-configs/environment-variable-configs) -1. [CLI flag](https://docs.getdbt.com/reference/global-configs/command-line-flags) +### Setting flags + +There are multiple ways of setting flags, which depend on the use case: +- **[Project-level `flags` (`dbt_project.yml`)](https://docs.getdbt.com/reference/global-configs/project-flags):** Define version-controlled defaults for everyone running this project. Preserve legacy behaviors until their slated deprecation. +- **[Environment variables](https://docs.getdbt.com/reference/global-configs/environment-variable-configs):** Define behavior that should be different in different runtime environments (development vs. production vs. [continuous integration](https://docs.getdbt.com/docs/deploy/continuous-integration), or different for different users in development (based on personal preferences). +- **[CLI options](https://docs.getdbt.com/reference/global-configs/command-line-options):** Define behavior specific to _this invocation_. Supported for all dbt commands. + +The precedence order is CLI > Env Var > Project. + +Most flags can be set in all three places: +```yaml +# dbt_project.yml +flags: + # set default for running this project -- anywhere, anytime, by anyone + fail_fast: true +``` +```bash +# set this environment variable to 'True' (bash syntax) +export DBT_FAIL_FAST=1 +dbt run +``` +```bash +dbt run --fail-fast # set to True for this specific invocation +dbt run --no-fail-fast # set to False +``` + +However, there are exceptions: +- Flags for impermanent file paths (e.g. `--log-path` or `--state-path`) cannot be set in `dbt_project.yml`. To override defaults, pass CLI options or set environment variables (`DBT_LOG_PATH`, `DBT_STATE_PATH`). +- Flags opting into legacy dbt behaviors can _only_ be defined in `dbt_project.yml`. These are intended to be set in version control, and migrated via pull/merge request. Their values should not diverge indefinitely across invocations, environments, or users. + +### Accessing flags + +Custom user-defined logic, written in Jinja, can check the values of flags using [the `flags` context variable](https://docs.getdbt.com/reference/dbt-jinja-functions/flags). + +```yaml +# dbt_project.yml + +on-run-start: + - '{{ log("I will stop at the first sign of trouble", info = true) if flags.FAIL_FAST }}' +``` + +Because the values of `flags` can differ across invocations, we strongly advise against using `flags` as an input to configurations or dependencies (`ref` + `source`) that dbt resolves [during parsing](https://docs.getdbt.com/reference/parsing#known-limitations). -Each config is prioritized over the previous one. For example, if all three are provided, then the CLI flag takes precedence. diff --git a/website/docs/reference/global-configs/command-line-flags.md b/website/docs/reference/global-configs/command-line-options.md similarity index 95% rename from website/docs/reference/global-configs/command-line-flags.md rename to website/docs/reference/global-configs/command-line-options.md index fbe89ce28f1..058aecb6cd5 100644 --- a/website/docs/reference/global-configs/command-line-flags.md +++ b/website/docs/reference/global-configs/command-line-options.md @@ -1,7 +1,7 @@ --- -title: "Command line flags" -id: "command-line-flags" -sidebar: "Command line flags" +title: "Command line options" +id: "command-line-options" +sidebar: "Command line options" --- For consistency, command-line interface (CLI) flags should come right after the `dbt` prefix and its subcommands. This includes "global" flags (supported for all commands). When set, CLI flags override environment variables and profile configs. diff --git a/website/docs/reference/global-configs/legacy-behaviors.md b/website/docs/reference/global-configs/legacy-behaviors.md new file mode 100644 index 00000000000..ca308fe3cd3 --- /dev/null +++ b/website/docs/reference/global-configs/legacy-behaviors.md @@ -0,0 +1,26 @@ +--- +title: "Legacy behaviors" +id: "legacy-behaviors" +sidebar: "Legacy behaviors" +--- + +Most flags exist to configure runtime behaviors with multiple valid choices. The right choice may vary based on the environment, user preference, or the specific invocation. + +Another category of flags provides existing projects with a migration window for runtime behaviors that are changing in newer releases of dbt. These flags help us achieve a balance between these goals, which can otherwise be in tension: +1. Providing a better, more sensible, and more consistent default behaviour for new users/projects going forward +2. Providing a migration window for existing users/projects — nothing changes overnight without warning +3. Maintainability of dbt software. Every fork in behaviour requires additional testing & cognitive overhead that slows future development. These flags exist to facilitate migration from "current" to "better," not to stick around forever. + +These flags go through three phases of development: +1. **Introduction:** The flag is introduced, and the logic added within dbt to support both 'old' + 'new' behaviours. +2. **Maturity:** The default value of the flag is switched, from 'old' to 'new.' +3. **Deprecation:** The flag, having been marked for deprecation (with user warnings), is removed. The logic to support the ‘old’ behaviour is removed from dbt codebases. + +These flags can _only_ be set in `dbt_project.yml`. They configure behaviors closely tied to project code. They should be defined in version control, and changed via pull/merge request, with the same testing and peer review. + +### Behaviors + +`source_freshness_run_project_hooks`: +- Should the 'dbt source freshness' command include on-run-start / on-run-end hooks? +- Planned introduced in v1.8, with default 'True' +- Planned removal in v1.9 diff --git a/website/docs/reference/global-configs/yaml-configurations.md b/website/docs/reference/global-configs/project-flags.md similarity index 74% rename from website/docs/reference/global-configs/yaml-configurations.md rename to website/docs/reference/global-configs/project-flags.md index 73b8f558a9f..b7d61655fe1 100644 --- a/website/docs/reference/global-configs/yaml-configurations.md +++ b/website/docs/reference/global-configs/project-flags.md @@ -1,9 +1,16 @@ --- -title: "YAML configurations" -id: "yaml-configurations" -sidebar: "YAML configurations" +title: "Project flags" +id: "project-flags" +sidebar: "Project flags" --- +Going forward, all + + + +:::warning Deprecated functionality +::: + For most global configurations, you can set "user profile" configurations in the `config:` block of `profiles.yml`. This style of configuration sets default values for all projects using this profile directory—usually, all projects running on your local machine. @@ -17,7 +24,9 @@ config: - + + + The exception: Some global configurations are actually set in `dbt_project.yml`, instead of `profiles.yml`, because they control where dbt places logs and artifacts. Those file paths are always relative to the location of `dbt_project.yml`. For more details, see ["Log and target paths"](#log-and-target-paths) below. diff --git a/website/docs/reference/global-configs/record-timing-info.md b/website/docs/reference/global-configs/record-timing-info.md new file mode 100644 index 00000000000..614941eede5 --- /dev/null +++ b/website/docs/reference/global-configs/record-timing-info.md @@ -0,0 +1,17 @@ +--- +title: "Record timing info" +id: "record-timing-info" +--- + +The `-r` or `--record-timing-info` flag saves performance profiling information to a file. This file can be visualized with `snakeviz` to understand the performance characteristics of a dbt invocation + + + +```text +$ dbt -r timing.txt run +... + +$ snakeviz timing.txt +``` + + diff --git a/website/docs/reference/global-configs/version-compatibility.md b/website/docs/reference/global-configs/version-compatibility.md index c7a1227adaf..8a659448879 100644 --- a/website/docs/reference/global-configs/version-compatibility.md +++ b/website/docs/reference/global-configs/version-compatibility.md @@ -4,7 +4,7 @@ id: "version-compatibility" sidebar: "Version compatibility" --- -Projects are recommended to set [dbt version requirements](/reference/project-configs/require-dbt-version), especially if they use features that are newer, or which may break in future versions of dbt Core. By default, if you run a project with an incompatible dbt version, dbt will raise an error. +In the first several years of dbt Core's development, breaking changes were more common. For this reason, we encouraged to set [dbt version requirements](/reference/project-configs/require-dbt-version) — especially if they use features that are newer, or which may break in future versions of dbt Core. By default, if you run a project with an incompatible dbt version, dbt will raise an error. You can use the `VERSION_CHECK` config to disable this check and suppress the error message: @@ -12,4 +12,4 @@ You can use the `VERSION_CHECK` config to disable this check and suppress the er dbt --no-version-check run Running with dbt=1.0.0 Found 13 models, 2 tests, 1 archives, 0 analyses, 204 macros, 2 operations.... -``` \ No newline at end of file +``` diff --git a/website/docs/reference/project-configs/require-dbt-version.md b/website/docs/reference/project-configs/require-dbt-version.md index 6b17bb46120..d619715f860 100644 --- a/website/docs/reference/project-configs/require-dbt-version.md +++ b/website/docs/reference/project-configs/require-dbt-version.md @@ -3,6 +3,11 @@ datatype: version-range | [version-range] description: "Read this guide to understand the require-dbt-version configuration in dbt." default_value: None --- + +:::info +As we prioritize the stability of ongoing dbt releases, this config is no longer recommended. +::: + ```yml diff --git a/website/sidebars.js b/website/sidebars.js index 0566ef8c3a6..44db30c6973 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -903,27 +903,39 @@ const sidebarSettings = { }, { type: "category", - label: "Global configs", + label: "Flags (global configs)", link: { type: "doc", id: "reference/global-configs/about-global-configs", }, items: [ - "reference/global-configs/command-line-flags", - "reference/global-configs/environment-variable-configs", - "reference/global-configs/logs", - "reference/global-configs/cache", - "reference/global-configs/failing-fast", - "reference/global-configs/json-artifacts", - "reference/global-configs/parsing", - "reference/global-configs/print-output", - "reference/global-configs/usage-stats", - "reference/global-configs/version-compatibility", - "reference/global-configs/warnings", - "reference/global-configs/yaml-configurations", + { + type: "category", + label: "Setting flags", + items: [ + "reference/global-configs/command-line-options", + "reference/global-configs/environment-variable-configs", + "reference/global-configs/project-flags", + ] + }, + { + type: "category", + label: "Available flags", + items: [ + "reference/global-configs/logs", + "reference/global-configs/cache", + "reference/global-configs/failing-fast", + "reference/global-configs/json-artifacts", + "reference/global-configs/legacy-behaviors", + "reference/global-configs/parsing", + "reference/global-configs/print-output", + "reference/global-configs/usage-stats", + "reference/global-configs/version-compatibility", + "reference/global-configs/warnings", + ] + }, ], }, - "reference/global-cli-flags", "reference/events-logging", "reference/exit-codes", "reference/parsing", diff --git a/website/vercel.json b/website/vercel.json index 1e4cc2fb021..cd30118efdc 100644 --- a/website/vercel.json +++ b/website/vercel.json @@ -852,6 +852,21 @@ "destination": "/reference/global-configs/about-global-configs", "permanent": true }, + { + "source": "/reference/global-cli-flags.yml", + "destination": "/reference/global-configs/about-global-configs.yml", + "permanent": true + }, + { + "source": "/reference/global-configs/command-line-flags", + "destination": "/reference/global-configs/command-line-options", + "permanent": true + }, + { + "source": "/reference/global-configs/yaml-configurations", + "destination": "/reference/global-configs/project-flags", + "permanent": true + }, { "source": "/docs/quickstarts/overview", "destination": "/quickstarts",