From 30545a9043fca16c4518ee68d6dadbef947cb8bc Mon Sep 17 00:00:00 2001 From: Natalie Fiann Date: Fri, 20 Sep 2024 12:04:46 +0100 Subject: [PATCH 1/7] Updated constraints doc to include the new recommended syntax --- .../resource-properties/constraints.md | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/website/docs/reference/resource-properties/constraints.md b/website/docs/reference/resource-properties/constraints.md index ff52a1fbcf4..7815fc9c7f1 100644 --- a/website/docs/reference/resource-properties/constraints.md +++ b/website/docs/reference/resource-properties/constraints.md @@ -21,7 +21,7 @@ The structure of a constraint is: - `type` (required): one of `not_null`, `unique`, `primary_key`, `foreign_key`, `check`, `custom` - `expression`: Free text input to qualify the constraint. Required for certain constraint types, and optional for others. - `name` (optional): Human-friendly name for this constraint. Supported by some data platforms. -- `columns` (model-level only): List of column names to apply the constraint over +- `columns` (model-level only): List of column names to apply the constraint over. @@ -29,8 +29,6 @@ When using `foreign_key`, you need to specify the referenced table's schema manu For example: `expression: "{{ target.schema }}.customers(customer_id)"` - - ```yml @@ -70,7 +68,54 @@ models: +In [dbt Cloud Versionless](/docs/dbt-versions/upgrade-dbt-version-in-cloud#versionless), you can use the improved syntax to define foreign keys, which uses `ref`. This feature will be available in the upcoming 1.9 release. For more information, update the documentation version to 1.9 and refer to the 'What's better about this syntax' section. + + + + + +In [dbt Cloud Versionless](/docs/dbt-versions/upgrade-dbt-version-in-cloud#versionless), you can use the improved syntax to define foreign keys, which uses `ref`. This feature will be available in the upcoming 1.9 release. + +#### What's better about this syntax + +The recommended syntax means dbt automatically resolves references to other models, ensuring they: + +- Capture dependencies +- Work across different environments + +Here's an example showing how to use the recommended syntax: + + + +```yml +models: + - name: + + # required + config: + contract: + enforced: true + + # Model-level constraints + constraints: + - type: foreign_key + columns: [id] + to: {{ ref('my_model_to') }} # or {{ source('source', 'source_table') }} + to_columns: [id] + + # Column-level definitions and constraints + columns: + - name: id + data_type: integer + constraints: + - type: foreign_key + to: {{ ref('my_model_to') }} # or {{ source('source', 'source_table') }} + to_columns: [id] +``` + + + ## Platform-specific support From c5bdccaeffa87db669b47328fcdd02a7a13a3efa Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Tue, 1 Oct 2024 14:56:38 +0200 Subject: [PATCH 2/7] Revise FK constraint docs --- .../resource-properties/constraints.md | 88 +++++++++---------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/website/docs/reference/resource-properties/constraints.md b/website/docs/reference/resource-properties/constraints.md index 7815fc9c7f1..b9fd460b44c 100644 --- a/website/docs/reference/resource-properties/constraints.md +++ b/website/docs/reference/resource-properties/constraints.md @@ -23,11 +23,11 @@ The structure of a constraint is: - `name` (optional): Human-friendly name for this constraint. Supported by some data platforms. - `columns` (model-level only): List of column names to apply the constraint over. - - -When using `foreign_key`, you need to specify the referenced table's schema manually. Use `{{ target.schema }}` in the `expression` field to automatically pass the schema used by the target environment. Note that later versions of dbt will have more efficient ways of handling this. +Foreign key constraints accept two additional inputs: +- `to`: A relation input, likely `ref()`, indicating the referenced table. +- `to_columns`: A list of column(s) in that table containing the corresponding primary or unique key. -For example: `expression: "{{ target.schema }}.customers(customer_id)"` +This syntax for defining foreign keys uses `ref`, meaning it will capture dependencies and works across different environments. It is available in [dbt Cloud Versionless](/docs/dbt-versions/upgrade-dbt-version-in-cloud#versionless), and versions of dbt Core starting with v1.9. @@ -37,80 +37,78 @@ models: # required config: - contract: - enforced: true + contract: {enforced: true} # model-level constraints constraints: - type: primary_key - columns: [FIRST_COLUMN, SECOND_COLUMN, ...] - - type: FOREIGN_KEY # multi_column - columns: [FIRST_COLUMN, SECOND_COLUMN, ...] - expression: "OTHER_MODEL_SCHEMA.OTHER_MODEL_NAME (OTHER_MODEL_FIRST_COLUMN, OTHER_MODEL_SECOND_COLUMN, ...)" + columns: [first_column, second_column, ...] + - type: foreign_key # multi_column + columns: [first_column, second_column, ...] + to: "{{ ref('other_model_name') }}" + to_columns: [other_model_first_column, other_model_second_columns, ...] - type: check - columns: [FIRST_COLUMN, SECOND_COLUMN, ...] - expression: "FIRST_COLUMN != SECOND_COLUMN" - name: HUMAN_FRIENDLY_NAME + columns: [first_column, second_column, ...] + expression: "first_column != second_column" + name: human_friendly_name - type: ... columns: - - name: FIRST_COLUMN - data_type: DATA_TYPE + - name: first_column + data_type: string # column-level constraints constraints: - type: not_null - type: unique - type: foreign_key - expression: OTHER_MODEL_SCHEMA.OTHER_MODEL_NAME (OTHER_MODEL_COLUMN) + to: "{{ ref('other_model_name') }}" + to_columns: other_model_column - type: ... ``` -In [dbt Cloud Versionless](/docs/dbt-versions/upgrade-dbt-version-in-cloud#versionless), you can use the improved syntax to define foreign keys, which uses `ref`. This feature will be available in the upcoming 1.9 release. For more information, update the documentation version to 1.9 and refer to the 'What's better about this syntax' section. - - - -In [dbt Cloud Versionless](/docs/dbt-versions/upgrade-dbt-version-in-cloud#versionless), you can use the improved syntax to define foreign keys, which uses `ref`. This feature will be available in the upcoming 1.9 release. - -#### What's better about this syntax - -The recommended syntax means dbt automatically resolves references to other models, ensuring they: - -- Capture dependencies -- Work across different environments + -Here's an example showing how to use the recommended syntax: +In older versions of dbt Core, when defining a `foreign_key` constraint, you need to manually specify the referenced table in the `expression` field. You can use `{{ target }}` variables to make this expression environment-aware, but the dependency between this model and the referenced table is not captured. Starting in dbt Core v1.9, you can specify the referenced table using the `ref()` function. ```yml models: - name: - - # required + + # required config: - contract: - enforced: true - - # Model-level constraints + contract: {enforced: true} + + # model-level constraints constraints: - - type: foreign_key - columns: [id] - to: {{ ref('my_model_to') }} # or {{ source('source', 'source_table') }} - to_columns: [id] - - # Column-level definitions and constraints + - type: primary_key + columns: [first_column, second_column, ...] + - type: foreign_key # multi_column + columns: [first_column, second_column, ...] + expression: "{{ target.schema }}.other_model_name (other_model_first_column, other_model_second_column, ...)" + - type: check + columns: [first_column, second_column, ...] + expression: "first_column != second_column" + name: human_friendly_name + - type: ... + columns: - - name: id - data_type: integer + - name: first_column + data_type: string + + # column-level constraints constraints: + - type: not_null + - type: unique - type: foreign_key - to: {{ ref('my_model_to') }} # or {{ source('source', 'source_table') }} - to_columns: [id] + expression: "{{ target.schema }}.other_model_name (other_model_column)" + - type: ... ``` From 6603fa44f87c72879f3174a5f77df33b1a2057c0 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Tue, 1 Oct 2024 15:04:14 +0200 Subject: [PATCH 3/7] Add note about Snowflake 'rely' for query optimization --- website/docs/reference/resource-properties/constraints.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/reference/resource-properties/constraints.md b/website/docs/reference/resource-properties/constraints.md index b9fd460b44c..ffaffc3d69d 100644 --- a/website/docs/reference/resource-properties/constraints.md +++ b/website/docs/reference/resource-properties/constraints.md @@ -119,7 +119,7 @@ models: In transactional databases, it is possible to define "constraints" on the allowed values of certain columns, stricter than just the data type of those values. For example, Postgres supports and enforces all the constraints in the ANSI SQL standard (`not null`, `unique`, `primary key`, `foreign key`), plus a flexible row-level `check` constraint that evaluates to a boolean expression. -Most analytical data platforms support and enforce a `not null` constraint, but they either do not support or do not enforce the rest. It is sometimes still desirable to add an "informational" constraint, knowing it is _not_ enforced, for the purpose of integrating with legacy data catalog or entity-relation diagram tools ([dbt-core#3295](https://github.com/dbt-labs/dbt-core/issues/3295)). +Most analytical data platforms support and enforce a `not null` constraint, but they either do not support or do not enforce the rest. It is sometimes still desirable to add an "informational" constraint, knowing it is _not_ enforced, for the purpose of integrating with legacy data catalog or entity-relation diagram tools ([dbt-core#3295](https://github.com/dbt-labs/dbt-core/issues/3295)). Some data platforms can optionally use primary or foreign key constraints for query optimization if you specify an additional keyword. To that end, there are two optional fields you can specify on any filter: - `warn_unenforced: False` to skip warning on constraints that are supported, but not enforced, by this data platform. The constraint will be included in templated DDL. @@ -287,7 +287,7 @@ select Snowflake suppports four types of constraints: `unique`, `not null`, `primary key`, and `foreign key`. It is important to note that only the `not null` (and the `not null` property of `primary key`) are actually checked at present. -The rest of the constraints are purely metadata, not verified when inserting data. +The rest of the constraints are purely metadata, not verified when inserting data. Although Snowflake does not validate `unique`, `primary`, or `foreign_key` constraints, you may optionally instruct Snowflake to use them for query optimization by specifying [`rely`](https://docs.snowflake.com/en/user-guide/join-elimination) in the constraint `expression` field. Currently, Snowflake doesn't support the `check` syntax and dbt will skip the `check` config and raise a warning message if it is set on some models in the dbt project. From aac0997b91593fc4ca29fcb85ee4b7f183c316c3 Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Tue, 1 Oct 2024 14:48:54 +0100 Subject: [PATCH 4/7] Update constraints.md fix error --- website/docs/reference/resource-properties/constraints.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/resource-properties/constraints.md b/website/docs/reference/resource-properties/constraints.md index ffaffc3d69d..211c25b916b 100644 --- a/website/docs/reference/resource-properties/constraints.md +++ b/website/docs/reference/resource-properties/constraints.md @@ -29,6 +29,7 @@ Foreign key constraints accept two additional inputs: This syntax for defining foreign keys uses `ref`, meaning it will capture dependencies and works across different environments. It is available in [dbt Cloud Versionless](/docs/dbt-versions/upgrade-dbt-version-in-cloud#versionless), and versions of dbt Core starting with v1.9. + ```yml @@ -68,7 +69,6 @@ models: ``` - From 957cc71730d81108c56c77aeb63f857dc64c6a3d Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:55:48 +0100 Subject: [PATCH 5/7] Update website/docs/reference/resource-properties/constraints.md --- website/docs/reference/resource-properties/constraints.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/resource-properties/constraints.md b/website/docs/reference/resource-properties/constraints.md index 211c25b916b..b952835679a 100644 --- a/website/docs/reference/resource-properties/constraints.md +++ b/website/docs/reference/resource-properties/constraints.md @@ -27,7 +27,7 @@ Foreign key constraints accept two additional inputs: - `to`: A relation input, likely `ref()`, indicating the referenced table. - `to_columns`: A list of column(s) in that table containing the corresponding primary or unique key. -This syntax for defining foreign keys uses `ref`, meaning it will capture dependencies and works across different environments. It is available in [dbt Cloud Versionless](/docs/dbt-versions/upgrade-dbt-version-in-cloud#versionless), and versions of dbt Core starting with v1.9. +This syntax for defining foreign keys uses `ref`, meaning it will capture dependencies and works across different environments. It's available in [dbt Cloud Versionless](/docs/dbt-versions/upgrade-dbt-version-in-cloud#versionless) and versions of dbt Core starting with v1.9. From d7e49aad71c5aef61829f24b27c9ff062667342f Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Tue, 1 Oct 2024 17:55:54 +0200 Subject: [PATCH 6/7] PR feedback --- website/docs/reference/resource-properties/constraints.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/docs/reference/resource-properties/constraints.md b/website/docs/reference/resource-properties/constraints.md index b952835679a..948fe223d68 100644 --- a/website/docs/reference/resource-properties/constraints.md +++ b/website/docs/reference/resource-properties/constraints.md @@ -23,13 +23,14 @@ The structure of a constraint is: - `name` (optional): Human-friendly name for this constraint. Supported by some data platforms. - `columns` (model-level only): List of column names to apply the constraint over. + + Foreign key constraints accept two additional inputs: - `to`: A relation input, likely `ref()`, indicating the referenced table. - `to_columns`: A list of column(s) in that table containing the corresponding primary or unique key. This syntax for defining foreign keys uses `ref`, meaning it will capture dependencies and works across different environments. It's available in [dbt Cloud Versionless](/docs/dbt-versions/upgrade-dbt-version-in-cloud#versionless) and versions of dbt Core starting with v1.9. - ```yml From c1cf55183f2b033cb3cd4f295e5cb58387b30402 Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:11:55 +0100 Subject: [PATCH 7/7] Update release-notes.md --- website/docs/docs/dbt-versions/release-notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/website/docs/docs/dbt-versions/release-notes.md b/website/docs/docs/dbt-versions/release-notes.md index 7c2614b2c10..d152fc92cf7 100644 --- a/website/docs/docs/dbt-versions/release-notes.md +++ b/website/docs/docs/dbt-versions/release-notes.md @@ -20,6 +20,7 @@ Release notes are grouped by month for both multi-tenant and virtual private clo ## September 2024 +- **New**: Use the new recommended syntax for [defining `foreign_key` constraints](/reference/resource-properties/constraints) using `refs`, available in dbt Cloud Versionless. This will soon be released in dbt Core v1.9. This new syntax will capture dependencies and works across different environments. - **Enhancement**: You can now run [Semantic Layer commands](/docs/build/metricflow-commands) commands in the [dbt Cloud IDE](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud). The supported commands are `dbt sl list`, `dbt sl list metrics`, `dbt sl list dimension-values`, `dbt sl list saved-queries`, `dbt sl query`, `dbt sl list dimensions`, `dbt sl list entities`, and `dbt sl validate`. - **New**: Microsoft Excel, a dbt Semantic Layer integration, is now generally available. The integration allows you to connect to Microsoft Excel to query metrics and collaborate with your team. Available for [Excel Desktop](https://pages.store.office.com/addinsinstallpage.aspx?assetid=WA200007100&rs=en-US&correlationId=4132ecd1-425d-982d-efb4-de94ebc83f26) or [Excel Online](https://pages.store.office.com/addinsinstallpage.aspx?assetid=WA200007100&rs=en-US&correlationid=4132ecd1-425d-982d-efb4-de94ebc83f26&isWac=True). For more information, refer to [Microsoft Excel](/docs/cloud-integrations/semantic-layer/excel). - **New**: [Data health tile](/docs/collaborate/data-tile) is now generally available in dbt Explorer. Data health tiles provide a quick at-a-glance view of your data quality, highlighting potential issues in your data. You can embed these tiles in your dashboards to quickly identify and address data quality issues in your dbt project.