diff --git a/website/dbt-versions.js b/website/dbt-versions.js index be55c893041..abba3307f6f 100644 --- a/website/dbt-versions.js +++ b/website/dbt-versions.js @@ -177,6 +177,10 @@ exports.versionedPages = [ { "page": "docs/build/saved-queries", "firstVersion": "1.7", + }, + { + "page": "reference/resource-configs/on_configuration_change", + "firstVersion": "1.6", } ] diff --git a/website/docs/docs/build/materializations.md b/website/docs/docs/build/materializations.md index 192284a31ca..67796afdbdb 100644 --- a/website/docs/docs/build/materializations.md +++ b/website/docs/docs/build/materializations.md @@ -109,27 +109,7 @@ When using the `table` materialization, your model is rebuilt as a ` | no | `apply` | n/a | -| [`cluster_by`](#clustering-clause) | `[]` | no | `none` | drop/create | -| [`partition_by`](#partition-clause) | `{}` | no | `none` | drop/create | -| [`enable_refresh`](#auto-refresh) | `` | no | `true` | alter | -| [`refresh_interval_minutes`](#auto-refresh) | `` | no | `30` | alter | -| [`max_staleness`](#auto-refresh) (in Preview) | `` | no | `none` | alter | -| [`description`](/reference/resource-properties/description) | `` | no | `none` | alter | -| [`labels`](#specifying-labels) | `{: }` | no | `none` | alter | -| [`hours_to_expiration`](#controlling-table-expiration) | `` | no | `none` | alter | -| [`kms_key_name`](#using-kms-encryption) | `` | no | `none` | alter | +| Parameter | Type | Required | Default | Change Monitoring Support | +|----------------------------------------------------------------------------------|------------------------|----------|---------|---------------------------| +| [`on_configuration_change`](/reference/resource-configs/on_configuration_change) | `` | no | `apply` | n/a | +| [`cluster_by`](#clustering-clause) | `[]` | no | `none` | drop/create | +| [`partition_by`](#partition-clause) | `{}` | no | `none` | drop/create | +| [`enable_refresh`](#auto-refresh) | `` | no | `true` | alter | +| [`refresh_interval_minutes`](#auto-refresh) | `` | no | `30` | alter | +| [`max_staleness`](#auto-refresh) (in Preview) | `` | no | `none` | alter | +| [`description`](/reference/resource-properties/description) | `` | no | `none` | alter | +| [`labels`](#specifying-labels) | `{: }` | no | `none` | alter | +| [`hours_to_expiration`](#controlling-table-expiration) | `` | no | `none` | alter | +| [`kms_key_name`](#using-kms-encryption) | `` | no | `none` | alter | ](/reference/resource-configs/resource-path): [+](/reference/resource-configs/plus-prefix)[materialized](/reference/resource-configs/materialized): materialized_view - [+](/reference/resource-configs/plus-prefix)on_configuration_change: apply | continue | fail + [+](/reference/resource-configs/plus-prefix)[on_configuration_change](/reference/resource-configs/on_configuration_change): apply | continue | fail [+](/reference/resource-configs/plus-prefix)[cluster_by](#clustering-clause): | [] [+](/reference/resource-configs/plus-prefix)[partition_by](#partition-clause): - field: @@ -794,7 +794,7 @@ models: - name: [] config: [materialized](/reference/resource-configs/materialized): materialized_view - on_configuration_change: apply | continue | fail + [on_configuration_change](/reference/resource-configs/on_configuration_change): apply | continue | fail [cluster_by](#clustering-clause): | [] [partition_by](#partition-clause): - field: @@ -827,7 +827,7 @@ models: ```jinja {{ config( [materialized](/reference/resource-configs/materialized)='materialized_view', - on_configuration_change="apply" | "continue" | "fail", + [on_configuration_change](/reference/resource-configs/on_configuration_change)="apply" | "continue" | "fail", [cluster_by](#clustering-clause)="" | [""], [partition_by](#partition-clause)={ "field": "", @@ -868,7 +868,7 @@ models: Many of these parameters correspond to their table counterparts and have been linked above. The set of parameters unique to materialized views covers [auto-refresh functionality](#auto-refresh). -Find more information about these parameters in the BigQuery docs: +Learn more about these parameters in BigQuery's docs: - [CREATE MATERIALIZED VIEW statement](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_materialized_view_statement) - [materialized_view_option_list](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#materialized_view_option_list) @@ -886,7 +886,7 @@ BigQuery only officially supports the configuration of the frequency (the "once however, there is a feature in preview that allows for the configuration of the staleness (the "5 minutes" refresh). dbt will monitor these parameters for changes and apply them using an `ALTER` statement. -Find more information about these parameters in the BigQuery docs: +Learn more about these parameters in BigQuery's docs: - [materialized_view_option_list](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#materialized_view_option_list) - [max_staleness](https://cloud.google.com/bigquery/docs/materialized-views-create#max_staleness) diff --git a/website/docs/reference/resource-configs/on_configuration_change.md b/website/docs/reference/resource-configs/on_configuration_change.md new file mode 100644 index 00000000000..68cfc7887b9 --- /dev/null +++ b/website/docs/reference/resource-configs/on_configuration_change.md @@ -0,0 +1,86 @@ +--- +resource_types: [models] +description: "on_configuration_change - Read this in-depth guide to learn about configuration change monitoring in dbt." +datatype: "string" +--- + +:::info +This functionality is currently only supported for [materialized views](/docs/build/materializations#materialized-view) on a subset of adapters. +::: + +The `on_configuration_change` config has three settings: +- `apply` (default) — Attempt to update the existing database object if possible, avoiding a complete rebuild. + - *Note:* If any individual configuration change requires a full refresh, a full refresh be performed in lieu of individual alter statements. +- `continue` — Allow runs to continue while also providing a warning that the object was left untouched. + - *Note:* This could result in downstream failures as those models may depend on these unimplemented changes. +- `fail` — Force the entire run to fail if a change is detected. + + + + + + + + +```yaml +models: + [](/reference/resource-configs/resource-path): + [+](/reference/resource-configs/plus-prefix)[materialized](/reference/resource-configs/materialized): + [+](/reference/resource-configs/plus-prefix)on_configuration_change: apply | continue | fail +``` + + + + + + + + + + +```yaml +version: 2 + +models: + - name: [] + config: + [materialized](/reference/resource-configs/materialized): + on_configuration_change: apply | continue | fail +``` + + + + + + + + + + +```jinja +{{ config( + [materialized](/reference/resource-configs/materialized)="", + on_configuration_change="apply" | "continue" | "fail" +) }} +``` + + + + + + + +Materializations are implemented following this "drop through" life cycle: +1. If a model does not exist with the provided path, create the new model. +2. If a model exists, but has a different type, drop the existing model and create the new model. +3. If `--full-refresh` is supplied, replace the existing model regardless of configuration changes and the `on_configuration_change` setting. +4. If there are no configuration changes, perform the default action for that type (e.g. apply refresh for a materialized view). +5. Determine whether to apply the configuration changes according to the `on_configuration_change` setting. diff --git a/website/docs/reference/resource-configs/postgres-configs.md b/website/docs/reference/resource-configs/postgres-configs.md index fcc0d91a47c..b7b88c525dd 100644 --- a/website/docs/reference/resource-configs/postgres-configs.md +++ b/website/docs/reference/resource-configs/postgres-configs.md @@ -111,10 +111,10 @@ models: The Postgres adapter supports [materialized views](https://www.postgresql.org/docs/current/rules-materializedviews.html) with the following configuration parameters: -| Parameter | Type | Required | Default | Change Monitoring Support | -|---------------------------|--------------------|----------|---------|---------------------------| -| `on_configuration_change` | `` | no | `apply` | n/a | -| [`indexes`](#indexes) | `[{}]` | no | `none` | alter | +| Parameter | Type | Required | Default | Change Monitoring Support | +|----------------------------------------------------------------------------------|--------------------|----------|---------|---------------------------| +| [`on_configuration_change`](/reference/resource-configs/on_configuration_change) | `` | no | `apply` | n/a | +| [`indexes`](#indexes) | `[{}]` | no | `none` | alter | ](/reference/resource-configs/resource-path): [+](/reference/resource-configs/plus-prefix)[materialized](/reference/resource-configs/materialized): materialized_view - [+](/reference/resource-configs/plus-prefix)on_configuration_change: apply | continue | fail + [+](/reference/resource-configs/plus-prefix)[on_configuration_change](/reference/resource-configs/on_configuration_change): apply | continue | fail [+](/reference/resource-configs/plus-prefix)[indexes](#indexes): - columns: [] unique: true | false @@ -158,7 +158,7 @@ models: - name: [] config: [materialized](/reference/resource-configs/materialized): materialized_view - on_configuration_change: apply | continue | fail + [on_configuration_change](/reference/resource-configs/on_configuration_change): apply | continue | fail [indexes](#indexes): - columns: [] unique: true | false @@ -177,7 +177,7 @@ models: ```jinja {{ config( [materialized](/reference/resource-configs/materialized)="materialized_view", - on_configuration_change="apply" | "continue" | "fail", + [on_configuration_change](/reference/resource-configs/on_configuration_change)="apply" | "continue" | "fail", [indexes](#indexes)=[ { "columns": [""], @@ -198,8 +198,7 @@ The [`indexes`](#indexes) parameter corresponds to that of a table, as explained It's worth noting that, unlike tables, dbt monitors this parameter for changes and applies the changes without dropping the materialized view. This happens via a `DROP/CREATE` of the indexes, which can be thought of as an `ALTER` of the materialized view. -Find more information about materialized view parameters in the Postgres docs: -- [CREATE MATERIALIZED VIEW](https://www.postgresql.org/docs/current/sql-creatematerializedview.html) +Learn more about these parameters in Postgres's [docs](https://www.postgresql.org/docs/current/sql-creatematerializedview.html). diff --git a/website/docs/reference/resource-configs/redshift-configs.md b/website/docs/reference/resource-configs/redshift-configs.md index 85b2af0c552..83e6efb05ca 100644 --- a/website/docs/reference/resource-configs/redshift-configs.md +++ b/website/docs/reference/resource-configs/redshift-configs.md @@ -114,14 +114,14 @@ models: The Redshift adapter supports [materialized views](https://docs.aws.amazon.com/redshift/latest/dg/materialized-view-overview.html) with the following configuration parameters: -| Parameter | Type | Required | Default | Change Monitoring Support | -|-------------------------------------------|--------------|----------|------------------------------------------------|---------------------------| -| `on_configuration_change` | `` | no | `apply` | n/a | -| [`dist`](#using-sortkey-and-distkey) | `` | no | `even` | drop/create | -| [`sort`](#using-sortkey-and-distkey) | `[]` | no | `none` | drop/create | -| [`sort_type`](#using-sortkey-and-distkey) | `` | no | `auto` if no `sort`
`compound` if `sort` | drop/create | -| [`auto_refresh`](#auto-refresh) | `` | no | `false` | alter | -| [`backup`](#backup) | `` | no | `true` | n/a | +| Parameter | Type | Required | Default | Change Monitoring Support | +|----------------------------------------------------------------------------------|--------------|----------|------------------------------------------------|---------------------------| +| [`on_configuration_change`](/reference/resource-configs/on_configuration_change) | `` | no | `apply` | n/a | +| [`dist`](#using-sortkey-and-distkey) | `` | no | `even` | drop/create | +| [`sort`](#using-sortkey-and-distkey) | `[]` | no | `none` | drop/create | +| [`sort_type`](#using-sortkey-and-distkey) | `` | no | `auto` if no `sort`
`compound` if `sort` | drop/create | +| [`auto_refresh`](#auto-refresh) | `` | no | `false` | alter | +| [`backup`](#backup) | `` | no | `true` | n/a | ](/reference/resource-configs/resource-path): [+](/reference/resource-configs/plus-prefix)[materialized](/reference/resource-configs/materialized): materialized_view - [+](/reference/resource-configs/plus-prefix)on_configuration_change: apply | continue | fail + [+](/reference/resource-configs/plus-prefix)[on_configuration_change](/reference/resource-configs/on_configuration_change): apply | continue | fail [+](/reference/resource-configs/plus-prefix)[dist](#using-sortkey-and-distkey): all | auto | even | [+](/reference/resource-configs/plus-prefix)[sort](#using-sortkey-and-distkey): | [] [+](/reference/resource-configs/plus-prefix)[sort_type](#using-sortkey-and-distkey): auto | compound | interleaved @@ -166,7 +166,7 @@ models: - name: [] config: [materialized](/reference/resource-configs/materialized): materialized_view - on_configuration_change: apply | continue | fail + [on_configuration_change](/reference/resource-configs/on_configuration_change): apply | continue | fail [dist](#using-sortkey-and-distkey): all | auto | even | [sort](#using-sortkey-and-distkey): | [] [sort_type](#using-sortkey-and-distkey): auto | compound | interleaved @@ -186,7 +186,7 @@ models: ```jinja {{ config( [materialized](/reference/resource-configs/materialized)="materialized_view", - on_configuration_change="apply" | "continue" | "fail", + [on_configuration_change](/reference/resource-configs/on_configuration_change)="apply" | "continue" | "fail", [dist](#using-sortkey-and-distkey)="all" | "auto" | "even" | "", [sort](#using-sortkey-and-distkey)=[""], [sort_type](#using-sortkey-and-distkey)="auto" | "compound" | "interleaved", @@ -204,7 +204,7 @@ models: Many of these parameters correspond to their table counterparts and have been linked above. The parameters unique to materialized views are the [auto-refresh](#auto-refresh) and [backup](#backup) functionality, which are covered below. -Find more information about the [CREATE MATERIALIZED VIEW](https://docs.aws.amazon.com/redshift/latest/dg/materialized-view-create-sql-command.html) parameters in the Redshift docs. +Learn more about these parameters in Redshift's [docs](https://docs.aws.amazon.com/redshift/latest/dg/materialized-view-create-sql-command.html). #### Auto-refresh @@ -230,7 +230,7 @@ By default, a materialized view will be backed up during a cluster snapshot. dbt cannot monitor this parameter as it is not queryable within Redshift. If the value is changed, the materialized view will need to go through a `--full-refresh` in order to set it. -Learn more about the [parameters](https://docs.aws.amazon.com/redshift/latest/dg/materialized-view-create-sql-command.html#mv_CREATE_MATERIALIZED_VIEW-parameters) in the Redshift docs. +Learn more about these parameters in Redshift's [docs](https://docs.aws.amazon.com/redshift/latest/dg/materialized-view-create-sql-command.html#mv_CREATE_MATERIALIZED_VIEW-parameters). ### Limitations diff --git a/website/docs/reference/resource-configs/snowflake-configs.md b/website/docs/reference/resource-configs/snowflake-configs.md index aa2bf370df6..ecc1e75ee94 100644 --- a/website/docs/reference/resource-configs/snowflake-configs.md +++ b/website/docs/reference/resource-configs/snowflake-configs.md @@ -355,11 +355,11 @@ of [materialized views](/docs/build/materializations#Materialized-View). In particular, dynamic tables have access to the `on_configuration_change` setting. Dynamic tables are supported with the following configuration parameters: -| Parameter | Type | Required | Default | Change Monitoring Support | -|----------------------------------------------------------|------------|----------|---------|---------------------------| -| `on_configuration_change` | `` | no | `apply` | n/a | -| [`target_lag`](#target-lag) | `` | yes | | alter | -| [`snowflake_warehouse`](#configuring-virtual-warehouses) | `` | yes | | alter | +| Parameter | Type | Required | Default | Change Monitoring Support | +|----------------------------------------------------------------------------------|------------|----------|---------|---------------------------| +| [`on_configuration_change`](/reference/resource-configs/on_configuration_change) | `` | no | `apply` | n/a | +| [`target_lag`](#target-lag) | `` | yes | | alter | +| [`snowflake_warehouse`](#configuring-virtual-warehouses) | `` | yes | | alter | ](/reference/resource-configs/resource-path): [+](/reference/resource-configs/plus-prefix)[materialized](/reference/resource-configs/materialized): dynamic_table - [+](/reference/resource-configs/plus-prefix)on_configuration_change: apply | continue | fail + [+](/reference/resource-configs/plus-prefix)[on_configuration_change](/reference/resource-configs/on_configuration_change): apply | continue | fail [+](/reference/resource-configs/plus-prefix)[target_lag](#target-lag): downstream | [+](/reference/resource-configs/plus-prefix)[snowflake_warehouse](#configuring-virtual-warehouses): ``` @@ -401,7 +401,7 @@ models: - name: [] config: [materialized](/reference/resource-configs/materialized): dynamic_table - on_configuration_change: apply | continue | fail + [on_configuration_change](/reference/resource-configs/on_configuration_change): apply | continue | fail [target_lag](#target-lag): downstream | [snowflake_warehouse](#configuring-virtual-warehouses): ``` @@ -418,7 +418,7 @@ models: ```jinja {{ config( [materialized](/reference/resource-configs/materialized)="dynamic_table", - on_configuration_change="apply" | "continue" | "fail", + [on_configuration_change](/reference/resource-configs/on_configuration_change)="apply" | "continue" | "fail", [target_lag](#target-lag)="downstream" | " seconds | minutes | hours | days", [snowflake_warehouse](#configuring-virtual-warehouses)="", ) }} @@ -430,7 +430,7 @@ models: -Find more information about these parameters in Snowflake's [docs](https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table): +Learn more about these parameters in Snowflake's [docs](https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table): ### Target lag @@ -438,7 +438,7 @@ Snowflake allows two configuration scenarios for scheduling automatic refreshes: - **Time-based** — Provide a value of the form ` { seconds | minutes | hours | days }`. For example, if the dynamic table needs to be updated every 30 minutes, use `target_lag='30 minutes'`. - **Downstream** — Applicable when the dynamic table is referenced by other dynamic tables. In this scenario, `target_lag='downstream'` allows for refreshes to be controlled at the target, instead of at each layer. -Find more information about `target_lag` in Snowflake's [docs](https://docs.snowflake.com/en/user-guide/dynamic-tables-refresh#understanding-target-lag). +Learn more about `target_lag` in Snowflake's [docs](https://docs.snowflake.com/en/user-guide/dynamic-tables-refresh#understanding-target-lag). ### Limitations diff --git a/website/sidebars.js b/website/sidebars.js index 9a247180ea3..a82b2e06ec2 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -775,6 +775,7 @@ const sidebarSettings = { "reference/model-properties", "reference/model-configs", "reference/resource-configs/materialized", + "reference/resource-configs/on_configuration_change", "reference/resource-configs/sql_header", ], },