diff --git a/website/docs/docs/build/snapshots.md b/website/docs/docs/build/snapshots.md index dd7a44fd48c..e5f33b11cab 100644 --- a/website/docs/docs/build/snapshots.md +++ b/website/docs/docs/build/snapshots.md @@ -83,7 +83,7 @@ The following table outlines the configurations available for snapshots: | [schema](/reference/resource-configs/schema) | Specify a custom schema for the snapshot | No | snapshots | | [alias](/reference/resource-configs/alias) | Specify an alias for the snapshot | No | your_custom_snapshot | | [strategy](/reference/resource-configs/strategy) | The snapshot strategy to use. Valid values: `timestamp` or `check` | Yes | timestamp | -| [unique_key](/reference/resource-configs/unique_key) | A column or expression for the record | Yes | id | +| [unique_key](/reference/resource-configs/unique_key) | A column(s) (string or array) or expression for the record | Yes | `id` or `[order_id, product_id]` | | [check_cols](/reference/resource-configs/check_cols) | If using the `check` strategy, then the columns to check | Only if using the `check` strategy | ["status"] | | [updated_at](/reference/resource-configs/updated_at) | If using the `timestamp` strategy, the timestamp column to compare | Only if using the `timestamp` strategy | updated_at | | [invalidate_hard_deletes](/reference/resource-configs/invalidate_hard_deletes) | Find hard deleted records in source and set `dbt_valid_to` to current time if the record no longer exists | No | True | diff --git a/website/docs/docs/cloud/connect-data-platform/connect-snowflake.md b/website/docs/docs/cloud/connect-data-platform/connect-snowflake.md index d8dd8dfec11..7e4bc7a9288 100644 --- a/website/docs/docs/cloud/connect-data-platform/connect-snowflake.md +++ b/website/docs/docs/cloud/connect-data-platform/connect-snowflake.md @@ -19,7 +19,7 @@ The following fields are required when creating a Snowflake connection ## Authentication methods -This section describes the different authentication methods available for connecting dbt Cloud to Snowflake. +This section describes the different authentication methods for connecting dbt Cloud to Snowflake. Configure Deployment environment (Production, Staging, General) credentials globally in the [**Connections**](/docs/deploy/deploy-environments#deployment-connection) area of **Account settings**. Individual users configure their development credentials in the [**Credentials**](/docs/cloud/dbt-cloud-ide/develop-in-the-cloud#get-started-with-the-cloud-ide) area of their user profile. ### Username / Password diff --git a/website/docs/reference/global-configs/about-global-configs.md b/website/docs/reference/global-configs/about-global-configs.md index 64d56d002fe..435a86d84ba 100644 --- a/website/docs/reference/global-configs/about-global-configs.md +++ b/website/docs/reference/global-configs/about-global-configs.md @@ -95,5 +95,5 @@ Because the values of `flags` can differ across invocations, we strongly advise | [use_experimental_parser](/reference/global-configs/parsing#experimental-parser) | boolean | False | ✅ | `DBT_USE_EXPERIMENTAL_PARSER` | `--use-experimental-parser`, `--no-use-experimental-parser` | ❌ | | [version_check](/reference/global-configs/version-compatibility) | boolean | varies | ✅ | `DBT_VERSION_CHECK` | `--version-check`, `--no-version-check` | ❌ | | [warn_error_options](/reference/global-configs/warnings) | dict | {} | ✅ | `DBT_WARN_ERROR_OPTIONS` | `--warn-error-options` | ✅ | -| [warn_error](/reference/global-configs/warnings) | boolean | False | ✅ | `DBT_WARN_ERROR` | `--warn-error`, `--no-warn-error` | ✅ | +| [warn_error](/reference/global-configs/warnings) | boolean | False | ✅ | `DBT_WARN_ERROR` | `--warn-error` | ✅ | | [write_json](/reference/global-configs/json-artifacts) | boolean | True | ✅ | `DBT_WRITE_JSON` | `--write-json`, `--no-write-json` | ✅ | diff --git a/website/docs/reference/global-configs/behavior-changes.md b/website/docs/reference/global-configs/behavior-changes.md index 299674ae9c1..94afa7c9cae 100644 --- a/website/docs/reference/global-configs/behavior-changes.md +++ b/website/docs/reference/global-configs/behavior-changes.md @@ -68,8 +68,8 @@ When we use dbt Cloud in the following table, we're referring to accounts that h | Flag | dbt Cloud: Intro | dbt Cloud: Maturity | dbt Core: Intro | dbt Core: Maturity | |-----------------------------------------------------------------|------------------|---------------------|-----------------|--------------------| | [require_explicit_package_overrides_for_builtin_materializations](#package-override-for-built-in-materialization) | 2024.04 | 2024.06 | 1.6.14, 1.7.14 | 1.8.0 | -| [require_resource_names_without_spaces](#no-spaces-in-resource-names) | 2024.05 | TBD* | 1.8.0 | 1.9.0 | -| [source_freshness_run_project_hooks](#project-hooks-with-source-freshness) | 2024.03 | TBD* | 1.8.0 | 1.9.0 | +| [require_resource_names_without_spaces](#no-spaces-in-resource-names) | 2024.05 | TBD* | 1.8.0 | 1.10.0 | +| [source_freshness_run_project_hooks](#project-hooks-with-source-freshness) | 2024.03 | TBD* | 1.8.0 | 1.10.0 | | [Redshift] [restrict_direct_pg_catalog_access](/reference/global-configs/redshift-changes#the-restrict_direct_pg_catalog_access-flag) | 2024.09 | TBD* | dbt-redshift v1.9.0 | 1.9.0 | | [skip_nodes_if_on_run_start_fails](#failures-in-on-run-start-hooks) | 2024.10 | TBD* | 1.9.0 | TBD* | | [state_modified_compare_more_unrendered_values](#source-definitions-for-state) | 2024.10 | TBD* | 1.9.0 | TBD* | diff --git a/website/docs/reference/resource-configs/unique_key.md b/website/docs/reference/resource-configs/unique_key.md index 996e7148292..41884e175d2 100644 --- a/website/docs/reference/resource-configs/unique_key.md +++ b/website/docs/reference/resource-configs/unique_key.md @@ -1,6 +1,6 @@ --- resource_types: [snapshots] -description: "Unique_key - Read this in-depth guide to learn about configurations in dbt." +description: "Learn more about unique_key configurations in dbt." datatype: column_name_or_expression --- @@ -14,7 +14,7 @@ snapshots: - name: orders_snapshot relation: source('my_source', 'my_table') [config](/reference/snapshot-configs): - unique_key: id + unique_key: order_id ``` @@ -52,7 +52,7 @@ snapshots: ## Description A column name or expression that is unique for the inputs of a snapshot. dbt uses this to match records between a result set and an existing snapshot, so that changes can be captured correctly. -In Versionless and dbt v1.9 and later, [snapshots](/docs/build/snapshots) are defined and configured in YAML files within your `snapshots/` directory. The `unique_key` is specified within the `config` block of your snapshot YAML file. +In Versionless and dbt v1.9 and later, [snapshots](/docs/build/snapshots) are defined and configured in YAML files within your `snapshots/` directory. You can specify one or multiple `unique_key` values within your snapshot YAML file's `config` key. :::caution @@ -114,29 +114,37 @@ snapshots: -### Use a combination of two columns as a unique key -This configuration accepts a valid column expression. As such, you can concatenate two columns together as a unique key if required. It's a good idea to use a separator (e.g. `'-'`) to ensure uniqueness. - +### Use multiple unique keys + +You can configure snapshots to use multiple unique keys for `primary_key` columns. + ```yaml snapshots: - - name: transaction_items_snapshot - relation: source('erp', 'transactions') + - name: orders_snapshot + relation: source('jaffle_shop', 'orders') config: schema: snapshots - unique_key: "transaction_id || '-' || line_item_id" + unique_key: + - order_id + - product_id strategy: timestamp updated_at: updated_at - + ``` + +### Use a combination of two columns as a unique key + +This configuration accepts a valid column expression. As such, you can concatenate two columns together as a unique key if required. It's a good idea to use a separator (for example, `'-'`) to ensure uniqueness. + ```jinja2 @@ -159,25 +167,9 @@ from {{ source('erp', 'transactions') }} ``` - Though, it's probably a better idea to construct this column in your query and use that as the `unique_key`: - - - - -```yaml -snapshots: - - name: transaction_items_snapshot - relation: {{ ref('transaction_items_ephemeral') }} - config: - schema: snapshots - unique_key: id - strategy: timestamp - updated_at: updated_at -``` - @@ -195,9 +187,6 @@ from {{ source('erp', 'transactions') }} In this example, we create an ephemeral model `transaction_items_ephemeral` that creates an `id` column that can be used as the `unique_key` our snapshot configuration. - - - ```jinja2 diff --git a/website/package-lock.json b/website/package-lock.json index 5fdf491f1cf..df0c9652529 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -27,6 +27,7 @@ "gray-matter": "^4.0.3", "hast-util-is-element": "^1.1.0", "js-yaml": "^4.1.0", + "markdown-to-jsx": "^7.5.0", "mobx": "^6.3.9", "node-polyfill-webpack-plugin": "^1.1.4", "papaparse": "^5.3.2", @@ -17038,6 +17039,17 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/markdown-to-jsx": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/markdown-to-jsx/-/markdown-to-jsx-7.5.0.tgz", + "integrity": "sha512-RrBNcMHiFPcz/iqIj0n3wclzHXjwS7mzjBNWecKKVhNTIxQepIix6Il/wZCn2Cg5Y1ow2Qi84+eJrryFRWBEWw==", + "engines": { + "node": ">= 10" + }, + "peerDependencies": { + "react": ">= 0.14.0" + } + }, "node_modules/marked": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", diff --git a/website/package.json b/website/package.json index 51edae4bf17..a16c8f9db9b 100644 --- a/website/package.json +++ b/website/package.json @@ -30,6 +30,7 @@ "gray-matter": "^4.0.3", "hast-util-is-element": "^1.1.0", "js-yaml": "^4.1.0", + "markdown-to-jsx": "^7.5.0", "mobx": "^6.3.9", "node-polyfill-webpack-plugin": "^1.1.4", "papaparse": "^5.3.2", diff --git a/website/snippets/_enterprise-permissions-table.md b/website/snippets/_enterprise-permissions-table.md index 688e8911bf4..a5b825d34d2 100644 --- a/website/snippets/_enterprise-permissions-table.md +++ b/website/snippets/_enterprise-permissions-table.md @@ -1,9 +1,4 @@ -Key: - -* (W)rite — Create new or modify existing. Includes `send`, `create`, `delete`, `allocate`, `modify`, `develop`, and `read`. -* (R)ead — Can view but can not create or change any fields. - Permissions: * Account-level permissions — Permissions related to the management of the dbt Cloud account. For example, billing and account settings. @@ -12,77 +7,114 @@ Permissions: ### Account roles Account roles enable you to manage the dbt Cloud account and manage the account settings (for example, generating service tokens, inviting users, and configuring SSO). They also provide project-level permissions. The **Account Admin** role is the highest level of access you can assign. +Key: + +* (W)rite — Create new or modify existing. Includes `send`, `create`, `delete`, `allocate`, `modify`, and `develop`. +* (R)ead — Can view but can not create or change any fields. + #### Account permissions for account roles -| Account-level permission| Account Admin | Billing admin | Manage

marketplace

apps | Project creator | Security admin | Viewer | + + + +{` +| Account-level permission| Account Admin | Billing admin | Manage marketplace apps | Project creator | Security admin | Viewer | |:-------------------------|:-------------:|:------------:|:-------------------------:|:---------------:|:--------------:|:------:| -| Account settings | W | | | R | R | R | -| Audit logs | R | | | | R | R | -| Auth provider | W | | | | W | R | -| Billing | W | W | | | | R | -| Connections | W | | | W | | | -| Groups | W | | | R | W | R | -| Invitations | W | | | W | W | R | -| IP restrictions | W | | | | W | R | -| Licenses | W | | | W | W | R | -| Marketplace app | | | W | | | | -| Members | W | | | W | W | R | -| Project (create) | W | | | W | | | -| Public models | R | R | | R | R | R | -| Service tokens | W | | | | R | R | -| Webhooks | W | | | | | | +| Account settings | W | - | - | R | R | R | +| Audit logs | R | - | - | - | R | R | +| Auth provider | W | - | - | - | W | R | +| Billing | W | W | - | - | - | R | +| Connections | W | - | - | W | - | - | +| Groups | W | - | - | R | W | R | +| Invitations | W | - | - | W | W | R | +| IP restrictions | W | - | - | - | W | R | +| Licenses | W | - | - | W | W | R | +| Marketplace app | - | - | W | - | - | - | +| Members | W | - | - | W | W | R | +| Project (create) | W | - | - | W | - | - | +| Public models | R | R | - | R | R | R | +| Service tokens | W | - | - | - | R | R | +| Webhooks | W | - | - | - | - | - | +`} + + #### Project permissions for account roles + + +{` |Project-level permission | Account Admin | Billing admin | Project creator | Security admin | Viewer | |:-------------------------|:-------------:|:-------------:|:---------------:|:--------------:|:------:| -| Environment credentials (deployment) | W | | W | | R | -| Custom env. variables | W | | W | | R | -| Data platform configurations | W | | W | | R | -| Develop (IDE or dbt Cloud CLI) | W | | W | | | -| Environments | W | | W | | R | -| Jobs | W | | W | | R | -| Metadata GraphQL API access | R | | R | | R | -| Permissions | W | | W | W | R | -| Projects | W | | W | R | R | -| Repositories | W | | W | | R | -| Runs | W | | W | | R | -| Semantic Layer config | W | | W | | R | +| Environment credentials | W | - | W | - | R | +| Custom env. variables | W | - | W | - | R | +| Data platform configurations| W | - | W | - | R | +| Develop (IDE or CLI) | W | - | W | - | - | +| Environments | W | - | W | - | R | +| Jobs | W | - | W | - | R | +| Metadata GraphQL API access | R | - | R | - | R | +| Permissions | W | - | W | W | R | +| Projects | W | - | W | R | R | +| Repositories | W | - | W | - | R | +| Runs | W | - | W | - | R | +| Semantic Layer config | W | - | W | v | R | +`} + ### Project role permissions The project roles enable you to work within the projects in various capacities. They primarily provide access to project-level permissions such as repos and the IDE or dbt Cloud CLI, but may also provide some account-level permissions. +Key: + +* (W)rite — Create new or modify existing. Includes `send`, `create`, `delete`, `allocate`, `modify`, and `develop`. +* (R)ead — Can view but can not create or change any fields. + #### Account permissions for project roles -| Account-level permission | Admin | Analyst | Database admin | Developer | Git Admin | Job admin | Job runner | Job viewer | Metadata

(Discovery API only) | Semantic Layer | Stakeholder | Team admin | Webhook | + + +{` +| Account-level permission | Admin | Analyst | Database admin | Developer | Git Admin | Job admin | Job runner | Job viewer | Metadata (Discovery API only) | Semantic Layer | Stakeholder | Team admin | Webhook | |--------------------------|:-----:|:-------:|:--------------:|:---------:|:---------:|:---------:|:-----------:|:-----------:|:--------:|:--------------:|:-----------:|:----------:|:-------:| -| Account settings | R | | R | | R | | | | | | | R | | -| Auth provider | | | | | | | | | | | | | | -| Billing | | | | | | | | | | | | | | -| Connections | R | R | R | R | R | R | | | | | R | R | | -| Groups | R | | R | R | R | | | | | | R | R | | -| Invitations | W | R | R | R | R | R | | R | | | R | R | | -| Licenses | W | R | R | R | R | R | | R | | | | R | | -| Members | W | | R | R | R | | | | | | R | R | | -| Project (create) | | | | | | | | | | | | | | -| Public models | R | R | R | R | R | R | | R | R | R | R | R | R | -| Service tokens | | | | | | | | | | | | | | -| Webhooks | W | | | W | | | | | | | | | W | +| Account settings | R | - | R | - | R | - | - | - | - | - | - | R | - | +| Auth provider | - | - | - | - | - | - | - | - | - | - | - | - | - | +| Billing | - | - | - | - | - | - | - | - | - | - | - | - | - | +| Connections | R | R | R | R | R | R | - | - | - | - | R | R | - | +| Groups | R | - | R | R | R | - | - | - | - | - | R | R | - | +| Invitations | W | R | R | R | R | R | - | R | - | - | R | R | - | +| Licenses | W | R | R | R | R | R | - | R | - | - | - | R | - | +| Members | W | - | R | R | R | - | - | - | - | - | R | R | - | +| Project (create) | - | - | - | - | - | - | - | - | - | - | - | - | - | +| Public models | R | R | R | R | R | R | - | R | R | R | R | R | R | +| Service tokens | - | - | - | - | - | - | - | - | - | - | - | - | - | +| Webhooks | W | - | - | W | - | - | - | - | - | - | - | - | W | +`} + + #### Project permissions for project roles - -|Project-level permission | Admin | Analyst | Database admin | Developer | Git Admin | Job admin | Job runner | Job viewer | Metadata

(Discovery API only) | Semantic Layer | Stakeholder | Team admin | Webhook | -|--------------------------|:-----:|:-------:|:--------------:|:---------:|:---------:|:---------:|:-----------:|:-----------:|:--------:|:--------------:|:-----------:|:----------:|:-------:| -| Environment credentials (deployment) | W | W | W | W | R | W | | | | | R | R | | -| Custom env. variables | W | W | W | W | W | W | | R | | | R | W | | -| Data platform configurations| W | W | W | W | R | W | | | | | R | R | | -| Develop
(IDE or dbt Cloud CLI) | W | W | | W | | | | | | | | | | -| Environments | W | R | R | R | R | W | | R | | | R | R | | -| Jobs | W | R | R | R | R | W | R | R | | | R | R | | -| Metadata GraphQL API access | R | R | R | R | R | R | | R | R | | R | R | | -| Permissions (Groups & Licenses) | W | | R | R | R | | | | | | | R | | | | | R | | | -| Projects | W | W | W | W | W | R | | R | | | R | W | | -| Repositories | W | | R | R | W | | | | | | R | R | | -| Runs | W | R | R | R | R | W | W | R | | | R | R | | -| Semantic Layer config | W | R | W | R | R | R | | | | W | R | R | | + + + +{` +|Project-level permission | Admin | Analyst | Database admin | Developer | Git Admin | Job admin | Job runner | Job viewer | Metadata (Discovery API only) | Semantic Layer | Stakeholder | Team admin | Webhook | +|--------------------------|:-----:|:-------:|:--------------:|:---------:|:---------:|:---------:|:-----------:|:-----------:|:---------------------------------------:|:--------------:|:-----------:|:----------:|:-------:| +| Environment credentials | W | W | W | W | R | W | - | - | - | - | R | R | - | +| Custom env. variables | W | W | W | W | W | W | - | R | - | - | R | W | - | +| Data platform configs | W | W | W | W | R | W | - | - | - | - | R | R | - | +| Develop (IDE or CLI) | W | W | - | W | - | - | - | - | - | - | - | - | - | +| Environments | W | R* | R* | R* | R* | W | - | R | - | - | R | R* | - | +| Jobs | W | R* | R* | R* | R* | W | R | R | - | - | R | R* | - | +| Metadata GraphQL API access| R | R | R | R | R | R | - | R | R | - | R | R | - | +| Permissions | W | - | R | R | R | - | - | - | - | - | - | R | - | +| Projects | W | W | W | W | W | R | - | R | - | - | R | W | - | +| Repositories | W | - | R | R | W | - | - | - | - | - | R | R | - | +| Runs | W | R* | R* | R* | R* | W | W | R | - | - | R | R* | - | +| Semantic Layer config | W | R | W | R | R | R | - | - | - | W | R | R | - | + +`} + + + +\* These permissions are `R`ead-only by default, but may be changed to `W`rite with [environment permissions](/docs/cloud/manage-access/environment-permissions#environments-and-roles). diff --git a/website/src/components/sortableTable/index.js b/website/src/components/sortableTable/index.js new file mode 100644 index 00000000000..93d54252c94 --- /dev/null +++ b/website/src/components/sortableTable/index.js @@ -0,0 +1,114 @@ +import React, { useState, useMemo } from 'react'; +import Markdown from 'markdown-to-jsx'; + +const stripMarkdown = (text) => { + let strippedText = text.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1'); + strippedText = strippedText.replace(/[_*`~]/g, ''); + return strippedText; +}; + +const parseMarkdownTable = (markdown) => { + const rows = markdown.trim().split('\n'); + const headers = rows[0].split('|').map((header) => header.trim()).filter(Boolean); + + const alignmentsRow = rows[1].split('|').map((align) => align.trim()).filter(Boolean); + const columnAlignments = alignmentsRow.map((alignment) => { + if (alignment.startsWith(':') && alignment.endsWith(':')) { + return 'center'; + } else if (alignment.startsWith(':')) { + return 'left'; + } else if (alignment.endsWith(':')) { + return 'right'; + } else { + return 'left'; + } + }); + + const data = rows.slice(2).map(row => row.split('|').map(cell => cell.trim()).filter(Boolean)); + + return { headers, data, columnAlignments }; +}; + +const SortableTable = ({ children }) => { + const { headers, data: initialData, columnAlignments } = useMemo( + () => parseMarkdownTable(children), + [children] + ); + + const [data, setData] = useState(initialData); + const [sortConfig, setSortConfig] = useState({ key: '', direction: 'asc' }); + + const sortTable = (keyIndex) => { + const newDirection = (sortConfig.key === keyIndex && sortConfig.direction === 'asc') ? 'desc' : 'asc'; + setSortConfig({ key: keyIndex, direction: newDirection }); + + const sortedData = [...data].sort((a, b) => { + const aVal = stripMarkdown(a[keyIndex]); + const bVal = stripMarkdown(b[keyIndex]); + if (aVal < bVal) return newDirection === 'asc' ? -1 : 1; + if (aVal > bVal) return newDirection === 'asc' ? 1 : -1; + return 0; + }); + + setData(sortedData); + }; + + return ( + + + + {headers.map((header, index) => ( + + ))} + + + + {data.map((row, rowIndex) => ( + + {row.map((cell, cellIndex) => ( + + ))} + + ))} + +
sortTable(index)} + style={{ + cursor: 'pointer', + position: 'relative', + textAlign: columnAlignments[index], + padding: '10px' + }} + > +
+ {header} + + ↑ + + + ↓ + +
+
+ {cell || '\u00A0'} +
+ ); +}; + +export default SortableTable; diff --git a/website/src/theme/MDXComponents/index.js b/website/src/theme/MDXComponents/index.js index d136222a0ce..422d6c99fab 100644 --- a/website/src/theme/MDXComponents/index.js +++ b/website/src/theme/MDXComponents/index.js @@ -13,6 +13,7 @@ import Mermaid from '@theme/Mermaid'; /* dbt Customizations: * Imports the following components below for export */ +import SortableTable from '@site/src/components/sortableTable'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem' import Changelog from '@site/src/components/changelog'; @@ -95,5 +96,6 @@ const MDXComponents = { DetailsToggle: DetailsToggle, Expandable: Expandable, ConfettiTrigger: ConfettiTrigger, + SortableTable: SortableTable, }; export default MDXComponents; diff --git a/website/static/img/blog/2024-05-07-unit-testing/unit-test-terminal-output.png b/website/static/img/blog/2024-05-07-unit-testing/unit-test-terminal-output.png new file mode 100644 index 00000000000..9e68587fa61 Binary files /dev/null and b/website/static/img/blog/2024-05-07-unit-testing/unit-test-terminal-output.png differ diff --git a/website/vercel.json b/website/vercel.json index 74f0eeff65b..3340a4ab684 100644 --- a/website/vercel.json +++ b/website/vercel.json @@ -2,6 +2,11 @@ "cleanUrls": true, "trailingSlash": false, "redirects": [ + { + "source": "/faqs/API/rotate-token", + "destination": "/docs/dbt-cloud-apis/service-tokens#service-token-update", + "permanent": true + }, { "source": "/styles", "destination": "https://github.com/dbt-labs/docs.getdbt.com/blob/current/contributing/adding-page-components.md",