From 669c49223baef0b2f81f4d5d2f32efaa611b38da Mon Sep 17 00:00:00 2001 From: hengm3467 <100685635+hengm3467@users.noreply.github.com> Date: Wed, 7 Feb 2024 01:47:08 +0800 Subject: [PATCH] Add documentation about the dbt-risingwave adapter (#4747) ## What are you changing in this pull request and why? Add documentation about the dbt-risingwave adapter. RisingWave is a streaming database that allows users to process and manage streaming data with a PostgreSQL-style experience. For more information about the company and product, see https://risingwave.com/. The purpose of this adapter is to enable RisingWave users to leverage the powerful capabilities of dbt to manage their stream processing jobs and pipelines in RisingWave. We consider dbt as the right way to unlock stream processing with our product. This adapter is available on [GitHub repo](https://github.com/risingwavelabs/dbt-risingwave) and [PyPl](https://pypi.org/project/dbt-risingwave/). ## Checklist - [ ] Review the [Content style guide](https://github.com/dbt-labs/docs.getdbt.com/blob/current/contributing/content-style-guide.md) so my content adheres to these guidelines. - [ ] For [docs versioning](https://github.com/dbt-labs/docs.getdbt.com/blob/current/contributing/single-sourcing-content.md#about-versioning), review how to [version a whole page](https://github.com/dbt-labs/docs.getdbt.com/blob/current/contributing/single-sourcing-content.md#adding-a-new-version) and [version a block of content](https://github.com/dbt-labs/docs.getdbt.com/blob/current/contributing/single-sourcing-content.md#versioning-blocks-of-content). - [ ] Add a checklist item for anything that needs to happen before this PR is merged, such as "needs technical review" or "change base branch." Adding or removing pages (delete if not applicable): - [x] Add page in `website/sidebars.js` - [x] Create setup file `/docs/core/connect-data-platform/risingwave-setup.md` - [x] Provide a unique filename for new pages --------- Co-authored-by: Leona B. Campbell <3880403+runleonarun@users.noreply.github.com> Co-authored-by: Amy Chen <46451573+amychen1776@users.noreply.github.com> --- website/docs/docs/community-adapters.md | 2 +- .../connect-data-platform/risingwave-setup.md | 87 +++++++++++++++++++ website/sidebars.js | 1 + 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 website/docs/docs/core/connect-data-platform/risingwave-setup.md diff --git a/website/docs/docs/community-adapters.md b/website/docs/docs/community-adapters.md index 1faf2fd9e25..39a906d7f60 100644 --- a/website/docs/docs/community-adapters.md +++ b/website/docs/docs/community-adapters.md @@ -17,4 +17,4 @@ Community adapters are adapter plugins contributed and maintained by members of | [TiDB](/docs/core/connect-data-platform/tidb-setup) | [Firebolt](/docs/core/connect-data-platform/firebolt-setup) | [MindsDB](/docs/core/connect-data-platform/mindsdb-setup) | [Vertica](/docs/core/connect-data-platform/vertica-setup) | [AWS Glue](/docs/core/connect-data-platform/glue-setup) | [MySQL](/docs/core/connect-data-platform/mysql-setup) | | [Upsolver](/docs/core/connect-data-platform/upsolver-setup) | [Databend Cloud](/docs/core/connect-data-platform/databend-setup) | [fal - Python models](/docs/core/connect-data-platform/fal-setup) | -| [TimescaleDB](https://dbt-timescaledb.debruyn.dev/) | [Extrica](/docs/core/connect-data-platform/extrica-setup) | | +| [TimescaleDB](https://dbt-timescaledb.debruyn.dev/) | [Extrica](/docs/core/connect-data-platform/extrica-setup) | [RisingWave](/docs/core/connect-data-platform/risingwave-setup) | diff --git a/website/docs/docs/core/connect-data-platform/risingwave-setup.md b/website/docs/docs/core/connect-data-platform/risingwave-setup.md new file mode 100644 index 00000000000..4e320d00f42 --- /dev/null +++ b/website/docs/docs/core/connect-data-platform/risingwave-setup.md @@ -0,0 +1,87 @@ +--- +title: "RisingWave setup" +id: "risingwave-setup" +description: "Read this guide to learn about how to set up RisingWave in dbt." +meta: + maintained_by: RisingWave + pypi_package: 'dbt-risingwave' + authors: 'Dylan Chen' + github_repo: 'risingwavelabs/dbt-risingwave' + min_core_version: 'v1.6.1' + min_supported_version: '' + cloud_support: Not Supported + slack_channel_name: 'N/A' + slack_channel_link: 'https://www.risingwave.com/slack' + platform_name: 'RisingWave' + config_page: '/reference/resource-configs/no-configs' +--- + +:::info Vendor-supported plugin + +Certain core functionality may vary. If you would like to report a bug, request a feature, or contribute, you can check out the linked repository and open an issue. + +::: + +import SetUpPages from '/snippets/_setup-pages-intro.md'; + + + +## Connecting to RisingWave with dbt-risingwave + +Before connecting to RisingWave, ensure that RisingWave is installed and running. For more information about how to get RisingWave up and running, see the [RisingWave quick start guide](https://docs.risingwave.com/docs/dev/get-started/). + +To connect to RisingWave with dbt, you need to add a RisingWave profile to your dbt profile file (`~/.dbt/profiles.yml`). Below is an example RisingWave profile. Revise the field values when necessary. + + + +```yaml +default: + outputs: + dev: + type: risingwave + host: [host name] + user: [user name] + pass: [password] + dbname: [database name] + port: [port] + schema: [dbt schema] + target: dev +``` + + + +|Field|Description| +|---|---| +|`host`| The host name or IP address of the RisingWave instance| +|`user`|The RisingWave database user you want to use| +|`pass`| The password of the database user| +|`dbname` | The RisingWave database name| +|`port` | The port number that RisingWave listens on| +|`schema`| The schema of the RisingWave database| + +To test the connection to RisingWave, run: + +```bash +dbt debug +``` + +## Materializations + +The dbt models for managing data transformations in RisingWave are similar to typical dbt SQL models. In the `dbt-risingwave` adapter, we have customized some of the materializations to align with the streaming data processing model of RisingWave. + +|Materializations| Supported|Notes| +|----|----|----| +|`table` |Yes |Creates a [table](https://docs.risingwave.com/docs/dev/sql-create-table/). To use this materialization, add `{{ config(materialized='table') }}` to your model SQL files. | +|`view`|Yes | Creates a [view](https://docs.risingwave.com/docs/dev/sql-create-view/). To use this materialization, add `{{ config(materialized='view') }}` to your model SQL files. | +|`ephemeral`|Yes| This materialization uses [common table expressions](https://docs.risingwave.com/docs/dev/query-syntax-with-clause/) in RisingWave under the hood. To use this materialization, add `{{ config(materialized='ephemeral') }}` to your model SQL files.| +|`materializedview`| To be deprecated. |It is available only for backward compatibility purposes (for v1.5.1 of the dbt-risingwave adapter plugin). If you are using v1.6.0 and later versions of the dbt-risingwave adapter plugin, use `materialized_view` instead.| +|`materialized_view`| Yes| Creates a [materialized view](https://docs.risingwave.com/docs/dev/sql-create-mv/). This materialization corresponds the `incremental` one in dbt. To use this materialization, add `{{ config(materialized='materialized_view') }}` to your model SQL files.| +| `incremental`|No|Please use `materialized_view` instead. Since RisingWave is designed to use materialized view to manage data transformation in an incremental way, you can just use the `materialized_view` materialization.| +|`source`| Yes| Creates a [source](https://docs.risingwave.com/docs/dev/sql-create-source/). To use this materialization, add {{ config(materialized='source') }} to your model SQL files. You need to provide your create source statement as a whole in this model. See [Example model files](https://docs.risingwave.com/docs/dev/use-dbt/#example-model-files) for details.| +|`table_with_connector`| Yes| Creates a table with connector settings. In RisingWave, a table with connector settings is similar to a source. The difference is that a table object with connector settings persists raw streaming data in the source, while a source object does not. To use this materialization, add `{{ config(materialized='table_with_connector') }}` to your model SQL files. You need to provide your create table with connector statement as a whole in this model (see [Example model files](https://docs.risingwave.com/docs/dev/use-dbt/#example-model-files) for details). Because dbt tables have their own semantics, RisingWave use `table_with_connector` to distinguish itself from a dbt table.| +|`sink`| Yes| Creates a [sink](https://docs.risingwave.com/docs/dev/sql-create-sink/). To use this materialization, add `{{ config(materialized='sink') }}` to your SQL files. You need to provide your create sink statement as a whole in this model. See [Example model files](https://docs.risingwave.com/docs/dev/use-dbt/#example-model-files) for details.| + +## Resources + +- [RisingWave's guide about using dbt for data transformations](https://docs.risingwave.com/docs/dev/use-dbt/) +- [A demo project using dbt to manage Nexmark benchmark queries in RisingWave](https://docs.risingwave.com/docs/dev/use-dbt/) diff --git a/website/sidebars.js b/website/sidebars.js index 6429c4679a2..ba78e0b7d8a 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -213,6 +213,7 @@ const sidebarSettings = { "docs/core/connect-data-platform/upsolver-setup", "docs/core/connect-data-platform/starrocks-setup", "docs/core/connect-data-platform/extrica-setup", + "docs/core/connect-data-platform/risingwave-setup", ], }, ],