From deae00fc3f3ada7830487fcd203834f196c293b5 Mon Sep 17 00:00:00 2001 From: Niranjan Subramanian Date: Tue, 8 Oct 2024 18:16:28 +0530 Subject: [PATCH 1/5] DBT-556 Added support for materializing Kudu table through impala adapter. --- CONTRIBUTING.md | 1 + KUDU_INTEGRATION.md | 57 ++++++++ README.md | 1 + dbt/adapters/impala/relation.py | 2 +- dbt/include/impala/macros/adapters.sql | 9 ++ test.env.example | 1 + tests/functional/adapter/test_kudu.py | 182 +++++++++++++++++++++++++ 7 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 KUDU_INTEGRATION.md create mode 100644 tests/functional/adapter/test_kudu.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1927652..48489d1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,6 +61,7 @@ When `dbt-impala` is installed this way, any changes you make to the `dbt-impala `dbt-impala` contains [functional](https://github.com/cloudera/dbt-impala/tree/master/tests/functional/) tests. Functional tests require an actual Impala warehouse to test against. - You can run functional tests "locally" by configuring a `test.env` file with appropriate `ENV` variables. +- To run `Kudu functional tests` as part of the test suite when underlying storage is `Kudu`, please set the `ENV` variable `DISABLE_KUDU_TEST` to `false`. ``` cp test.env.example test.env diff --git a/KUDU_INTEGRATION.md b/KUDU_INTEGRATION.md new file mode 100644 index 0000000..20c7e3f --- /dev/null +++ b/KUDU_INTEGRATION.md @@ -0,0 +1,57 @@ +# Kudu Integration using dbt-impala + +The `dbt-impala` adapter allows you to use [dbt](https://www.getdbt.com/) along with [Apache Kudu](https://kudu.apache.org) and [Cloudera Data Platform](https://cloudera.com) + + +## Getting started + +- [Install dbt](https://docs.getdbt.com/docs/installation) +- Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) + +### Requirements + +- In a CDP public cloud deployment, Kudu is available as one of the many Cloudera Runtime services within the Real-time Data Mart template. +- To use Kudu, you can create a Data Hub cluster by selecting Real-time Data Mart template template in the Management Console. +- Follow this [article](https://blog.cloudera.com/integrating-cloudera-data-warehouse-with-kudu-clusters) on integrating the created Kudu service with Impala CDW. + + +For general instructions, please follow [Readme](README.md) guidelines. + +## Supported features +| Name | Supported | Kudu | +|------|-----------|---------| +|Materialization: View|Yes| N/A | +|Materialization: Table|Yes| Yes | +|Materialization: Table with Partitions |Yes| No | +|Materialization: Incremental - Append|Yes| Yes | +|Materialization: Incremental - Append with Partitions |Yes| No | +|Materialization: Incremental - Insert+Overwrite |Yes| No | +|Materialization: Incremental - Insert+Overwrite with Partition |Yes| No | +|Materialization: Incremental - Merge|No| No | +|Materialization: Ephemeral|Yes| No | +|Seeds|Yes| Yes | +|Tests|Yes| Yes | +|Snapshots|No| No | +|Documentation|Yes| Yes | +|Authentication: LDAP|Yes| Yes | +|Authentication: Kerberos|Yes| No | + +### Tests Coverage + +#### Functional Tests +| Name | Base | Kudu | +|------|------|---------| +|Materialization: View|Yes| N/A | +|Materialization: Table|Yes| Yes | +|Materialization: Table with Partitions |Yes| No | +|Materialization: Incremental - Append|Yes| Yes | +|Materialization: Incremental - Append with Partitions |Yes| No | +|Materialization: Incremental - Insert+Overwrite |Yes| Yes | +|Materialization: Incremental - Insert+Overwrite with Partition |Yes| No | +|Materialization: Ephemeral|Yes| Yes | +|Seeds|Yes| Yes | +|Tests|Yes| Yes | +|Snapshots|No| No | +|Documentation| Yes | Yes | +|Authentication: LDAP|Yes| Yes | +|Authentication: Kerberos|No| No | diff --git a/README.md b/README.md index ed21de3..f93ba77 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ The `dbt-impala` adapter allows you to use [dbt](https://www.getdbt.com/) along - [Install dbt](https://docs.getdbt.com/docs/installation) - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) +- For using `dbt-impala` adapter against [Apache Kudu](https://kudu.apache.org), please follow [Kudu Integration](KUDU_INTEGRATION.md) guidelines. ### Requirements diff --git a/dbt/adapters/impala/relation.py b/dbt/adapters/impala/relation.py index 02e3aaf..8222b9b 100644 --- a/dbt/adapters/impala/relation.py +++ b/dbt/adapters/impala/relation.py @@ -37,7 +37,7 @@ class ImpalaIncludePolicy(Policy): class ImpalaRelation(BaseRelation): quote_policy: ImpalaQuotePolicy = field(default_factory=lambda: ImpalaQuotePolicy()) include_policy: ImpalaIncludePolicy = field(default_factory=lambda: ImpalaIncludePolicy()) - quote_character: str = None + quote_character: str = "`" information: str = None def __post_init__(self): diff --git a/dbt/include/impala/macros/adapters.sql b/dbt/include/impala/macros/adapters.sql index 8cbe597..6e0b36b 100644 --- a/dbt/include/impala/macros/adapters.sql +++ b/dbt/include/impala/macros/adapters.sql @@ -68,6 +68,14 @@ {%- endif %} {%- endmacro -%} +{% macro ct_option_primary_key(label, required=false) %} + {%- set primaryKey = config.get('primary_key', validator=validation[basestring]) -%} + + {%- if primaryKey is not none %} + {{label}} {{primaryKey}} + {%- endif %} +{%- endmacro -%} + {% macro ct_option_stored_as(label, required=false) %} {%- set storedAs = config.get('stored_as', validator=validation[basestring]) -%} @@ -180,6 +188,7 @@ {{ ct_option_row_format(label="row format") }} {{ ct_option_with_serdeproperties(label="with serdeproperties") }} {%- if table_type == 'iceberg' -%} STORED BY ICEBERG {%- endif -%} + {{ ct_option_primary_key(label="PRIMARY KEY") }} {{ ct_option_stored_as(label="stored as") }} {{ ct_option_location_clause(label="location") }} {{ ct_option_cached_in(label="cached in") }} diff --git a/test.env.example b/test.env.example index 1339ba5..6d98bb1 100644 --- a/test.env.example +++ b/test.env.example @@ -17,3 +17,4 @@ IMPALA_SCHEMA=my_schema IMPALA_USER=my_user IMPALA_PASSWORD=my_password IMPALA_HTTP_PATH=my_http_path +DISABLE_KUDU_TEST=true diff --git a/tests/functional/adapter/test_kudu.py b/tests/functional/adapter/test_kudu.py new file mode 100644 index 0000000..a91da52 --- /dev/null +++ b/tests/functional/adapter/test_kudu.py @@ -0,0 +1,182 @@ +# Copyright 2024 Cloudera Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os +from dbt.tests.util import run_dbt, relation_from_name, check_relations_equal + +from dbt.tests.adapter.basic.test_incremental import ( + BaseIncremental, + BaseIncrementalNotSchemaChange, +) + +from dbt.tests.adapter.basic.files import ( + schema_base_yml, + model_incremental, +) + +pytestmark = pytest.mark.skipif( + os.getenv(key="DISABLE_KUDU_TEST", default="true") == "true", + reason="Kudu tests will be run when DISABLE_KUDU_TEST is set to false in test.env", +) + +incremental_kudu_sql = ( + """ + {{ + config( + materialized="incremental", + stored_as="kudu", + primary_key="(id)" + ) +}} +""".strip() + + model_incremental +) + + +class TestIncrementalKudu(BaseIncremental): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"name": "incremental_test_model"} + + @pytest.fixture(scope="class") + def models(self): + return {"incremental_test_model.sql": incremental_kudu_sql, "schema.yml": schema_base_yml} + + def test_incremental(self, project): + # seed command + results = run_dbt(["seed"]) + assert len(results) == 2 + + # base table rowcount + relation = relation_from_name(project.adapter, "base") + result = project.run_sql(f"select count(*) as num_rows from {relation}", fetch="one") + assert result[0] == 10 + + # added table rowcount + relation = relation_from_name(project.adapter, "added") + result = project.run_sql(f"select count(*) as num_rows from {relation}", fetch="one") + assert result[0] == 20 + + # run command + # the "seed_name" var changes the seed identifier in the schema file + results = run_dbt(["run", "--vars", "seed_name: base"]) + assert len(results) == 1 + + # check relations equal + check_relations_equal(project.adapter, ["base", "incremental_test_model"]) + + # change seed_name var + # the "seed_name" var changes the seed identifier in the schema file + results = run_dbt(["run", "--vars", "seed_name: added"]) + assert len(results) == 1 + + # check relations equal + check_relations_equal(project.adapter, ["added", "incremental_test_model"]) + + # run full-refresh and compare with base table again + results = run_dbt( + [ + "run", + "--select", + "incremental_test_model", + "--full-refresh", + "--vars", + "seed_name: base", + ] + ) + assert len(results) == 1 + + check_relations_equal(project.adapter, ["base", "incremental_test_model"]) + + # get catalog from docs generate + catalog = run_dbt(["docs", "generate"]) + assert len(catalog.nodes) == 3 + assert len(catalog.sources) == 1 + + +insertoverwrite_sql = """ + {{ + config( + materialized="incremental", + incremental_strategy="insert_overwrite", + partition_by="id_partition", + stored_as="kudu", + primary_key="(id)" + ) +}} + select *, id as id_partition from {{ source('raw', 'seed') }} + {% if is_incremental() %} + where id > (select max(id) from {{ this }}) + {% endif %} +""".strip() + + +@pytest.mark.skip(reason="Need to fix partition by syntax for Kudu") +class TestInsertoverwriteKudu(TestIncrementalKudu): + @pytest.fixture(scope="class") + def models(self): + return {"incremental_test_model.sql": insertoverwrite_sql, "schema.yml": schema_base_yml} + + +incremental_single_partitionby_sql = """ + {{ + config( + materialized="incremental", + partition_by="id_partition", + stored_as="kudu", + primary_key="(id)" + ) +}} + select *, id as id_partition from {{ source('raw', 'seed') }} + {% if is_incremental() %} + where id > (select max(id) from {{ this }}) + {% endif %} +""".strip() + + +@pytest.mark.skip(reason="Need to fix partition by syntax for Kudu") +class TestIncrementalWithSinglePartitionKeyKudu(TestIncrementalKudu): + @pytest.fixture(scope="class") + def models(self): + return { + "incremental_test_model.sql": incremental_single_partitionby_sql, + "schema.yml": schema_base_yml, + } + + +incremental_multiple_partitionby_sql = """ + {{ + config( + materialized="incremental", + partition_by=["id_partition1", "id_partition2"], + stored_as="kudu", + primary_key="(id)" + ) + }} + select *, id as id_partition1, id as id_partition2 from {{ source('raw', 'seed') }} + {% if is_incremental() %} + where id > (select max(id) from {{ this }}) + {% endif %} +""".strip() + + +@pytest.mark.skip(reason="Need to fix partition by syntax for Kudu") +class TestIncrementalWithMultiplePartitionKeyKudu(TestIncrementalKudu): + @pytest.fixture(scope="class") + def models(self): + return { + "incremental_test_model.sql": incremental_multiple_partitionby_sql, + "schema.yml": schema_base_yml, + } From bb1331d10ef7da8f90d1ab9f85e3b482d98353db Mon Sep 17 00:00:00 2001 From: Niranjan Subramanian Date: Thu, 10 Oct 2024 23:37:14 +0530 Subject: [PATCH 2/5] DBT-556 Addressed review comment. --- KUDU_INTEGRATION.md | 39 --------------------------- README.md | 66 ++++++++++++++++++++++----------------------- 2 files changed, 33 insertions(+), 72 deletions(-) diff --git a/KUDU_INTEGRATION.md b/KUDU_INTEGRATION.md index 20c7e3f..d88b978 100644 --- a/KUDU_INTEGRATION.md +++ b/KUDU_INTEGRATION.md @@ -16,42 +16,3 @@ The `dbt-impala` adapter allows you to use [dbt](https://www.getdbt.com/) along For general instructions, please follow [Readme](README.md) guidelines. - -## Supported features -| Name | Supported | Kudu | -|------|-----------|---------| -|Materialization: View|Yes| N/A | -|Materialization: Table|Yes| Yes | -|Materialization: Table with Partitions |Yes| No | -|Materialization: Incremental - Append|Yes| Yes | -|Materialization: Incremental - Append with Partitions |Yes| No | -|Materialization: Incremental - Insert+Overwrite |Yes| No | -|Materialization: Incremental - Insert+Overwrite with Partition |Yes| No | -|Materialization: Incremental - Merge|No| No | -|Materialization: Ephemeral|Yes| No | -|Seeds|Yes| Yes | -|Tests|Yes| Yes | -|Snapshots|No| No | -|Documentation|Yes| Yes | -|Authentication: LDAP|Yes| Yes | -|Authentication: Kerberos|Yes| No | - -### Tests Coverage - -#### Functional Tests -| Name | Base | Kudu | -|------|------|---------| -|Materialization: View|Yes| N/A | -|Materialization: Table|Yes| Yes | -|Materialization: Table with Partitions |Yes| No | -|Materialization: Incremental - Append|Yes| Yes | -|Materialization: Incremental - Append with Partitions |Yes| No | -|Materialization: Incremental - Insert+Overwrite |Yes| Yes | -|Materialization: Incremental - Insert+Overwrite with Partition |Yes| No | -|Materialization: Ephemeral|Yes| Yes | -|Seeds|Yes| Yes | -|Tests|Yes| Yes | -|Snapshots|No| No | -|Documentation| Yes | Yes | -|Authentication: LDAP|Yes| Yes | -|Authentication: Kerberos|No| No | diff --git a/README.md b/README.md index f93ba77..8e9c03b 100644 --- a/README.md +++ b/README.md @@ -41,40 +41,40 @@ demo_project: ``` ## Supported features -| Name | Supported | Iceberg | -|------|-----------|---------| -|Materialization: View|Yes| N/A | -|Materialization: Table|Yes| Yes | -|Materialization: Table with Partitions |Yes| Yes | -|Materialization: Incremental - Append|Yes| Yes | -|Materialization: Incremental - Append with Partitions |Yes| Yes | -|Materialization: Incremental - Insert+Overwrite |Yes| Yes | -|Materialization: Incremental - Insert+Overwrite with Partition |Yes| Yes | -|Materialization: Incremental - Merge|No| No | -|Materialization: Ephemeral|Yes| Yes | -|Seeds|Yes| Yes | -|Tests|Yes| Yes | -|Snapshots|No| No | -|Documentation|Yes| Yes | -|Authentication: LDAP|Yes| Yes | -|Authentication: Kerberos|Yes| No | +| Name | Supported | Iceberg | Kudu | +|------|-----------|---------|------| +|Materialization: View|Yes| N/A | N/A | +|Materialization: Table|Yes| Yes | Yes | +|Materialization: Table with Partitions |Yes| Yes | No | +|Materialization: Incremental - Append|Yes| Yes | Yes | +|Materialization: Incremental - Append with Partitions |Yes| Yes | No | +|Materialization: Incremental - Insert+Overwrite |Yes| Yes | Yes | +|Materialization: Incremental - Insert+Overwrite with Partition |Yes| Yes | No | +|Materialization: Incremental - Merge|No| No | No | +|Materialization: Ephemeral|Yes| Yes | No | +|Seeds|Yes| Yes | Yes | +|Tests|Yes| Yes | Yes | +|Snapshots|No| No | No | +|Documentation|Yes| Yes | Yes | +|Authentication: LDAP|Yes| Yes | Yes | +|Authentication: Kerberos|Yes| No | No | ### Tests Coverage #### Functional Tests -| Name | Base | Iceberg | -|------|------|---------| -|Materialization: View|Yes| N/A | -|Materialization: Table|Yes| Yes | -|Materialization: Table with Partitions |Yes| Yes | -|Materialization: Incremental - Append|Yes| Yes | -|Materialization: Incremental - Append with Partitions |Yes| Yes | -|Materialization: Incremental - Insert+Overwrite |Yes| Yes | -|Materialization: Incremental - Insert+Overwrite with Partition |Yes| Yes | -|Materialization: Ephemeral|Yes| Yes | -|Seeds|Yes| Yes | -|Tests|Yes| Yes | -|Snapshots|No| No | -|Documentation| Yes | Yes | -|Authentication: LDAP|Yes| Yes | -|Authentication: Kerberos|No| No | +| Name | Base | Iceberg | Kudu | +|------|------|---------|------| +|Materialization: View|Yes| N/A | N/A | +|Materialization: Table|Yes| Yes | Yes | +|Materialization: Table with Partitions |Yes| Yes | No | +|Materialization: Incremental - Append|Yes| Yes | Yes | +|Materialization: Incremental - Append with Partitions |Yes| Yes | No | +|Materialization: Incremental - Insert+Overwrite |Yes| Yes | Yes | +|Materialization: Incremental - Insert+Overwrite with Partition |Yes| Yes | No | +|Materialization: Ephemeral|Yes| Yes | No | +|Seeds|Yes| Yes | Yes | +|Tests|Yes| Yes | Yes | +|Snapshots|No| No | No | +|Documentation| Yes | Yes | Yes | +|Authentication: LDAP|Yes| Yes | Yes | +|Authentication: Kerberos|No| No | No | From 297aa2b7fb991ce26eb43b79858660a03f60186c Mon Sep 17 00:00:00 2001 From: Niranjan Subramanian Date: Fri, 11 Oct 2024 14:59:04 +0530 Subject: [PATCH 3/5] DBT-556 Incorporated a review comment in CONTRIBUTING.md file. --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 48489d1..38906ca 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,7 +61,7 @@ When `dbt-impala` is installed this way, any changes you make to the `dbt-impala `dbt-impala` contains [functional](https://github.com/cloudera/dbt-impala/tree/master/tests/functional/) tests. Functional tests require an actual Impala warehouse to test against. - You can run functional tests "locally" by configuring a `test.env` file with appropriate `ENV` variables. -- To run `Kudu functional tests` as part of the test suite when underlying storage is `Kudu`, please set the `ENV` variable `DISABLE_KUDU_TEST` to `false`. +- To run `Kudu functional tests` as part of the test suite when underlying storage is `Kudu`, please set the `ENV` variable `DISABLE_KUDU_TEST` to `false`. Kudu tests are disabled by default as this `ENV` variable is set to true. ``` cp test.env.example test.env From 1c2e96162457e038078709a5cebedee1e1824182 Mon Sep 17 00:00:00 2001 From: Niranjan Subramanian Date: Mon, 21 Oct 2024 17:17:33 +0530 Subject: [PATCH 4/5] DBT-556 Updated README.md with correct set of available tests. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e9c03b..a5651be 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ demo_project: |Materialization: Table with Partitions |Yes| Yes | No | |Materialization: Incremental - Append|Yes| Yes | Yes | |Materialization: Incremental - Append with Partitions |Yes| Yes | No | -|Materialization: Incremental - Insert+Overwrite |Yes| Yes | Yes | +|Materialization: Incremental - Insert+Overwrite |Yes| No | No | |Materialization: Incremental - Insert+Overwrite with Partition |Yes| Yes | No | |Materialization: Ephemeral|Yes| Yes | No | |Seeds|Yes| Yes | Yes | From 490078050ff6b6d4eb1b656b25ccc98fa0aedc70 Mon Sep 17 00:00:00 2001 From: Niranjan Subramanian Date: Tue, 12 Nov 2024 20:16:05 +0530 Subject: [PATCH 5/5] DBT-803 Fix-1.8.0-impala Added partition support for materializing Kudu tables. --- README.md | 6 +-- dbt/include/impala/macros/adapters.sql | 18 +++++++- tests/functional/adapter/test_kudu.py | 64 ++++++-------------------- 3 files changed, 33 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index a5651be..76acf71 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,10 @@ demo_project: |------|-----------|---------|------| |Materialization: View|Yes| N/A | N/A | |Materialization: Table|Yes| Yes | Yes | -|Materialization: Table with Partitions |Yes| Yes | No | +|Materialization: Table with Partitions |Yes| Yes | Yes | |Materialization: Incremental - Append|Yes| Yes | Yes | |Materialization: Incremental - Append with Partitions |Yes| Yes | No | -|Materialization: Incremental - Insert+Overwrite |Yes| Yes | Yes | +|Materialization: Incremental - Insert+Overwrite |Yes| Yes | No | |Materialization: Incremental - Insert+Overwrite with Partition |Yes| Yes | No | |Materialization: Incremental - Merge|No| No | No | |Materialization: Ephemeral|Yes| Yes | No | @@ -66,7 +66,7 @@ demo_project: |------|------|---------|------| |Materialization: View|Yes| N/A | N/A | |Materialization: Table|Yes| Yes | Yes | -|Materialization: Table with Partitions |Yes| Yes | No | +|Materialization: Table with Partitions |Yes| Yes | Yes | |Materialization: Incremental - Append|Yes| Yes | Yes | |Materialization: Incremental - Append with Partitions |Yes| Yes | No | |Materialization: Incremental - Insert+Overwrite |Yes| No | No | diff --git a/dbt/include/impala/macros/adapters.sql b/dbt/include/impala/macros/adapters.sql index 6e0b36b..4074087 100644 --- a/dbt/include/impala/macros/adapters.sql +++ b/dbt/include/impala/macros/adapters.sql @@ -29,6 +29,15 @@ {%- endif %} {%- endmacro -%} +{% macro ct_option_kudu_partition_cols(label, required=false) %} + {%- set cols = config.get('partition_by', validator=validation[basestring]) -%} + {%- if cols is not none %} + {%- if cols is string -%} + {{ label }} {{ cols }} + {%- endif -%} + {%- endif %} +{%- endmacro -%} + {% macro ct_option_sort_cols(label, required=false) %} {%- set cols = config.get('sort_by', validator=validation.any[list, basestring]) -%} {%- if cols is not none %} @@ -173,22 +182,27 @@ {%- set sql_header = config.get('sql_header', none) -%} {%- set is_external = config.get('external') -%} {%- set table_type = config.get('table_type') -%} + {%- set stored_as = config.get('stored_as', none) -%} {{ sql_header if sql_header is not none }} create {% if is_external == true -%}external{%- endif %} table {{ relation.include(schema=true) }} + {{ ct_option_primary_key(label="PRIMARY KEY") }} {% if table_type == 'iceberg' -%} {{ ct_option_partition_cols(label="partitioned by spec") }} {% else %} - {{ ct_option_partition_cols(label="partitioned by") }} + {% if stored_as == 'kudu' -%} + {{ ct_option_kudu_partition_cols(label="partition by") }} + {% else %} + {{ ct_option_partition_cols(label="partitioned by") }} + {%- endif %} {%- endif %} {{ ct_option_sort_cols(label="sort by") }} {{ ct_option_comment_relation(label="comment") }} {{ ct_option_row_format(label="row format") }} {{ ct_option_with_serdeproperties(label="with serdeproperties") }} {%- if table_type == 'iceberg' -%} STORED BY ICEBERG {%- endif -%} - {{ ct_option_primary_key(label="PRIMARY KEY") }} {{ ct_option_stored_as(label="stored as") }} {{ ct_option_location_clause(label="location") }} {{ ct_option_cached_in(label="cached in") }} diff --git a/tests/functional/adapter/test_kudu.py b/tests/functional/adapter/test_kudu.py index a91da52..73d7cdd 100644 --- a/tests/functional/adapter/test_kudu.py +++ b/tests/functional/adapter/test_kudu.py @@ -26,6 +26,8 @@ model_incremental, ) +from dbt.tests.adapter.basic.test_table_materialization import BaseTableMaterialization + pytestmark = pytest.mark.skipif( os.getenv(key="DISABLE_KUDU_TEST", default="true") == "true", reason="Kudu tests will be run when DISABLE_KUDU_TEST is set to false in test.env", @@ -106,77 +108,39 @@ def test_incremental(self, project): assert len(catalog.sources) == 1 -insertoverwrite_sql = """ +materialization_hash_partitionby_sql = """ {{ config( - materialized="incremental", - incremental_strategy="insert_overwrite", - partition_by="id_partition", + materialized="table", + partition_by="HASH (id) PARTITIONS 2", stored_as="kudu", primary_key="(id)" ) }} - select *, id as id_partition from {{ source('raw', 'seed') }} - {% if is_incremental() %} - where id > (select max(id) from {{ this }}) - {% endif %} +select * from {{ this.schema }}.seed """.strip() -@pytest.mark.skip(reason="Need to fix partition by syntax for Kudu") -class TestInsertoverwriteKudu(TestIncrementalKudu): +class TestMaterializationWithHashPartitionKudu(BaseTableMaterialization): @pytest.fixture(scope="class") def models(self): - return {"incremental_test_model.sql": insertoverwrite_sql, "schema.yml": schema_base_yml} + return {"materialized.sql": materialization_hash_partitionby_sql} -incremental_single_partitionby_sql = """ +materialization_range_partitionby_sql = """ {{ config( - materialized="incremental", - partition_by="id_partition", + materialized="table", + partition_by="Range (id) (PARTITION VALUES < 5, PARTITION 5 <= VALUES)", stored_as="kudu", primary_key="(id)" ) }} - select *, id as id_partition from {{ source('raw', 'seed') }} - {% if is_incremental() %} - where id > (select max(id) from {{ this }}) - {% endif %} -""".strip() - - -@pytest.mark.skip(reason="Need to fix partition by syntax for Kudu") -class TestIncrementalWithSinglePartitionKeyKudu(TestIncrementalKudu): - @pytest.fixture(scope="class") - def models(self): - return { - "incremental_test_model.sql": incremental_single_partitionby_sql, - "schema.yml": schema_base_yml, - } - - -incremental_multiple_partitionby_sql = """ - {{ - config( - materialized="incremental", - partition_by=["id_partition1", "id_partition2"], - stored_as="kudu", - primary_key="(id)" - ) - }} - select *, id as id_partition1, id as id_partition2 from {{ source('raw', 'seed') }} - {% if is_incremental() %} - where id > (select max(id) from {{ this }}) - {% endif %} +select * from {{ this.schema }}.seed """.strip() -@pytest.mark.skip(reason="Need to fix partition by syntax for Kudu") -class TestIncrementalWithMultiplePartitionKeyKudu(TestIncrementalKudu): +class TestMaterializationWithRangePartitionKudu(BaseTableMaterialization): @pytest.fixture(scope="class") def models(self): - return { - "incremental_test_model.sql": incremental_multiple_partitionby_sql, - "schema.yml": schema_base_yml, - } + return {"materialized.sql": materialization_range_partitionby_sql}