From 1f8382ec83dad9b8eed664624fdb8170fac7876e Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Fri, 8 Sep 2023 13:36:01 -0700 Subject: [PATCH 1/9] Add `date_spine` macro (and macros it depends on) from dbt-utils to core The macros added are - date_spine - get_intervals_between - generate_series - get_powers_of_two We're adding these to core because they are becoming more prevalently used with the increase usage in the semantic layer. Basically if you are using the semantic layer currently, then it is almost a requirement to use dbt-utils, which is undesireable given the SL is supported directly in core. The primary focus of this was to just add `date_spine`. However, because `date_spine` depends on other macros, these other macros were also moved. --- .../macros/utils/date_spine.sql | 75 +++++++++++++++++++ .../macros/utils/generate_series.sql | 53 +++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 core/dbt/include/global_project/macros/utils/date_spine.sql create mode 100644 core/dbt/include/global_project/macros/utils/generate_series.sql diff --git a/core/dbt/include/global_project/macros/utils/date_spine.sql b/core/dbt/include/global_project/macros/utils/date_spine.sql new file mode 100644 index 00000000000..833fbcc575b --- /dev/null +++ b/core/dbt/include/global_project/macros/utils/date_spine.sql @@ -0,0 +1,75 @@ +{% macro get_intervals_between(start_date, end_date, datepart) -%} + {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }} +{%- endmacro %} + +{% macro default__get_intervals_between(start_date, end_date, datepart) -%} + {%- call statement('get_intervals_between', fetch_result=True) %} + + select {{ dbt.datediff(start_date, end_date, datepart) }} + + {%- endcall -%} + + {%- set value_list = load_result('get_intervals_between') -%} + + {%- if value_list and value_list['data'] -%} + {%- set values = value_list['data'] | map(attribute=0) | list %} + {{ return(values[0]) }} + {%- else -%} + {{ return(1) }} + {%- endif -%} + +{%- endmacro %} + + + + +{% macro date_spine(datepart, start_date, end_date) %} + {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }} +{%- endmacro %} + +{% macro default__date_spine(datepart, start_date, end_date) %} + + + {# call as follows: + + date_spine( + "day", + "to_date('01/01/2016', 'mm/dd/yyyy')", + "dbt.dateadd(week, 1, current_date)" + ) #} + + + with rawdata as ( + + {{dbt.generate_series( + dbt.get_intervals_between(start_date, end_date, datepart) + )}} + + ), + + all_periods as ( + + select ( + {{ + dbt.dateadd( + datepart, + "row_number() over (order by 1) - 1", + start_date + ) + }} + ) as date_{{datepart}} + from rawdata + + ), + + filtered as ( + + select * + from all_periods + where date_{{datepart}} <= {{ end_date }} + + ) + + select * from filtered + +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/utils/generate_series.sql b/core/dbt/include/global_project/macros/utils/generate_series.sql new file mode 100644 index 00000000000..f6a09605af3 --- /dev/null +++ b/core/dbt/include/global_project/macros/utils/generate_series.sql @@ -0,0 +1,53 @@ +{% macro get_powers_of_two(upper_bound) %} + {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }} +{% endmacro %} + +{% macro default__get_powers_of_two(upper_bound) %} + + {% if upper_bound <= 0 %} + {{ exceptions.raise_compiler_error("upper bound must be positive") }} + {% endif %} + + {% for _ in range(1, 100) %} + {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %} + {% endfor %} + +{% endmacro %} + + +{% macro generate_series(upper_bound) %} + {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }} +{% endmacro %} + +{% macro default__generate_series(upper_bound) %} + + {% set n = dbt.get_powers_of_two(upper_bound) %} + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + {% for i in range(n) %} + p{{i}}.generated_number * power(2, {{i}}) + {% if not loop.last %} + {% endif %} + {% endfor %} + + 1 + as generated_number + + from + + {% for i in range(n) %} + p as p{{i}} + {% if not loop.last %} cross join {% endif %} + {% endfor %} + + ) + + select * + from unioned + where generated_number <= {{upper_bound}} + order by generated_number + +{% endmacro %} From 8317db0de0bcd812e0dbd976054a7c77b57fe8a3 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Mon, 11 Sep 2023 10:01:52 -0700 Subject: [PATCH 2/9] Add adapter tests for `get_powers_of_two` macro --- .../utils/fixture_get_powers_of_two.py | 39 +++++++++++++++++++ .../adapter/utils/test_get_powers_of_two.py | 21 ++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tests/adapter/dbt/tests/adapter/utils/fixture_get_powers_of_two.py create mode 100644 tests/adapter/dbt/tests/adapter/utils/test_get_powers_of_two.py diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_get_powers_of_two.py b/tests/adapter/dbt/tests/adapter/utils/fixture_get_powers_of_two.py new file mode 100644 index 00000000000..04ace2062f6 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_get_powers_of_two.py @@ -0,0 +1,39 @@ +# get_powers_of_two + +models__test_get_powers_of_two_sql = """ +select {{ get_powers_of_two(1) }} as actual, 1 as expected + +union all + +select {{ get_powers_of_two(4) }} as actual, 2 as expected + +union all + +select {{ get_powers_of_two(27) }} as actual, 5 as expected + +union all + +select {{ get_powers_of_two(256) }} as actual, 8 as expected + +union all + +select {{ get_powers_of_two(3125) }} as actual, 12 as expected + +union all + +select {{ get_powers_of_two(46656) }} as actual, 16 as expected + +union all + +select {{ get_powers_of_two(823543) }} as actual, 20 as expected +""" + +models__test_get_powers_of_two_yml = """ +version: 2 +models: + - name: test_powers_of_two + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/adapter/dbt/tests/adapter/utils/test_get_powers_of_two.py b/tests/adapter/dbt/tests/adapter/utils/test_get_powers_of_two.py new file mode 100644 index 00000000000..aa6f4d1a196 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/test_get_powers_of_two.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.utils.base_utils import BaseUtils +from dbt.tests.adapter.utils.fixture_get_powers_of_two import ( + models__test_get_powers_of_two_sql, + models__test_get_powers_of_two_yml, +) + + +class BaseGetPowersOfTwo(BaseUtils): + @pytest.fixture(scope="class") + def models(self): + return { + "test_get_powers_of_two.yml": models__test_get_powers_of_two_yml, + "test_get_powers_of_two.sql": self.interpolate_macro_namespace( + models__test_get_powers_of_two_sql, "get_powers_of_two" + ), + } + + +class TestGetPowersOfTwo(BaseGetPowersOfTwo): + pass From 073cdf4c9546c3f7f072712c01604a6b6e7fa213 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Mon, 11 Sep 2023 13:56:40 -0700 Subject: [PATCH 3/9] Add adapter tests for `generate_series` macro --- .../adapter/utils/fixture_generate_series.py | 45 +++++++++++++++++++ .../adapter/utils/test_generate_series.py | 21 +++++++++ 2 files changed, 66 insertions(+) create mode 100644 tests/adapter/dbt/tests/adapter/utils/fixture_generate_series.py create mode 100644 tests/adapter/dbt/tests/adapter/utils/test_generate_series.py diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_generate_series.py b/tests/adapter/dbt/tests/adapter/utils/fixture_generate_series.py new file mode 100644 index 00000000000..0ce578df9b2 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_generate_series.py @@ -0,0 +1,45 @@ +# If generate_series works properly, there should be no `null` values in the resulting model + +models__test_generate_series_sql = """ +with generated_numbers as ( + {{ dbt.generate_series(10) }} +), expected_numbers as ( + select 1 as expected + union all + select 2 as expected + union all + select 3 as expected + union all + select 4 as expected + union all + select 5 as expected + union all + select 6 as expected + union all + select 7 as expected + union all + select 8 as expected + union all + select 9 as expected + union all + select 10 as expected +), joined as ( + select + generated_numbers.generated_number, + expected_numbers.expected + from generated_numbers + left join expected_numbers on generated_numbers.generated_number = expected_numbers.expected +) + +SELECT * from joined +""" + +models__test_generate_series_yml = """ +version: 2 +models: + - name: test_generate_series + tests: + - assert_equal: + actual: generated_number + expected: expected +""" diff --git a/tests/adapter/dbt/tests/adapter/utils/test_generate_series.py b/tests/adapter/dbt/tests/adapter/utils/test_generate_series.py new file mode 100644 index 00000000000..afc8d77dd3b --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/test_generate_series.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.utils.base_utils import BaseUtils +from dbt.tests.adapter.utils.fixture_generate_series import ( + models__test_generate_series_sql, + models__test_generate_series_yml, +) + + +class BaseGenerateSeries(BaseUtils): + @pytest.fixture(scope="class") + def models(self): + return { + "test_generate_series.yml": models__test_generate_series_yml, + "test_generate_series.sql": self.interpolate_macro_namespace( + models__test_generate_series_sql, "generate_series" + ), + } + + +class TestGenerateSeries(BaseGenerateSeries): + pass From e3fb225ad3f9fc9db04555b425de047027ed5fec Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Wed, 13 Sep 2023 09:54:08 -0700 Subject: [PATCH 4/9] Add adapter tests for `get_intervals_between` macro --- .../utils/fixture_get_intervals_between.py | 15 +++++++++++++ .../utils/test_get_intervals_between.py | 21 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py create mode 100644 tests/adapter/dbt/tests/adapter/utils/test_get_intervals_between.py diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py b/tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py new file mode 100644 index 00000000000..167e360d779 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py @@ -0,0 +1,15 @@ +models__test_get_intervals_between_sql = """ +SELECT + {{ get_intervals_between("'09/01/2023'::date", "'09/12/2023'::date", "day") }} as intervals, + 11 as expected +""" + +models__test_get_intervals_between_yml = """ +version: 2 +models: + - name: test_get_intervals_between + tests: + - assert_equal: + actual: intervals + expected: expected +""" diff --git a/tests/adapter/dbt/tests/adapter/utils/test_get_intervals_between.py b/tests/adapter/dbt/tests/adapter/utils/test_get_intervals_between.py new file mode 100644 index 00000000000..588d2c538d7 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/test_get_intervals_between.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.utils.base_utils import BaseUtils +from dbt.tests.adapter.utils.fixture_get_intervals_between import ( + models__test_get_intervals_between_sql, + models__test_get_intervals_between_yml, +) + + +class BaseGetIntervalsBetween(BaseUtils): + @pytest.fixture(scope="class") + def models(self): + return { + "test_get_intervals_between.yml": models__test_get_intervals_between_yml, + "test_get_intervals_between.sql": self.interpolate_macro_namespace( + models__test_get_intervals_between_sql, "get_intervals_between" + ), + } + + +class TestGetIntervalsBetween(BaseGetIntervalsBetween): + pass From be7ffb3a6f45b573208d18e5013fbacfcd7356a4 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Wed, 13 Sep 2023 10:48:38 -0700 Subject: [PATCH 5/9] Add adapter tests for `date_spine` macro --- .../tests/adapter/utils/fixture_date_spine.py | 43 +++++++++++++++++++ .../tests/adapter/utils/test_date_spine.py | 21 +++++++++ 2 files changed, 64 insertions(+) create mode 100644 tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py create mode 100644 tests/adapter/dbt/tests/adapter/utils/test_date_spine.py diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py b/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py new file mode 100644 index 00000000000..75492aa3445 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py @@ -0,0 +1,43 @@ +# If date_spine works properly, there should be no `null` values in the resulting model + +models__test_date_spine_sql = """ +with generated_dates as ( + {{ date_spine("day", "'2023-09-01'::date", "'2023-09-10'::date") }} +), expected_dates as ( + select '2023-09-01'::date as expected + union all + select '2023-09-02'::date as expected + union all + select '2023-09-03'::date as expected + union all + select '2023-09-04'::date as expected + union all + select '2023-09-05'::date as expected + union all + select '2023-09-06'::date as expected + union all + select '2023-09-07'::date as expected + union all + select '2023-09-08'::date as expected + union all + select '2023-09-09'::date as expected +), joined as ( + select + generated_dates.date_day, + expected_dates.expected + from generated_dates + left join expected_dates on generated_dates.date_day = expected_dates.expected +) + +SELECT * from joined +""" + +models__test_date_spine_yml = """ +version: 2 +models: + - name: test_date_spine + tests: + - assert_equal: + actual: date_day + expected: expected +""" diff --git a/tests/adapter/dbt/tests/adapter/utils/test_date_spine.py b/tests/adapter/dbt/tests/adapter/utils/test_date_spine.py new file mode 100644 index 00000000000..0a5d7b7d29f --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/test_date_spine.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.utils.base_utils import BaseUtils +from dbt.tests.adapter.utils.fixture_date_spine import ( + models__test_date_spine_sql, + models__test_date_spine_yml, +) + + +class BaseDateSpine(BaseUtils): + @pytest.fixture(scope="class") + def models(self): + return { + "test_date_spine.yml": models__test_date_spine_yml, + "test_date_spine.sql": self.interpolate_macro_namespace( + models__test_date_spine_sql, "date_spine" + ), + } + + +class TestDateSpine(BaseDateSpine): + pass From ac54a12b5daf9972fc02a322f995b8fb7b9a60c1 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Thu, 14 Sep 2023 14:44:33 -0700 Subject: [PATCH 6/9] Improve test fixture for `date_spine` macro to work with multiple adapters --- .../tests/adapter/utils/fixture_date_spine.py | 85 +++++++++++++++---- 1 file changed, 67 insertions(+), 18 deletions(-) diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py b/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py index 75492aa3445..e8692d4290a 100644 --- a/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py @@ -2,25 +2,74 @@ models__test_date_spine_sql = """ with generated_dates as ( - {{ date_spine("day", "'2023-09-01'::date", "'2023-09-10'::date") }} + {% if target.type == 'postgres' %} + {{ date_spine("day", "'2023-09-01'::date", "'2023-09-10'::date") }} + + {% elif target.type == 'bigquery' %} + select cast(date_day as date) as date_day + from ({{ date_spine("day", "'2023-09-01'", "'2023-09-10'") }}) + + {% else %} + {{ date_spine("day", "'2023-09-01'", "'2023-09-10'") }} + {% endif %} ), expected_dates as ( - select '2023-09-01'::date as expected - union all - select '2023-09-02'::date as expected - union all - select '2023-09-03'::date as expected - union all - select '2023-09-04'::date as expected - union all - select '2023-09-05'::date as expected - union all - select '2023-09-06'::date as expected - union all - select '2023-09-07'::date as expected - union all - select '2023-09-08'::date as expected - union all - select '2023-09-09'::date as expected + {% if target.type == 'postgres' %} + select '2023-09-01'::date as expected + union all + select '2023-09-02'::date as expected + union all + select '2023-09-03'::date as expected + union all + select '2023-09-04'::date as expected + union all + select '2023-09-05'::date as expected + union all + select '2023-09-06'::date as expected + union all + select '2023-09-07'::date as expected + union all + select '2023-09-08'::date as expected + union all + select '2023-09-09'::date as expected + + {% elif target.type == 'bigquery' %} + select cast('2023-09-01' as date) as expected + union all + select cast('2023-09-02' as date) as expected + union all + select cast('2023-09-03' as date) as expected + union all + select cast('2023-09-04' as date) as expected + union all + select cast('2023-09-05' as date) as expected + union all + select cast('2023-09-06' as date) as expected + union all + select cast('2023-09-07' as date) as expected + union all + select cast('2023-09-08' as date) as expected + union all + select cast('2023-09-09' as date) as expected + + {% else %} + select '2023-09-01' as expected + union all + select '2023-09-02' as expected + union all + select '2023-09-03' as expected + union all + select '2023-09-04' as expected + union all + select '2023-09-05' as expected + union all + select '2023-09-06' as expected + union all + select '2023-09-07' as expected + union all + select '2023-09-08' as expected + union all + select '2023-09-09' as expected + {% endif %} ), joined as ( select generated_dates.date_day, From db1684cfcbb3e2fbe86809a3dff9c6638558d2f5 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Thu, 21 Sep 2023 15:37:56 -0700 Subject: [PATCH 7/9] Cast to types to date in fixture_date_spine when targeting redshift --- tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py b/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py index e8692d4290a..3e987aebb28 100644 --- a/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py @@ -5,7 +5,7 @@ {% if target.type == 'postgres' %} {{ date_spine("day", "'2023-09-01'::date", "'2023-09-10'::date") }} - {% elif target.type == 'bigquery' %} + {% elif target.type == 'bigquery' or target.type == 'redshift' %} select cast(date_day as date) as date_day from ({{ date_spine("day", "'2023-09-01'", "'2023-09-10'") }}) @@ -32,7 +32,7 @@ union all select '2023-09-09'::date as expected - {% elif target.type == 'bigquery' %} + {% elif target.type == 'bigquery' or target.type == 'redshift' %} select cast('2023-09-01' as date) as expected union all select cast('2023-09-02' as date) as expected From afb2e48eed2544894d9fd5a6db091173cb415d4e Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Fri, 22 Sep 2023 10:39:44 -0700 Subject: [PATCH 8/9] Improve test fixture for `get_intervals_between` macro to work with multiple adapters --- .../tests/adapter/utils/fixture_get_intervals_between.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py b/tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py index 167e360d779..bd1b0ddea4b 100644 --- a/tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py @@ -1,7 +1,12 @@ models__test_get_intervals_between_sql = """ SELECT - {{ get_intervals_between("'09/01/2023'::date", "'09/12/2023'::date", "day") }} as intervals, + {% if target.type == 'postgres' %} + {{ get_intervals_between("'09/01/2023'::date", "'09/12/2023'::date", "day") }} as intervals, + {% else %} + {{ get_intervals_between("'09/01/2023'", "'09/12/2023'", "day") }} as intervals, + {% endif %} 11 as expected + """ models__test_get_intervals_between_yml = """ From b1dbd34bd506a56ce20a7b441fa8ff8bf1743fe8 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Fri, 22 Sep 2023 11:26:03 -0700 Subject: [PATCH 9/9] changie doc for adding date_spine macro --- .changes/unreleased/Features-20230922-112531.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20230922-112531.yaml diff --git a/.changes/unreleased/Features-20230922-112531.yaml b/.changes/unreleased/Features-20230922-112531.yaml new file mode 100644 index 00000000000..99af15448f5 --- /dev/null +++ b/.changes/unreleased/Features-20230922-112531.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Adding `date_spine` macro (and supporting macros) from dbt-utils to dbt-core +time: 2023-09-22T11:25:31.383445-07:00 +custom: + Author: QMalcolm + Issue: "8172"