From aa86fdfe71690e76873facba89c49c3284398855 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Mon, 25 Sep 2023 12:20:05 -0700 Subject: [PATCH] Add date spine macros to core (#8616) * Add `date_spine` macro (and macros it depends on) from dbt-utils to core The macros added are - date_spine - get_intervals_between - generate_series - get_powers_of_two We're adding these to core because they are becoming more prevalently used with the increase usage in the semantic layer. Basically if you are using the semantic layer currently, then it is almost a requirement to use dbt-utils, which is undesireable given the SL is supported directly in core. The primary focus of this was to just add `date_spine`. However, because `date_spine` depends on other macros, these other macros were also moved. * Add adapter tests for `get_powers_of_two` macro * Add adapter tests for `generate_series` macro * Add adapter tests for `get_intervals_between` macro * Add adapter tests for `date_spine` macro * Improve test fixture for `date_spine` macro to work with multiple adapters * Cast to types to date in fixture_date_spine when targeting redshift * Improve test fixture for `get_intervals_between` macro to work with multiple adapters * changie doc for adding date_spine macro --- .../unreleased/Features-20230922-112531.yaml | 6 ++ .../macros/utils/date_spine.sql | 75 +++++++++++++++ .../macros/utils/generate_series.sql | 53 +++++++++++ .../tests/adapter/utils/fixture_date_spine.py | 92 +++++++++++++++++++ .../adapter/utils/fixture_generate_series.py | 45 +++++++++ .../utils/fixture_get_intervals_between.py | 20 ++++ .../utils/fixture_get_powers_of_two.py | 39 ++++++++ .../tests/adapter/utils/test_date_spine.py | 21 +++++ .../adapter/utils/test_generate_series.py | 21 +++++ .../utils/test_get_intervals_between.py | 21 +++++ .../adapter/utils/test_get_powers_of_two.py | 21 +++++ 11 files changed, 414 insertions(+) create mode 100644 .changes/unreleased/Features-20230922-112531.yaml create mode 100644 core/dbt/include/global_project/macros/utils/date_spine.sql create mode 100644 core/dbt/include/global_project/macros/utils/generate_series.sql create mode 100644 tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py create mode 100644 tests/adapter/dbt/tests/adapter/utils/fixture_generate_series.py create mode 100644 tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py create mode 100644 tests/adapter/dbt/tests/adapter/utils/fixture_get_powers_of_two.py create mode 100644 tests/adapter/dbt/tests/adapter/utils/test_date_spine.py create mode 100644 tests/adapter/dbt/tests/adapter/utils/test_generate_series.py create mode 100644 tests/adapter/dbt/tests/adapter/utils/test_get_intervals_between.py create mode 100644 tests/adapter/dbt/tests/adapter/utils/test_get_powers_of_two.py diff --git a/.changes/unreleased/Features-20230922-112531.yaml b/.changes/unreleased/Features-20230922-112531.yaml new file mode 100644 index 00000000000..99af15448f5 --- /dev/null +++ b/.changes/unreleased/Features-20230922-112531.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Adding `date_spine` macro (and supporting macros) from dbt-utils to dbt-core +time: 2023-09-22T11:25:31.383445-07:00 +custom: + Author: QMalcolm + Issue: "8172" diff --git a/core/dbt/include/global_project/macros/utils/date_spine.sql b/core/dbt/include/global_project/macros/utils/date_spine.sql new file mode 100644 index 00000000000..833fbcc575b --- /dev/null +++ b/core/dbt/include/global_project/macros/utils/date_spine.sql @@ -0,0 +1,75 @@ +{% macro get_intervals_between(start_date, end_date, datepart) -%} + {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }} +{%- endmacro %} + +{% macro default__get_intervals_between(start_date, end_date, datepart) -%} + {%- call statement('get_intervals_between', fetch_result=True) %} + + select {{ dbt.datediff(start_date, end_date, datepart) }} + + {%- endcall -%} + + {%- set value_list = load_result('get_intervals_between') -%} + + {%- if value_list and value_list['data'] -%} + {%- set values = value_list['data'] | map(attribute=0) | list %} + {{ return(values[0]) }} + {%- else -%} + {{ return(1) }} + {%- endif -%} + +{%- endmacro %} + + + + +{% macro date_spine(datepart, start_date, end_date) %} + {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }} +{%- endmacro %} + +{% macro default__date_spine(datepart, start_date, end_date) %} + + + {# call as follows: + + date_spine( + "day", + "to_date('01/01/2016', 'mm/dd/yyyy')", + "dbt.dateadd(week, 1, current_date)" + ) #} + + + with rawdata as ( + + {{dbt.generate_series( + dbt.get_intervals_between(start_date, end_date, datepart) + )}} + + ), + + all_periods as ( + + select ( + {{ + dbt.dateadd( + datepart, + "row_number() over (order by 1) - 1", + start_date + ) + }} + ) as date_{{datepart}} + from rawdata + + ), + + filtered as ( + + select * + from all_periods + where date_{{datepart}} <= {{ end_date }} + + ) + + select * from filtered + +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/utils/generate_series.sql b/core/dbt/include/global_project/macros/utils/generate_series.sql new file mode 100644 index 00000000000..f6a09605af3 --- /dev/null +++ b/core/dbt/include/global_project/macros/utils/generate_series.sql @@ -0,0 +1,53 @@ +{% macro get_powers_of_two(upper_bound) %} + {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }} +{% endmacro %} + +{% macro default__get_powers_of_two(upper_bound) %} + + {% if upper_bound <= 0 %} + {{ exceptions.raise_compiler_error("upper bound must be positive") }} + {% endif %} + + {% for _ in range(1, 100) %} + {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %} + {% endfor %} + +{% endmacro %} + + +{% macro generate_series(upper_bound) %} + {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }} +{% endmacro %} + +{% macro default__generate_series(upper_bound) %} + + {% set n = dbt.get_powers_of_two(upper_bound) %} + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + {% for i in range(n) %} + p{{i}}.generated_number * power(2, {{i}}) + {% if not loop.last %} + {% endif %} + {% endfor %} + + 1 + as generated_number + + from + + {% for i in range(n) %} + p as p{{i}} + {% if not loop.last %} cross join {% endif %} + {% endfor %} + + ) + + select * + from unioned + where generated_number <= {{upper_bound}} + order by generated_number + +{% endmacro %} diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py b/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py new file mode 100644 index 00000000000..3e987aebb28 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py @@ -0,0 +1,92 @@ +# If date_spine works properly, there should be no `null` values in the resulting model + +models__test_date_spine_sql = """ +with generated_dates as ( + {% if target.type == 'postgres' %} + {{ date_spine("day", "'2023-09-01'::date", "'2023-09-10'::date") }} + + {% elif target.type == 'bigquery' or target.type == 'redshift' %} + select cast(date_day as date) as date_day + from ({{ date_spine("day", "'2023-09-01'", "'2023-09-10'") }}) + + {% else %} + {{ date_spine("day", "'2023-09-01'", "'2023-09-10'") }} + {% endif %} +), expected_dates as ( + {% if target.type == 'postgres' %} + select '2023-09-01'::date as expected + union all + select '2023-09-02'::date as expected + union all + select '2023-09-03'::date as expected + union all + select '2023-09-04'::date as expected + union all + select '2023-09-05'::date as expected + union all + select '2023-09-06'::date as expected + union all + select '2023-09-07'::date as expected + union all + select '2023-09-08'::date as expected + union all + select '2023-09-09'::date as expected + + {% elif target.type == 'bigquery' or target.type == 'redshift' %} + select cast('2023-09-01' as date) as expected + union all + select cast('2023-09-02' as date) as expected + union all + select cast('2023-09-03' as date) as expected + union all + select cast('2023-09-04' as date) as expected + union all + select cast('2023-09-05' as date) as expected + union all + select cast('2023-09-06' as date) as expected + union all + select cast('2023-09-07' as date) as expected + union all + select cast('2023-09-08' as date) as expected + union all + select cast('2023-09-09' as date) as expected + + {% else %} + select '2023-09-01' as expected + union all + select '2023-09-02' as expected + union all + select '2023-09-03' as expected + union all + select '2023-09-04' as expected + union all + select '2023-09-05' as expected + union all + select '2023-09-06' as expected + union all + select '2023-09-07' as expected + union all + select '2023-09-08' as expected + union all + select '2023-09-09' as expected + {% endif %} +), joined as ( + select + generated_dates.date_day, + expected_dates.expected + from generated_dates + left join expected_dates on generated_dates.date_day = expected_dates.expected +) + +SELECT * from joined +""" + +models__test_date_spine_yml = """ +version: 2 +models: + - name: test_date_spine + tests: + - assert_equal: + actual: date_day + expected: expected +""" diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_generate_series.py b/tests/adapter/dbt/tests/adapter/utils/fixture_generate_series.py new file mode 100644 index 00000000000..0ce578df9b2 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_generate_series.py @@ -0,0 +1,45 @@ +# If generate_series works properly, there should be no `null` values in the resulting model + +models__test_generate_series_sql = """ +with generated_numbers as ( + {{ dbt.generate_series(10) }} +), expected_numbers as ( + select 1 as expected + union all + select 2 as expected + union all + select 3 as expected + union all + select 4 as expected + union all + select 5 as expected + union all + select 6 as expected + union all + select 7 as expected + union all + select 8 as expected + union all + select 9 as expected + union all + select 10 as expected +), joined as ( + select + generated_numbers.generated_number, + expected_numbers.expected + from generated_numbers + left join expected_numbers on generated_numbers.generated_number = expected_numbers.expected +) + +SELECT * from joined +""" + +models__test_generate_series_yml = """ +version: 2 +models: + - name: test_generate_series + tests: + - assert_equal: + actual: generated_number + expected: expected +""" diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py b/tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py new file mode 100644 index 00000000000..bd1b0ddea4b --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py @@ -0,0 +1,20 @@ +models__test_get_intervals_between_sql = """ +SELECT + {% if target.type == 'postgres' %} + {{ get_intervals_between("'09/01/2023'::date", "'09/12/2023'::date", "day") }} as intervals, + {% else %} + {{ get_intervals_between("'09/01/2023'", "'09/12/2023'", "day") }} as intervals, + {% endif %} + 11 as expected + +""" + +models__test_get_intervals_between_yml = """ +version: 2 +models: + - name: test_get_intervals_between + tests: + - assert_equal: + actual: intervals + expected: expected +""" diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_get_powers_of_two.py b/tests/adapter/dbt/tests/adapter/utils/fixture_get_powers_of_two.py new file mode 100644 index 00000000000..04ace2062f6 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_get_powers_of_two.py @@ -0,0 +1,39 @@ +# get_powers_of_two + +models__test_get_powers_of_two_sql = """ +select {{ get_powers_of_two(1) }} as actual, 1 as expected + +union all + +select {{ get_powers_of_two(4) }} as actual, 2 as expected + +union all + +select {{ get_powers_of_two(27) }} as actual, 5 as expected + +union all + +select {{ get_powers_of_two(256) }} as actual, 8 as expected + +union all + +select {{ get_powers_of_two(3125) }} as actual, 12 as expected + +union all + +select {{ get_powers_of_two(46656) }} as actual, 16 as expected + +union all + +select {{ get_powers_of_two(823543) }} as actual, 20 as expected +""" + +models__test_get_powers_of_two_yml = """ +version: 2 +models: + - name: test_powers_of_two + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/adapter/dbt/tests/adapter/utils/test_date_spine.py b/tests/adapter/dbt/tests/adapter/utils/test_date_spine.py new file mode 100644 index 00000000000..0a5d7b7d29f --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/test_date_spine.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.utils.base_utils import BaseUtils +from dbt.tests.adapter.utils.fixture_date_spine import ( + models__test_date_spine_sql, + models__test_date_spine_yml, +) + + +class BaseDateSpine(BaseUtils): + @pytest.fixture(scope="class") + def models(self): + return { + "test_date_spine.yml": models__test_date_spine_yml, + "test_date_spine.sql": self.interpolate_macro_namespace( + models__test_date_spine_sql, "date_spine" + ), + } + + +class TestDateSpine(BaseDateSpine): + pass diff --git a/tests/adapter/dbt/tests/adapter/utils/test_generate_series.py b/tests/adapter/dbt/tests/adapter/utils/test_generate_series.py new file mode 100644 index 00000000000..afc8d77dd3b --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/test_generate_series.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.utils.base_utils import BaseUtils +from dbt.tests.adapter.utils.fixture_generate_series import ( + models__test_generate_series_sql, + models__test_generate_series_yml, +) + + +class BaseGenerateSeries(BaseUtils): + @pytest.fixture(scope="class") + def models(self): + return { + "test_generate_series.yml": models__test_generate_series_yml, + "test_generate_series.sql": self.interpolate_macro_namespace( + models__test_generate_series_sql, "generate_series" + ), + } + + +class TestGenerateSeries(BaseGenerateSeries): + pass diff --git a/tests/adapter/dbt/tests/adapter/utils/test_get_intervals_between.py b/tests/adapter/dbt/tests/adapter/utils/test_get_intervals_between.py new file mode 100644 index 00000000000..588d2c538d7 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/test_get_intervals_between.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.utils.base_utils import BaseUtils +from dbt.tests.adapter.utils.fixture_get_intervals_between import ( + models__test_get_intervals_between_sql, + models__test_get_intervals_between_yml, +) + + +class BaseGetIntervalsBetween(BaseUtils): + @pytest.fixture(scope="class") + def models(self): + return { + "test_get_intervals_between.yml": models__test_get_intervals_between_yml, + "test_get_intervals_between.sql": self.interpolate_macro_namespace( + models__test_get_intervals_between_sql, "get_intervals_between" + ), + } + + +class TestGetIntervalsBetween(BaseGetIntervalsBetween): + pass diff --git a/tests/adapter/dbt/tests/adapter/utils/test_get_powers_of_two.py b/tests/adapter/dbt/tests/adapter/utils/test_get_powers_of_two.py new file mode 100644 index 00000000000..aa6f4d1a196 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/test_get_powers_of_two.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.utils.base_utils import BaseUtils +from dbt.tests.adapter.utils.fixture_get_powers_of_two import ( + models__test_get_powers_of_two_sql, + models__test_get_powers_of_two_yml, +) + + +class BaseGetPowersOfTwo(BaseUtils): + @pytest.fixture(scope="class") + def models(self): + return { + "test_get_powers_of_two.yml": models__test_get_powers_of_two_yml, + "test_get_powers_of_two.sql": self.interpolate_macro_namespace( + models__test_get_powers_of_two_sql, "get_powers_of_two" + ), + } + + +class TestGetPowersOfTwo(BaseGetPowersOfTwo): + pass