Add date spine macros to core

.changes/unreleased/Features-20230922-112531.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Adding `date_spine` macro (and supporting macros) from dbt-utils to dbt-core
+time: 2023-09-22T11:25:31.383445-07:00
+custom:
+  Author: QMalcolm
+  Issue: "8172"

core/dbt/include/global_project/macros/utils/date_spine.sql

@@ -0,0 +1,75 @@
+{% macro get_intervals_between(start_date, end_date, datepart) -%}
+    {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }}
+{%- endmacro %}
+
+{% macro default__get_intervals_between(start_date, end_date, datepart) -%}
+    {%- call statement('get_intervals_between', fetch_result=True) %}
+
+        select {{ dbt.datediff(start_date, end_date, datepart) }}
+
+    {%- endcall -%}
+
+    {%- set value_list = load_result('get_intervals_between') -%}
+
+    {%- if value_list and value_list['data'] -%}
+        {%- set values = value_list['data'] | map(attribute=0) | list %}
+        {{ return(values[0]) }}
+    {%- else -%}
+        {{ return(1) }}
+    {%- endif -%}
+
+{%- endmacro %}
+
+
+
+
+{% macro date_spine(datepart, start_date, end_date) %}
+    {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }}
+{%- endmacro %}
+
+{% macro default__date_spine(datepart, start_date, end_date) %}
+
+
+    {# call as follows:
+
+    date_spine(
+        "day",
+        "to_date('01/01/2016', 'mm/dd/yyyy')",
+        "dbt.dateadd(week, 1, current_date)"
+    ) #}
+
+
+    with rawdata as (
+
+        {{dbt.generate_series(
+            dbt.get_intervals_between(start_date, end_date, datepart)
+        )}}
+
+    ),
+
+    all_periods as (
+
+        select (
+            {{
+                dbt.dateadd(
+                    datepart,
+                    "row_number() over (order by 1) - 1",
+                    start_date
+                )
+            }}
+        ) as date_{{datepart}}
+        from rawdata
+
+    ),
+
+    filtered as (
+
+        select *
+        from all_periods
+        where date_{{datepart}} <= {{ end_date }}
+
+    )
+
+    select * from filtered
+
+{% endmacro %}

core/dbt/include/global_project/macros/utils/generate_series.sql

@@ -0,0 +1,53 @@
+{% macro get_powers_of_two(upper_bound) %}
+    {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }}
+{% endmacro %}
+
+{% macro default__get_powers_of_two(upper_bound) %}
+
+    {% if upper_bound <= 0 %}
+    {{ exceptions.raise_compiler_error("upper bound must be positive") }}
+    {% endif %}
+
+    {% for _ in range(1, 100) %}
+       {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}
+    {% endfor %}
+
+{% endmacro %}
+
+
+{% macro generate_series(upper_bound) %}
+    {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }}
+{% endmacro %}
+
+{% macro default__generate_series(upper_bound) %}
+
+    {% set n = dbt.get_powers_of_two(upper_bound) %}
+
+    with p as (
+        select 0 as generated_number union all select 1
+    ), unioned as (
+
+    select
+
+    {% for i in range(n) %}
+    p{{i}}.generated_number * power(2, {{i}})
+    {% if not loop.last %} + {% endif %}
+    {% endfor %}
+    + 1
+    as generated_number
+
+    from
+
+    {% for i in range(n) %}
+    p as p{{i}}
+    {% if not loop.last %} cross join {% endif %}
+    {% endfor %}
+
+    )
+
+    select *
+    from unioned
+    where generated_number <= {{upper_bound}}
+    order by generated_number
+
+{% endmacro %}

tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py

@@ -0,0 +1,92 @@
+# If date_spine works properly, there should be no `null` values in the resulting model
+
+models__test_date_spine_sql = """
+with generated_dates as (
+    {% if target.type == 'postgres' %}
+        {{ date_spine("day", "'2023-09-01'::date", "'2023-09-10'::date") }}
+
+    {% elif target.type == 'bigquery' or target.type == 'redshift' %}
+        select cast(date_day as date) as date_day
+        from ({{ date_spine("day", "'2023-09-01'", "'2023-09-10'") }})
+
+    {% else %}
+        {{ date_spine("day", "'2023-09-01'", "'2023-09-10'") }}
+    {% endif %}
+), expected_dates as (
+    {% if target.type == 'postgres' %}
+        select '2023-09-01'::date as expected
+        union all
+        select '2023-09-02'::date as expected
+        union all
+        select '2023-09-03'::date as expected
+        union all
+        select '2023-09-04'::date as expected
+        union all
+        select '2023-09-05'::date as expected
+        union all
+        select '2023-09-06'::date as expected
+        union all
+        select '2023-09-07'::date as expected
+        union all
+        select '2023-09-08'::date as expected
+        union all
+        select '2023-09-09'::date as expected
+
+    {% elif target.type == 'bigquery' or target.type == 'redshift' %}
+        select cast('2023-09-01' as date) as expected
+        union all
+        select cast('2023-09-02' as date) as expected
+        union all
+        select cast('2023-09-03' as date) as expected
+        union all
+        select cast('2023-09-04' as date) as expected
+        union all
+        select cast('2023-09-05' as date) as expected
+        union all
+        select cast('2023-09-06' as date) as expected
+        union all
+        select cast('2023-09-07' as date) as expected
+        union all
+        select cast('2023-09-08' as date) as expected
+        union all
+        select cast('2023-09-09' as date) as expected
+
+    {% else %}
+        select '2023-09-01' as expected
+        union all
+        select '2023-09-02' as expected
+        union all
+        select '2023-09-03' as expected
+        union all
+        select '2023-09-04' as expected
+        union all
+        select '2023-09-05' as expected
+        union all
+        select '2023-09-06' as expected
+        union all
+        select '2023-09-07' as expected
+        union all
+        select '2023-09-08' as expected
+        union all
+        select '2023-09-09' as expected
+    {% endif %}
+), joined as (
+    select
+        generated_dates.date_day,
+        expected_dates.expected
+    from generated_dates
+    left join expected_dates on generated_dates.date_day = expected_dates.expected
+)
+
+SELECT * from joined
+"""
+
+models__test_date_spine_yml = """
+version: 2
+models:
+  - name: test_date_spine
+    tests:
+      - assert_equal:
+          actual: date_day
+          expected: expected
+"""

tests/adapter/dbt/tests/adapter/utils/fixture_generate_series.py

@@ -0,0 +1,45 @@
+# If generate_series works properly, there should be no `null` values in the resulting model
+
+models__test_generate_series_sql = """
+with generated_numbers as (
+    {{ dbt.generate_series(10) }}
+), expected_numbers as (
+    select 1 as expected
+    union all
+    select 2 as expected
+    union all
+    select 3 as expected
+    union all
+    select 4 as expected
+    union all
+    select 5 as expected
+    union all
+    select 6 as expected
+    union all
+    select 7 as expected
+    union all
+    select 8 as expected
+    union all
+    select 9 as expected
+    union all
+    select 10 as expected
+), joined as (
+    select
+        generated_numbers.generated_number,
+        expected_numbers.expected
+    from generated_numbers
+    left join expected_numbers on generated_numbers.generated_number = expected_numbers.expected
+)
+
+SELECT * from joined
+"""
+
+models__test_generate_series_yml = """
+version: 2
+models:
+  - name: test_generate_series
+    tests:
+      - assert_equal:
+          actual: generated_number
+          expected: expected
+"""

tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py

@@ -0,0 +1,20 @@
+models__test_get_intervals_between_sql = """
+SELECT
+  {% if target.type == 'postgres' %}
+    {{ get_intervals_between("'09/01/2023'::date", "'09/12/2023'::date", "day") }} as intervals,
+  {% else %}
+      {{ get_intervals_between("'09/01/2023'", "'09/12/2023'", "day") }} as intervals,
+  {% endif %}
+  11 as expected
+
+"""
+
+models__test_get_intervals_between_yml = """
+version: 2
+models:
+  - name: test_get_intervals_between
+    tests:
+      - assert_equal:
+          actual: intervals
+          expected: expected
+"""

tests/adapter/dbt/tests/adapter/utils/fixture_get_powers_of_two.py

@@ -0,0 +1,39 @@
+# get_powers_of_two
+
+models__test_get_powers_of_two_sql = """
+select {{ get_powers_of_two(1) }} as actual, 1 as expected
+
+union all
+
+select {{ get_powers_of_two(4) }} as actual, 2 as expected
+
+union all
+
+select {{ get_powers_of_two(27) }} as actual, 5 as expected
+
+union all
+
+select {{ get_powers_of_two(256) }} as actual, 8 as expected
+
+union all
+
+select {{ get_powers_of_two(3125) }} as actual, 12 as expected
+
+union all
+
+select {{ get_powers_of_two(46656) }} as actual, 16 as expected
+
+union all
+
+select {{ get_powers_of_two(823543) }} as actual, 20 as expected
+"""
+
+models__test_get_powers_of_two_yml = """
+version: 2
+models:
+  - name: test_powers_of_two
+    tests:
+      - assert_equal:
+          actual: actual
+          expected: expected
+"""

tests/adapter/dbt/tests/adapter/utils/test_date_spine.py

@@ -0,0 +1,21 @@
+import pytest
+from dbt.tests.adapter.utils.base_utils import BaseUtils
+from dbt.tests.adapter.utils.fixture_date_spine import (
+    models__test_date_spine_sql,
+    models__test_date_spine_yml,
+)
+
+
+class BaseDateSpine(BaseUtils):
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "test_date_spine.yml": models__test_date_spine_yml,
+            "test_date_spine.sql": self.interpolate_macro_namespace(
+                models__test_date_spine_sql, "date_spine"
+            ),
+        }
+
+
+class TestDateSpine(BaseDateSpine):
+    pass

tests/adapter/dbt/tests/adapter/utils/test_generate_series.py

@@ -0,0 +1,21 @@
+import pytest
+from dbt.tests.adapter.utils.base_utils import BaseUtils
+from dbt.tests.adapter.utils.fixture_generate_series import (
+    models__test_generate_series_sql,
+    models__test_generate_series_yml,
+)
+
+
+class BaseGenerateSeries(BaseUtils):
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "test_generate_series.yml": models__test_generate_series_yml,
+            "test_generate_series.sql": self.interpolate_macro_namespace(
+                models__test_generate_series_sql, "generate_series"
+            ),
+        }
+
+
+class TestGenerateSeries(BaseGenerateSeries):
+    pass