From 1db32281a6e3e94cc423eb0c61d80b3edd4135c7 Mon Sep 17 00:00:00 2001 From: Austin Clooney Date: Mon, 1 Jul 2024 16:48:16 -0400 Subject: [PATCH] Update BQ deduplicate macro to support partition pruning downstream. --- macros/sql/deduplicate.sql | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/macros/sql/deduplicate.sql b/macros/sql/deduplicate.sql index 3e75579c..eaed0daf 100644 --- a/macros/sql/deduplicate.sql +++ b/macros/sql/deduplicate.sql @@ -93,12 +93,19 @@ -- clause in BigQuery: -- https://github.com/dbt-labs/dbt-utils/issues/335#issuecomment-788157572 #} -{%- macro bigquery__deduplicate(relation, partition_by, order_by) -%} +{%- macro bigquery__deduplicate(relation, partition_by, order_by, partition_by_pass_thru=false) -%} select unique.* + {%- if partition_by_pass_thru %} + except({{ partition_by }}), + {{ partition_by }} + {%- endif %} from ( select - array_agg ( + {%- if partition_by_pass_thru %} + {{ partition_by }}, + {%- endif %} + array_agg( original order by {{ order_by }} limit 1