fix delete in project_by_project_table

bqbooster · Jan 13, 2025 · 6469685 · 6469685
1 parent c189789
commit 6469685
Showing 1 changed file with 49 additions and 36 deletions.
diff --git a/macros/project_by_project_table.sql b/macros/project_by_project_table.sql
@@ -1,5 +1,8 @@
 {%- materialization project_by_project_table, adapter='bigquery' -%}
 
+-- If we have projects, process them one by one
+-- This materialization should only be used in project mode if it isn't the case please report a bug
+
 {% set target_relation = this %}
 {% set existing_relation = load_relation(this) %}
 {% set projects = project_list() %}
@@ -44,49 +47,59 @@
   {% endif %}
 {% endif %}
 
--- If we have projects, process them one by one
-{% if projects|length > 0 %}
-  {% set all_insert_sql = [] %}
-  {% for project in projects %}
-    {% set project_sql = sql | replace('`region-', '`' ~ project | trim ~ '`.`region-') %}
-    {#- Incremental case -#}
-    {% if existing_relation is not none %}
-    {#- with partitioned data special where condition #}
-      {% if partition_config is not none and max_partition_value is not none and max_partition_value | length > 0 %}
-        {% set where_condition = 'WHERE ' ~ partition_config.field ~ ' >= TIMESTAMP_TRUNC("' ~ max_partition_value ~ '", HOUR)' %}
-      {% endif %}
+{% set main_sql = [] %}
 
-      {#- with regular data where condition #}
-      {% set insert_sql %}
-        DELETE FROM {{ target_relation }}
-        {{ where_condition }};
+{#- Incremental case -#}
+  {% if existing_relation is not none %}
+    {#- with partitioned data special where condition #}
+    {% if partition_config is not none and max_partition_value is not none and max_partition_value | length > 0 %}
+      {% set where_condition = 'WHERE ' ~ partition_config.field ~ ' >= TIMESTAMP_TRUNC("' ~ max_partition_value ~ '", HOUR)' %}
+    {% else %}
+      {% set where_condition = 'WHERE TRUE' %}
+    {% endif %}
 
-        INSERT INTO {{ target_relation }}
-        {{ project_sql }}
-        {{ where_condition }}
-      {% endset %}
+    {#- Delete from statement when incremental case -#}
+    {#- That statement is common to all projects -#}
+    {% set delete_sql %}
+          DELETE FROM {{ target_relation }}
+          {{ where_condition }}
+    {% endset %}
+    {% do main_sql.append(delete_sql) %}
+{% endif %}
 
-    {% else %}
-      {#- "Full-refresh" case -#}
-      {% if partition_config is not none %}
-       {#- bigquery doesn't allow more than 4000 partitions per insert so if we have hourly tables it's ~ 166 days -#}
-      {% set project_sql = project_sql + ' WHERE ' ~ partition_config.field ~ ' >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 166 DAY)' %}
-      {% endif %}
+{% for project in projects %}
+  {% set project_sql = sql | replace('`region-', '`' ~ project | trim ~ '`.`region-') %}
+  {#- Incremental case -#}
+  {% if existing_relation is not none %}
+  {#- with partitioned data special where condition #}
 
-      {% set insert_sql %}
-        INSERT INTO {{ target_relation }}
-        {{ project_sql }}
-      {% endset %}
+    {#- with regular data where condition #}
+    {% set insert_sql %}
+      INSERT INTO {{ target_relation }}
+      {{ project_sql }}
+      {{ where_condition }}
+    {% endset %}
 
+  {% else %}
+    {#- "Full-refresh" case -#}
+    {% if partition_config is not none %}
+      {#- bigquery doesn't allow more than 4000 partitions per insert so if we have hourly tables it's ~ 166 days -#}
+    {% set project_sql = project_sql + ' WHERE ' ~ partition_config.field ~ ' >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 166 DAY)' %}
     {% endif %}
-    {% do all_insert_sql.append(insert_sql) %}
-  {% endfor %}
 
-  {% call statement('main') -%}
-    {{ all_insert_sql | join(';\n') }}
-  {%- endcall %}
-
-{% endif %}
+    {% set insert_sql %}
+      INSERT INTO {{ target_relation }}
+      {{ project_sql }}
+    {% endset %}
+
+  {% endif %}
+  {% do main_sql.append(insert_sql) %}
+{% endfor %}
+
+{% call statement('main') -%}
+  {{ main_sql | join(';\n') }}
+{%- endcall %}
+
 
 {{ run_hooks(post_hooks) }}
 {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}