From 4f9380c162ba8f2aa27be044c08a7289c066e0d1 Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Mon, 26 Dec 2022 10:31:20 +0100 Subject: [PATCH 1/2] base is_incremental --- macros/dune/is_incremental.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 macros/dune/is_incremental.sql diff --git a/macros/dune/is_incremental.sql b/macros/dune/is_incremental.sql new file mode 100644 index 00000000000..8ef6541901a --- /dev/null +++ b/macros/dune/is_incremental.sql @@ -0,0 +1,12 @@ +{% macro is_incremental() %} + {#-- do not run introspective queries in parsing #} + {% if not execute %} + {{ return(False) }} + {% else %} + {% set relation = adapter.get_relation(this.database, this.schema, this.table) %} + {{ return(relation is not none + and relation.type == 'table' + and model.config.materialized == 'incremental' + and not should_full_refresh()) }} + {% endif %} +{% endmacro %} From b7a80b5e4c427adf803db3dd0647af0d0684c353 Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Mon, 26 Dec 2022 10:56:57 +0100 Subject: [PATCH 2/2] add force-incremental variable --- macros/dune/is_incremental.sql | 6 ++++++ macros/dune/macros_schema.yml | 10 ++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/macros/dune/is_incremental.sql b/macros/dune/is_incremental.sql index 8ef6541901a..39cd360ec05 100644 --- a/macros/dune/is_incremental.sql +++ b/macros/dune/is_incremental.sql @@ -1,4 +1,10 @@ {% macro is_incremental() %} + {#-- allow for command line compile of incremental models #} + {#-- Usage: dbt compile --vars '{force-incremental: True}' #} + {% if var('force-incremental', False) %} + {{ return(True) }} + {% endif %} + {#-- do not run introspective queries in parsing #} {% if not execute %} {{ return(False) }} diff --git a/macros/dune/macros_schema.yml b/macros/dune/macros_schema.yml index d3213da0d95..7870486a38a 100644 --- a/macros/dune/macros_schema.yml +++ b/macros/dune/macros_schema.yml @@ -9,7 +9,7 @@ macros: description: "Custom schema name" - name: node type: column name or expression - description: "Node" + description: "Node" - name: generate_alias_name description: "This overrides the dbt-core function for setting aliases. For PR test, the alias is set to the user schema and file name. " @@ -38,4 +38,10 @@ macros: - name: spark__create_csv_table description: > - Same as create_table_as but for seeds with spark package \ No newline at end of file + Same as create_table_as but for seeds with spark package + + - name: is_incremental + description: > + Overrides the core is_incremental marco to allow the usage of a force-incremental command line variable. + This enables us to easily get the compiled incremental models. + Example usage: dbt compile --vars '{force-incremental: True}'