From 4ddace857ee232acb9a15552831b543b66c5e353 Mon Sep 17 00:00:00 2001 From: Alex Higgs Date: Mon, 21 Jun 2021 14:19:13 +0100 Subject: [PATCH 1/2] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8da46c0e5..1667f2fe8 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@

- Documentation Status Date: Tue, 13 Jul 2021 15:23:02 +0000 Subject: [PATCH 2/2] Update to dbt 0.20.0 - Removed built-in docs for now. Re-factoring these. Update README --- README.md | 4 +- dbt_project.yml | 4 +- docs/internal.md | 151 ------------------ .../helpers/check_placeholder.md | 21 --- .../helpers/get_period_boundaries.md | 37 ----- .../helpers/get_period_filter_sql.md | 21 --- .../helpers/get_period_of_load.md | 12 -- .../helpers/get_start_stop_dates.md | 14 -- .../replace_placeholder_with_filter.md | 13 -- .../helpers/shared_definitions.md | 47 ------ docs/materialisations/materialisations.md | 14 -- docs/overview.md | 26 --- docs/platforms.md | 4 - docs/staging.md | 119 -------------- docs/supporting.md | 80 ---------- docs/tables/shared_definitions.md | 79 --------- docs/tables/tables.md | 66 -------- macros/internal/alias.sql | 2 +- macros/internal/alias_all.sql | 2 +- macros/internal/as_constant.sql | 2 +- macros/internal/internal_schema.yml | 55 ------- macros/internal/multikey.sql | 2 +- macros/materialisations/helpers_schema.yml | 107 ------------- .../helpers_snowflake_schema.yml | 96 ----------- .../materialisations_schema.yml | 5 - .../materialisations/period_mat_helpers.sql | 44 ++--- macros/materialisations/rank_mat_helpers.sql | 6 +- macros/staging/derive_columns.sql | 2 +- macros/staging/hash_columns.sql | 2 +- macros/staging/rank_columns.sql | 2 +- macros/staging/stage.sql | 10 +- macros/staging/staging_schema.yml | 35 ---- macros/staging/staging_snowflake_schema.yml | 45 ------ macros/supporting/hash.sql | 2 +- macros/supporting/prefix.sql | 6 +- macros/supporting/supporting_schema.yml | 29 ---- .../supporting_snowflake_schema.yml | 35 ---- macros/tables/eff_sat.sql | 14 +- macros/tables/hub.sql | 6 +- macros/tables/link.sql | 6 +- macros/tables/ma_sat.sql | 6 +- macros/tables/sat.sql | 6 +- macros/tables/t_link.sql | 6 +- macros/tables/tables_schema.yml | 121 -------------- macros/tables/tables_snowflake_schema.yml | 136 ---------------- packages.yml | 5 +- 46 files changed, 69 insertions(+), 1438 deletions(-) delete mode 100644 docs/internal.md delete mode 100644 docs/materialisations/helpers/check_placeholder.md delete mode 100644 docs/materialisations/helpers/get_period_boundaries.md delete mode 100644 docs/materialisations/helpers/get_period_filter_sql.md delete mode 100644 docs/materialisations/helpers/get_period_of_load.md delete mode 100644 docs/materialisations/helpers/get_start_stop_dates.md delete mode 100644 docs/materialisations/helpers/replace_placeholder_with_filter.md delete mode 100644 docs/materialisations/helpers/shared_definitions.md delete mode 100644 docs/materialisations/materialisations.md delete mode 100644 docs/overview.md delete mode 100644 docs/platforms.md delete mode 100644 docs/staging.md delete mode 100644 docs/supporting.md delete mode 100644 docs/tables/shared_definitions.md delete mode 100644 docs/tables/tables.md delete mode 100644 macros/internal/internal_schema.yml delete mode 100644 macros/materialisations/helpers_schema.yml delete mode 100644 macros/materialisations/helpers_snowflake_schema.yml delete mode 100644 macros/materialisations/materialisations_schema.yml delete mode 100644 macros/staging/staging_schema.yml delete mode 100644 macros/staging/staging_snowflake_schema.yml delete mode 100644 macros/supporting/supporting_schema.yml delete mode 100644 macros/supporting/supporting_snowflake_schema.yml delete mode 100644 macros/tables/tables_schema.yml delete mode 100644 macros/tables/tables_snowflake_schema.yml diff --git a/README.md b/README.md index 1667f2fe8..c2f81418c 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@

- Documentation Status =0.18.0", "<0.20.0"] +version: '0.7.6' +require-dbt-version: [">=0.18.0", "<0.21.0"] config-version: 2 source-paths: ["models"] diff --git a/docs/internal.md b/docs/internal.md deleted file mode 100644 index c3ea0f26a..000000000 --- a/docs/internal.md +++ /dev/null @@ -1,151 +0,0 @@ -{% docs macro__alias %} - -Perform aliasing on a mapping and optionally prefix the string as well. - -See also: -[alias_all](#!/macro/macro.dbtvault.alias_all) - -{% enddocs %} - - -{% docs arg__alias__alias_config %} - -A mapping, containing a configuration for the aliasing. - -| Key | Description | Type | -| ------------- | -------------------- | ------ | -| source_column | Column being aliased | string | -| alias | Column alias | string | - -{% enddocs %} - - -{% docs arg__alias__prefix %} - -A string to prefix the column with. - -{% enddocs %} - - - - -{% docs macro__alias_all %} - -Perform aliasing on a mapping and optionally prefix the string as well. - -{% enddocs %} - - -{% docs arg__alias_all__columns %} - -A list of columns, as strings or mappings. - -e.g. - -``` -src_hashdiff: - source_column: "CUSTOMER_HASHDIFF" - alias: "HASHDIFF" -``` - -{% enddocs %} - -{% docs arg__alias_all__prefix %} - -A string to prefix all columns with. - -{% enddocs %} - - - - -{% docs macro__as_constant %} - -Render a string as a constant value if it is prefixed with an exclamation mark (`!`) otherwise, return as provided. - -{% enddocs %} - - -{% docs arg__as_constant__column_str %} - -The string to parse as a constant. - -{% enddocs %} - - - - -{% docs macro__expand_column_list %} - -Flatten a nested list structure into a single list so that it may be rendered in CSV format or provided to other macros. - -{% enddocs %} - - -{% docs arg__expand_column_list__columns %} - -A list of lists to flatten. May contain strings as well, these will be added as single items in the returned list. - -{% enddocs %} - - - - -{% docs macro__multikey %} - -Apply the same conditions and comparisons to a list of columns/keys. - -e.g. Given the following argument values: - -columns = ['CUSTOMER_ID', 'NATION_ID'] -prefix = ['a', 'b'] -condition = '=' -operator = 'AND' - -The macro would render this as: - -``` -a.CUSTOMER_ID = b.CUSTOMER_ID -AND a.NATION_ID = b.NATION_ID -``` - -{% enddocs %} - - -{% docs arg__multikey__columns %} - -A list of columns to generate comparisons for. - -{% enddocs %} - - -{% docs arg__multikey__prefix %} - -A pair of prefixes, one for each side of the comparison. - -{% enddocs %} - - -{% docs arg__multikey__condition %} - -The comparison to make between the keys, should be one of: - -'<>', '!=', '=' - -{% enddocs %} - - -{% docs arg__multikey__operator %} - -The operator to join the conditions with, defaults to 'AND', but could also be 'OR'. - -{% enddocs %} - - - - -{% docs macro__prepend_generated_by %} - -A convenience macro to print a `-- Generated by dbtvault.` string. - -{% enddocs %} \ No newline at end of file diff --git a/docs/materialisations/helpers/check_placeholder.md b/docs/materialisations/helpers/check_placeholder.md deleted file mode 100644 index 5b794d90d..000000000 --- a/docs/materialisations/helpers/check_placeholder.md +++ /dev/null @@ -1,21 +0,0 @@ -{% docs macro__check_placeholder %} - -Searches the given SQL string for an expected placeholder, throwing an error if it is not found. - -{% enddocs %} - - -{% docs arg__check_placeholder__model_sql %} - -The SQL string to search. - -{% enddocs %} - - -{% docs arg__check_placeholder__placeholder %} - -Optional. Default: `__PERIOD_FILTER__` - -The placeholder to search for. - -{% enddocs %} \ No newline at end of file diff --git a/docs/materialisations/helpers/get_period_boundaries.md b/docs/materialisations/helpers/get_period_boundaries.md deleted file mode 100644 index 72135677e..000000000 --- a/docs/materialisations/helpers/get_period_boundaries.md +++ /dev/null @@ -1,37 +0,0 @@ -{% docs macro__get_period_boundaries %} - -Get the start and stop timestamp, as well as the number of periods/iterations which need to be made to do the full load. -It is important to note that this materialisation handles the idempotent nature of the materialisation by running a `COALESCE` -on the maximal date found in the target table if it already exists, and the provided `start_date`. - -This also allows the materialisation to handle aborted loads. - -{% enddocs %} - - -{% docs arg__get_period_boundaries__target_schema %} - -The schema that the target table is materialised in. - -{% enddocs %} - - -{% docs arg__get_period_boundaries__target_table %} - -The name of the materialised target table. - -{% enddocs %} - - -{% docs arg__get_period_boundaries__start_date %} - -The date stamp to start loading from. Must be in the format 'YYYY-MM-DD' - -{% enddocs %} - - -{% docs arg__get_period_boundaries__stop_date %} - -THe date stamp to stop loading on. Must be in the format 'YYYY-MM-DD' - -{% enddocs %} \ No newline at end of file diff --git a/docs/materialisations/helpers/get_period_filter_sql.md b/docs/materialisations/helpers/get_period_filter_sql.md deleted file mode 100644 index e6ca30657..000000000 --- a/docs/materialisations/helpers/get_period_filter_sql.md +++ /dev/null @@ -1,21 +0,0 @@ -{% docs macro__get_period_filter_sql %} - -A wrapper around the `replace_placeholder_with_period_filter` macro which creates a query designed to -build a temporary table, to select the necessary records for the given load cycle. - -{% enddocs %} - - -{% docs arg__get_period_filter_sql__target_cols_csv %} - -A CSV string of the columns to be created in the target table -(the table the model is creating with this materialisation) - -{% enddocs %} - - -{% docs arg__get_period_filter_sql__base_sql %} - -The SQL provided by the model, prior to any manipulation. - -{% enddocs %} \ No newline at end of file diff --git a/docs/materialisations/helpers/get_period_of_load.md b/docs/materialisations/helpers/get_period_of_load.md deleted file mode 100644 index 5ff70f161..000000000 --- a/docs/materialisations/helpers/get_period_of_load.md +++ /dev/null @@ -1,12 +0,0 @@ -{% docs macro__get_period_of_load %} - -A helper macro to fetch the date of the current load cycle. - -{% enddocs %} - - -{% docs arg__get_period_of_load__start_timestamp %} - -The `start_timestamp` of the load, derived from the `start_date`. - -{% enddocs %} \ No newline at end of file diff --git a/docs/materialisations/helpers/get_start_stop_dates.md b/docs/materialisations/helpers/get_start_stop_dates.md deleted file mode 100644 index 5d00d139e..000000000 --- a/docs/materialisations/helpers/get_start_stop_dates.md +++ /dev/null @@ -1,14 +0,0 @@ -{% docs macro__get_start_stop_dates %} - -A helper macro to fetch the start and stop dates to load with. It will either infer the date range from the min and max -dates present in the tables in `date_source_models` list, or alternatively use the `start_date` and `stop_date` -config options. The config options take precedence if both are provided. A suitable error is raised if neither is provided. - -{% enddocs %} - - -{% docs arg__get_start_stop_dates__date_source_models %} - -A list of models to union together and extract min and max dates from, which will be used as the range to load records with. - -{% enddocs %} \ No newline at end of file diff --git a/docs/materialisations/helpers/replace_placeholder_with_filter.md b/docs/materialisations/helpers/replace_placeholder_with_filter.md deleted file mode 100644 index f2261e8bc..000000000 --- a/docs/materialisations/helpers/replace_placeholder_with_filter.md +++ /dev/null @@ -1,13 +0,0 @@ -{% docs macro__replace_placeholder_with_period_filter %} - -Replace the `__PERIOD_FILTER__` string present in the given SQL, with a `WHERE` clause which filters data by a -specific `period` of time, `offset` from the `start_date`. - -{% enddocs %} - - -{% docs arg__replace_placeholder_with_period_filter__core_sql %} - -SQL string containing the `__PERIOD_FILTER__` string. - -{% enddocs %} \ No newline at end of file diff --git a/docs/materialisations/helpers/shared_definitions.md b/docs/materialisations/helpers/shared_definitions.md deleted file mode 100644 index 24e980364..000000000 --- a/docs/materialisations/helpers/shared_definitions.md +++ /dev/null @@ -1,47 +0,0 @@ -{% docs arg__period_materialisation__timestamp_field %} - -The field to reference and extract timestamps and dates from. - -This should be the same as the `src_ldts` attribute if using a table macro. - -{% enddocs %} - - -{% docs arg__period_materialisation__offset %} - -The period of time to offset the start of the load from. For example, if period is set to `day` and the offset is `1`, then -this will evaluate to `start + 1 day` - -{% enddocs %} - - -{% docs arg__period_materialisation__period %} - -The period of time to iterate through. The naming varies per platform, though some common examples are: - -- hour -- day -- month -- year - -See below for platform specific documentation. - -[Snowflake](https://docs.snowflake.com/en/sql-reference/functions-date-time.html#supported-date-and-time-parts) - -{% enddocs %} - - -{% docs arg__period_materialisation__start_timestamp %} - -The starting timestamp for the range of records to be loaded. -Records must have a timestamp greater or equal to this value to be included. - -{% enddocs %} - - -{% docs arg__period_materialisation__stop_timestamp %} - -The stopping timestamp for the range of records to be loaded. -Records must have a timestamp less than this value to be included. - -{% enddocs %} \ No newline at end of file diff --git a/docs/materialisations/materialisations.md b/docs/materialisations/materialisations.md deleted file mode 100644 index 3d81ec06b..000000000 --- a/docs/materialisations/materialisations.md +++ /dev/null @@ -1,14 +0,0 @@ -{% docs materialization__vault_insert_by_period__default %} - -A materialisation designed to iterate through source data and load it in discrete periods of time, configurable by the user. - -{% enddocs %} - - - - -{% docs macro__is_vault_insert_by_period %} - -Check that a model is using the `vault_insert_by_period` materialisation. - -{% enddocs %} \ No newline at end of file diff --git a/docs/overview.md b/docs/overview.md deleted file mode 100644 index 548addd40..000000000 --- a/docs/overview.md +++ /dev/null @@ -1,26 +0,0 @@ -{% docs __dbtvault__ %} - -

- -

- -[![Documentation Status](https://readthedocs.org/projects/dbtvault/badge/?version=stable)](https://dbtvault.readthedocs.io/en/latest/?badge=stable) - -[![Join our Slack](https://img.shields.io/badge/Slack-Join-yellow?style=flat&logo=slack)](https://join.slack.com/t/dbtvault/shared_invite/enQtODY5MTY3OTIyMzg2LWJlZDMyNzM4YzAzYjgzYTY0MTMzNTNjN2EyZDRjOTljYjY0NDYyYzEwMTlhODMzNGY3MmU2ODNhYWUxYmM2NjA) - -# dbtvault by [Datavault](https://www.data-vault.co.uk) - -Build your own Data Vault data warehouse! dbtvault is a free to use dbt package that generates & executes the ETL you need to run a Data Vault 2.0 Data Warehouse on a Snowflake database. - -What does dbtvault offer? -- productivity gains, fewer errors -- multi-threaded execution of the generated SQL -- your data modeller can generate most of the ETL code directly from their mapping metadata -- your ETL developers can focus on the 5% of the SQL code that is different -- dbt generates documentation and data flow diagrams - -powered by [dbt](https://www.getdbt.com/), a registered trademark of [Fishtown Analytics](https://www.fishtownanalytics.com/) - -[See the github repo for more details!](https://github.com/Datavault-UK/dbtvault/) - -{% enddocs %} \ No newline at end of file diff --git a/docs/platforms.md b/docs/platforms.md deleted file mode 100644 index 6a357885d..000000000 --- a/docs/platforms.md +++ /dev/null @@ -1,4 +0,0 @@ -{% docs platform__snowflake %} - -Snowflake implementation -{% enddocs %} \ No newline at end of file diff --git a/docs/staging.md b/docs/staging.md deleted file mode 100644 index 7db16fcf9..000000000 --- a/docs/staging.md +++ /dev/null @@ -1,119 +0,0 @@ -{% docs macro__stage %} - -A macro to aid in generating a staging layer for the raw vault. Allows users to: - -- Create new columns from already existing columns (Derived columns) -- Create new hashed columns from already existing columns and provided derived columns (Hashed columns) - -[Read more online](https://dbtvault.readthedocs.io/en/latest/macros/#stage) - -{% enddocs %} - - -{% docs arg__stage__include_source_columns %} - -True by default. If true, all columns included in the source model for the stage layer, will be propagated to the stage layer. - -If false, only derived and hash columns (if configured) will be present in the resulting stage layer. - -{% enddocs %} - - -{% docs arg__stage__source_model %} - -The dbt model name or source to build a staging layer from. Can be provided in the following formats: - -``` -[REF STYLE] -source_model: model_name -OR -[SOURCES STYLE] -source_model: - source_name: source_table_name" -``` - -{% enddocs %} - - -{% docs arg__stage__hashed_columns %} - -A mapping of hash key names to column names which should be hashed to create that key. - -e.g. - -``` -hashed_columns: - SUPPLIER_PK: 'SUPPLIERKEY' - SUPPLIER_NATION_PK: 'SUPPLIER_NATION_KEY' - SUPPLIER_REGION_PK: 'SUPPLIER_REGION_KEY' - REGION_PK: 'SUPPLIER_REGION_KEY' - NATION_PK: 'SUPPLIER_NATION_KEY' - NATION_REGION_PK: - - 'SUPPLIER_NATION_KEY' - - 'SUPPLIER_REGION_KEY' - LINK_SUPPLIER_NATION_PK: - - 'SUPPLIERKEY' - - 'SUPPLIER_NATION_KEY' - PART_PK: 'PARTKEY' - INVENTORY_PK: - - 'PARTKEY' - - 'SUPPLIERKEY' - INVENTORY_HASHDIFF: - is_hashdiff: true - columns: - - 'PARTKEY' - - 'SUPPLIERKEY' - - 'AVAILQTY' - - 'SUPPLYCOST' - - 'PART_SUPPLY_COMMENT' -``` - -{% enddocs %} - - -{% docs arg__stage__derived_columns %} - -A mapping of new column names to existing columns which should be hashed to create that key. - -e.g. - -``` -derived_columns: - NATION_KEY: 'SUPPLIER_NATION_KEY' - REGION_KEY: 'SUPPLIER_REGION_KEY' - SOURCE: '!TPCH-INVENTORY' -``` - -{% enddocs %} - - - - -{% docs macro__derive_columns %} - -A macro used by the `stage` macro internally, which processes a mapping of new columns to source columns, in order to generate new columns. - -See also: -[stage](#!/macro/macro.dbtvault.stage) -[Online docs](https://dbtvault.readthedocs.io/en/latest/macros/#derive_columns) - -{% enddocs %} - - -{% docs arg__derive_columns__source_relation %} - -The source relation to extract columns from, for deriving from. - -{% enddocs %} - - - -{% docs macro__hash_columns %} - -A macro used by the `stage` macro internally, which processes a mapping of hash key names to source columns, in order to generate hash keys. - -See also: -[stage](#!/macro/macro.dbtvault.stage) -[Online docs](https://dbtvault.readthedocs.io/en/latest/macros/#hash_columns) - -{% enddocs %} \ No newline at end of file diff --git a/docs/supporting.md b/docs/supporting.md deleted file mode 100644 index f2db10bdb..000000000 --- a/docs/supporting.md +++ /dev/null @@ -1,80 +0,0 @@ -{% docs macro__hash %} - -Generate SQL to hash one or more columns using MD5 or SHA256. - -See [How do we hash?](https://dbtvault.readthedocs.io/en/latest/best_practices/#how-do-we-hash) for an in-depth look at how dbtvault does hashing. - -[Read more online](https://dbtvault.readthedocs.io/en/latest/macros/#hash) - -{% enddocs %} - - -{% docs arg__hash__columns %} - -A list of one or more columns to hash. - -{% enddocs %} - - -{% docs arg__hash__alias %} - -The alias (name) for the new column output using the hash macro. - -{% enddocs %} - - -{% docs arg__hash__is_hashdiff %} - -Boolean flag. If true, sort the column names in alphabetical order prior to hashing. -This is required for hashdiffs to ensure consistent hashing. - -{% enddocs %} - - - - -{% docs macro__prefix %} - -Prefix one or more strings with a given string and print each one. - -[Read more online](https://dbtvault.readthedocs.io/en/latest/macros/#prefix) - -{% enddocs %} - - -{% docs arg__prefix__columns %} - -A list of columns (string or mapping) to prefix. - -If a column is specified using an alias mapping as follows: - -{'source_column': <'column name'>, 'alias': <'alias string'>} - -Then it will also be aliased using `AS `. - -{% enddocs %} - - -{% docs arg__prefix__prefix_str %} - -The string to prepend to each column/string. - -{% enddocs %} - - - -{% docs arg__prefix__alias_target %} - -Switch the aliasing target. `source` by default. - -If a column is specified using an alias mapping as follows: - -`{'source_column': <'column name'>, 'alias': <'alias string'>}` - -Then it will also be aliased using `AS `. - -However, if the `alias_target` is `target` instead of `source`, the column will be rendered as follows: - -`AS ` - -{% enddocs %} \ No newline at end of file diff --git a/docs/tables/shared_definitions.md b/docs/tables/shared_definitions.md deleted file mode 100644 index 0971a81b7..000000000 --- a/docs/tables/shared_definitions.md +++ /dev/null @@ -1,79 +0,0 @@ -{% docs arg__tables__src_pk %} - -The column used as the primary key of the table. This must be a hash key generated from a natural key in the staging layer. -In future versions of dbtvault, hashing will not be a requirement. - - -{% enddocs %} - - -{% docs arg__tables__src_nk %} - -The column used as the natural or business key of the table. This must be the non-hashed version of the column used for the `src_pk`. - -{% enddocs %} - - -{% docs arg__tables__src_fk %} - -A single column or list of columns which are the primary key columns of other, related tables. Used in links to shows hubs associated with the link. - -{% enddocs %} - - -{% docs arg__tables__src_hashdiff %} - -A column which contains a single hash, composed of a list of columns and alpha-sorted. A hashdiff is used as a kind of checksum, to detect changes in records. -If any of the columns which form the hashdiff change their value, then the hashdiff itself will change. This is used in satellites to detect changes in the payload. - - -{% enddocs %} - - -{% docs arg__tables__src_payload_sat %} - -A list or single list of columns which contains the payload of the satellite. -A satellite payload should contain the concrete attributes for entity descried in the corresponding hub record. - - -{% enddocs %} - - -{% docs arg__tables__src_payload__t_link %} - -A list or single list of columns which contains the payload of the t-link. -A t-link payload should contain the transactional/event attributes for entity descried in the corresponding hub record. - -{% enddocs %} - - -{% docs arg__tables__src_eff %} - -The effective from column for a record. This is the business-effective date of a record. - -- For a transactional link, this would be the time at which the transaction occurred. -- For a satellite, this would be the time at which we first saw this data in the system (i.e when the payload changed) in a given form. -- For an effectivity satellite, this is the time at which we first saw the link relationship in a given form. - -{% enddocs %} - - -{% docs arg__tables__src_ldts %} - -The load datetime stamp of the record. When this record appeared/was loaded into the database. - -{% enddocs %} - - -{% docs arg__tables__src_source %} - -The source for a given record. This can be a code which corresponds to a lookup table or simply a string with a named system. - -{% enddocs %} - - -{% docs arg__tables__source_model %} - -The name of the model which contains the data which needs to be loaded. This can be a list for Hubs and Links, which could have multiple sources. - -{% enddocs %} \ No newline at end of file diff --git a/docs/tables/tables.md b/docs/tables/tables.md deleted file mode 100644 index ecafcb402..000000000 --- a/docs/tables/tables.md +++ /dev/null @@ -1,66 +0,0 @@ -{% docs macro__hub %} - -A Hub contains a distinct set of keys for a given top-level business concept, for example a `HUB_CUSTOMER` hub may contain a distinct list -of Customer IDs. - -[Read more online](https://dbtvault.readthedocs.io/en/latest/macros/#hub) - -{% enddocs %} - - - - -{% docs macro__link %} - -A Link contains a distinct set of relationships between top-level business concepts. -These structures 'link' hubs together based on a business relationship between the two. - -[Read more online](https://dbtvault.readthedocs.io/en/latest/macros/#link) - -{% enddocs %} - - - - -{% docs macro__sat %} - -A Satellite contains records corresponding to Hub or Link records which provide concrete attributes for those records. For example a `SAT_CUSTOMER_DETAILS` Satellite -would contain details about the customer, by using the same primary key as the corresponding hub record. -The payload for this example may contain `CUSTOMER_DOB`, `CUSTOMER_GIVEN_NAME`, `CUSTOMER_SURNAME` columns. - -[Read more online](https://dbtvault.readthedocs.io/en/latest/macros/#sat) - -{% enddocs %} - - - - -{% docs macro__eff_sat %} - -An Effectivity Satellite keeps track of the effective dates of relationships contained in links. -If a relationship changes (for example, a Customer moves country, changing a customer and nation relation) -then an effectivity satellite will record this change as a new entry, and when it happened. - -When a new relationship is loaded from the source, a new record will be created with the new relation and an open end date (the max date, `9999-12-31`). -If auto end-dating is enabled and a relationship changes which is already recorded in the effectivity satellite, then effectivity satellites in dbtvault will -automatically create a record as a copy of the old record. This record will be created with the effective date of the new relation. - -If auto end-dating is not enabled, a new record with open end date will still be created, but additional business rules will need to be applied to work out the -end dates manually. This may be useful when there is external business logic which describes under what situations a relationship is considered effective or not. - -[Read more online](https://dbtvault.readthedocs.io/en/latest/macros/#eff_sat) - -{% enddocs %} - - - - -{% docs macro__t_link %} - -A transactional link is an immutable list of transaction records. By definition transactions are never modified: -if a transaction needs to be updated, then a new transaction occurs. Transactional links contain a payload of columns which contain -details about the transaction, usually consisting of payments, location, type and more. - -[Read more online](https://dbtvault.readthedocs.io/en/latest/macros/#t_link) - -{% enddocs %} \ No newline at end of file diff --git a/macros/internal/alias.sql b/macros/internal/alias.sql index 9a786cb13..95cd6a9fd 100644 --- a/macros/internal/alias.sql +++ b/macros/internal/alias.sql @@ -1,6 +1,6 @@ {%- macro alias(alias_config=none, prefix=none) -%} - {{- adapter.dispatch('alias', packages = dbtvault.get_dbtvault_namespaces())(alias_config=alias_config, prefix=prefix) -}} + {{- adapter.dispatch('alias', 'dbtvault')(alias_config=alias_config, prefix=prefix) -}} {%- endmacro %} diff --git a/macros/internal/alias_all.sql b/macros/internal/alias_all.sql index 987b02ea4..4db7eeea8 100644 --- a/macros/internal/alias_all.sql +++ b/macros/internal/alias_all.sql @@ -1,6 +1,6 @@ {%- macro alias_all(columns=none, prefix=none) -%} - {{- adapter.dispatch('alias_all', packages = dbtvault.get_dbtvault_namespaces())(columns=columns, prefix=prefix) -}} + {{- adapter.dispatch('alias_all', 'dbtvault')(columns=columns, prefix=prefix) -}} {%- endmacro %} diff --git a/macros/internal/as_constant.sql b/macros/internal/as_constant.sql index 69e0e7d62..6e7d3633f 100644 --- a/macros/internal/as_constant.sql +++ b/macros/internal/as_constant.sql @@ -1,6 +1,6 @@ {%- macro as_constant(column_str=none) -%} - {{- adapter.dispatch('as_constant', packages = dbtvault.get_dbtvault_namespaces())(column_str=column_str) -}} + {{- adapter.dispatch('as_constant', 'dbtvault')(column_str=column_str) -}} {%- endmacro %} diff --git a/macros/internal/internal_schema.yml b/macros/internal/internal_schema.yml deleted file mode 100644 index 788aeb314..000000000 --- a/macros/internal/internal_schema.yml +++ /dev/null @@ -1,55 +0,0 @@ -version: 2 - -macros: - - name: alias - description: '{{ doc("macro__alias") }}' - arguments: - - name: alias_config - type: Mapping - description: '{{ doc("arg__alias__alias_config") }}' - - name: prefix - type: String - description: '{{ doc("arg__alias__prefix") }}' - - - name: alias_all - description: '{{ doc("macro__alias_all") }}' - arguments: - - name: columns - type: string - description: '{{ doc("arg__alias_all__columns") }}' - - name: prefix - type: string - description: '{{ doc("arg__alias_all__prefix") }}' - - - name: as_constant - description: '{{ doc("macro__as_constant") }}' - arguments: - - name: column_str - type: string - description: '{{ doc("arg__as_constant__column_str") }}' - - - name: expand_column_list - description: '{{ doc("macro__expand_column_list") }}' - arguments: - - name: columns - type: list - description: '{{ doc("arg__expand_column_list__columns") }}' - - - name: multikey - description: '{{ doc("macro__multikey") }}' - arguments: - - name: columns - type: list - description: '{{ doc("arg__multikey__columns") }}' - - name: prefix - type: string/list - description: '{{ doc("arg__multikey__prefix") }}' - - name: condition - type: string - description: '{{ doc("arg__multikey__condition") }}' - - name: operator - type: string - description: '{{ doc("arg__multikey__operator") }}' - - - name: prepend_generated_by - description: '{{ doc("macro__prepend_generated_by") }}' \ No newline at end of file diff --git a/macros/internal/multikey.sql b/macros/internal/multikey.sql index f0506a907..52ca23f4d 100644 --- a/macros/internal/multikey.sql +++ b/macros/internal/multikey.sql @@ -1,6 +1,6 @@ {%- macro multikey(columns, prefix=none, condition=none, operator='AND') -%} - {{- adapter.dispatch('multikey', packages = dbtvault.get_dbtvault_namespaces())(columns=columns, prefix=prefix, condition=condition, operator=operator) -}} + {{- adapter.dispatch('multikey', 'dbtvault')(columns=columns, prefix=prefix, condition=condition, operator=operator) -}} {%- endmacro %} diff --git a/macros/materialisations/helpers_schema.yml b/macros/materialisations/helpers_schema.yml deleted file mode 100644 index 3dee27854..000000000 --- a/macros/materialisations/helpers_schema.yml +++ /dev/null @@ -1,107 +0,0 @@ -version: 2 - -macros: - - name: replace_placeholder_with_period_filter - description: '{{ doc("macro__replace_placeholder_with_period_filter") }}' - arguments: - - name: core_sql - type: string - description: '{{ doc("arg__replace_placeholder_with_period_filter__core_sql") }}' - - name: timestamp_field - type: string - description: '{{ doc("arg__period_materialisation__timestamp_field") }}' - - name: start_timestamp - type: string - description: '{{ doc("arg__period_materialisation__start_timestamp") }}' - - name: stop_timestamp - type: string - description: '{{ doc("arg__period_materialisation__stop_timestamp") }}' - - name: offset - type: integer - description: '{{ doc("arg__period_materialisation__offset") }}' - - name: period - type: string - description: '{{ doc("arg__period_materialisation__period") }}' - - - name: get_period_filter_sql - description: '{{ doc("macro__get_period_filter_sql") }}' - arguments: - - name: target_cols_csv - type: string - description: '{{ doc("arg__get_period_filter_sql__target_cols_csv") }}' - - name: base_sql - type: string - description: '{{ doc("arg__get_period_filter_sql__base_sql") }}' - - name: timestamp_field - type: string - description: '{{ doc("arg__period_materialisation__timestamp_field") }}' - - name: period - type: string - description: '{{ doc("arg__period_materialisation__period") }}' - - name: start_timestamp - type: string - description: '{{ doc("arg__period_materialisation__start_timestamp") }}' - - name: stop_timestamp - type: string - description: '{{ doc("arg__period_materialisation__stop_timestamp") }}' - - name: offset - type: string - description: '{{ doc("arg__period_materialisation__offset") }}' - - - name: get_period_boundaries - description: '{{ doc("macro__get_period_boundaries") }}' - arguments: - - name: target_schema - type: string - description: '{{ doc("arg__get_period_boundaries__target_schema") }}' - - name: target_table - type: string - description: '{{ doc("arg__get_period_boundaries__target_table") }}' - - name: timestamp_field - type: string - description: '{{ doc("arg__period_materialisation__timestamp_field") }}' - - name: start_date - type: string - description: '{{ doc("arg__get_period_boundaries__start_date") }}' - - name: stop_date - type: string - description: '{{ doc("arg__get_period_boundaries__stop_date") }}' - - name: period - type: string - description: '{{ doc("arg__period_materialisation__period") }}' - - - name: get_period_of_load - description: '{{ doc("macro__get_period_of_load") }}' - arguments: - - name: period - type: string - description: '{{ doc("arg__period_materialisation__period") }}' - - name: offset - type: string - description: '{{ doc("arg__period_materialisation__offset") }}' - - name: start_timestamp - type: string - description: '{{ doc("arg__get_period_of_load__start_timestamp") }}' - - - name: is_vault_insert_by_period - description: '{{ doc("macro__is_vault_insert_by_period") }}' - - - name: check_placeholder - description: '{{ doc("macro__check_placeholder") }}' - arguments: - - name: model_sql - type: string - description: '{{ doc("arg__check_placeholder__model_sql") }}' - - name: placeholder - type: string - description: '{{ doc("arg__check_placeholder__placeholder") }}' - - - name: get_start_stop_dates - description: '{{ doc("macro__get_start_stop_dates") }}' - arguments: - - name: timestamp_field - type: string - description: '{{ doc("arg__period_materialisation__timestamp_field") }}' - - name: date_source_models - type: string/list - description: '{{ doc("arg__get_start_stop_dates__date_source_models") }}' \ No newline at end of file diff --git a/macros/materialisations/helpers_snowflake_schema.yml b/macros/materialisations/helpers_snowflake_schema.yml deleted file mode 100644 index 099849275..000000000 --- a/macros/materialisations/helpers_snowflake_schema.yml +++ /dev/null @@ -1,96 +0,0 @@ -version: 2 - -macros: - - name: default__replace_placeholder_with_period_filter - description: | - {{ doc("macro__replace_placeholder_with_period_filter") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: core_sql - type: string - description: '{{ doc("arg__replace_placeholder_with_period_filter__core_sql") }}' - - name: timestamp_field - type: string - description: '{{ doc("arg__period_materialisation__timestamp_field") }}' - - name: start_timestamp - type: string - description: '{{ doc("arg__period_materialisation__start_timestamp") }}' - - name: stop_timestamp - type: string - description: '{{ doc("arg__period_materialisation__stop_timestamp") }}' - - name: offset - type: integer - description: '{{ doc("arg__period_materialisation__offset") }}' - - name: period - type: string - description: '{{ doc("arg__period_materialisation__period") }}' - - - name: default__get_period_filter_sql - description: | - {{ doc("macro__get_period_filter_sql") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: target_cols_csv - type: string - description: '{{ doc("arg__get_period_filter_sql__target_cols_csv") }}' - - name: base_sql - type: string - description: '{{ doc("arg__get_period_filter_sql__base_sql") }}' - - name: timestamp_field - type: string - description: '{{ doc("arg__period_materialisation__timestamp_field") }}' - - name: period - type: string - description: '{{ doc("arg__period_materialisation__period") }}' - - name: start_timestamp - type: string - description: '{{ doc("arg__period_materialisation__start_timestamp") }}' - - name: stop_timestamp - type: string - description: '{{ doc("arg__period_materialisation__stop_timestamp") }}' - - name: offset - type: string - description: '{{ doc("arg__period_materialisation__offset") }}' - - - name: default__get_period_boundaries - description: | - {{ doc("macro__get_period_boundaries") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: target_schema - type: string - description: '{{ doc("arg__get_period_boundaries__target_schema") }}' - - name: target_table - type: string - description: '{{ doc("arg__get_period_boundaries__target_table") }}' - - name: timestamp_field - type: string - description: '{{ doc("arg__period_materialisation__timestamp_field") }}' - - name: start_date - type: string - description: '{{ doc("arg__get_period_boundaries__start_date") }}' - - name: stop_date - type: string - description: '{{ doc("arg__get_period_boundaries__stop_date") }}' - - name: period - type: string - description: '{{ doc("arg__period_materialisation__period") }}' - - - name: default__get_period_of_load - description: | - {{ doc("macro__get_period_of_load") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: period - type: string - description: '{{ doc("arg__period_materialisation__period") }}' - - name: offset - type: string - description: '{{ doc("arg__period_materialisation__offset") }}' - - name: start_timestamp - type: string - description: '{{ doc("arg__get_period_of_load__start_timestamp") }}' \ No newline at end of file diff --git a/macros/materialisations/materialisations_schema.yml b/macros/materialisations/materialisations_schema.yml deleted file mode 100644 index 67f8edf59..000000000 --- a/macros/materialisations/materialisations_schema.yml +++ /dev/null @@ -1,5 +0,0 @@ -version: 2 - -macros: - - name: materialization_vault_insert_by_period_default - description: '{{ doc("materialization__vault_insert_by_period__default") }}' \ No newline at end of file diff --git a/macros/materialisations/period_mat_helpers.sql b/macros/materialisations/period_mat_helpers.sql index e3532e7a3..20e8f80ac 100644 --- a/macros/materialisations/period_mat_helpers.sql +++ b/macros/materialisations/period_mat_helpers.sql @@ -7,12 +7,12 @@ {%- macro replace_placeholder_with_period_filter(core_sql, timestamp_field, start_timestamp, stop_timestamp, offset, period) -%} {% set macro = adapter.dispatch('replace_placeholder_with_period_filter', - packages = dbtvault.get_dbtvault_namespaces())(core_sql=core_sql, - timestamp_field=timestamp_field, - start_timestamp=start_timestamp, - stop_timestamp=stop_timestamp, - offset=offset, - period=period) %} + 'dbtvault')(core_sql=core_sql, + timestamp_field=timestamp_field, + start_timestamp=start_timestamp, + stop_timestamp=stop_timestamp, + offset=offset, + period=period) %} {% do return(macro) %} {%- endmacro %} @@ -35,13 +35,13 @@ {%- macro get_period_filter_sql(target_cols_csv, base_sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} {% set macro = adapter.dispatch('get_period_filter_sql', - packages = dbtvault.get_dbtvault_namespaces())(target_cols_csv=target_cols_csv, - base_sql=base_sql, - timestamp_field=timestamp_field, - period=period, - start_timestamp=start_timestamp, - stop_timestamp=stop_timestamp, - offset=offset) %} + 'dbtvault')(target_cols_csv=target_cols_csv, + base_sql=base_sql, + timestamp_field=timestamp_field, + period=period, + start_timestamp=start_timestamp, + stop_timestamp=stop_timestamp, + offset=offset) %} {% do return(macro) %} {%- endmacro %} @@ -63,12 +63,12 @@ {%- macro get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%} {% set macro = adapter.dispatch('get_period_boundaries', - packages = dbtvault.get_dbtvault_namespaces())(target_schema=target_schema, - target_table=target_table, - timestamp_field=timestamp_field, - start_date=start_date, - stop_date=stop_date, - period=period) %} + 'dbtvault')(target_schema=target_schema, + target_table=target_table, + timestamp_field=timestamp_field, + start_date=start_date, + stop_date=stop_date, + period=period) %} {% do return(macro) %} {%- endmacro %} @@ -107,9 +107,9 @@ {%- macro get_period_of_load(period, offset, start_timestamp) -%} {% set macro = adapter.dispatch('get_period_of_load', - packages = dbtvault.get_dbtvault_namespaces())(period=period, - offset=offset, - start_timestamp=start_timestamp) %} + 'dbtvault')(period=period, + offset=offset, + start_timestamp=start_timestamp) %} {% do return(macro) %} {%- endmacro %} diff --git a/macros/materialisations/rank_mat_helpers.sql b/macros/materialisations/rank_mat_helpers.sql index 78fd9b3ab..2febf0e51 100644 --- a/macros/materialisations/rank_mat_helpers.sql +++ b/macros/materialisations/rank_mat_helpers.sql @@ -7,9 +7,9 @@ {%- macro replace_placeholder_with_rank_filter(core_sql, rank_column, rank_iteration) -%} {% set macro = adapter.dispatch('replace_placeholder_with_rank_filter', - packages = dbtvault.get_dbtvault_namespaces())(core_sql=core_sql, - rank_column=rank_column, - rank_iteration=rank_iteration) %} + 'dbtvault')(core_sql=core_sql, + rank_column=rank_column, + rank_iteration=rank_iteration) %} {% do return(macro) %} {%- endmacro %} diff --git a/macros/staging/derive_columns.sql b/macros/staging/derive_columns.sql index 1a31e971a..62667042d 100644 --- a/macros/staging/derive_columns.sql +++ b/macros/staging/derive_columns.sql @@ -1,6 +1,6 @@ {%- macro derive_columns(source_relation=none, columns=none) -%} - {{- adapter.dispatch('derive_columns', packages = dbtvault.get_dbtvault_namespaces())(source_relation=source_relation, columns=columns) -}} + {{- adapter.dispatch('derive_columns', 'dbtvault')(source_relation=source_relation, columns=columns) -}} {%- endmacro %} diff --git a/macros/staging/hash_columns.sql b/macros/staging/hash_columns.sql index bfd4f3104..7ac75530e 100644 --- a/macros/staging/hash_columns.sql +++ b/macros/staging/hash_columns.sql @@ -1,6 +1,6 @@ {%- macro hash_columns(columns=none) -%} - {{- adapter.dispatch('hash_columns', packages = dbtvault.get_dbtvault_namespaces())(columns=columns) -}} + {{- adapter.dispatch('hash_columns', 'dbtvault')(columns=columns) -}} {%- endmacro %} diff --git a/macros/staging/rank_columns.sql b/macros/staging/rank_columns.sql index 0694daff4..772ed19e1 100644 --- a/macros/staging/rank_columns.sql +++ b/macros/staging/rank_columns.sql @@ -1,6 +1,6 @@ {%- macro rank_columns(columns=none) -%} - {{- adapter.dispatch('rank_columns', packages = dbtvault.get_dbtvault_namespaces())(columns=columns) -}} + {{- adapter.dispatch('rank_columns', 'dbtvault')(columns=columns) -}} {%- endmacro %} diff --git a/macros/staging/stage.sql b/macros/staging/stage.sql index 67ecfdda7..85085ef99 100644 --- a/macros/staging/stage.sql +++ b/macros/staging/stage.sql @@ -4,11 +4,11 @@ {%- set include_source_columns = true -%} {%- endif -%} - {{- adapter.dispatch('stage', packages = dbtvault.get_dbtvault_namespaces())(include_source_columns=include_source_columns, - source_model=source_model, - hashed_columns=hashed_columns, - derived_columns=derived_columns, - ranked_columns=ranked_columns) -}} + {{- adapter.dispatch('stage', 'dbtvault')(include_source_columns=include_source_columns, + source_model=source_model, + hashed_columns=hashed_columns, + derived_columns=derived_columns, + ranked_columns=ranked_columns) -}} {%- endmacro -%} {%- macro default__stage(include_source_columns, source_model, hashed_columns, derived_columns, ranked_columns) -%} diff --git a/macros/staging/staging_schema.yml b/macros/staging/staging_schema.yml deleted file mode 100644 index 0c3ebeae5..000000000 --- a/macros/staging/staging_schema.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: 2 - -macros: - - name: stage - description: '{{ doc("macro__stage") }}' - arguments: - - name: include_source_columns - type: boolean - description: '{{ doc("arg__stage__include_source_columns") }}' - - name: source_model - type: string - description: '{{ doc("arg__stage__source_model") }}' - - name: hashed_columns - type: Mapping - description: '{{ doc("arg__stage__hashed_columns") }}' - - name: derived_columns - type: Mapping - description: '{{ doc("arg__stage__derived_columns") }}' - - - name: hash_columns - description: '{{ doc("macro__hash_columns") }}' - arguments: - - name: columns - type: list - description: '{{ doc("arg__stage__hashed_columns") }}' - - - name: derive_columns - description: '{{ doc("macro__derive_columns") }}' - arguments: - - name: source_relation - type: Relation - description: '{{ doc("arg__derive_columns__source_relation") }}' - - name: columns - type: list - description: '{{ doc("arg__stage__hashed_columns") }}' \ No newline at end of file diff --git a/macros/staging/staging_snowflake_schema.yml b/macros/staging/staging_snowflake_schema.yml deleted file mode 100644 index 7cac4792a..000000000 --- a/macros/staging/staging_snowflake_schema.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: 2 - -macros: - - name: default__stage - description: | - {{ doc("macro__stage") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: include_source_columns - type: boolean - description: '{{ doc("arg__stage__include_source_columns") }}' - - name: source_model - type: string - description: '{{ doc("arg__stage__source_model") }}' - - name: hashed_columns - type: Mapping - description: '{{ doc("arg__stage__hashed_columns") }}' - - name: derived_columns - type: Mapping - description: '{{ doc("arg__stage__derived_columns") }}' - - - name: default__hash_columns - description: | - {{ doc("macro__hash_columns") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: columns - type: list - description: '{{ doc("arg__stage__hashed_columns") }}' - - - name: default__derive_columns - description: | - {{ doc("macro__derive_columns") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: source_relation - type: Relation - description: '{{ doc("arg__derive_columns__source_relation") }}' - - name: columns - type: list - description: '{{ doc("arg__stage__hashed_columns") }}' - diff --git a/macros/supporting/hash.sql b/macros/supporting/hash.sql index ffc485e2f..ebd5c4164 100644 --- a/macros/supporting/hash.sql +++ b/macros/supporting/hash.sql @@ -4,7 +4,7 @@ {%- set is_hashdiff = false -%} {% endif %} - {{- adapter.dispatch('hash', packages = dbtvault.get_dbtvault_namespaces())(columns=columns, alias=alias, is_hashdiff=is_hashdiff) -}} + {{- adapter.dispatch('hash', 'dbtvault')(columns=columns, alias=alias, is_hashdiff=is_hashdiff) -}} {%- endmacro %} diff --git a/macros/supporting/prefix.sql b/macros/supporting/prefix.sql index d72bae973..9fa006fb8 100644 --- a/macros/supporting/prefix.sql +++ b/macros/supporting/prefix.sql @@ -1,8 +1,8 @@ {%- macro prefix(columns, prefix_str, alias_target) -%} - {{- adapter.dispatch('prefix', packages = dbtvault.get_dbtvault_namespaces())(columns=columns, - prefix_str=prefix_str, - alias_target=alias_target) -}} + {{- adapter.dispatch('prefix', 'dbtvault')(columns=columns, + prefix_str=prefix_str, + alias_target=alias_target) -}} {%- endmacro -%} diff --git a/macros/supporting/supporting_schema.yml b/macros/supporting/supporting_schema.yml deleted file mode 100644 index fa68f5dd6..000000000 --- a/macros/supporting/supporting_schema.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: 2 - -macros: - - name: hash - description: '{{ doc("macro__hash") }}' - arguments: - - name: columns - type: list - description: '{{ doc("arg__hash__columns") }}' - - name: alias - type: string - description: '{{ doc("arg__hash__alias") }}' - - name: is_hashdiff - type: boolean - description: '{{ doc("arg__hash__is_hashdiff") }}' - - - name: prefix - description: '{{ doc("macro__prefix") }}' - arguments: - - name: columns - type: list - description: '{{ doc("arg__prefix__columns") }}' - - name: prefix_str - type: string - description: '{{ doc("arg__prefix__prefix_str") }}' - - name: alias_target - type: source | target - description: '{{ doc("arg__prefix__alias_target") }}' - \ No newline at end of file diff --git a/macros/supporting/supporting_snowflake_schema.yml b/macros/supporting/supporting_snowflake_schema.yml deleted file mode 100644 index d39c2b69f..000000000 --- a/macros/supporting/supporting_snowflake_schema.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: 2 - -macros: - - name: default__hash - description: | - {{ doc("macro__hash") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: columns - type: list - description: '{{ doc("arg__hash__columns") }}' - - name: alias - type: string - description: '{{ doc("arg__hash__alias") }}' - - name: is_hashdiff - type: boolean - description: '{{ doc("arg__hash__is_hashdiff") }}' - - - name: default__prefix - description: | - {{ doc("macro__prefix") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: columns - type: list - description: '{{ doc("arg__prefix__columns") }}' - - name: prefix_str - type: string - description: '{{ doc("arg__prefix__prefix_str") }}' - - name: alias_target - type: source | target - description: '{{ doc("arg__prefix__alias_target") }}' - \ No newline at end of file diff --git a/macros/tables/eff_sat.sql b/macros/tables/eff_sat.sql index be696b4b8..ce200159e 100644 --- a/macros/tables/eff_sat.sql +++ b/macros/tables/eff_sat.sql @@ -1,9 +1,9 @@ {%- macro eff_sat(src_pk, src_dfk, src_sfk, src_start_date, src_end_date, src_eff, src_ldts, src_source, source_model) -%} - {{- adapter.dispatch('eff_sat', packages = dbtvault.get_dbtvault_namespaces())(src_pk=src_pk, src_dfk=src_dfk, src_sfk=src_sfk, - src_start_date=src_start_date, src_end_date=src_end_date, - src_eff=src_eff, src_ldts=src_ldts, src_source=src_source, - source_model=source_model) -}} + {{- adapter.dispatch('eff_sat', 'dbtvault')(src_pk=src_pk, src_dfk=src_dfk, src_sfk=src_sfk, + src_start_date=src_start_date, src_end_date=src_end_date, + src_eff=src_eff, src_ldts=src_ldts, src_source=src_source, + source_model=source_model) -}} {%- endmacro -%} {%- macro default__eff_sat(src_pk, src_dfk, src_sfk, src_start_date, src_end_date, src_eff, src_ldts, src_source, source_model) -%} @@ -80,7 +80,7 @@ new_reopened_records AS ( g.{{ src_ldts }}, g.{{ src_source }} FROM source_data AS g - INNER JOIN latest_closed lc + INNER JOIN latest_closed AS lc ON g.{{ src_pk }} = lc.{{ src_pk }} ), @@ -103,7 +103,7 @@ new_closed_records AS ( WHERE ({{ dbtvault.multikey(src_sfk, prefix=['lo', 'h'], condition='<>', operator='OR') }}) ), -{#- if is_auto_end_dating -#} +{#- end if is_auto_end_dating -#} {%- endif %} records_to_insert AS ( @@ -123,7 +123,7 @@ records_to_insert AS ( FROM source_data AS i ) -{#- if not dbtvault.is_any_incremental() -#} +{#- end if not dbtvault.is_any_incremental() -#} {%- endif %} SELECT * FROM records_to_insert diff --git a/macros/tables/hub.sql b/macros/tables/hub.sql index b708cb0ac..18308e5b1 100644 --- a/macros/tables/hub.sql +++ b/macros/tables/hub.sql @@ -1,8 +1,8 @@ {%- macro hub(src_pk, src_nk, src_ldts, src_source, source_model) -%} - {{- adapter.dispatch('hub', packages = dbtvault.get_dbtvault_namespaces())(src_pk=src_pk, src_nk=src_nk, - src_ldts=src_ldts, src_source=src_source, - source_model=source_model) -}} + {{- adapter.dispatch('hub', 'dbtvault')(src_pk=src_pk, src_nk=src_nk, + src_ldts=src_ldts, src_source=src_source, + source_model=source_model) -}} {%- endmacro -%} diff --git a/macros/tables/link.sql b/macros/tables/link.sql index bb7891326..a2f987027 100644 --- a/macros/tables/link.sql +++ b/macros/tables/link.sql @@ -1,8 +1,8 @@ {%- macro link(src_pk, src_fk, src_ldts, src_source, source_model) -%} - {{- adapter.dispatch('link', packages = dbtvault.get_dbtvault_namespaces())(src_pk=src_pk, src_fk=src_fk, - src_ldts=src_ldts, src_source=src_source, - source_model=source_model) -}} + {{- adapter.dispatch('link', 'dbtvault')(src_pk=src_pk, src_fk=src_fk, + src_ldts=src_ldts, src_source=src_source, + source_model=source_model) -}} {%- endmacro -%} diff --git a/macros/tables/ma_sat.sql b/macros/tables/ma_sat.sql index a96eb40f4..38fee7691 100644 --- a/macros/tables/ma_sat.sql +++ b/macros/tables/ma_sat.sql @@ -1,8 +1,8 @@ {%- macro ma_sat(src_pk, src_cdk, src_hashdiff, src_payload, src_eff, src_ldts, src_source, source_model) -%} - {{- adapter.dispatch('ma_sat', packages = dbtvault.get_dbtvault_namespaces())(src_pk=src_pk, src_cdk=src_cdk, src_hashdiff=src_hashdiff, - src_payload=src_payload, src_eff=src_eff, src_ldts=src_ldts, - src_source=src_source, source_model=source_model) -}} + {{- adapter.dispatch('ma_sat', 'dbtvault')(src_pk=src_pk, src_cdk=src_cdk, src_hashdiff=src_hashdiff, + src_payload=src_payload, src_eff=src_eff, src_ldts=src_ldts, + src_source=src_source, source_model=source_model) -}} {%- endmacro %} diff --git a/macros/tables/sat.sql b/macros/tables/sat.sql index ca5c09e22..f4249cc6c 100644 --- a/macros/tables/sat.sql +++ b/macros/tables/sat.sql @@ -1,8 +1,8 @@ {%- macro sat(src_pk, src_hashdiff, src_payload, src_eff, src_ldts, src_source, source_model) -%} - {{- adapter.dispatch('sat', packages = dbtvault.get_dbtvault_namespaces())(src_pk=src_pk, src_hashdiff=src_hashdiff, - src_payload=src_payload, src_eff=src_eff, src_ldts=src_ldts, - src_source=src_source, source_model=source_model) -}} + {{- adapter.dispatch('sat', 'dbtvault')(src_pk=src_pk, src_hashdiff=src_hashdiff, + src_payload=src_payload, src_eff=src_eff, src_ldts=src_ldts, + src_source=src_source, source_model=source_model) -}} {%- endmacro %} diff --git a/macros/tables/t_link.sql b/macros/tables/t_link.sql index 80a20765c..109194e73 100644 --- a/macros/tables/t_link.sql +++ b/macros/tables/t_link.sql @@ -1,8 +1,8 @@ {%- macro t_link(src_pk, src_fk, src_payload, src_eff, src_ldts, src_source, source_model) -%} - {{- adapter.dispatch('t_link', packages = dbtvault.get_dbtvault_namespaces())(src_pk=src_pk, src_fk=src_fk, src_payload=src_payload, - src_eff=src_eff, src_ldts=src_ldts, src_source=src_source, - source_model=source_model) -}} + {{- adapter.dispatch('t_link', 'dbtvault')(src_pk=src_pk, src_fk=src_fk, src_payload=src_payload, + src_eff=src_eff, src_ldts=src_ldts, src_source=src_source, + source_model=source_model) -}} {%- endmacro %} diff --git a/macros/tables/tables_schema.yml b/macros/tables/tables_schema.yml deleted file mode 100644 index 36ff65daa..000000000 --- a/macros/tables/tables_schema.yml +++ /dev/null @@ -1,121 +0,0 @@ -version: 2 - -macros: - - name: hub - description: '{{ doc("macro__hub") }}' - arguments: - - name: src_pk - type: mapping/string - description: '{{ doc("arg__tables__src_pk") }}' - - name: src_nk - type: mapping/string - description: '{{ doc("arg__tables__src_nk") }}' - - name: src_ldts - type: string - description: '{{ doc("arg__tables__src_ldts") }}' - - name: src_source - type: string - description: '{{ doc("arg__tables__src_source") }}' - - name: source_model - type: string - description: '{{ doc("arg__tables__source_model") }}' - - - name: link - description: '{{ doc("macro__link") }}' - arguments: - - name: src_pk - type: mapping/string - description: '{{ doc("arg__tables__src_pk") }}' - - name: src_fk - type: mapping/string - description: '{{ doc("arg__tables__src_fk") }}' - - name: src_ldts - type: string - description: '{{ doc("arg__tables__src_ldts") }}' - - name: src_source - type: string - description: '{{ doc("arg__tables__src_source") }}' - - name: source_model - type: string - description: '{{ doc("arg__tables__source_model") }}' - - - name: sat - description: '{{ doc("macro__sat") }}' - arguments: - - name: src_pk - type: mapping/string - description: '{{ doc("arg__tables__src_pk") }}' - - name: src_hashdiff - type: string - description: '{{ doc("arg__tables__src_hashdiff") }}' - - name: src_payload - type: string - description: '{{ doc("arg__tables__src_payload_sat") }}' - - name: src_eff - type: string - description: '{{ doc("arg__tables__src_eff") }}' - - name: src_ldts - type: string - description: '{{ doc("arg__tables__src_ldts") }}' - - name: src_source - type: string - description: '{{ doc("arg__tables__src_source") }}' - - name: source_model - type: string - description: '{{ doc("arg__tables__source_model") }}' - - - name: t_link - description: '{{ doc("macro__t_link") }}' - arguments: - - name: src_pk - type: mapping/string - description: '{{ doc("arg__tables__src_pk") }}' - - name: src_fk - type: mapping/string - description: '{{ doc("arg__tables__src_fk") }}' - - name: src_payload - type: string - description: '{{ doc("arg__tables__src_payload__t_link") }}' - - name: src_eff - type: string - description: '{{ doc("arg__tables__src_eff") }}' - - name: src_ldts - type: string - description: '{{ doc("arg__tables__src_ldts") }}' - - name: src_source - type: string - description: '{{ doc("arg__tables__src_source") }}' - - name: source_model - type: string - description: '{{ doc("arg__tables__source_model") }}' - - - name: eff_sat - description: '{{ doc("macro__eff_sat") }}' - arguments: - - name: src_pk - type: mapping/string - description: '{{ doc("arg__tables__src_pk") }}' - - name: src_dfk - type: list/string - description: '' - - name: src_sfk - type: list/string - description: '' - - name: src_start_date - type: string - description: '' - - name: src_end_date - type: string - description: '' - - name: src_eff - type: string - description: '{{ doc("arg__tables__src_eff") }}' - - name: src_ldts - type: string - description: '{{ doc("arg__tables__src_ldts") }}' - - name: src_source - type: string - description: '{{ doc("arg__tables__src_source") }}' - - name: source_model - type: string - description: '{{ doc("arg__tables__source_model") }}' \ No newline at end of file diff --git a/macros/tables/tables_snowflake_schema.yml b/macros/tables/tables_snowflake_schema.yml deleted file mode 100644 index dbc860053..000000000 --- a/macros/tables/tables_snowflake_schema.yml +++ /dev/null @@ -1,136 +0,0 @@ -version: 2 - -macros: - - name: default__hub - description: | - {{ doc("macro__hub") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: src_pk - type: mapping/string - description: '{{ doc("arg__tables__src_pk") }}' - - name: src_nk - type: mapping/string - description: '{{ doc("arg__tables__src_nk") }}' - - name: src_ldts - type: string - description: '{{ doc("arg__tables__src_ldts") }}' - - name: src_source - type: string - description: '{{ doc("arg__tables__src_source") }}' - - name: source_model - type: string - description: '{{ doc("arg__tables__source_model") }}' - - - name: default__link - description: | - {{ doc("macro__link") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: src_pk - type: mapping/string - description: '{{ doc("arg__tables__src_pk") }}' - - name: src_fk - type: mapping/string - description: '{{ doc("arg__tables__src_fk") }}' - - name: src_ldts - type: string - description: '{{ doc("arg__tables__src_ldts") }}' - - name: src_source - type: string - description: '{{ doc("arg__tables__src_source") }}' - - name: source_model - type: string - description: '{{ doc("arg__tables__source_model") }}' - - - name: default__sat - description: | - {{ doc("macro__sat") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: src_pk - type: mapping/string - description: '{{ doc("arg__tables__src_pk") }}' - - name: src_hashdiff - type: string - description: '{{ doc("arg__tables__src_hashdiff") }}' - - name: src_payload - type: string - description: '{{ doc("arg__tables__src_payload_sat") }}' - - name: src_eff - type: string - description: '{{ doc("arg__tables__src_eff") }}' - - name: src_ldts - type: string - description: '{{ doc("arg__tables__src_ldts") }}' - - name: src_source - type: string - description: '{{ doc("arg__tables__src_source") }}' - - name: source_model - type: string - description: '{{ doc("arg__tables__source_model") }}' - - - name: default__t_link - description: | - {{ doc("macro__t_link") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: src_pk - type: mapping/string - description: '{{ doc("arg__tables__src_pk") }}' - - name: src_fk - type: mapping/string - description: '{{ doc("arg__tables__src_fk") }}' - - name: src_payload - type: string - description: '{{ doc("arg__tables__src_payload__t_link") }}' - - name: src_eff - type: string - description: '{{ doc("arg__tables__src_eff") }}' - - name: src_ldts - type: string - description: '{{ doc("arg__tables__src_ldts") }}' - - name: src_source - type: string - description: '{{ doc("arg__tables__src_source") }}' - - name: source_model - type: string - description: '{{ doc("arg__tables__source_model") }}' - - - name: default__eff_sat - description: | - {{ doc("macro__eff_sat") }} - - {{ doc("platform__snowflake") }} - arguments: - - name: src_pk - type: mapping/string - description: '{{ doc("arg__tables__src_pk") }}' - - name: src_dfk - type: list/string - description: '' - - name: src_sfk - type: list/string - description: '' - - name: src_start_date - type: string - description: '' - - name: src_end_date - type: string - description: '' - - name: src_eff - type: string - description: '{{ doc("arg__tables__src_eff") }}' - - name: src_ldts - type: string - description: '{{ doc("arg__tables__src_ldts") }}' - - name: src_source - type: string - description: '{{ doc("arg__tables__src_source") }}' - - name: source_model - type: string - description: '{{ doc("arg__tables__source_model") }}' \ No newline at end of file diff --git a/packages.yml b/packages.yml index 09a6842db..9b3a08c05 100644 --- a/packages.yml +++ b/packages.yml @@ -1,4 +1,3 @@ packages: - - - package: fishtown-analytics/dbt_utils - version: 0.6.4 \ No newline at end of file + - package: dbt-labs/dbt_utils + version: 0.7.0 \ No newline at end of file