diff --git a/macros/get_filtered_columns_in_relation.sql b/macros/get_filtered_columns_in_relation.sql new file mode 100644 index 00000000..a64622bb --- /dev/null +++ b/macros/get_filtered_columns_in_relation.sql @@ -0,0 +1,26 @@ +{% macro get_filtered_columns_in_relation(from, except=[]) -%} + {{ return(adapter.dispatch('get_filtered_columns_in_relation', 'dbt_utils')(from, except)) }} +{% endmacro %} + +{% macro default__get_filtered_columns_in_relation(from, except=[]) -%} + {%- if not execute -%} + {{ return('') }} + {% endif %} + {%- do dbt_utils._is_relation(from, 'get_filtered_columns_in_relation') -%} + {%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%} + + {# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + + + {%- set include_cols = [] %} + {%- set cols = adapter.get_columns_in_relation(from) -%} + {%- set except = except | map("lower") | list %} + {%- for col in cols -%} + {%- if col.column|lower not in except -%} + {% do include_cols.append(col.column) %} + {%- endif %} + {%- endfor %} + + {{ return(include_cols) }} + +{%- endmacro %} \ No newline at end of file diff --git a/models/_samples/snapshot/example_join_snapshots.sql b/models/_samples/snapshot/example_join_snapshots.sql index 201d1c73..8ec428c8 100644 --- a/models/_samples/snapshot/example_join_snapshots.sql +++ b/models/_samples/snapshot/example_join_snapshots.sql @@ -46,4 +46,4 @@ final as ( from joined ) -select * from final +select * from final order by order_id diff --git a/models/audit_helper_play.sql b/models/audit_helper_play.sql new file mode 100644 index 00000000..c5602816 --- /dev/null +++ b/models/audit_helper_play.sql @@ -0,0 +1,17 @@ +{% set dbt_relation=ref('snapshot_stg_payments') %} + +{% if execute %} + {%- set old_etl_relation = adapter.get_relation( + database="RAW", + schema="STRIPE", + identifier="PAYMENT") -%} +with audit_results as ( + {{ audit_helper.compare_relations( + a_relation=old_etl_relation, + b_relation=dbt_relation, + primary_key="orderid", + summarize = false + ) }} ) +{% endif %} +--if there are any rows where in_a and in_b are not the same, there are mismatching rows between db objects +select * from audit_results where in_a != in_b \ No newline at end of file diff --git a/models/check_payment_columns.sql b/models/check_payment_columns.sql new file mode 100644 index 00000000..2989bb25 --- /dev/null +++ b/models/check_payment_columns.sql @@ -0,0 +1,4 @@ +SELECT column_name +FROM raw.information_schema.columns +WHERE table_schema = 'STRIPE' + AND table_name = 'PAYMENT' diff --git a/models/marts/finance/fct_payments.sql b/models/marts/finance/fct_payments.sql new file mode 100644 index 00000000..b93daf73 --- /dev/null +++ b/models/marts/finance/fct_payments.sql @@ -0,0 +1,6 @@ +select + *, + null as test_col, + 10 as constant +from + {{ ref('stg_stripe__payments') }} \ No newline at end of file diff --git a/models/marts/operations/_operations__models.yml b/models/marts/operations/_operations__models.yml index 3c8f1766..0b7f2e85 100644 --- a/models/marts/operations/_operations__models.yml +++ b/models/marts/operations/_operations__models.yml @@ -17,6 +17,8 @@ models: - not_null - name: manufacturer description: manufacturer of the part + data_tests: + - not_null - name: name description: name of the part - name: brand diff --git a/models/marts/operations/dim_parts.sql b/models/marts/operations/dim_parts.sql index 61b56458..679ab86d 100644 --- a/models/marts/operations/dim_parts.sql +++ b/models/marts/operations/dim_parts.sql @@ -13,7 +13,8 @@ final as ( type, size, container, - retail_price + retail_price, + retail_price + 30 as inflation_price from part ) diff --git a/models/staging/stripe/stg_stripe__payments.sql b/models/staging/stripe/stg_stripe__payments.sql index 61ede9e5..d668f0d4 100644 --- a/models/staging/stripe/stg_stripe__payments.sql +++ b/models/staging/stripe/stg_stripe__payments.sql @@ -1,5 +1,6 @@ -- example showing staging model after snapshot + select -- ids id as payment_id, diff --git a/models/staging/stripe/stg_stripe_payments_non_snapshot.sql b/models/staging/stripe/stg_stripe_payments_non_snapshot.sql new file mode 100644 index 00000000..53ea1189 --- /dev/null +++ b/models/staging/stripe/stg_stripe_payments_non_snapshot.sql @@ -0,0 +1 @@ +select * from {{ source('stripe', 'payment') }} \ No newline at end of file diff --git a/models/staging/stripe/stg_stripe_payments_non_snapshot.yml b/models/staging/stripe/stg_stripe_payments_non_snapshot.yml new file mode 100644 index 00000000..268babd2 --- /dev/null +++ b/models/staging/stripe/stg_stripe_payments_non_snapshot.yml @@ -0,0 +1,25 @@ +models: + - name: stg_stripe_payments_non_snapshot + description: This model extracts all payment data from the Stripe source without applying any snapshot logic, providing a real-time view of payment transactions. + data_tests: + - audit_compare_relations: + dbt_model: ref('stg_stripe_payments_non_snapshot') + database: 'RAW' # Replace with the target database name + schema: 'STRIPE' # Replace with the target schema name + identifier: 'PAYMENT' # Replace with the target table name + primary_key: 'order_id' + columns: + - name: id + description: A unique identifier for each payment transaction. + - name: amount + description: The total amount of the payment transaction. + - name: currency + description: The currency in which the payment was made. + - name: created + description: The timestamp when the payment was created. + - name: status + description: The current status of the payment transaction, such as succeeded or failed. + - name: customer_id + description: A unique identifier for the customer associated with the payment. + - name: payment_method + description: The method used for the payment, such as credit card or bank transfer. \ No newline at end of file diff --git a/package-lock.yml b/package-lock.yml index 891b671a..ebb103a7 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -1,16 +1,16 @@ packages: -- package: dbt-labs/codegen - version: 0.9.0 -- package: dbt-labs/dbt_utils - version: 1.1.0 -- package: dbt-labs/audit_helper - version: 0.9.0 -- package: brooklyn-data/dbt_artifacts - version: 2.6.2 -- package: dbt-labs/dbt_project_evaluator - version: 0.6.0 -- package: calogica/dbt_expectations - version: 0.8.5 -- package: calogica/dbt_date - version: 0.7.2 -sha1_hash: a17b7bdcdb4efda257369bdfd548f5104eb383a3 + - package: dbt-labs/codegen + version: 0.9.0 + - package: dbt-labs/dbt_utils + version: 1.1.0 + - package: dbt-labs/audit_helper + version: 0.12.0 + - package: brooklyn-data/dbt_artifacts + version: 2.6.2 + - package: dbt-labs/dbt_project_evaluator + version: 0.6.0 + - package: calogica/dbt_expectations + version: 0.8.5 + - package: calogica/dbt_date + version: 0.7.2 +sha1_hash: 63b5c93e38778479feac12cf895f51e16d1454e6 diff --git a/packages.yml b/packages.yml index ebd40712..c4ede965 100644 --- a/packages.yml +++ b/packages.yml @@ -4,7 +4,7 @@ packages: - package: dbt-labs/dbt_utils version: 1.1.0 - package: dbt-labs/audit_helper - version: 0.9.0 + version: 0.12.0 - package: brooklyn-data/dbt_artifacts version: 2.6.2 - package: dbt-labs/dbt_project_evaluator diff --git a/tests/generic/audit_compare_relations.sql b/tests/generic/audit_compare_relations.sql new file mode 100644 index 00000000..4ae1e032 --- /dev/null +++ b/tests/generic/audit_compare_relations.sql @@ -0,0 +1,23 @@ + + +{% test audit_compare_relations(dbt_model, database=none, schema=none, identifier=none, primary_key=none) %} + +{% set dbt_relation= dbt_model %} + +{% if execute %} + {%- set old_etl_relation = adapter.get_relation( + database= database, + schema= schema, + identifier= identifier ) -%} +with audit_results as ( + {{ audit_helper.compare_relations( + a_relation=old_etl_relation, + b_relation=dbt_relation, + primary_key= primary_key , + summarize = false + ) }} ) +{% endif %} +--if there are any rows where in_a and in_b are not the same, there are mismatching rows between db objects +select * from audit_results where in_a != in_b + +{% endtest %}