commit a bunch

macros/get_filtered_columns_in_relation.sql

@@ -0,0 +1,26 @@
+{% macro get_filtered_columns_in_relation(from, except=[]) -%}
+    {{ return(adapter.dispatch('get_filtered_columns_in_relation', 'dbt_utils')(from, except)) }}
+{% endmacro %}
+
+{% macro default__get_filtered_columns_in_relation(from, except=[]) -%}
+    {%- if not execute -%}
+        {{ return('') }}
+    {% endif %}
+    {%- do dbt_utils._is_relation(from, 'get_filtered_columns_in_relation') -%}
+    {%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%}
+
+    {# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #}
+
+
+    {%- set include_cols = [] %}
+    {%- set cols = adapter.get_columns_in_relation(from) -%}
+    {%- set except = except | map("lower") | list %}
+    {%- for col in cols -%}
+        {%- if col.column|lower not in except -%}
+            {% do include_cols.append(col.column) %}
+        {%- endif %}
+    {%- endfor %}
+
+    {{ return(include_cols) }}
+
+{%- endmacro %}

models/_samples/snapshot/example_join_snapshots.sql

@@ -46,4 +46,4 @@ final as (
     from joined
 )
 
-select * from final
+select * from final order by order_id

models/audit_helper_play.sql

@@ -0,0 +1,17 @@
+{% set dbt_relation=ref('snapshot_stg_payments') %}
+
+{% if execute %}
+    {%- set old_etl_relation = adapter.get_relation(
+        database="RAW",
+        schema="STRIPE",
+        identifier="PAYMENT") -%}
+with audit_results as (
+    {{ audit_helper.compare_relations(
+        a_relation=old_etl_relation,
+        b_relation=dbt_relation,
+        primary_key="orderid",
+        summarize = false
+    ) }} )
+{% endif %}
+--if there are any rows where in_a and in_b are not the same, there are mismatching rows between db objects
+select * from audit_results where in_a != in_b

models/check_payment_columns.sql

@@ -0,0 +1,4 @@
+SELECT column_name
+FROM raw.information_schema.columns
+WHERE table_schema = 'STRIPE'
+  AND table_name = 'PAYMENT'

models/marts/finance/fct_payments.sql

@@ -0,0 +1,6 @@
+select
+    *,
+    null as test_col,
+    10 as constant
+from    
+    {{ ref('stg_stripe__payments') }}

models/staging/stripe/stg_stripe__payments.sql

@@ -1,5 +1,6 @@
 -- example showing staging model after snapshot
 
+
 select
     -- ids
     id as payment_id,

models/staging/stripe/stg_stripe_payments_non_snapshot.sql

@@ -0,0 +1 @@
+select * from {{ source('stripe', 'payment') }}

models/staging/stripe/stg_stripe_payments_non_snapshot.yml

@@ -0,0 +1,25 @@
+models:
+  - name: stg_stripe_payments_non_snapshot
+    description: This model extracts all payment data from the Stripe source without applying any snapshot logic, providing a real-time view of payment transactions.
+    data_tests:
+      - audit_compare_relations:
+          dbt_model: ref('stg_stripe_payments_non_snapshot')
+          database: 'RAW'  # Replace with the target database name
+          schema: 'STRIPE'  # Replace with the target schema name
+          identifier: 'PAYMENT'  # Replace with the target table name
+          primary_key: 'order_id'
+    columns:
+      - name: id
+        description: A unique identifier for each payment transaction.
+      - name: amount
+        description: The total amount of the payment transaction.
+      - name: currency
+        description: The currency in which the payment was made.
+      - name: created
+        description: The timestamp when the payment was created.
+      - name: status
+        description: The current status of the payment transaction, such as succeeded or failed.
+      - name: customer_id
+        description: A unique identifier for the customer associated with the payment.
+      - name: payment_method
+        description: The method used for the payment, such as credit card or bank transfer.

package-lock.yml

@@ -1,16 +1,16 @@
 packages:
-- package: dbt-labs/codegen
-  version: 0.9.0
-- package: dbt-labs/dbt_utils
-  version: 1.1.0
-- package: dbt-labs/audit_helper
-  version: 0.9.0
-- package: brooklyn-data/dbt_artifacts
-  version: 2.6.2
-- package: dbt-labs/dbt_project_evaluator
-  version: 0.6.0
-- package: calogica/dbt_expectations
-  version: 0.8.5
-- package: calogica/dbt_date
-  version: 0.7.2
-sha1_hash: a17b7bdcdb4efda257369bdfd548f5104eb383a3
+  - package: dbt-labs/codegen
+    version: 0.9.0
+  - package: dbt-labs/dbt_utils
+    version: 1.1.0
+  - package: dbt-labs/audit_helper
+    version: 0.12.0
+  - package: brooklyn-data/dbt_artifacts
+    version: 2.6.2
+  - package: dbt-labs/dbt_project_evaluator
+    version: 0.6.0
+  - package: calogica/dbt_expectations
+    version: 0.8.5
+  - package: calogica/dbt_date
+    version: 0.7.2
+sha1_hash: 63b5c93e38778479feac12cf895f51e16d1454e6

packages.yml

@@ -4,7 +4,7 @@ packages:
   - package: dbt-labs/dbt_utils
     version: 1.1.0
   - package: dbt-labs/audit_helper
-    version: 0.9.0
+    version: 0.12.0
   - package: brooklyn-data/dbt_artifacts
     version: 2.6.2
   - package: dbt-labs/dbt_project_evaluator

tests/generic/audit_compare_relations.sql

@@ -0,0 +1,23 @@
+
+
+{% test audit_compare_relations(dbt_model, database=none, schema=none, identifier=none, primary_key=none) %}
+
+{% set dbt_relation= dbt_model %}
+
+{% if execute %}
+    {%- set old_etl_relation = adapter.get_relation(
+        database= database,
+        schema= schema,
+        identifier= identifier ) -%}
+with audit_results as (
+    {{ audit_helper.compare_relations(
+        a_relation=old_etl_relation,
+        b_relation=dbt_relation,
+        primary_key= primary_key ,
+        summarize = false
+    ) }} )
+{% endif %}
+--if there are any rows where in_a and in_b are not the same, there are mismatching rows between db objects
+select * from audit_results where in_a != in_b
+
+{% endtest %}