From b8aac1ece1eb90995e961dff3b2888e2e9ac3b5a Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Mon, 16 Sep 2024 23:48:11 +0000 Subject: [PATCH 01/28] Anything with commit changes --- analysis/test_file.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 analysis/test_file.sql diff --git a/analysis/test_file.sql b/analysis/test_file.sql new file mode 100644 index 00000000..e69de29b From 673b5d12cd04b0b9b4f99af87d13b8e022aa87fd Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 18:48:24 +0000 Subject: [PATCH 02/28] refactoring snapshot_stg_payments and snapshot_stg_payments to match new snapshot strategy --- snapshots/jaffle_snapshots.yml | 24 ++++++++++++++++++++++++ snapshots/snapshot_int_line_items.sql | 22 ---------------------- snapshots/snapshot_stg_payments.sql | 16 ---------------- 3 files changed, 24 insertions(+), 38 deletions(-) create mode 100644 snapshots/jaffle_snapshots.yml delete mode 100644 snapshots/snapshot_int_line_items.sql delete mode 100644 snapshots/snapshot_stg_payments.sql diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml new file mode 100644 index 00000000..707111d5 --- /dev/null +++ b/snapshots/jaffle_snapshots.yml @@ -0,0 +1,24 @@ +version: 2 + +snapshots: + - name: snapshot_stg_payments + relation: source('stripe', 'payment') + config: + enabled: true + strategy: timestamp + updated_at: _BATCHED_AT + unique_key: ID + persist_docs: + relation: true + columns: true + - name: snapshot_int_line_items + relation: ref('int_line_items_amounts_calculated') + config: + enabled: true + strategy: check + check_cols: ['gross_item_sales_amount', 'net_item_sales_amount'] + unique_key: order_item_id + persist_docs: + relation: true + columns: true + diff --git a/snapshots/snapshot_int_line_items.sql b/snapshots/snapshot_int_line_items.sql deleted file mode 100644 index 90bc5502..00000000 --- a/snapshots/snapshot_int_line_items.sql +++ /dev/null @@ -1,22 +0,0 @@ -{% snapshot snapshot_int_line_items %} - --- Example showing check columns strategy and using custom schemas (this is rare) - --- if you don't want to write to the same schema in development and prod, --- use environment-aware variable from dbt_project.yml - --- if you do not have a reliable updated at timestamp, checek columns for changes. - -{{ - config( - target_database='analytics', - target_schema=var('example_target_snapshot_schema'), - unique_key='order_item_id', - strategy='check', - check_cols=['gross_item_sales_amount', 'net_item_sales_amount'], - ) -}} - -select * from {{ ref('int_line_items_amounts_calculated') }} - -{% endsnapshot %} \ No newline at end of file diff --git a/snapshots/snapshot_stg_payments.sql b/snapshots/snapshot_stg_payments.sql deleted file mode 100644 index adc61b2f..00000000 --- a/snapshots/snapshot_stg_payments.sql +++ /dev/null @@ -1,16 +0,0 @@ -{% snapshot snapshot_stg_payments %} - -{{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='ID', - - strategy='timestamp', - updated_at='_BATCHED_AT', - ) -}} - -select * from {{ source('stripe', 'payment') }} - -{% endsnapshot %} \ No newline at end of file From 910cccc4e6de35d25db9cf91797270ad64ad244a Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 18:49:41 +0000 Subject: [PATCH 03/28] deleting generate schema snapshot--i don't think this is relevant anymore given that snapshots now build to your dev schema then whatever your target is in prod. --- .../_samples/example_generate_schema_snapshot.sql | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 snapshots/_samples/example_generate_schema_snapshot.sql diff --git a/snapshots/_samples/example_generate_schema_snapshot.sql b/snapshots/_samples/example_generate_schema_snapshot.sql deleted file mode 100644 index 2174d7b7..00000000 --- a/snapshots/_samples/example_generate_schema_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_generate_schema_snapshot %} - {{ - config( - target_database='analytics', - target_schema=generate_schema_name('snapshots'), - unique_key='id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_line_items_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file From c11e92d4214ce783cee4d686f01d35efb8334553 Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 18:55:03 +0000 Subject: [PATCH 04/28] deleting old orders_line_items snapshot, replacing it with new config. also commenting out example join snapshot. --- .../_samples/snapshot/example_join_snapshots.sql | 3 ++- .../example_orders_line_items_snapshot.sql | 14 -------------- snapshots/jaffle_snapshots.yml | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 15 deletions(-) delete mode 100644 snapshots/_samples/example_orders_line_items_snapshot.sql diff --git a/models/_samples/snapshot/example_join_snapshots.sql b/models/_samples/snapshot/example_join_snapshots.sql index 201d1c73..9ea20d45 100644 --- a/models/_samples/snapshot/example_join_snapshots.sql +++ b/models/_samples/snapshot/example_join_snapshots.sql @@ -1,4 +1,4 @@ -with order_snapshot as ( +{# with order_snapshot as ( select * exclude dbt_valid_to, coalesce(dbt_valid_to, cast('{{ var("future_proof_date") }}' as timestamp)) as dbt_valid_to @@ -47,3 +47,4 @@ final as ( ) select * from final +#} \ No newline at end of file diff --git a/snapshots/_samples/example_orders_line_items_snapshot.sql b/snapshots/_samples/example_orders_line_items_snapshot.sql deleted file mode 100644 index 63603f50..00000000 --- a/snapshots/_samples/example_orders_line_items_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_orders_line_items_snapshot %} - {{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_line_items_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml index 707111d5..eebfaba7 100644 --- a/snapshots/jaffle_snapshots.yml +++ b/snapshots/jaffle_snapshots.yml @@ -1,6 +1,7 @@ version: 2 snapshots: +# This is a pretty standard snapshot - name: snapshot_stg_payments relation: source('stripe', 'payment') config: @@ -11,6 +12,7 @@ snapshots: persist_docs: relation: true columns: true +# This snapshot has the check_cols example - name: snapshot_int_line_items relation: ref('int_line_items_amounts_calculated') config: @@ -21,4 +23,16 @@ snapshots: persist_docs: relation: true columns: true +# This snapshot has invalidate hard deletes as an example + - name: snapshot_orders_line_items + relation: ref('example_orders_line_items_source_for_snapshot') + config: + enabled: true + strategy: timestamp + unique_key: id + updated_at: order_updated_at + invalidate_hard_deletes: true + persist_docs: + relation: true + columns: true From 080bb9468f9756b2c7b8cf84e4053db58911a8ae Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 18:58:33 +0000 Subject: [PATCH 05/28] refactoring example_orders_snapshot into new way --- snapshots/_samples/example_orders_snapshot.sql | 14 -------------- snapshots/jaffle_snapshots.yml | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 15 deletions(-) delete mode 100644 snapshots/_samples/example_orders_snapshot.sql diff --git a/snapshots/_samples/example_orders_snapshot.sql b/snapshots/_samples/example_orders_snapshot.sql deleted file mode 100644 index e4e7ad6d..00000000 --- a/snapshots/_samples/example_orders_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_orders_snapshot %} - {{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='order_id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml index eebfaba7..afa84f23 100644 --- a/snapshots/jaffle_snapshots.yml +++ b/snapshots/jaffle_snapshots.yml @@ -12,6 +12,19 @@ snapshots: persist_docs: relation: true columns: true +# Another standard snapshot but this time with a different table. + - name: snapshot_example_orders + relation: ref('example_orders_source_for_snapshot') + config: + enabled: true + strategy: timestamp + updated_at: order_updated_at + unique_key: ID + persist_docs: + relation: true + columns: true + invalidate_hard_deletes: true + # This snapshot has the check_cols example - name: snapshot_int_line_items relation: ref('int_line_items_amounts_calculated') @@ -19,7 +32,7 @@ snapshots: enabled: true strategy: check check_cols: ['gross_item_sales_amount', 'net_item_sales_amount'] - unique_key: order_item_id + unique_key: order_id persist_docs: relation: true columns: true From 2c8532a0cd1f96a045b2df26035aaed1346885b3 Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 20:02:37 +0000 Subject: [PATCH 06/28] convert example join snapshot to markdown till better fix avail --- .../{example_join_snapshots.sql => example_join_snapshots.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/_samples/snapshot/{example_join_snapshots.sql => example_join_snapshots.md} (100%) diff --git a/models/_samples/snapshot/example_join_snapshots.sql b/models/_samples/snapshot/example_join_snapshots.md similarity index 100% rename from models/_samples/snapshot/example_join_snapshots.sql rename to models/_samples/snapshot/example_join_snapshots.md From ef2f5474aa2aa97d03bab1f439670e7cec68c571 Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 20:04:18 +0000 Subject: [PATCH 07/28] fixing order snapshot --- snapshots/jaffle_snapshots.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml index afa84f23..c7bd414a 100644 --- a/snapshots/jaffle_snapshots.yml +++ b/snapshots/jaffle_snapshots.yml @@ -19,7 +19,7 @@ snapshots: enabled: true strategy: timestamp updated_at: order_updated_at - unique_key: ID + unique_key: order_id persist_docs: relation: true columns: true From 833d85cdf5cae3607cd121300de36108f2c7cdec Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 20:11:36 +0000 Subject: [PATCH 08/28] fixing snapshot line items --- snapshots/jaffle_snapshots.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml index c7bd414a..98b9695a 100644 --- a/snapshots/jaffle_snapshots.yml +++ b/snapshots/jaffle_snapshots.yml @@ -32,7 +32,7 @@ snapshots: enabled: true strategy: check check_cols: ['gross_item_sales_amount', 'net_item_sales_amount'] - unique_key: order_id + unique_key: order_item_id persist_docs: relation: true columns: true From ba8118a332db57181a03ca010d185cc5bbdda41d Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Mon, 18 Nov 2024 19:22:31 +0000 Subject: [PATCH 09/28] Commit changes --- macros/test_macro.sql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 macros/test_macro.sql diff --git a/macros/test_macro.sql b/macros/test_macro.sql new file mode 100644 index 00000000..0db90691 --- /dev/null +++ b/macros/test_macro.sql @@ -0,0 +1,5 @@ +{% macro test_macro() %} + + {{ log("This is a message by log macro", info=True) }} + +{% endmacro %} \ No newline at end of file From 6f267b4ab12f1ecc1e68c3081740843e80fc4f56 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Tue, 26 Nov 2024 02:31:23 +0000 Subject: [PATCH 10/28] Commit changes for tests --- dbt_project.yml | 5 +-- .../orders/int_daily_orders_fact.sql | 36 ++++++++++++++++++ models/marts/dims/_dims__models.yml | 38 +++++++++++++++++++ models/marts/dims/_dims__unit_tests | 12 ++++++ models/marts/dims/dim_countries.sql | 20 ++++++++++ models/marts/dims/dim_customer_info.sql | 16 ++++++++ models/marts/dims/dim_dates.sql | 19 ++++++++++ models/marts/dims/dim_orders.sql | 21 ++++++++++ models/marts/facts/_facts__models.yml | 10 +++++ models/marts/facts/fct_daily_orders.sql | 25 ++++++++++++ models/test_model.sql | 18 +++++++++ models/test_query.sql | 0 seeds/_seeds_properties.yml | 7 ++++ seeds/test_seed.csv | 2 + 14 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 models/intermediate/orders/int_daily_orders_fact.sql create mode 100644 models/marts/dims/_dims__models.yml create mode 100644 models/marts/dims/_dims__unit_tests create mode 100644 models/marts/dims/dim_countries.sql create mode 100644 models/marts/dims/dim_customer_info.sql create mode 100644 models/marts/dims/dim_dates.sql create mode 100644 models/marts/dims/dim_orders.sql create mode 100644 models/marts/facts/_facts__models.yml create mode 100644 models/marts/facts/fct_daily_orders.sql create mode 100644 models/test_model.sql create mode 100644 models/test_query.sql create mode 100644 seeds/_seeds_properties.yml create mode 100644 seeds/test_seed.csv diff --git a/dbt_project.yml b/dbt_project.yml index 0801a25d..bc8f141f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -65,7 +65,4 @@ models: +materialized: ephemeral tests: - rapid_onboarding_exemplar: - _samples: - staging: - +store_failures: "{{ true if target.name == 'prod' else false }}" + +store_failures: false diff --git a/models/intermediate/orders/int_daily_orders_fact.sql b/models/intermediate/orders/int_daily_orders_fact.sql new file mode 100644 index 00000000..afe02ff2 --- /dev/null +++ b/models/intermediate/orders/int_daily_orders_fact.sql @@ -0,0 +1,36 @@ +with + +dates as ( + + select * from {{ ref('dim_dates') }} + +), + +orders as ( + + select * from {{ ref('dim_orders') }} + +), + +countries as ( + + select * from {{ ref('dim_countries') }} + +), + +join_sources as ( + + select + ordr.order_sk, + country.country_sk as order_country_origin_sk, + date.date_sk as order_date_sk, + ordr.order_quantity + from orders ordr + left join dates date + on ordr.order_date = date.calendar_date + left join countries country + on ordr.order_country_origin = country.country_name + +) + +select * from join_sources diff --git a/models/marts/dims/_dims__models.yml b/models/marts/dims/_dims__models.yml new file mode 100644 index 00000000..934e5581 --- /dev/null +++ b/models/marts/dims/_dims__models.yml @@ -0,0 +1,38 @@ +version: 2 + +models: + - name: dim_orders + columns: + - name: order_sk + data_tests: + - unique + - not_null + + - name: dim_countries + columns: + - name: country_name + data_tests: + - accepted_values: + values: + - "United States" + - "France" + - "Italy" + - "Philippines" + +unit_tests: + - name: test_if_full_name_format_is_correct + description: "Checks if the format for customer full name is correct" + model: dim_customer_info + config: + tags: | + {%- if target.name == 'prod' -%} unit_test + {%- else -%} do_not_run + {% endif %} + given: + - input: ref('dim_orders') + rows: + - {"cust_fname": "Johnny", "cust_lname": "Bravo"} + + expect: + rows: + - {cust_fullname: "Bravo, Johnny"} \ No newline at end of file diff --git a/models/marts/dims/_dims__unit_tests b/models/marts/dims/_dims__unit_tests new file mode 100644 index 00000000..4ee81973 --- /dev/null +++ b/models/marts/dims/_dims__unit_tests @@ -0,0 +1,12 @@ +unit_tests: + - name: test_if_full_name_format_is_correct + description: "Checks if the format for customer full name is correct" + model: dim_customer_info + given: + - input: ref('dim_orders') + rows: + - {"cust_fname": "Johnny", "cust_lname": "Bravo"} + + expect: + rows: + - {cust_fullname: "Bravo, Johnny"} diff --git a/models/marts/dims/dim_countries.sql b/models/marts/dims/dim_countries.sql new file mode 100644 index 00000000..d70eda81 --- /dev/null +++ b/models/marts/dims/dim_countries.sql @@ -0,0 +1,20 @@ +{{ + config( + materialized='table' + ) +}} + +with + +countries as ( + + select 1 as country_sk, 'United States'::varchar(100) as country_name union all + select 2 as country_sk, 'France'::varchar(100) as country_name union all + select 3 as country_sk, 'Italy'::varchar(100) as country_name union all + select 4 as country_sk, 'Philippines'::varchar(100) as country_name + -- union all + -- select 5 as country_sk, 'Italia'::varchar(100) as country_name + +) + +select * from countries \ No newline at end of file diff --git a/models/marts/dims/dim_customer_info.sql b/models/marts/dims/dim_customer_info.sql new file mode 100644 index 00000000..a81f8407 --- /dev/null +++ b/models/marts/dims/dim_customer_info.sql @@ -0,0 +1,16 @@ +with + +dim_orders as ( + + select * from {{ ref('dim_orders') }} + +), + +final as ( + + select + cust_lname || ', ' || cust_fname as cust_fullname, + from dim_orders +) + +select * from final diff --git a/models/marts/dims/dim_dates.sql b/models/marts/dims/dim_dates.sql new file mode 100644 index 00000000..af9975e9 --- /dev/null +++ b/models/marts/dims/dim_dates.sql @@ -0,0 +1,19 @@ +{{ + config( + materialized='table' + ) +}} + +with + +dates as ( + + select '20240101'::decimal(18,2) as date_sk, '2024-01-01'::date as calendar_date union all + select '20240102'::decimal(18,2) as date_sk, '2024-01-02'::date as calendar_date union all + select '20240103'::decimal(18,2) as date_sk, '2024-01-03'::date as calendar_date union all + select '20240104'::decimal(18,2) as date_sk, '2024-01-04'::date as calendar_date union all + select '20240105'::decimal(18,2) as date_sk, '2024-01-05'::date as calendar_date + +) + +select * from dates \ No newline at end of file diff --git a/models/marts/dims/dim_orders.sql b/models/marts/dims/dim_orders.sql new file mode 100644 index 00000000..1f0f1476 --- /dev/null +++ b/models/marts/dims/dim_orders.sql @@ -0,0 +1,21 @@ +{{ + config( + materialized='table' + ) +}} + + +with + +orders as ( + + select '1'::decimal(18,2) as order_sk, 'order-001' as order_id, '2024-01-01'::date as order_date, 'United States'::varchar(100) as order_country_origin,'John'::varchar(10) as cust_fname, 'Lennon'::varchar(10) as cust_lname, 1 as order_quantity union all + select '2'::decimal(18,2) as order_sk, 'order-002' as order_id, '2024-01-03'::date as order_date, 'France'::varchar(100) as order_country_origin,'Ringo'::varchar(10) as cust_fname, 'Starr'::varchar(10) as cust_lname, 2 as order_quantity union all + select '3'::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Paul'::varchar(10) as cust_fname, 'McCartney'::varchar(10) as cust_lname, 3 as order_quantity + -- union all + -- select NULL::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Johnny'::varchar(10) as cust_fname, 'English'::varchar(10) as cust_lname, 3 as order_quantity union all + -- select '3'::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Mister'::varchar(10) as cust_fname, 'Bean'::varchar(10) as cust_lname, 3 as order_quantity + +) + +select * from orders \ No newline at end of file diff --git a/models/marts/facts/_facts__models.yml b/models/marts/facts/_facts__models.yml new file mode 100644 index 00000000..553f0f74 --- /dev/null +++ b/models/marts/facts/_facts__models.yml @@ -0,0 +1,10 @@ +version: 2 + +models: + - name: fct_daily_orders + columns: + - name: order_sk + tests: + - relationships: + to: ref('dim_orders') + field: order_sk \ No newline at end of file diff --git a/models/marts/facts/fct_daily_orders.sql b/models/marts/facts/fct_daily_orders.sql new file mode 100644 index 00000000..a3d5bd59 --- /dev/null +++ b/models/marts/facts/fct_daily_orders.sql @@ -0,0 +1,25 @@ +with + +int_daily_orders_fact as ( + + select * from {{ ref('int_daily_orders_fact') }} + +), + +agg_order_quantity as ( + + select + order_sk, + order_date_sk, + order_country_origin_sk, + sum(order_quantity) as total_order_quantity + from int_daily_orders_fact + group by + order_sk, + order_date_sk, + order_country_origin_sk + +) + +select * from agg_order_quantity + diff --git a/models/test_model.sql b/models/test_model.sql new file mode 100644 index 00000000..e36a24f6 --- /dev/null +++ b/models/test_model.sql @@ -0,0 +1,18 @@ +{{ + config( + materialized='table' + ) +}} + + +with + +src as ( + + select 1 as col_id + +) + +select * from src + +{{ log("This is a log inside the model", info=True) }} \ No newline at end of file diff --git a/models/test_query.sql b/models/test_query.sql new file mode 100644 index 00000000..e69de29b diff --git a/seeds/_seeds_properties.yml b/seeds/_seeds_properties.yml new file mode 100644 index 00000000..c4c5956a --- /dev/null +++ b/seeds/_seeds_properties.yml @@ -0,0 +1,7 @@ +version: 2 + +seeds: + - name: seeds_test + config: + database: analytics + schema: dbt_kfajardo \ No newline at end of file diff --git a/seeds/test_seed.csv b/seeds/test_seed.csv new file mode 100644 index 00000000..dbd888af --- /dev/null +++ b/seeds/test_seed.csv @@ -0,0 +1,2 @@ +col1,col2,col3 +1,2,3 \ No newline at end of file From 9d6fef6b9ba26065ceaaf041b585e54419f7daec Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Tue, 26 Nov 2024 02:41:33 +0000 Subject: [PATCH 11/28] Added singular test --- tests/assert_fct_daily_orders.sql | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/assert_fct_daily_orders.sql diff --git a/tests/assert_fct_daily_orders.sql b/tests/assert_fct_daily_orders.sql new file mode 100644 index 00000000..517e7cd5 --- /dev/null +++ b/tests/assert_fct_daily_orders.sql @@ -0,0 +1,21 @@ +with + + +source as ( + + select * from {{ ref('fct_daily_orders') }} + +), + +validations as ( + + select + order_sk, + count(*) as rec_cnt + from {{ ref('fct_daily_orders') }} + group by order_sk + having count(*) > 1 + +) + +select * from validations \ No newline at end of file From 08a594ffe0f4b7d4039c3cf56d5640b99613213b Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Tue, 26 Nov 2024 13:44:59 +0000 Subject: [PATCH 12/28] Commit changes. Added contracts. --- models/marts/dims/_dims__models.yml | 16 ++++++++++++++++ models/marts/dims/dim_orders.sql | 6 +++--- ...l => assert_duplicates__fct_daily_orders.sql} | 0 tests/assert_null__payment.sql | 12 ++++++++++++ 4 files changed, 31 insertions(+), 3 deletions(-) rename tests/{assert_fct_daily_orders.sql => assert_duplicates__fct_daily_orders.sql} (100%) create mode 100644 tests/assert_null__payment.sql diff --git a/models/marts/dims/_dims__models.yml b/models/marts/dims/_dims__models.yml index 934e5581..d2c69aa8 100644 --- a/models/marts/dims/_dims__models.yml +++ b/models/marts/dims/_dims__models.yml @@ -2,11 +2,27 @@ version: 2 models: - name: dim_orders + config: + contract: + enforced: true columns: - name: order_sk + data_type: decimal(10,2) data_tests: - unique - not_null + - name: cust_fname + data_type: string + - name: cust_lname + data_type: string + - name: order_country_origin + data_type: string + - name: order_date + data_type: date + - name: order_id + data_type: string + - name: order_quantity + data_type: decimal(15,2) - name: dim_countries columns: diff --git a/models/marts/dims/dim_orders.sql b/models/marts/dims/dim_orders.sql index 1f0f1476..6e72fbb0 100644 --- a/models/marts/dims/dim_orders.sql +++ b/models/marts/dims/dim_orders.sql @@ -9,9 +9,9 @@ with orders as ( - select '1'::decimal(18,2) as order_sk, 'order-001' as order_id, '2024-01-01'::date as order_date, 'United States'::varchar(100) as order_country_origin,'John'::varchar(10) as cust_fname, 'Lennon'::varchar(10) as cust_lname, 1 as order_quantity union all - select '2'::decimal(18,2) as order_sk, 'order-002' as order_id, '2024-01-03'::date as order_date, 'France'::varchar(100) as order_country_origin,'Ringo'::varchar(10) as cust_fname, 'Starr'::varchar(10) as cust_lname, 2 as order_quantity union all - select '3'::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Paul'::varchar(10) as cust_fname, 'McCartney'::varchar(10) as cust_lname, 3 as order_quantity + select '1'::numeric(10,2) as order_sk, 'order-001' as order_id, '2024-01-01'::date as order_date, 'United States'::varchar(100) as order_country_origin,'John'::varchar(10) as cust_fname, 'Lennon'::varchar(10) as cust_lname, 1::decimal(17,3) as order_quantity union all + select '2'::numeric(10,2) as order_sk, 'order-002' as order_id, '2024-01-03'::date as order_date, 'France'::varchar(100) as order_country_origin,'Ringo'::varchar(10) as cust_fname, 'Starr'::varchar(10) as cust_lname, 2::decimal(17,2) as order_quantity union all + select '3'::numeric(10,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Paul'::varchar(10) as cust_fname, 'McCartney'::varchar(10) as cust_lname, 3::decimal(17,3) as order_quantity -- union all -- select NULL::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Johnny'::varchar(10) as cust_fname, 'English'::varchar(10) as cust_lname, 3 as order_quantity union all -- select '3'::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Mister'::varchar(10) as cust_fname, 'Bean'::varchar(10) as cust_lname, 3 as order_quantity diff --git a/tests/assert_fct_daily_orders.sql b/tests/assert_duplicates__fct_daily_orders.sql similarity index 100% rename from tests/assert_fct_daily_orders.sql rename to tests/assert_duplicates__fct_daily_orders.sql diff --git a/tests/assert_null__payment.sql b/tests/assert_null__payment.sql new file mode 100644 index 00000000..3627f80c --- /dev/null +++ b/tests/assert_null__payment.sql @@ -0,0 +1,12 @@ +with + +source as ( + + select * from {{ source('stripe', 'payment') }} + +) + +select +id +from source +where id is null \ No newline at end of file From 62a15a7a87ea76ed466305816eed2c4c73bd5a69 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Wed, 27 Nov 2024 23:41:15 +0000 Subject: [PATCH 13/28] Commit changes; unit test --- models/marts/dims/_dims__models.yml | 18 ------------------ ..._dims__unit_tests => _dims__unit_tests.yml} | 9 ++++++++- 2 files changed, 8 insertions(+), 19 deletions(-) rename models/marts/dims/{_dims__unit_tests => _dims__unit_tests.yml} (62%) diff --git a/models/marts/dims/_dims__models.yml b/models/marts/dims/_dims__models.yml index d2c69aa8..e3c0a8c6 100644 --- a/models/marts/dims/_dims__models.yml +++ b/models/marts/dims/_dims__models.yml @@ -34,21 +34,3 @@ models: - "France" - "Italy" - "Philippines" - -unit_tests: - - name: test_if_full_name_format_is_correct - description: "Checks if the format for customer full name is correct" - model: dim_customer_info - config: - tags: | - {%- if target.name == 'prod' -%} unit_test - {%- else -%} do_not_run - {% endif %} - given: - - input: ref('dim_orders') - rows: - - {"cust_fname": "Johnny", "cust_lname": "Bravo"} - - expect: - rows: - - {cust_fullname: "Bravo, Johnny"} \ No newline at end of file diff --git a/models/marts/dims/_dims__unit_tests b/models/marts/dims/_dims__unit_tests.yml similarity index 62% rename from models/marts/dims/_dims__unit_tests rename to models/marts/dims/_dims__unit_tests.yml index 4ee81973..ab82c254 100644 --- a/models/marts/dims/_dims__unit_tests +++ b/models/marts/dims/_dims__unit_tests.yml @@ -1,7 +1,14 @@ +version: 2 + unit_tests: - name: test_if_full_name_format_is_correct description: "Checks if the format for customer full name is correct" model: dim_customer_info + config: + tags: | + {%- if target.name == 'prod' -%} unit_test + {%- else -%} do_not_run + {% endif %} given: - input: ref('dim_orders') rows: @@ -9,4 +16,4 @@ unit_tests: expect: rows: - - {cust_fullname: "Bravo, Johnny"} + - {cust_fullname: "Bravo, Johnny"} \ No newline at end of file From 5485517953cc5e68f512d959a70896b7cc1d651a Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Thu, 28 Nov 2024 00:54:29 +0000 Subject: [PATCH 14/28] Commit changes for unit test --- dbt_project.yml | 8 ++- models/intermediate/cfo/int_revenue_fct.sql | 67 +++++++++++++++++++ ...dims__models.yml => _dims__data_tests.yml} | 0 models/marts/dims/_dims__unit_tests.yml | 62 ++++++++++++++++- .../dims/cfo/dim_depreciation_accounts.sql | 16 +++++ .../marts/dims/cfo/dim_interests_accounts.sql | 16 +++++ models/marts/dims/cfo/dim_net_income.sql | 16 +++++ models/marts/dims/cfo/dim_tax_accounts.sql | 15 +++++ .../dims/{ => my_orders}/dim_countries.sql | 0 .../{ => my_orders}/dim_customer_info.sql | 0 .../marts/dims/{ => my_orders}/dim_dates.sql | 0 .../marts/dims/{ => my_orders}/dim_orders.sql | 0 ...cts__models.yml => _facts__data_tests.yml} | 0 models/marts/facts/cfo/fct_revenue.sql | 14 ++++ .../{ => my_orders}/fct_daily_orders.sql | 0 15 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 models/intermediate/cfo/int_revenue_fct.sql rename models/marts/dims/{_dims__models.yml => _dims__data_tests.yml} (100%) create mode 100644 models/marts/dims/cfo/dim_depreciation_accounts.sql create mode 100644 models/marts/dims/cfo/dim_interests_accounts.sql create mode 100644 models/marts/dims/cfo/dim_net_income.sql create mode 100644 models/marts/dims/cfo/dim_tax_accounts.sql rename models/marts/dims/{ => my_orders}/dim_countries.sql (100%) rename models/marts/dims/{ => my_orders}/dim_customer_info.sql (100%) rename models/marts/dims/{ => my_orders}/dim_dates.sql (100%) rename models/marts/dims/{ => my_orders}/dim_orders.sql (100%) rename models/marts/facts/{_facts__models.yml => _facts__data_tests.yml} (100%) create mode 100644 models/marts/facts/cfo/fct_revenue.sql rename models/marts/facts/{ => my_orders}/fct_daily_orders.sql (100%) diff --git a/dbt_project.yml b/dbt_project.yml index bc8f141f..ebf91ff7 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -31,8 +31,8 @@ vars: incremental_lookback_period: 'hour' future_proof_date: '9999-12-31' -on-run-end: - - "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}" +# on-run-end: +# - "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}" # Configuring models @@ -42,6 +42,10 @@ models: marts: +materialized: table +tags: ['tag'] + dims: + +tags: ["myorders"] + facts: + +tags: ["myorders"] finance: +group: finance marketing: diff --git a/models/intermediate/cfo/int_revenue_fct.sql b/models/intermediate/cfo/int_revenue_fct.sql new file mode 100644 index 00000000..22ebbec8 --- /dev/null +++ b/models/intermediate/cfo/int_revenue_fct.sql @@ -0,0 +1,67 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + +with + +depreciations as ( + + select * from {{ ref('dim_depreciation_accounts') }} + where account_type = 'Major' + +), + +interests as ( + + select * from {{ ref('dim_interests_accounts') }} + where account_type = 'Major' + +), + +net_income as ( + + select * from {{ ref('dim_net_income') }} + where account_type = 'Major' + +), + + +taxes as ( + + select * from {{ ref('dim_tax_accounts') }} + where account_type = 'Major' + +), + +join_sources as ( + + select + net_inc.account_id, + net_inc.net_income_usd, + deprec.depreciation_cost_usd, + ints.interests_amount_usd, + tax.tax_usd, + net_inc.account_type + from net_income net_inc + left join depreciations deprec + on net_inc.account_id = deprec.account_id + left join interests ints + on net_inc.account_id = ints.account_id + left join taxes tax + on net_inc.account_id = tax.account_id + +), + +final as ( + + select + account_id, + account_type, + (net_income_usd + interests_amount_usd + tax_usd + depreciation_cost_usd)::numeric(10, 3) as ebitda + from join_sources + +) + +select * from final + diff --git a/models/marts/dims/_dims__models.yml b/models/marts/dims/_dims__data_tests.yml similarity index 100% rename from models/marts/dims/_dims__models.yml rename to models/marts/dims/_dims__data_tests.yml diff --git a/models/marts/dims/_dims__unit_tests.yml b/models/marts/dims/_dims__unit_tests.yml index ab82c254..60b54778 100644 --- a/models/marts/dims/_dims__unit_tests.yml +++ b/models/marts/dims/_dims__unit_tests.yml @@ -16,4 +16,64 @@ unit_tests: expect: rows: - - {cust_fullname: "Bravo, Johnny"} \ No newline at end of file + - {cust_fullname: "Bravo, Johnny"} + + - name: test_accuracy_of_ebitda_calculation + description: "Test if the EBITDA calculation is correct" + model: int_revenue_fct + config: + tags: | + {%- if target.name != 'prod' -%} unit_test + {%- else -%} do_not_run + {% endif %} + + given: + - input: ref('dim_depreciation_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "depreciation_cost_usd": "-10"} + + - input: ref('dim_interests_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "interests_amount_usd": "10"} + + - input: ref('dim_net_income') + rows: + - {"account_id": "1", "account_type": "Major", "net_income_usd": "10"} + + - input: ref('dim_tax_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "tax_usd": "-10"} + + expect: + rows: + - {"account_id": "1", "account_type": "Major", "ebitda": "0"} + + - name: test_accuracy_of_ebitda_calculation__sql_format + description: "Test if the EBITDA calculation is correct using SQL format" + model: int_revenue_fct + config: + tags: | + {%- if target.name != 'prod' -%} unit_test + {%- else -%} do_not_run + {% endif %} + + given: + - input: ref('dim_depreciation_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "depreciation_cost_usd": "-10"} + + - input: ref('dim_interests_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "interests_amount_usd": "10"} + + - input: ref('dim_net_income') + rows: + - {"account_id": "1", "account_type": "Major", "net_income_usd": "10"} + + - input: ref('dim_tax_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "tax_usd": "-10"} + + expect: + rows: + - {"account_id": "1", "account_type": "Major", "ebitda": "0"} \ No newline at end of file diff --git a/models/marts/dims/cfo/dim_depreciation_accounts.sql b/models/marts/dims/cfo/dim_depreciation_accounts.sql new file mode 100644 index 00000000..80bf71c1 --- /dev/null +++ b/models/marts/dims/cfo/dim_depreciation_accounts.sql @@ -0,0 +1,16 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + + +with + +source as ( + + select '1'::numeric(10, 3) as account_id, '-1000.00'::numeric(10, 3) depreciation_cost_usd,'Major'::varchar as account_type, '2024-01-01'::date as record_updated_ts union all + select '2'::numeric(10, 3) as account_id, '-1000.00'::numeric(10, 3) depreciation_cost_usd,'Minor'::varchar as account_type, '2024-01-01'::date as record_updated_ts + +) + +select * from source diff --git a/models/marts/dims/cfo/dim_interests_accounts.sql b/models/marts/dims/cfo/dim_interests_accounts.sql new file mode 100644 index 00000000..6fadd7fe --- /dev/null +++ b/models/marts/dims/cfo/dim_interests_accounts.sql @@ -0,0 +1,16 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + + +with + +source as ( + + select '1'::numeric(10, 3) as account_id, '10000.00'::numeric(10, 3) interests_amount_usd,'Major'::varchar as account_type, '2024-01-01'::date as record_updated_ts union all + select '2'::numeric(10, 3) as account_id, '10000.00'::numeric(10, 3) interests_amount_usd,'Minor'::varchar as account_type, '2024-01-01'::date as record_updated_ts + +) + +select * from source diff --git a/models/marts/dims/cfo/dim_net_income.sql b/models/marts/dims/cfo/dim_net_income.sql new file mode 100644 index 00000000..b434f507 --- /dev/null +++ b/models/marts/dims/cfo/dim_net_income.sql @@ -0,0 +1,16 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + + +with + +source as ( + + select '1'::numeric(10, 3) as account_id, '10000.00'::numeric(10, 3) net_income_usd,'Major'::varchar as account_type, '2024-01-01'::date as record_updated_ts union all + select '2'::numeric(10, 3) as account_id, '10000.00'::numeric(10, 3) net_income_usd,'Minor'::varchar as account_type, '2024-01-01'::date as record_updated_ts + +) + +select * from source diff --git a/models/marts/dims/cfo/dim_tax_accounts.sql b/models/marts/dims/cfo/dim_tax_accounts.sql new file mode 100644 index 00000000..92087d9b --- /dev/null +++ b/models/marts/dims/cfo/dim_tax_accounts.sql @@ -0,0 +1,15 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + +with + +source as ( + + select '1'::numeric(10, 3) as account_id, '-2000.00'::numeric(10, 3) tax_usd,'Major'::varchar as account_type, '2024-01-01'::date as record_updated_ts union all + select '2'::numeric(10, 3) as account_id, '-2000.00'::numeric(10, 3) tax_usd,'Minor'::varchar as account_type, '2024-01-01'::date as record_updated_ts + +) + +select * from source diff --git a/models/marts/dims/dim_countries.sql b/models/marts/dims/my_orders/dim_countries.sql similarity index 100% rename from models/marts/dims/dim_countries.sql rename to models/marts/dims/my_orders/dim_countries.sql diff --git a/models/marts/dims/dim_customer_info.sql b/models/marts/dims/my_orders/dim_customer_info.sql similarity index 100% rename from models/marts/dims/dim_customer_info.sql rename to models/marts/dims/my_orders/dim_customer_info.sql diff --git a/models/marts/dims/dim_dates.sql b/models/marts/dims/my_orders/dim_dates.sql similarity index 100% rename from models/marts/dims/dim_dates.sql rename to models/marts/dims/my_orders/dim_dates.sql diff --git a/models/marts/dims/dim_orders.sql b/models/marts/dims/my_orders/dim_orders.sql similarity index 100% rename from models/marts/dims/dim_orders.sql rename to models/marts/dims/my_orders/dim_orders.sql diff --git a/models/marts/facts/_facts__models.yml b/models/marts/facts/_facts__data_tests.yml similarity index 100% rename from models/marts/facts/_facts__models.yml rename to models/marts/facts/_facts__data_tests.yml diff --git a/models/marts/facts/cfo/fct_revenue.sql b/models/marts/facts/cfo/fct_revenue.sql new file mode 100644 index 00000000..3e7fe56e --- /dev/null +++ b/models/marts/facts/cfo/fct_revenue.sql @@ -0,0 +1,14 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + +with + +sources as ( + + select * from {{ ref('int_revenue_fct') }} + +) + +select * from sources diff --git a/models/marts/facts/fct_daily_orders.sql b/models/marts/facts/my_orders/fct_daily_orders.sql similarity index 100% rename from models/marts/facts/fct_daily_orders.sql rename to models/marts/facts/my_orders/fct_daily_orders.sql From 2cbb34b6db3e80f0d1f62a7cbdead9fe694c1cd5 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Thu, 28 Nov 2024 01:10:02 +0000 Subject: [PATCH 15/28] commit changes for unit test --- models/marts/dims/_dims__unit_tests.yml | 60 ++++++++++++++++++++----- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/models/marts/dims/_dims__unit_tests.yml b/models/marts/dims/_dims__unit_tests.yml index 60b54778..92b6551e 100644 --- a/models/marts/dims/_dims__unit_tests.yml +++ b/models/marts/dims/_dims__unit_tests.yml @@ -59,21 +59,61 @@ unit_tests: given: - input: ref('dim_depreciation_accounts') - rows: - - {"account_id": "1", "account_type": "Major", "depreciation_cost_usd": "-10"} + format: sql + rows: | + with + main as ( + select + '1'::numeric(10, 3) as account_id, + 'Major'::varchar as account_type, + '-10'::numeric(10, 3) as depreciation_cost_usd + ) + + select * from main - input: ref('dim_interests_accounts') - rows: - - {"account_id": "1", "account_type": "Major", "interests_amount_usd": "10"} + format: sql + rows: | + with + main as ( + select + '1'::numeric(10, 3) as account_id, + 'Major'::varchar as account_type, + '10'::numeric(10, 3) as interests_amount_usd + ) + + select * from main - input: ref('dim_net_income') - rows: - - {"account_id": "1", "account_type": "Major", "net_income_usd": "10"} + format: sql + rows: | + with + main as ( + select + '1'::numeric(10, 3) as account_id, + 'Major'::varchar as account_type, + '10'::numeric(10, 3) as net_income_usd + ) + + select * from main - input: ref('dim_tax_accounts') - rows: - - {"account_id": "1", "account_type": "Major", "tax_usd": "-10"} + format: sql + rows: | + with + main as ( + select + '1'::numeric(10, 3) as account_id, + 'Major'::varchar as account_type, + '-10'::numeric(10, 3) as tax_usd + ) + + select * from main expect: - rows: - - {"account_id": "1", "account_type": "Major", "ebitda": "0"} \ No newline at end of file + format: sql + rows: | + select + 1::numeric(10, 3) as account_id, + 'Major' as account_type, + 0::numeric(10, 3) as ebitda \ No newline at end of file From 2624cb35024763b008f0b18c72d53e3f3450149b Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Tue, 3 Dec 2024 02:46:46 +0000 Subject: [PATCH 16/28] commit updates --- dbt_project.yml | 7 ++ models/_groups.yml | 10 +++ .../incremental/example_incremental_model.sql | 1 + .../int_daily_orders_fact.sql | 0 .../accounting}/int_revenue_fct.sql | 0 models/marts/{facts => }/cfo/fct_revenue.sql | 0 models/marts/dims/_dims__data_tests.yml | 36 ---------- models/marts/dims/_dims__unit_tests.yml | 2 +- models/marts/dims/_properties.yml | 65 +++++++++++++++++++ .../{dim_orders.sql => dim_orders_v1.sql} | 9 ++- models/marts/dims/my_orders/dim_orders_v2.sql | 19 ++++++ ..._data_tests.yml => _facts__properties.yml} | 0 models/python/dim_orders_ml.py | 19 ++++++ models/python/python_transformations.py | 19 ++++++ tests/fixtures/1_fixture.sql | 0 .../dim_depreciation_accounts__fixture.sql | 0 .../input/dim_net_income__fixture.sql | 0 17 files changed, 149 insertions(+), 38 deletions(-) create mode 100644 models/_groups.yml rename models/intermediate/{orders => finance}/int_daily_orders_fact.sql (100%) rename models/{intermediate/cfo => marts/accounting}/int_revenue_fct.sql (100%) rename models/marts/{facts => }/cfo/fct_revenue.sql (100%) delete mode 100644 models/marts/dims/_dims__data_tests.yml create mode 100644 models/marts/dims/_properties.yml rename models/marts/dims/my_orders/{dim_orders.sql => dim_orders_v1.sql} (64%) create mode 100644 models/marts/dims/my_orders/dim_orders_v2.sql rename models/marts/facts/{_facts__data_tests.yml => _facts__properties.yml} (100%) create mode 100644 models/python/dim_orders_ml.py create mode 100644 models/python/python_transformations.py create mode 100644 tests/fixtures/1_fixture.sql create mode 100644 tests/fixtures/input/dim_depreciation_accounts__fixture.sql create mode 100644 tests/fixtures/input/dim_net_income__fixture.sql diff --git a/dbt_project.yml b/dbt_project.yml index ebf91ff7..45fab5f0 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -52,6 +52,13 @@ models: +group: marketing operations: +group: operations + accounting: + +group: accounting + +access: public + cfo: + +group: cfo + +access: public + intermediate: +materialized: view staging: diff --git a/models/_groups.yml b/models/_groups.yml new file mode 100644 index 00000000..5dbec80a --- /dev/null +++ b/models/_groups.yml @@ -0,0 +1,10 @@ +groups: + - name: cfo + owner: + # 'name' or 'email' is required; additional properties allowed + email: cfo@mycompany.com + + - name: accounting + owner: + email: finance@mycompany.com + \ No newline at end of file diff --git a/models/_samples/incremental/example_incremental_model.sql b/models/_samples/incremental/example_incremental_model.sql index cf8cb7bc..032a1b4d 100644 --- a/models/_samples/incremental/example_incremental_model.sql +++ b/models/_samples/incremental/example_incremental_model.sql @@ -7,6 +7,7 @@ with source as ( select * from {{ ref('example_source_for_incremental') }} + {% if is_incremental() %} -- this filter will only be applied on an incremental run where _etl_loaded_at > (select max(_etl_loaded_at) from {{ this }}) diff --git a/models/intermediate/orders/int_daily_orders_fact.sql b/models/intermediate/finance/int_daily_orders_fact.sql similarity index 100% rename from models/intermediate/orders/int_daily_orders_fact.sql rename to models/intermediate/finance/int_daily_orders_fact.sql diff --git a/models/intermediate/cfo/int_revenue_fct.sql b/models/marts/accounting/int_revenue_fct.sql similarity index 100% rename from models/intermediate/cfo/int_revenue_fct.sql rename to models/marts/accounting/int_revenue_fct.sql diff --git a/models/marts/facts/cfo/fct_revenue.sql b/models/marts/cfo/fct_revenue.sql similarity index 100% rename from models/marts/facts/cfo/fct_revenue.sql rename to models/marts/cfo/fct_revenue.sql diff --git a/models/marts/dims/_dims__data_tests.yml b/models/marts/dims/_dims__data_tests.yml deleted file mode 100644 index e3c0a8c6..00000000 --- a/models/marts/dims/_dims__data_tests.yml +++ /dev/null @@ -1,36 +0,0 @@ -version: 2 - -models: - - name: dim_orders - config: - contract: - enforced: true - columns: - - name: order_sk - data_type: decimal(10,2) - data_tests: - - unique - - not_null - - name: cust_fname - data_type: string - - name: cust_lname - data_type: string - - name: order_country_origin - data_type: string - - name: order_date - data_type: date - - name: order_id - data_type: string - - name: order_quantity - data_type: decimal(15,2) - - - name: dim_countries - columns: - - name: country_name - data_tests: - - accepted_values: - values: - - "United States" - - "France" - - "Italy" - - "Philippines" diff --git a/models/marts/dims/_dims__unit_tests.yml b/models/marts/dims/_dims__unit_tests.yml index 92b6551e..70ea8a8a 100644 --- a/models/marts/dims/_dims__unit_tests.yml +++ b/models/marts/dims/_dims__unit_tests.yml @@ -46,7 +46,7 @@ unit_tests: expect: rows: - - {"account_id": "1", "account_type": "Major", "ebitda": "0"} + - {"account_id": "1", "account_type": "Major", "ebitda": "-1"} - name: test_accuracy_of_ebitda_calculation__sql_format description: "Test if the EBITDA calculation is correct using SQL format" diff --git a/models/marts/dims/_properties.yml b/models/marts/dims/_properties.yml new file mode 100644 index 00000000..adf796ac --- /dev/null +++ b/models/marts/dims/_properties.yml @@ -0,0 +1,65 @@ +version: 2 + + + +models: + - name: dim_orders + latest_version: 2 + + versions: + - v: 1 + config: + columns: + - name: order_sk + data_type: decimal(10,2) + data_tests: + - unique + - not_null + - name: cust_fname + data_type: string + - name: cust_lname + data_type: string + - name: order_country_origin + data_type: string + - name: order_date + data_type: date + - name: order_id + data_type: string + - name: order_quantity + data_type: decimal(15,2) + + - v: 2 + config: + alias: dim_orders + #group: accounting + columns: + - name: order_sk + data_type: decimal(10,2) + data_tests: + - unique + - not_null + - name: credit_card_number + data_type: varchar(100) + - name: cust_fname + data_type: string + - name: cust_lname + data_type: string + - name: order_country_origin + data_type: string + - name: order_date + data_type: date + - name: + data_type: string + + - name: dim_countries + config: + group: accounting + columns: + - name: country_name + data_tests: + - accepted_values: + values: + - "United States" + - "France" + - "Italy" + - "Philippines" diff --git a/models/marts/dims/my_orders/dim_orders.sql b/models/marts/dims/my_orders/dim_orders_v1.sql similarity index 64% rename from models/marts/dims/my_orders/dim_orders.sql rename to models/marts/dims/my_orders/dim_orders_v1.sql index 6e72fbb0..9d3f1e86 100644 --- a/models/marts/dims/my_orders/dim_orders.sql +++ b/models/marts/dims/my_orders/dim_orders_v1.sql @@ -12,10 +12,17 @@ orders as ( select '1'::numeric(10,2) as order_sk, 'order-001' as order_id, '2024-01-01'::date as order_date, 'United States'::varchar(100) as order_country_origin,'John'::varchar(10) as cust_fname, 'Lennon'::varchar(10) as cust_lname, 1::decimal(17,3) as order_quantity union all select '2'::numeric(10,2) as order_sk, 'order-002' as order_id, '2024-01-03'::date as order_date, 'France'::varchar(100) as order_country_origin,'Ringo'::varchar(10) as cust_fname, 'Starr'::varchar(10) as cust_lname, 2::decimal(17,2) as order_quantity union all select '3'::numeric(10,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Paul'::varchar(10) as cust_fname, 'McCartney'::varchar(10) as cust_lname, 3::decimal(17,3) as order_quantity + +-- model contract example: missing column order_quantity + -- select '1'::numeric(10,2) as order_sk, 'order-001' as order_id, '2024-01-01'::date as order_date, 'United States'::varchar(100) as order_country_origin,'John'::varchar(10) as cust_fname, 'Lennon'::varchar(10) as cust_lname union all + -- select '2'::numeric(10,2) as order_sk, 'order-002' as order_id, '2024-01-03'::date as order_date, 'France'::varchar(100) as order_country_origin,'Ringo'::varchar(10) as cust_fname, 'Starr'::varchar(10) as cust_lname union all + -- select '3'::numeric(10,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Paul'::varchar(10) as cust_fname, 'McCartney'::varchar(10) as cust_lname -- union all -- select NULL::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Johnny'::varchar(10) as cust_fname, 'English'::varchar(10) as cust_lname, 3 as order_quantity union all -- select '3'::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Mister'::varchar(10) as cust_fname, 'Bean'::varchar(10) as cust_lname, 3 as order_quantity ) -select * from orders \ No newline at end of file +select * from orders + + diff --git a/models/marts/dims/my_orders/dim_orders_v2.sql b/models/marts/dims/my_orders/dim_orders_v2.sql new file mode 100644 index 00000000..068f5b3d --- /dev/null +++ b/models/marts/dims/my_orders/dim_orders_v2.sql @@ -0,0 +1,19 @@ +{{ + config( + materialized='table' + ) +}} + + +with + +orders as ( + + select '1'::numeric(10,2) as order_sk,'1234-567-00'::varchar(100) as credit_card_number ,'order-001' as order_id, '2024-01-01'::date as order_date, 'United States'::varchar(100) as order_country_origin,'John'::varchar(10) as cust_fname, 'Lennon'::varchar(10) as cust_lname union all + select '2'::numeric(10,2) as order_sk,'1234-567-00'::varchar(100) as credit_card_number ,'order-002' as order_id, '2024-01-03'::date as order_date, 'France'::varchar(100) as order_country_origin,'Ringo'::varchar(10) as cust_fname, 'Starr'::varchar(10) as cust_lname union all + select '3'::numeric(10,2) as order_sk,'1234-567-00'::varchar(100) as credit_card_number ,'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Paul'::varchar(10) as cust_fname, 'McCartney'::varchar(10) as cust_lname + +) +select * from orders + + diff --git a/models/marts/facts/_facts__data_tests.yml b/models/marts/facts/_facts__properties.yml similarity index 100% rename from models/marts/facts/_facts__data_tests.yml rename to models/marts/facts/_facts__properties.yml diff --git a/models/python/dim_orders_ml.py b/models/python/dim_orders_ml.py new file mode 100644 index 00000000..95d5d8ea --- /dev/null +++ b/models/python/dim_orders_ml.py @@ -0,0 +1,19 @@ +import numpy +import pandas as pd + +def model(dbt, session): + + dbt.config(materialized="table") + + upstream_model = dbt.ref("dim_orders") + sorted = upstream_model.sort_values(by="CUST_LNAME") + + print("This is a line in python") + + # Process data with the external package + + + # Return the DataFrame as the model output + return sorted + + \ No newline at end of file diff --git a/models/python/python_transformations.py b/models/python/python_transformations.py new file mode 100644 index 00000000..d885ef09 --- /dev/null +++ b/models/python/python_transformations.py @@ -0,0 +1,19 @@ +import string +import random +import pandas as pd + +def model(dbt, session): + dbt.config(materialized="table") + + upstream_model = dbt.ref("dim_orders") + + letters = string.ascii_lowercase + string.ascii_uppercase + string.digits + default_salt = (''.join(random.choice(letters) for i in range(30))) + + columns_without_cc_number = upstream_model[['order_sk', 'order_id', 'order_date', 'order_country_origin', 'cust_fname', 'cust_lname']] + + # Create DataFrame with the string as a single row in one column + df = pd.DataFrame([default_salt], columns=['hashed_id']) + + return columns_without_cc_number + \ No newline at end of file diff --git a/tests/fixtures/1_fixture.sql b/tests/fixtures/1_fixture.sql new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/input/dim_depreciation_accounts__fixture.sql b/tests/fixtures/input/dim_depreciation_accounts__fixture.sql new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/input/dim_net_income__fixture.sql b/tests/fixtures/input/dim_net_income__fixture.sql new file mode 100644 index 00000000..e69de29b From 3b4efda66f166799e36e301bc420891b28c4bd4f Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Tue, 3 Dec 2024 02:49:03 +0000 Subject: [PATCH 17/28] Update changes --- models/python/python_transformations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/python/python_transformations.py b/models/python/python_transformations.py index d885ef09..0c00fdd3 100644 --- a/models/python/python_transformations.py +++ b/models/python/python_transformations.py @@ -13,7 +13,7 @@ def model(dbt, session): columns_without_cc_number = upstream_model[['order_sk', 'order_id', 'order_date', 'order_country_origin', 'cust_fname', 'cust_lname']] # Create DataFrame with the string as a single row in one column - df = pd.DataFrame([default_salt], columns=['hashed_id']) + hashed_value = pd.DataFrame([default_salt], columns=['hashed_id']) return columns_without_cc_number \ No newline at end of file From d84881a8b92c1321273aff8305071563027a3523 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Wed, 4 Dec 2024 20:14:53 +0000 Subject: [PATCH 18/28] commit changes --- models/_groups.yml | 3 +- .../orders}/int_revenue_fct.sql | 0 models/marts/dims/_properties.yml | 52 ++----------------- models/marts/dims/my_orders/dim_countries.sql | 12 +++-- .../{dim_orders_v1.sql => dim_orders.sql} | 0 models/marts/facts/_facts__properties.yml | 5 ++ .../{my_orders => }/fct_daily_orders.sql | 0 models/metrics/metricflow_time_spine.sql | 19 +++++++ models/python/_stripe__sources.sql | 22 ++++++++ models/sem_orders.yml | 15 ++++++ 10 files changed, 75 insertions(+), 53 deletions(-) rename models/{marts/accounting => intermediate/orders}/int_revenue_fct.sql (100%) rename models/marts/dims/my_orders/{dim_orders_v1.sql => dim_orders.sql} (100%) rename models/marts/facts/{my_orders => }/fct_daily_orders.sql (100%) create mode 100644 models/metrics/metricflow_time_spine.sql create mode 100644 models/python/_stripe__sources.sql create mode 100644 models/sem_orders.yml diff --git a/models/_groups.yml b/models/_groups.yml index 5dbec80a..80736aed 100644 --- a/models/_groups.yml +++ b/models/_groups.yml @@ -7,4 +7,5 @@ groups: - name: accounting owner: email: finance@mycompany.com - \ No newline at end of file + + diff --git a/models/marts/accounting/int_revenue_fct.sql b/models/intermediate/orders/int_revenue_fct.sql similarity index 100% rename from models/marts/accounting/int_revenue_fct.sql rename to models/intermediate/orders/int_revenue_fct.sql diff --git a/models/marts/dims/_properties.yml b/models/marts/dims/_properties.yml index adf796ac..506b19a4 100644 --- a/models/marts/dims/_properties.yml +++ b/models/marts/dims/_properties.yml @@ -4,58 +4,16 @@ version: 2 models: - name: dim_orders - latest_version: 2 - - versions: - - v: 1 - config: - columns: - - name: order_sk - data_type: decimal(10,2) - data_tests: - - unique - - not_null - - name: cust_fname - data_type: string - - name: cust_lname - data_type: string - - name: order_country_origin - data_type: string - - name: order_date - data_type: date - - name: order_id - data_type: string - - name: order_quantity - data_type: decimal(15,2) - - - v: 2 - config: - alias: dim_orders - #group: accounting - columns: - - name: order_sk - data_type: decimal(10,2) - data_tests: - - unique - - not_null - - name: credit_card_number - data_type: varchar(100) - - name: cust_fname - data_type: string - - name: cust_lname - data_type: string - - name: order_country_origin - data_type: string - - name: order_date - data_type: date - - name: - data_type: string - name: dim_countries config: - group: accounting + contract: + enforced: true columns: + - name: country_sk + data_type: numeric(13, 2) - name: country_name + data_type: string data_tests: - accepted_values: values: diff --git a/models/marts/dims/my_orders/dim_countries.sql b/models/marts/dims/my_orders/dim_countries.sql index d70eda81..30c1b070 100644 --- a/models/marts/dims/my_orders/dim_countries.sql +++ b/models/marts/dims/my_orders/dim_countries.sql @@ -8,11 +8,13 @@ with countries as ( - select 1 as country_sk, 'United States'::varchar(100) as country_name union all - select 2 as country_sk, 'France'::varchar(100) as country_name union all - select 3 as country_sk, 'Italy'::varchar(100) as country_name union all - select 4 as country_sk, 'Philippines'::varchar(100) as country_name - -- union all + select 'United States'::varchar(100) as country_name + + -- select 1 as country_sk, 'United States'::varchar(100) as country_name union all + -- select 2 as country_sk, 'France'::varchar(100) as country_name union all + -- select 3 as country_sk, 'Italy'::varchar(100) as country_name union all + -- select 4 as country_sk, 'Philippines'::varchar(100) as country_name + -- -- union all -- select 5 as country_sk, 'Italia'::varchar(100) as country_name ) diff --git a/models/marts/dims/my_orders/dim_orders_v1.sql b/models/marts/dims/my_orders/dim_orders.sql similarity index 100% rename from models/marts/dims/my_orders/dim_orders_v1.sql rename to models/marts/dims/my_orders/dim_orders.sql diff --git a/models/marts/facts/_facts__properties.yml b/models/marts/facts/_facts__properties.yml index 553f0f74..4886a355 100644 --- a/models/marts/facts/_facts__properties.yml +++ b/models/marts/facts/_facts__properties.yml @@ -2,8 +2,13 @@ version: 2 models: - name: fct_daily_orders + description: | + "This is a fact table about orders." + columns: - name: order_sk + description: "Order SK" + tests: - relationships: to: ref('dim_orders') diff --git a/models/marts/facts/my_orders/fct_daily_orders.sql b/models/marts/facts/fct_daily_orders.sql similarity index 100% rename from models/marts/facts/my_orders/fct_daily_orders.sql rename to models/marts/facts/fct_daily_orders.sql diff --git a/models/metrics/metricflow_time_spine.sql b/models/metrics/metricflow_time_spine.sql new file mode 100644 index 00000000..ac4d3339 --- /dev/null +++ b/models/metrics/metricflow_time_spine.sql @@ -0,0 +1,19 @@ +{{ + config( + materialized = 'table', + ) +}} +with days as ( + {{ + dbt_utils.date_spine( + 'day', + "to_date('01/01/2000','mm/dd/yyyy')", + "to_date('01/01/2027','mm/dd/yyyy')" + ) + }} +), +final as ( + select cast(date_day as date) as date_day + from days +) +select * from final diff --git a/models/python/_stripe__sources.sql b/models/python/_stripe__sources.sql new file mode 100644 index 00000000..c419c3a7 --- /dev/null +++ b/models/python/_stripe__sources.sql @@ -0,0 +1,22 @@ +version: 2 + +sources: + - name: stripe2 + description: Incoming stripe payment data. + database: raw + schema: stripe + tables: + - name: payment + columns: + - name: id + data_tests: + - not_null + - unique + - name: paymentmethod + data_tests: + - accepted_values: + values: ['credit_card', 'bank_transfer', 'gift_card', 'coupon'] + - name: status + data_tests: + - accepted_values: + values: ['success', 'fail'] \ No newline at end of file diff --git a/models/sem_orders.yml b/models/sem_orders.yml new file mode 100644 index 00000000..f88b9395 --- /dev/null +++ b/models/sem_orders.yml @@ -0,0 +1,15 @@ +semantic_models: + - name: orders + model: ref('dim_orders') # References the dbt model named `fact_transactions` + description: "This table contains information about orders." + defaults: + agg_time_dimension: order_date + + entities: # Entities included in the table are defined here. MetricFlow will use these columns as join keys. + - name: transaction + type: primary + expr: transaction_id + - name: customer + type: foreign + expr: customer_id + From f803d7df087767c0e54957ac546b51000c5a9bbe Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Wed, 4 Dec 2024 21:52:52 +0000 Subject: [PATCH 19/28] Commit changes --- models/marts/dims/my_orders/dim_countries.sql | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/models/marts/dims/my_orders/dim_countries.sql b/models/marts/dims/my_orders/dim_countries.sql index 30c1b070..1d780829 100644 --- a/models/marts/dims/my_orders/dim_countries.sql +++ b/models/marts/dims/my_orders/dim_countries.sql @@ -8,12 +8,10 @@ with countries as ( - select 'United States'::varchar(100) as country_name - - -- select 1 as country_sk, 'United States'::varchar(100) as country_name union all - -- select 2 as country_sk, 'France'::varchar(100) as country_name union all - -- select 3 as country_sk, 'Italy'::varchar(100) as country_name union all - -- select 4 as country_sk, 'Philippines'::varchar(100) as country_name + select 1 as country_sk, 'United States'::varchar(100) as country_name union all + select 2 as country_sk, 'France'::varchar(100) as country_name union all + select 3 as country_sk, 'Italy'::varchar(100) as country_name union all + select 4 as country_sk, 'Philippines'::varchar(100) as country_name -- -- union all -- select 5 as country_sk, 'Italia'::varchar(100) as country_name From b76018bdd29b74c405d209b3d6ece929f33359e3 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Wed, 4 Dec 2024 21:54:36 +0000 Subject: [PATCH 20/28] Update changes --- models/marts/dims/my_orders/dim_countries.sql | 3 ++- models/marts/dims/my_orders/dim_customer_info.sql | 7 +++++++ models/marts/dims/my_orders/dim_dates.sql | 5 ++++- models/marts/dims/my_orders/dim_orders.sql | 4 +++- models/marts/dims/my_orders/dim_orders_v2.sql | 4 +++- 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/models/marts/dims/my_orders/dim_countries.sql b/models/marts/dims/my_orders/dim_countries.sql index 1d780829..2fa22f5c 100644 --- a/models/marts/dims/my_orders/dim_countries.sql +++ b/models/marts/dims/my_orders/dim_countries.sql @@ -1,6 +1,7 @@ {{ config( - materialized='table' + materialized='table', + tags=["orders"] ) }} diff --git a/models/marts/dims/my_orders/dim_customer_info.sql b/models/marts/dims/my_orders/dim_customer_info.sql index a81f8407..a4ee3d82 100644 --- a/models/marts/dims/my_orders/dim_customer_info.sql +++ b/models/marts/dims/my_orders/dim_customer_info.sql @@ -1,3 +1,10 @@ +{{ + config( + materialized='table', + tags=["orders"] + ) +}} + with dim_orders as ( diff --git a/models/marts/dims/my_orders/dim_dates.sql b/models/marts/dims/my_orders/dim_dates.sql index af9975e9..71d678ba 100644 --- a/models/marts/dims/my_orders/dim_dates.sql +++ b/models/marts/dims/my_orders/dim_dates.sql @@ -1,9 +1,12 @@ {{ config( - materialized='table' + materialized='table', + tags=["orders"] ) }} + + with dates as ( diff --git a/models/marts/dims/my_orders/dim_orders.sql b/models/marts/dims/my_orders/dim_orders.sql index 9d3f1e86..e709c22b 100644 --- a/models/marts/dims/my_orders/dim_orders.sql +++ b/models/marts/dims/my_orders/dim_orders.sql @@ -1,10 +1,12 @@ {{ config( - materialized='table' + materialized='table', + tags=["orders"] ) }} + with orders as ( diff --git a/models/marts/dims/my_orders/dim_orders_v2.sql b/models/marts/dims/my_orders/dim_orders_v2.sql index 068f5b3d..372882d8 100644 --- a/models/marts/dims/my_orders/dim_orders_v2.sql +++ b/models/marts/dims/my_orders/dim_orders_v2.sql @@ -1,10 +1,12 @@ {{ config( - materialized='table' + materialized='table', + tags=["orders"] ) }} + with orders as ( From baa0d3f32a62bbc56921263c9630e6efbbb42d8e Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Wed, 4 Dec 2024 22:16:33 +0000 Subject: [PATCH 21/28] Update tags --- models/marts/facts/fct_daily_orders.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/models/marts/facts/fct_daily_orders.sql b/models/marts/facts/fct_daily_orders.sql index a3d5bd59..4117bedd 100644 --- a/models/marts/facts/fct_daily_orders.sql +++ b/models/marts/facts/fct_daily_orders.sql @@ -1,3 +1,8 @@ +{{ config( + materialized="table", + tags="orders" +) }} + with int_daily_orders_fact as ( From 419f3580784a64e2d507606fd02f7911e6318d78 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Thu, 5 Dec 2024 01:11:20 +0000 Subject: [PATCH 22/28] commit changes --- models/python/python_transformations.py | 19 --------------- .../_stripe__sources.sql | 0 .../demo/_data_tests_py__models.yml | 9 +++++++ models/python_demo/demo/date_spine.sql | 14 +++++++++++ models/python_demo/demo/is_holiday_2024.py | 24 +++++++++++++++++++ .../others}/dim_orders_ml.py | 0 .../others/python_transformations.py | 24 +++++++++++++++++++ 7 files changed, 71 insertions(+), 19 deletions(-) delete mode 100644 models/python/python_transformations.py rename models/{python => python_demo}/_stripe__sources.sql (100%) create mode 100644 models/python_demo/demo/_data_tests_py__models.yml create mode 100644 models/python_demo/demo/date_spine.sql create mode 100644 models/python_demo/demo/is_holiday_2024.py rename models/{python => python_demo/others}/dim_orders_ml.py (100%) create mode 100644 models/python_demo/others/python_transformations.py diff --git a/models/python/python_transformations.py b/models/python/python_transformations.py deleted file mode 100644 index 0c00fdd3..00000000 --- a/models/python/python_transformations.py +++ /dev/null @@ -1,19 +0,0 @@ -import string -import random -import pandas as pd - -def model(dbt, session): - dbt.config(materialized="table") - - upstream_model = dbt.ref("dim_orders") - - letters = string.ascii_lowercase + string.ascii_uppercase + string.digits - default_salt = (''.join(random.choice(letters) for i in range(30))) - - columns_without_cc_number = upstream_model[['order_sk', 'order_id', 'order_date', 'order_country_origin', 'cust_fname', 'cust_lname']] - - # Create DataFrame with the string as a single row in one column - hashed_value = pd.DataFrame([default_salt], columns=['hashed_id']) - - return columns_without_cc_number - \ No newline at end of file diff --git a/models/python/_stripe__sources.sql b/models/python_demo/_stripe__sources.sql similarity index 100% rename from models/python/_stripe__sources.sql rename to models/python_demo/_stripe__sources.sql diff --git a/models/python_demo/demo/_data_tests_py__models.yml b/models/python_demo/demo/_data_tests_py__models.yml new file mode 100644 index 00000000..ee30f0bf --- /dev/null +++ b/models/python_demo/demo/_data_tests_py__models.yml @@ -0,0 +1,9 @@ +version: 2 + +models: + - name: is_holiday_2024 + columns: + - name: date_day + data_tests: + - unique + \ No newline at end of file diff --git a/models/python_demo/demo/date_spine.sql b/models/python_demo/demo/date_spine.sql new file mode 100644 index 00000000..14b6b1ec --- /dev/null +++ b/models/python_demo/demo/date_spine.sql @@ -0,0 +1,14 @@ +with + +dates as ( + + {{ dbt_utils.date_spine( + datepart="day", + start_date="cast('2024-01-01' as date)", + end_date="cast('2024-12-31' as date)" + ) + }} + +) + +select * from dates \ No newline at end of file diff --git a/models/python_demo/demo/is_holiday_2024.py b/models/python_demo/demo/is_holiday_2024.py new file mode 100644 index 00000000..898e5928 --- /dev/null +++ b/models/python_demo/demo/is_holiday_2024.py @@ -0,0 +1,24 @@ +import holidays +import pandas + +#all python models need to be defined at the start with this specific syntax +def model(dbt, session): + + #python models don't use Jinja. Here we are using dbt.config to create model configurations + #be sure to materialize python models as tables, and to specify the packages that were imported above + dbt.config( + materialized="table", + tags=["py_models"], + packages=['pandas', 'holidays'] + ) + + us_holidays = holidays.US() + + #python models don't use Jinja. Here we are using dbt.ref to create model references + df = dbt.ref('date_spine').to_pandas() + + # if you are using snowpark columns need to be uppercase + df['IS_HOLIDAY'] = df['DATE_DAY'].apply(lambda date: date in us_holidays) + + #in dbt, you always need to return your data frame at the end of your models + return df diff --git a/models/python/dim_orders_ml.py b/models/python_demo/others/dim_orders_ml.py similarity index 100% rename from models/python/dim_orders_ml.py rename to models/python_demo/others/dim_orders_ml.py diff --git a/models/python_demo/others/python_transformations.py b/models/python_demo/others/python_transformations.py new file mode 100644 index 00000000..e00ca6b1 --- /dev/null +++ b/models/python_demo/others/python_transformations.py @@ -0,0 +1,24 @@ +import string +import random +import pandas +import hashlib + +def model(dbt, session): + dbt.config( + materialized="table", + packages=['pandas'] + ) + + df = dbt.ref("dim_orders").to_pandas() + + letters = string.ascii_lowercase + string.ascii_uppercase + string.digits + default_salt = (''.join(random.choice(letters) for i in range(30))) + + # Define the variable to concatenate with + variable_hash = 'some_secret_salt' + + # Use lambda to concatenate the column with variable_hash, and then hash the result + df = df.assign(HASHED = lambda x: (x['ORDER_COUNTRY_ORIGIN'] + default_salt)) + + return df + \ No newline at end of file From 961a2430508fb2056d58fab16ac03fe771d2f86e Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Thu, 5 Dec 2024 01:49:01 +0000 Subject: [PATCH 23/28] Commit changes --- .../stg_jaffle_shop__customers.sql | 53 ------------------- .../jaffle_shop/stg_jaffle_shop__orders.sql | 48 ----------------- .../_semantic_layer_sources.yml | 15 ++++++ .../marts/dim_customer.sql | 29 ++++++++++ .../semantic_layer_demo/marts/fact_orders.sql | 43 +++++++++++++++ .../jaffle_shop/_jaffle_shop__models.yml | 0 .../jaffle_shop/_jaffle_shop__sources.yml | 0 .../stg_jaffle_shop__customers.sql | 11 ++++ .../jaffle_shop/stg_jaffle_shop__orders.sql | 12 +++++ .../staging/stripe/stg_stripe__payment.sql | 17 ++++++ .../staging/stripe/stg_stripe__payments.sql | 2 +- 11 files changed, 128 insertions(+), 102 deletions(-) delete mode 100644 models/_samples/staging/jaffle_shop/stg_jaffle_shop__customers.sql delete mode 100644 models/_samples/staging/jaffle_shop/stg_jaffle_shop__orders.sql create mode 100644 models/semantic_layer_demo/_semantic_layer_sources.yml create mode 100644 models/semantic_layer_demo/marts/dim_customer.sql create mode 100644 models/semantic_layer_demo/marts/fact_orders.sql rename models/{_samples => semantic_layer_demo}/staging/jaffle_shop/_jaffle_shop__models.yml (100%) rename models/{_samples => semantic_layer_demo}/staging/jaffle_shop/_jaffle_shop__sources.yml (100%) create mode 100644 models/semantic_layer_demo/staging/jaffle_shop/stg_jaffle_shop__customers.sql create mode 100644 models/semantic_layer_demo/staging/jaffle_shop/stg_jaffle_shop__orders.sql create mode 100644 models/semantic_layer_demo/staging/stripe/stg_stripe__payment.sql diff --git a/models/_samples/staging/jaffle_shop/stg_jaffle_shop__customers.sql b/models/_samples/staging/jaffle_shop/stg_jaffle_shop__customers.sql deleted file mode 100644 index 0e10767e..00000000 --- a/models/_samples/staging/jaffle_shop/stg_jaffle_shop__customers.sql +++ /dev/null @@ -1,53 +0,0 @@ -{#- -Explaining Configurations - - grants: shows how to grant a specific privilege (in this case select) on the object to a role in Snowflake (in this case transformer) - - post_hook: shows you can run a statement after your model is built - - alias: sets the name of the object that will be created in the warehouse and overrides the file name. in this case, the model - will b created as staging_jaffle_shop_customers - - materialized: overwrites the materialization strategy from the dbt_project.yml file to be table instead of view - - persist_docs: persists the documentation into the database - - schema: adjusting the schema that the model gets built in to be a concatenation of the target schema and "jaffle_shop" - - database: example of how you can overwrite the target database by setting the database in the config. in this example, - we're overriding to the analytics database (even though that is the same as the target database, just an example) - --#} - -{{ - config( - grants = {'select': ['transformer']}, - post_hook = 'select * from {{ this }} limit 1', - alias = 'staging_jaffle_shop_customers', - materialized = 'table', - persist_docs = {"relation": true, "columns": true}, - tags = ['finance'], - schema = 'jaffle_shop', - database = 'analytics' - ) -}} - -{{ - config( - materialized='table' - ) -}} - - - -with source as ( - - select * from {{ source('jaffle_shop', 'customers') }} - -), - -renamed as ( - - select - id as customer_id, - first_name, - last_name - - from source - -) - -select * from renamed diff --git a/models/_samples/staging/jaffle_shop/stg_jaffle_shop__orders.sql b/models/_samples/staging/jaffle_shop/stg_jaffle_shop__orders.sql deleted file mode 100644 index afa5cf86..00000000 --- a/models/_samples/staging/jaffle_shop/stg_jaffle_shop__orders.sql +++ /dev/null @@ -1,48 +0,0 @@ -{#- -Explaining Configurations - - grants: shows how to grant a specific privilege (in this case select) on the object to a role in Snowflake (in this case transformer) - - post_hook: shows you can run a statement after your model is built - - alias: sets the name of the object that will be created in the warehouse and overrides the file name. in this case, the model - will b created as staging_jaffle_shop_orders - - materialized: overwrites the materialization strategy from the dbt_project.yml file to be table instead of view - - persist_docs: persists the documentation into the database - - schema: adjusting the schema that the model gets built in to be a concatenation of the target schema and "jaffle_shop" - - database: example of how you can overwrite the target database by setting the database in the config. in this example, - we're overriding to the analytics database (even though that is the same as the target database, just an example) - --#} - -{{ - config( - grants = {'select': ['transformer']}, - post_hook = 'select * from {{ this }} limit 1', - alias = 'staging_jaffle_shop_orders', - materialized = 'table', - persist_docs = {"relation": true, "columns": true}, - tags = ['finance', 'orders'], - schema = 'jaffle_shop', - database = 'analytics' - ) -}} - - -with source as ( - - select * from {{ source('jaffle_shop', 'orders') }} - -), - -renamed as ( - - select - id as order_id, - user_id as customer_id, - order_date, - status, - _etl_loaded_at - - from source - -) - -select * from renamed diff --git a/models/semantic_layer_demo/_semantic_layer_sources.yml b/models/semantic_layer_demo/_semantic_layer_sources.yml new file mode 100644 index 00000000..aff5be45 --- /dev/null +++ b/models/semantic_layer_demo/_semantic_layer_sources.yml @@ -0,0 +1,15 @@ +version: 2 + +sources: + - name: orders + database: raw + schema: jaffle_shop + tables: + - name: orders + - name: customers + + - name: payments + database: raw + schema: stripe + tables: + - name: payment \ No newline at end of file diff --git a/models/semantic_layer_demo/marts/dim_customer.sql b/models/semantic_layer_demo/marts/dim_customer.sql new file mode 100644 index 00000000..e7bc0723 --- /dev/null +++ b/models/semantic_layer_demo/marts/dim_customer.sql @@ -0,0 +1,29 @@ +with customers as ( + select * from {{ ref('stg_jaffle_shop__customers')}} +), +orders as ( + select * from {{ ref('fact_orders')}} +), +customer_orders as ( + select + customer_id, + min(order_date) as first_order_date, + max(order_date) as most_recent_order_date, + count(order_id) as number_of_orders, + sum(amount) as lifetime_value + from orders + group by 1 +), +final as ( + select + customers.customer_id, + customers.first_name, + customers.last_name, + customer_orders.first_order_date, + customer_orders.most_recent_order_date, + coalesce(customer_orders.number_of_orders, 0) as number_of_orders, + customer_orders.lifetime_value + from customers + left join customer_orders using (customer_id) +) +select * from final diff --git a/models/semantic_layer_demo/marts/fact_orders.sql b/models/semantic_layer_demo/marts/fact_orders.sql new file mode 100644 index 00000000..53ae30e5 --- /dev/null +++ b/models/semantic_layer_demo/marts/fact_orders.sql @@ -0,0 +1,43 @@ +{{ + config( + tags=["semantic_layer_demo"] + ) +}} + +with orders as ( + select * from {{ ref('stg_jaffle_shop__orders' )}} +), + + +payments as ( + select * from {{ ref('stg_stripe__payment') }} +), + + +order_payments as ( + select + order_id, + sum(case when status = 'success' then amount end) as amount + + + from payments + group by 1 +), + + +final as ( + + + select + orders.order_id, + orders.customer_id, + orders.order_date, + coalesce(order_payments.amount, 0) as amount + + + from orders + left join order_payments using (order_id) +) + + +select * from final diff --git a/models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml b/models/semantic_layer_demo/staging/jaffle_shop/_jaffle_shop__models.yml similarity index 100% rename from models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml rename to models/semantic_layer_demo/staging/jaffle_shop/_jaffle_shop__models.yml diff --git a/models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml b/models/semantic_layer_demo/staging/jaffle_shop/_jaffle_shop__sources.yml similarity index 100% rename from models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml rename to models/semantic_layer_demo/staging/jaffle_shop/_jaffle_shop__sources.yml diff --git a/models/semantic_layer_demo/staging/jaffle_shop/stg_jaffle_shop__customers.sql b/models/semantic_layer_demo/staging/jaffle_shop/stg_jaffle_shop__customers.sql new file mode 100644 index 00000000..6284f71a --- /dev/null +++ b/models/semantic_layer_demo/staging/jaffle_shop/stg_jaffle_shop__customers.sql @@ -0,0 +1,11 @@ +{{ + config( + tags=['semantic_layer_demo'] + ) +}} + + select + id as customer_id, + first_name, + last_name +from {{ source('jaffle_shop', 'customers') }} \ No newline at end of file diff --git a/models/semantic_layer_demo/staging/jaffle_shop/stg_jaffle_shop__orders.sql b/models/semantic_layer_demo/staging/jaffle_shop/stg_jaffle_shop__orders.sql new file mode 100644 index 00000000..6be9048d --- /dev/null +++ b/models/semantic_layer_demo/staging/jaffle_shop/stg_jaffle_shop__orders.sql @@ -0,0 +1,12 @@ +{{ + config( + tags=['semantic_layer_demo'] + ) +}} + + select + id as order_id, + user_id as customer_id, + order_date, + status + from {{ source('jaffle_shop', 'orders') }} \ No newline at end of file diff --git a/models/semantic_layer_demo/staging/stripe/stg_stripe__payment.sql b/models/semantic_layer_demo/staging/stripe/stg_stripe__payment.sql new file mode 100644 index 00000000..e0bc13ff --- /dev/null +++ b/models/semantic_layer_demo/staging/stripe/stg_stripe__payment.sql @@ -0,0 +1,17 @@ +{{ + config( + tags=['semantic_layer_demo'] + ) +}} + +select + id as payment_id, + orderid as order_id, + paymentmethod as payment_method, + status, + -- amount is stored in cents, convert it to dollars + amount / 100 as amount, + created as created_at + + +from {{ source('stripe', 'payment') }} \ No newline at end of file diff --git a/models/staging/stripe/stg_stripe__payments.sql b/models/staging/stripe/stg_stripe__payments.sql index 61ede9e5..61fd513d 100644 --- a/models/staging/stripe/stg_stripe__payments.sql +++ b/models/staging/stripe/stg_stripe__payments.sql @@ -13,6 +13,6 @@ select -- datetimes created as created_at -from {{ ref('snapshot_stg_payments') }} +from {{ source('stripe', 'payment') }} -- pull only the most recent update for each unique record where dbt_valid_to is null From 9bf0cc0917b878f290b3eee7b16f58fe0bec9161 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Thu, 5 Dec 2024 02:10:16 +0000 Subject: [PATCH 24/28] commit changes --- models/metrics/customers_metrics.yml | 0 models/metrics/dim_customers.yml | 53 ++++++++++++++++++ models/metrics/fact_orders.yml | 45 +++++++++++++++ models/metrics/orders_metrics.yml | 56 +++++++++++++++++++ models/sem_orders.yml | 15 ----- .../marts/dim_customer.sql | 7 +++ 6 files changed, 161 insertions(+), 15 deletions(-) create mode 100644 models/metrics/customers_metrics.yml create mode 100644 models/metrics/dim_customers.yml create mode 100644 models/metrics/fact_orders.yml create mode 100644 models/metrics/orders_metrics.yml delete mode 100644 models/sem_orders.yml diff --git a/models/metrics/customers_metrics.yml b/models/metrics/customers_metrics.yml new file mode 100644 index 00000000..e69de29b diff --git a/models/metrics/dim_customers.yml b/models/metrics/dim_customers.yml new file mode 100644 index 00000000..b843609c --- /dev/null +++ b/models/metrics/dim_customers.yml @@ -0,0 +1,53 @@ +semantic_models: + + - name: customers + defaults: + agg_time_dimension: most_recent_order_date + description: | + semantic model for dim_customers + + model: ref('dim_customers') + + entities: + - name: customer + expr: customer_id + type: primary + + dimensions: + - name: customer_name + type: categorical + expr: first_name + + - name: first_order_date + type: time + type_params: + time_granularity: day + + - name: most_recent_order_date + type: time + type_params: + time_granularity: day + + measures: + - name: count_lifetime_orders + description: Total count of orders per customer. + agg: sum + expr: number_of_orders + + - name: lifetime_spend + agg: sum + expr: lifetime_value + description: Gross customer lifetime spend inclusive of taxes. + + - name: customers + expr: customer_id + agg: count_distinct + +metrics: + - name: "customers_with_orders" + label: "customers_with_orders" + description: "Unique count of customers placing orders" + type: simple + type_params: + measure: + name: customers \ No newline at end of file diff --git a/models/metrics/fact_orders.yml b/models/metrics/fact_orders.yml new file mode 100644 index 00000000..19fdd2e4 --- /dev/null +++ b/models/metrics/fact_orders.yml @@ -0,0 +1,45 @@ +semantic_models: + + - name: orders + defaults: + agg_time_dimension: order_date + description: | + Order fact table. This table's grain is one row per order. + model: ref('fact_orders') + + entities: + - name: order_id + type: primary + + - name: customer + expr: customer_id + type: foreign + + dimensions: + - name: order_date + type: time + type_params: + time_granularity: day + + measures: + - name: order_total + description: The total amount for each order including taxes. + agg: sum + expr: amount + + - name: order_count + expr: 1 + agg: sum + + - name: customers_with_orders + description: Distinct count of customers placing orders + agg: count_distinct + expr: customer_id + + - name: order_value_p99 ## The 99th percentile order value + expr: amount + agg: percentile + agg_params: + percentile: 0.99 + use_discrete_percentile: True + use_approximate_percentile: False \ No newline at end of file diff --git a/models/metrics/orders_metrics.yml b/models/metrics/orders_metrics.yml new file mode 100644 index 00000000..facac76e --- /dev/null +++ b/models/metrics/orders_metrics.yml @@ -0,0 +1,56 @@ +metrics: + # Simple type metrics + - name: "order_total" + description: "Sum of orders value" + type: simple + label: "order_total" + type_params: + measure: + name: order_total + + - name: "order_count" + description: "number of orders" + type: simple + label: "order_count" + type_params: + measure: + name: order_count + + - name: large_orders + description: "Count of orders with order total over 20." + type: simple + label: "Large Orders" + type_params: + measure: + name: order_count + filter: | + {{ Metric('order_total', group_by=['order_id']) }} >= 20 + + # Ratio type metric + - name: "avg_order_value" + label: "avg_order_value" + description: "average value of each order" + type: ratio + type_params: + numerator: order_total + denominator: order_count + + # Cumulative type metrics + - name: "cumulative_order_amount_mtd" + label: "cumulative_order_amount_mtd" + description: "The month to date value of all orders" + type: cumulative + type_params: + measure: + name: order_total + grain_to_date: month + # Derived metric + - name: "pct_of_orders_that_are_large" + label: "pct_of_orders_that_are_large" + description: "percent of orders that are large" + type: derived + type_params: + expr: large_orders/order_count + metrics: + - name: large_orders + - name: order_count \ No newline at end of file diff --git a/models/sem_orders.yml b/models/sem_orders.yml deleted file mode 100644 index f88b9395..00000000 --- a/models/sem_orders.yml +++ /dev/null @@ -1,15 +0,0 @@ -semantic_models: - - name: orders - model: ref('dim_orders') # References the dbt model named `fact_transactions` - description: "This table contains information about orders." - defaults: - agg_time_dimension: order_date - - entities: # Entities included in the table are defined here. MetricFlow will use these columns as join keys. - - name: transaction - type: primary - expr: transaction_id - - name: customer - type: foreign - expr: customer_id - diff --git a/models/semantic_layer_demo/marts/dim_customer.sql b/models/semantic_layer_demo/marts/dim_customer.sql index e7bc0723..b6a21647 100644 --- a/models/semantic_layer_demo/marts/dim_customer.sql +++ b/models/semantic_layer_demo/marts/dim_customer.sql @@ -1,3 +1,10 @@ +{{ + config( + materialized='table', + tags=["semantic_layer_demo"] + ) +}} + with customers as ( select * from {{ ref('stg_jaffle_shop__customers')}} ), From 3f36a3aa2e2e60299f3137549573d01913098d15 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Thu, 5 Dec 2024 02:14:49 +0000 Subject: [PATCH 25/28] Commit changes --- models/metrics/customers_metrics.yml | 8 ++++++++ models/metrics/dim_customers.yml | 10 +--------- models/metrics/orders_metrics.yml | 1 + 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/models/metrics/customers_metrics.yml b/models/metrics/customers_metrics.yml index e69de29b..ab0aefef 100644 --- a/models/metrics/customers_metrics.yml +++ b/models/metrics/customers_metrics.yml @@ -0,0 +1,8 @@ +metrics: + - name: "customers_with_orders" + label: "customers_with_orders" + description: "Unique count of customers placing orders" + type: simple + type_params: + measure: + name: customers \ No newline at end of file diff --git a/models/metrics/dim_customers.yml b/models/metrics/dim_customers.yml index b843609c..68101680 100644 --- a/models/metrics/dim_customers.yml +++ b/models/metrics/dim_customers.yml @@ -42,12 +42,4 @@ semantic_models: - name: customers expr: customer_id agg: count_distinct - -metrics: - - name: "customers_with_orders" - label: "customers_with_orders" - description: "Unique count of customers placing orders" - type: simple - type_params: - measure: - name: customers \ No newline at end of file + \ No newline at end of file diff --git a/models/metrics/orders_metrics.yml b/models/metrics/orders_metrics.yml index facac76e..b5b7d680 100644 --- a/models/metrics/orders_metrics.yml +++ b/models/metrics/orders_metrics.yml @@ -44,6 +44,7 @@ metrics: measure: name: order_total grain_to_date: month + # Derived metric - name: "pct_of_orders_that_are_large" label: "pct_of_orders_that_are_large" From 60cf77937cf9edb75a8d6f135ff7308dd2ea99f3 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Thu, 5 Dec 2024 02:16:07 +0000 Subject: [PATCH 26/28] Commit changes --- .../jaffle_shop/_jaffle_shop__models.yml | 136 +++++++++--------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/models/semantic_layer_demo/staging/jaffle_shop/_jaffle_shop__models.yml b/models/semantic_layer_demo/staging/jaffle_shop/_jaffle_shop__models.yml index 838684c8..eba0774d 100644 --- a/models/semantic_layer_demo/staging/jaffle_shop/_jaffle_shop__models.yml +++ b/models/semantic_layer_demo/staging/jaffle_shop/_jaffle_shop__models.yml @@ -1,72 +1,72 @@ -version: 2 +# version: 2 -models: +# models: - - name: stg_jaffle_shop__orders - description: > - multi line description - of my staging orders model from - jaffle shop - data_tests: - - dbt_utils.recency: - datepart: day - field: _etl_loaded_at - interval: 7 - columns: - - name: order_id - data_tests: - - not_null: - config: - where: _etl_loaded_at >= dateadd('day', -3, current_timestamp()) - severity: error - error_if: ">20" - warn_if: ">10" - limit: 25 - store_failures: true - - filtered_unique: - records_to_check: _etl_loaded_at >= dateadd('day', -3, current_timestamp()) - - name: order_date - description: | - multi line description - of my order date column from my - jaffle shop orders model - data_tests: - - dbt_expectations.expect_column_values_to_be_of_type: - column_type: date +# - name: stg_jaffle_shop__orders +# description: > +# multi line description +# of my staging orders model from +# jaffle shop +# data_tests: +# - dbt_utils.recency: +# datepart: day +# field: _etl_loaded_at +# interval: 7 +# columns: +# - name: order_id +# data_tests: +# - not_null: +# config: +# where: _etl_loaded_at >= dateadd('day', -3, current_timestamp()) +# severity: error +# error_if: ">20" +# warn_if: ">10" +# limit: 25 +# store_failures: true +# - filtered_unique: +# records_to_check: _etl_loaded_at >= dateadd('day', -3, current_timestamp()) +# - name: order_date +# description: | +# multi line description +# of my order date column from my +# jaffle shop orders model +# data_tests: +# - dbt_expectations.expect_column_values_to_be_of_type: +# column_type: date - - name: stg_jaffle_shop__customers - description: | - multi line description - of my staging - jaffle shop customers model - data_tests: - - not_empty - - dbt_utils.equal_rowcount: - compare_model: source('jaffle_shop', 'customers') - - dbt_utils.equality: - compare_model: source('jaffle_shop', 'customers') - compare_columns: - - first_name - - last_name - - dbt_utils.expression_is_true: - expression: "len(first_name) > 1" - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 +# - name: stg_jaffle_shop__customers +# description: | +# multi line description +# of my staging +# jaffle shop customers model +# data_tests: +# - not_empty +# - dbt_utils.equal_rowcount: +# compare_model: source('jaffle_shop', 'customers') +# - dbt_utils.equality: +# compare_model: source('jaffle_shop', 'customers') +# compare_columns: +# - first_name +# - last_name +# - dbt_utils.expression_is_true: +# expression: "len(first_name) > 1" +# - dbt_expectations.expect_table_row_count_to_be_between: +# min_value: 1 - columns: - - name: customer_id - description: > - multi line description - of my customer id column from my - jaffle shop customers model - data_tests: - - unique - - not_null - - unique_and_not_null - - name: first_name - data_tests: - - dbt_utils.at_least_one - - dbt_utils.not_constant - - dbt_utils.not_empty_string - - dbt_utils.not_accepted_values: - values: ['test'] +# columns: +# - name: customer_id +# description: > +# multi line description +# of my customer id column from my +# jaffle shop customers model +# data_tests: +# - unique +# - not_null +# - unique_and_not_null +# - name: first_name +# data_tests: +# - dbt_utils.at_least_one +# - dbt_utils.not_constant +# - dbt_utils.not_empty_string +# - dbt_utils.not_accepted_values: +# values: ['test'] From 03dc3d3e13abea662b284eb10ab53ff392cc63e7 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Thu, 5 Dec 2024 02:30:24 +0000 Subject: [PATCH 27/28] Commit changes --- models/metrics/dim_customers.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/models/metrics/dim_customers.yml b/models/metrics/dim_customers.yml index 68101680..11721250 100644 --- a/models/metrics/dim_customers.yml +++ b/models/metrics/dim_customers.yml @@ -6,7 +6,7 @@ semantic_models: description: | semantic model for dim_customers - model: ref('dim_customers') + model: ref('dim_customer') entities: - name: customer @@ -42,4 +42,3 @@ semantic_models: - name: customers expr: customer_id agg: count_distinct - \ No newline at end of file From 893fcb9ed1e632fb5b204fa5db22fabdb54e4f89 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Thu, 5 Dec 2024 03:21:08 +0000 Subject: [PATCH 28/28] commit updates --- models/metrics/{customers_metrics.yml => met_customers.yml} | 0 models/metrics/{orders_metrics.yml => met_orders.yml} | 0 models/metrics/{dim_customers.yml => sem_dim_customers.yml} | 0 models/metrics/{fact_orders.yml => sem_fact_orders.yml} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename models/metrics/{customers_metrics.yml => met_customers.yml} (100%) rename models/metrics/{orders_metrics.yml => met_orders.yml} (100%) rename models/metrics/{dim_customers.yml => sem_dim_customers.yml} (100%) rename models/metrics/{fact_orders.yml => sem_fact_orders.yml} (100%) diff --git a/models/metrics/customers_metrics.yml b/models/metrics/met_customers.yml similarity index 100% rename from models/metrics/customers_metrics.yml rename to models/metrics/met_customers.yml diff --git a/models/metrics/orders_metrics.yml b/models/metrics/met_orders.yml similarity index 100% rename from models/metrics/orders_metrics.yml rename to models/metrics/met_orders.yml diff --git a/models/metrics/dim_customers.yml b/models/metrics/sem_dim_customers.yml similarity index 100% rename from models/metrics/dim_customers.yml rename to models/metrics/sem_dim_customers.yml diff --git a/models/metrics/fact_orders.yml b/models/metrics/sem_fact_orders.yml similarity index 100% rename from models/metrics/fact_orders.yml rename to models/metrics/sem_fact_orders.yml