From 732a87493ee1ee7a71cad9928459007b802683c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 15:54:32 -0800 Subject: [PATCH 01/29] Add avg_by_grade model --- .../L2_core/mayrapena1324_avg_by_grade.sql | 20 +++++++++++++++++++ .../L2_core/mayrapena1324_avg_by_grade.yml | 9 +++++++++ 2 files changed, 29 insertions(+) create mode 100644 transform/models/L2_core/mayrapena1324_avg_by_grade.sql create mode 100644 transform/models/L2_core/mayrapena1324_avg_by_grade.yml diff --git a/transform/models/L2_core/mayrapena1324_avg_by_grade.sql b/transform/models/L2_core/mayrapena1324_avg_by_grade.sql new file mode 100644 index 0000000..61625c0 --- /dev/null +++ b/transform/models/L2_core/mayrapena1324_avg_by_grade.sql @@ -0,0 +1,20 @@ +with raw_source as ( + + select * from {{ ref('stg_mayrapena1324_personal_loans') }} + +), + +final as ( + + select + grade, + avg(loan_amnt) as avg_loan_amount, + count(*) as total_loans + from raw_source + where loan_status = 'Fully Paid' + group by grade + order by grade + +) + +select * from final diff --git a/transform/models/L2_core/mayrapena1324_avg_by_grade.yml b/transform/models/L2_core/mayrapena1324_avg_by_grade.yml new file mode 100644 index 0000000..143e30c --- /dev/null +++ b/transform/models/L2_core/mayrapena1324_avg_by_grade.yml @@ -0,0 +1,9 @@ +version: 2 + +models: + - name: mayrapena1324_avg_by_grade + description: '' + columns: + - name: grade + - name: avg_loan_amount + - name: total_loans From 0e95536c7b51754b76d431a2ba3969b203d1da0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 15:54:46 -0800 Subject: [PATCH 02/29] Add staging models --- .../L1_staging/loans/_loans_mayrapena1324.yml | 8 +++ .../stg_mayrapena1324_personal_loans.sql | 71 +++++++++++++++++++ .../stg_mayrapena1324_personal_loans.yml | 61 ++++++++++++++++ 3 files changed, 140 insertions(+) create mode 100644 transform/models/L1_staging/loans/_loans_mayrapena1324.yml create mode 100644 transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql create mode 100644 transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml diff --git a/transform/models/L1_staging/loans/_loans_mayrapena1324.yml b/transform/models/L1_staging/loans/_loans_mayrapena1324.yml new file mode 100644 index 0000000..9f4f208 --- /dev/null +++ b/transform/models/L1_staging/loans/_loans_mayrapena1324.yml @@ -0,0 +1,8 @@ +version: 2 + +sources: + - name: MAYRAPENA1324 + database: RAW + tables: + - name: PERSONAL_LOANS + description: '' diff --git a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql new file mode 100644 index 0000000..3e9d375 --- /dev/null +++ b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql @@ -0,0 +1,71 @@ +with raw_source as ( + + select * + from {{ source('MAYRAPENA1324', 'PERSONAL_LOANS') }} + +), + +final as ( + + select + "_AIRBYTE_RAW_ID"::varchar as airbyte_raw_id, + "_AIRBYTE_EXTRACTED_AT"::timestamp_tz as airbyte_extracted_at, + "_AIRBYTE_META"::variant as airbyte_meta, + "TOTAL_ACC"::float as total_acc, + "ANNUAL_INC"::float as annual_inc, + "EMP_LENGTH"::varchar as emp_length, + "DESC"::varchar as desc, + "TOTAL_PYMNT"::float as total_pymnt, + "LAST_PYMNT_D"::varchar as last_pymnt_d, + "ADDR_STATE"::varchar as addr_state, + "NEXT_PYMNT_D"::varchar as next_pymnt_d, + "EMP_TITLE"::varchar as emp_title, + "COLLECTION_RECOVERY_FEE"::float as collection_recovery_fee, + "MTHS_SINCE_LAST_MAJOR_DEROG"::float as mths_since_last_major_derog, + "INQ_LAST_6MTHS"::float as inq_last_6mths, + "SUB_GRADE"::varchar as sub_grade, + "FUNDED_AMNT_INV"::float as funded_amnt_inv, + "DELINQ_2YRS"::float as delinq_2yrs, + "LOAN_ID"::varchar as loan_id, + "FUNDED_AMNT"::float as funded_amnt, + "VERIFICATION_STATUS"::varchar as verification_status, + "DTI"::float as dti, + "TOTAL_REC_PRNCP"::float as total_rec_prncp, + "GRADE"::varchar as grade, + "HOME_OWNERSHIP"::varchar as home_ownership, + "ISSUE_D"::varchar as issue_d, + "MTHS_SINCE_LAST_DELINQ"::float as mths_since_last_delinq, + "OUT_PRNCP"::float as out_prncp, + "PUB_REC"::float as pub_rec, + "INT_RATE"::float as int_rate, + "ZIP_CODE"::varchar as zip_code, + "OPEN_ACC"::float as open_acc, + "TERM"::varchar as term, + "PYMNT_PLAN"::varchar as pymnt_plan, + "URL"::varchar as url, + "REVOL_BAL"::float as revol_bal, + "RECOVERIES"::float as recoveries, + "LAST_PYMNT_AMNT"::float as last_pymnt_amnt, + "LOAN_AMNT"::float as loan_amnt, + "PURPOSE"::varchar as purpose, + "INITIAL_LIST_STATUS"::varchar as initial_list_status, + "TOTAL_REC_INT"::float as total_rec_int, + "TOTAL_PYMNT_INV"::float as total_pymnt_inv, + "MTHS_SINCE_LAST_RECORD"::float as mths_since_last_record, + "LAST_CREDIT_PULL_D"::varchar as last_credit_pull_d, + "TOTAL_REC_LATE_FEE"::float as total_rec_late_fee, + "MEMBER_ID"::float as member_id, + "POLICY_CODE"::float as policy_code, + "TITLE"::varchar as title, + "LOAN_STATUS"::varchar as loan_status, + "INSTALLMENT"::float as installment, + "EARLIEST_CR_LINE"::varchar as earliest_cr_line, + "REVOL_UTIL"::varchar as revol_util, + "OUT_PRNCP_INV"::float as out_prncp_inv, + "COLLECTIONS_12_MTHS_EX_MED"::float as collections_12_mths_ex_med + + from raw_source + +) + +select * from final diff --git a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml new file mode 100644 index 0000000..a4581f0 --- /dev/null +++ b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml @@ -0,0 +1,61 @@ +version: 2 + +models: + - name: PERSONAL_LOANS + description: '' + columns: + - name: airbyte_raw_id + - name: airbyte_extracted_at + - name: airbyte_meta + - name: total_acc + - name: annual_inc + - name: emp_length + - name: desc + - name: total_pymnt + - name: last_pymnt_d + - name: addr_state + - name: next_pymnt_d + - name: emp_title + - name: collection_recovery_fee + - name: mths_since_last_major_derog + - name: inq_last_6mths + - name: sub_grade + - name: funded_amnt_inv + - name: delinq_2yrs + - name: loan_id + - name: funded_amnt + - name: verification_status + - name: dti + - name: total_rec_prncp + - name: grade + - name: home_ownership + - name: issue_d + - name: mths_since_last_delinq + - name: out_prncp + - name: pub_rec + - name: int_rate + - name: zip_code + - name: open_acc + - name: term + - name: pymnt_plan + - name: url + - name: revol_bal + - name: recoveries + - name: last_pymnt_amnt + - name: loan_amnt + - name: purpose + - name: initial_list_status + - name: total_rec_int + - name: total_pymnt_inv + - name: mths_since_last_record + - name: last_credit_pull_d + - name: total_rec_late_fee + - name: member_id + - name: policy_code + - name: title + - name: loan_status + - name: installment + - name: earliest_cr_line + - name: revol_util + - name: out_prncp_inv + - name: collections_12_mths_ex_med From d2503b11bdd96d7dba99caa51513377afb23780b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 16:02:00 -0800 Subject: [PATCH 03/29] Add country populations description --- .../models/L1_staging/country_data/stg_country_populations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transform/models/L1_staging/country_data/stg_country_populations.yml b/transform/models/L1_staging/country_data/stg_country_populations.yml index de9a565..8407b4f 100644 --- a/transform/models/L1_staging/country_data/stg_country_populations.yml +++ b/transform/models/L1_staging/country_data/stg_country_populations.yml @@ -2,7 +2,7 @@ version: 2 models: - name: COUNTRY_POPULATIONS - description: '' + description: 'A country populations table' columns: - name: year description: The year for which the population value is recorded From 79b77d2f03e52786f97bf7d23f885d6c77556def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 16:35:07 -0800 Subject: [PATCH 04/29] Add test --- .../L1_staging/country_data/stg_country_populations.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transform/models/L1_staging/country_data/stg_country_populations.yml b/transform/models/L1_staging/country_data/stg_country_populations.yml index 63d6f1c..01b7c01 100644 --- a/transform/models/L1_staging/country_data/stg_country_populations.yml +++ b/transform/models/L1_staging/country_data/stg_country_populations.yml @@ -6,10 +6,10 @@ models: columns: - name: year description: The year for which the population value is recorded - data_tests: - - not_null - name: country_name description: The name of the country + data_tests: + - unique - name: value description: The population value for a particular year and country - name: country_code From 0cf3648152e9d45891507923f0eaddaab7cafdfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 16:39:50 -0800 Subject: [PATCH 05/29] Test changing name --- .../models/L1_staging/country_data/stg_country_populations.sql | 2 +- .../models/L1_staging/country_data/stg_country_populations.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/transform/models/L1_staging/country_data/stg_country_populations.sql b/transform/models/L1_staging/country_data/stg_country_populations.sql index 688f17d..d89717f 100644 --- a/transform/models/L1_staging/country_data/stg_country_populations.sql +++ b/transform/models/L1_staging/country_data/stg_country_populations.sql @@ -1,7 +1,7 @@ with raw_source as ( select * - from {{ source('COUNTRY_DATA', 'COUNTRY_POPULATIONS') }} + from {{ source('COUNTRY_DATA', 'STG_COUNTRY_POPULATIONS') }} ), diff --git a/transform/models/L1_staging/country_data/stg_country_populations.yml b/transform/models/L1_staging/country_data/stg_country_populations.yml index 01b7c01..c366d62 100644 --- a/transform/models/L1_staging/country_data/stg_country_populations.yml +++ b/transform/models/L1_staging/country_data/stg_country_populations.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: COUNTRY_POPULATIONS + - name: STG_COUNTRY_POPULATIONS description: 'Raw population information from Github Datasets repository' columns: - name: year From 6975c6881577cb0aeb8ef204a237126c52618de3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 13:15:16 -0800 Subject: [PATCH 06/29] Fix yml reference --- .../models/L1_staging/country_data/stg_country_populations.sql | 2 +- .../L1_staging/loans/stg_mayrapena1324_personal_loans.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/transform/models/L1_staging/country_data/stg_country_populations.sql b/transform/models/L1_staging/country_data/stg_country_populations.sql index d89717f..688f17d 100644 --- a/transform/models/L1_staging/country_data/stg_country_populations.sql +++ b/transform/models/L1_staging/country_data/stg_country_populations.sql @@ -1,7 +1,7 @@ with raw_source as ( select * - from {{ source('COUNTRY_DATA', 'STG_COUNTRY_POPULATIONS') }} + from {{ source('COUNTRY_DATA', 'COUNTRY_POPULATIONS') }} ), diff --git a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml index a4581f0..08ad9e1 100644 --- a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml +++ b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: PERSONAL_LOANS + - name: STG_MAYRAPENA1324_PERSONAL_LOANS description: '' columns: - name: airbyte_raw_id From c6d18ba2045704e356756db48d6eb578d60a35fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 13:38:52 -0800 Subject: [PATCH 07/29] Change role for macro --- transform/macros/create_database.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transform/macros/create_database.sql b/transform/macros/create_database.sql index 9245dd2..795211a 100644 --- a/transform/macros/create_database.sql +++ b/transform/macros/create_database.sql @@ -12,7 +12,7 @@ identifier="tables") -%} {% if not database_exists %} {% set create_db_sql %} - use role transformer_dbt; + use role analyst; create database {{ target.database }}; grant ownership on database {{ target.database }} to role {{ target.role }}; use role {{ target.role }}; From ba351f1482caf20219b08a87fc90c4200a1e5669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 13:43:34 -0800 Subject: [PATCH 08/29] Make scripts executable --- automate/dbt/get_artifacts.sh | 0 automate/dbt/remove_test_databases.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 automate/dbt/get_artifacts.sh mode change 100644 => 100755 automate/dbt/remove_test_databases.sh diff --git a/automate/dbt/get_artifacts.sh b/automate/dbt/get_artifacts.sh old mode 100644 new mode 100755 diff --git a/automate/dbt/remove_test_databases.sh b/automate/dbt/remove_test_databases.sh old mode 100644 new mode 100755 From 1d1aac1787f67cddea2b85c36680fae612ed5488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 13:55:04 -0800 Subject: [PATCH 09/29] Update get_last_artifacts --- transform/macros/cicd/get_last_artifacts.sql | 40 ++++++++++++++------ 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/transform/macros/cicd/get_last_artifacts.sql b/transform/macros/cicd/get_last_artifacts.sql index 80331a2..0ee6104 100644 --- a/transform/macros/cicd/get_last_artifacts.sql +++ b/transform/macros/cicd/get_last_artifacts.sql @@ -1,30 +1,48 @@ {# Macro for returning dbt manifest from a snowflake stage. #} {# dbt run-operation get_last_artifacts -#} + #} {# Once this is completed, deferral and state modifiers are available using --state logs #} -{% macro get_last_artifacts(stage = 'RAW.DBT_ARTIFACTS.ARTIFACTS') %} - {# we will put the manifest.json in the log directory and use the with the --state param in dbt #} +{% macro get_last_artifacts() %} + {# Fallback variable used to run/debug macro in vscode #} + {% set stage_name = 'RAW.DBT_ARTIFACTS.ARTIFACTS' %} + + {# We will put the manifest.json in the log directory and use it with the --state param in dbt #} {% set logs_dir = env_var('DATACOVES__DBT_HOME') ~ "/logs/" %} {# List only the .json files in the root folder (excludes archive dir) #} {% set list_stage_query %} - LIST @{{ stage }} PATTERN = '^((?!(archive/)).)*.json$'; + LIST @{{ stage_name }} PATTERN = '^((?!(archive/)).)*.json$'; {% endset %} - {{ print("\nCurrent items in stage " ~ stage) }} + {{ print("\nCurrent items in stage " ~ stage_name) }} {% set results = run_query(list_stage_query) %} {{ results.exclude('md5').print_table(max_column_width=40) }} {{ print("\n" ~ "="*85) }} - {% set artifacts_destination = "file://" + logs_dir %} + {% if results and results.rows %} - {% set get_query %} - get @{{ stage }}/manifest.json {{ artifacts_destination }}; - get @{{ stage }}/catalog.json {{ artifacts_destination }}; - {% endset %} + {% set artifacts_destination = "file://" + logs_dir %} + + {# Download and print manifest.json #} + {% set get_manifest_query %} + get @{{ stage_name }}/manifest.json {{ artifacts_destination }}; + {% endset %} + {% set download_manifest_results = run_query(get_manifest_query) %}x + {{ print("Manifest Downloaded") }} + {{ download_manifest_results.print_table(max_column_width=40) }} + + {# Download and print catalog.json #} + {% set get_catalog_query %} + get @{{ stage_name }}/catalog.json {{ artifacts_destination }}; + {% endset %} + {% set download_catalog_results = run_query(get_catalog_query) %} + {{ print("Catalog Downloaded") }} + {{ download_catalog_results.print_table(max_column_width=40) }} - {% set results = run_query(get_query) %} + {% else %} + {{ print("No artifacts found in stage " ~ stage_name ~ ". Skipping file download.") }} + {% endif %} {% endmacro %} From ac0031ddee16835ca676a189044ba575468476bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 14:17:43 -0800 Subject: [PATCH 10/29] Update get_artifacts.sh to handle no manifest directory. --- automate/dbt/get_artifacts.sh | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/automate/dbt/get_artifacts.sh b/automate/dbt/get_artifacts.sh index 0c537bf..485cd01 100755 --- a/automate/dbt/get_artifacts.sh +++ b/automate/dbt/get_artifacts.sh @@ -1,30 +1,35 @@ - #! /bin/bash -# Cause script to exit on error +# Cause script to exit on errors except for controlled checks set -e +# Navigate to the DBT project directory cd $DATACOVES__DBT_HOME mkdir -p logs dbt run-operation get_last_artifacts -LINES_IN_MANIFEST="$(grep -c '^' logs/manifest.json)" -if [ $LINES_IN_MANIFEST -eq 0 ] -then - echo "Manifest not found in Snowflake stage, contact the Snowflake administrator to load a updated manifest to snowflake." +# Check if manifest,son exist, count lines if does or set to 0 +if [ -e "logs/manifest.json" ]; then + LINES_IN_MANIFEST="$(grep -c '^' logs/manifest.json)" +else + LINES_IN_MANIFEST="0" +fi + +if [ $LINES_IN_MANIFEST -eq 0 ]; then + echo "Production manifest for dbt was not found in Snowflake." # This is used by github actions - echo "::set-output name=manifest_found::false" + # echo "::set-output name=manifest_found::false" # This is used by Jenkins - # echo "false" > temp_MANIFEST_FOUND.txt + echo "false" >> temp_MANIFEST_FOUND.txt else echo "Updated manifest from production" # This is used by github actions - echo "::set-output name=manifest_found::true" + # echo "::set-output name=manifest_found::true" # This is used by Jenkins - # echo "true" > temp_MANIFEST_FOUND.txt + echo "true" >> temp_MANIFEST_FOUND.txt fi From ffa78a866bc7036bab43f889293a79b2d8c49782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 14:29:08 -0800 Subject: [PATCH 11/29] Testing Personal_loans --- .../L1_staging/loans/{_loans_mayrapena1324.yml => _loans.yml} | 0 ...{stg_mayrapena1324_personal_loans.sql => personal_loans.sql} | 0 ...{stg_mayrapena1324_personal_loans.yml => personal_loans.yml} | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename transform/models/L1_staging/loans/{_loans_mayrapena1324.yml => _loans.yml} (100%) rename transform/models/L1_staging/loans/{stg_mayrapena1324_personal_loans.sql => personal_loans.sql} (100%) rename transform/models/L1_staging/loans/{stg_mayrapena1324_personal_loans.yml => personal_loans.yml} (97%) diff --git a/transform/models/L1_staging/loans/_loans_mayrapena1324.yml b/transform/models/L1_staging/loans/_loans.yml similarity index 100% rename from transform/models/L1_staging/loans/_loans_mayrapena1324.yml rename to transform/models/L1_staging/loans/_loans.yml diff --git a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql b/transform/models/L1_staging/loans/personal_loans.sql similarity index 100% rename from transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql rename to transform/models/L1_staging/loans/personal_loans.sql diff --git a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml b/transform/models/L1_staging/loans/personal_loans.yml similarity index 97% rename from transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml rename to transform/models/L1_staging/loans/personal_loans.yml index 08ad9e1..a4581f0 100644 --- a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml +++ b/transform/models/L1_staging/loans/personal_loans.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: STG_MAYRAPENA1324_PERSONAL_LOANS + - name: PERSONAL_LOANS description: '' columns: - name: airbyte_raw_id From 7f5711fda726c2266d6aec673b4b307e2ab70bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 14:30:33 -0800 Subject: [PATCH 12/29] Fix personal_loans ref --- transform/models/L2_core/mayrapena1324_avg_by_grade.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transform/models/L2_core/mayrapena1324_avg_by_grade.sql b/transform/models/L2_core/mayrapena1324_avg_by_grade.sql index 61625c0..f2e10c3 100644 --- a/transform/models/L2_core/mayrapena1324_avg_by_grade.sql +++ b/transform/models/L2_core/mayrapena1324_avg_by_grade.sql @@ -1,6 +1,6 @@ with raw_source as ( - select * from {{ ref('stg_mayrapena1324_personal_loans') }} + select * from {{ ref('personal_loans') }} ), From 57e498cdc54d0f5a777f276481567bb85b3bf0bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 14:38:49 -0800 Subject: [PATCH 13/29] Testing --- .github/workflows/pull_request_build.yml | 5 +++++ transform/dbt_project.yml | 4 ++++ transform/models/L1_staging/loans/personal_loans.yml | 2 +- ...rsonal_loans.sql => stg_mayrapena1324_personal_loans.sql} | 0 transform/models/L2_core/mayrapena1324_avg_by_grade.sql | 2 +- 5 files changed, 11 insertions(+), 2 deletions(-) rename transform/models/L1_staging/loans/{personal_loans.sql => stg_mayrapena1324_personal_loans.sql} (100%) diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index d7d5d05..04ef306 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -87,6 +87,11 @@ jobs: run: "dbt build --fail-fast --empty" - name: Generate Docs Combining Prod and branch catalog.json + if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' && contains(github.event.pull_request.labels.*.name, 'full-refresh') != true }} + run: "dbt-coves generate docs --merge-deferred --state logs" + + - name: Generate Docs Combining Prod and branch catalog.json + if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' || contains(github.event.pull_request.labels.*.name, 'full-refresh') }} run: "dbt-coves generate docs --merge-deferred --state logs" - name: Run governance checks diff --git a/transform/dbt_project.yml b/transform/dbt_project.yml index 9ce6984..a558ad5 100644 --- a/transform/dbt_project.yml +++ b/transform/dbt_project.yml @@ -37,6 +37,10 @@ models: datacoves_starter_project: L1_staging: +materialized: view + loans: + +materialized: view + country_data: + +materialized: view L2_core: +materialized: view L3_marts: diff --git a/transform/models/L1_staging/loans/personal_loans.yml b/transform/models/L1_staging/loans/personal_loans.yml index a4581f0..08ad9e1 100644 --- a/transform/models/L1_staging/loans/personal_loans.yml +++ b/transform/models/L1_staging/loans/personal_loans.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: PERSONAL_LOANS + - name: STG_MAYRAPENA1324_PERSONAL_LOANS description: '' columns: - name: airbyte_raw_id diff --git a/transform/models/L1_staging/loans/personal_loans.sql b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql similarity index 100% rename from transform/models/L1_staging/loans/personal_loans.sql rename to transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql diff --git a/transform/models/L2_core/mayrapena1324_avg_by_grade.sql b/transform/models/L2_core/mayrapena1324_avg_by_grade.sql index f2e10c3..61625c0 100644 --- a/transform/models/L2_core/mayrapena1324_avg_by_grade.sql +++ b/transform/models/L2_core/mayrapena1324_avg_by_grade.sql @@ -1,6 +1,6 @@ with raw_source as ( - select * from {{ ref('personal_loans') }} + select * from {{ ref('stg_mayrapena1324_personal_loans') }} ), From d9f1cec74e0330cc156036b506ffe8729ad883eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 14:43:32 -0800 Subject: [PATCH 14/29] Add descriptions. --- transform/models/L1_staging/loans/_loans.yml | 2 +- transform/models/L1_staging/loans/personal_loans.yml | 2 +- transform/models/L2_core/mayrapena1324_avg_by_grade.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/transform/models/L1_staging/loans/_loans.yml b/transform/models/L1_staging/loans/_loans.yml index 9f4f208..26b9fd6 100644 --- a/transform/models/L1_staging/loans/_loans.yml +++ b/transform/models/L1_staging/loans/_loans.yml @@ -5,4 +5,4 @@ sources: database: RAW tables: - name: PERSONAL_LOANS - description: '' + description: 'A personal loans source table' diff --git a/transform/models/L1_staging/loans/personal_loans.yml b/transform/models/L1_staging/loans/personal_loans.yml index 08ad9e1..102a97c 100644 --- a/transform/models/L1_staging/loans/personal_loans.yml +++ b/transform/models/L1_staging/loans/personal_loans.yml @@ -2,7 +2,7 @@ version: 2 models: - name: STG_MAYRAPENA1324_PERSONAL_LOANS - description: '' + description: 'A staging model for personal loans' columns: - name: airbyte_raw_id - name: airbyte_extracted_at diff --git a/transform/models/L2_core/mayrapena1324_avg_by_grade.yml b/transform/models/L2_core/mayrapena1324_avg_by_grade.yml index 143e30c..dfc2b70 100644 --- a/transform/models/L2_core/mayrapena1324_avg_by_grade.yml +++ b/transform/models/L2_core/mayrapena1324_avg_by_grade.yml @@ -2,7 +2,7 @@ version: 2 models: - name: mayrapena1324_avg_by_grade - description: '' + description: 'An average loan amount by grade' columns: - name: grade - name: avg_loan_amount From 0e51c07ba7cb3c88a569eac53b0fe9056fbd1d10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 14:45:38 -0800 Subject: [PATCH 15/29] Rename stg yml --- .../{personal_loans.yml => stg_mayrapena1324_personal_loans.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename transform/models/L1_staging/loans/{personal_loans.yml => stg_mayrapena1324_personal_loans.yml} (100%) diff --git a/transform/models/L1_staging/loans/personal_loans.yml b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml similarity index 100% rename from transform/models/L1_staging/loans/personal_loans.yml rename to transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml From aa509a698358e8d3f0cd2aa580326ce7dd381036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 14:50:22 -0800 Subject: [PATCH 16/29] Change caps --- .../L1_staging/loans/stg_mayrapena1324_personal_loans.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml index 102a97c..3e2ffe1 100644 --- a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml +++ b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: STG_MAYRAPENA1324_PERSONAL_LOANS + - name: stg_mayrapena1324_personal_loans description: 'A staging model for personal loans' columns: - name: airbyte_raw_id From cbe6747971ef63e513afd74f06a0f6cbfcc00a25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 14:54:44 -0800 Subject: [PATCH 17/29] Change target --- automate/dbt/profiles.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/automate/dbt/profiles.yml b/automate/dbt/profiles.yml index d91e94b..840aaa7 100644 --- a/automate/dbt/profiles.yml +++ b/automate/dbt/profiles.yml @@ -1,11 +1,11 @@ default: - target: default_target + target: dev outputs: - default_target: + dev: type: snowflake threads: 16 client_session_keep_alive: true - + account: "{{ env_var('DATACOVES__MAIN__ACCOUNT') }}" database: "{{ env_var('DATACOVES__MAIN__DATABASE') }}" schema: "{{ env_var('DATACOVES__MAIN__SCHEMA') }}" @@ -13,4 +13,3 @@ default: password: "{{ env_var('DATACOVES__MAIN__PASSWORD') }}" role: "{{ env_var('DATACOVES__MAIN__ROLE') }}" warehouse: "{{ env_var('DATACOVES__MAIN__WAREHOUSE') }}" - From f5b7bc2c6139664dbbb4a8059c5a42770685d543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 14:57:16 -0800 Subject: [PATCH 18/29] Change back to default_target and remove has columns --- .pre-commit-config.yaml | 3 +-- automate/dbt/profiles.yml | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index be781c8..bc1580d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ repos: - id: check-script-ref-and-source - id: check-model-has-description - id: check-model-has-properties-file - - id: check-model-has-all-columns + # - id: check-model-has-all-columns - repo: https://github.com/sqlfluff/sqlfluff @@ -40,4 +40,3 @@ repos: - id: yamllint args: [-c=.yamllint] exclude: ^transform/.dbt_coves/templates - diff --git a/automate/dbt/profiles.yml b/automate/dbt/profiles.yml index 840aaa7..b4d437c 100644 --- a/automate/dbt/profiles.yml +++ b/automate/dbt/profiles.yml @@ -1,7 +1,7 @@ default: - target: dev + target: default_target outputs: - dev: + default_target: type: snowflake threads: 16 client_session_keep_alive: true From 61f67d3d80d8330ca347973b8e8a1c4a3847e067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 15:09:07 -0800 Subject: [PATCH 19/29] Update pre-commit-config.yml --- .pre-commit-config.yaml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bc1580d..151bfd7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,8 @@ files: ^transform/models/ repos: - - repo: https://github.com/dbt-checkpoint/dbt-checkpoint - rev: v2.0.1 + rev: v2.0.5 hooks: - id: check-source-table-has-description @@ -12,12 +11,13 @@ repos: - id: check-script-ref-and-source - id: check-model-has-description - id: check-model-has-properties-file - # - id: check-model-has-all-columns - + - id: check-model-has-all-columns + - id: check-database-casing-consistency + always_run: true - repo: https://github.com/sqlfluff/sqlfluff # this is the version of sqlfluff, needs to be updated when using a new sqlfluff version (pip show sqlfluff) - rev: 2.3.2 + rev: 3.1.1 hooks: - id: sqlfluff-lint language: python @@ -25,17 +25,15 @@ repos: # sqlfluff-templater-dbt should match the version of sqlfluff above in rev (pip show sqlfluff-templater-dbt) # dbt-snowflake needs to match the version in transform tab of Datacoves (pip show dbt-snowflake) additional_dependencies: - # ["sqlfluff-templater-dbt==2.3.2", "dbt-snowflake==1.6.8", dbt-core==1.6.9] [ - "sqlfluff-templater-dbt==2.3.2", - "dbt-redshift==1.6.7", - dbt-core==1.6.9, + "sqlfluff-templater-dbt==3.1.1", + "dbt-core==1.8.7", + "dbt-snowflake==1.8.4", ] - args: [--config, "transform/.sqlfluff"] - + args: [--config, transform/.sqlfluff] - repo: https://github.com/adrienverge/yamllint.git - rev: v1.17.0 + rev: v1.35.1 hooks: - id: yamllint args: [-c=.yamllint] From ea59aecc8f62f9bf67a43e40d0f7d35325c6d2db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 15:14:06 -0800 Subject: [PATCH 20/29] Remove checks --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 151bfd7..ef6f3d0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,8 +11,8 @@ repos: - id: check-script-ref-and-source - id: check-model-has-description - id: check-model-has-properties-file - - id: check-model-has-all-columns - - id: check-database-casing-consistency + # - id: check-model-has-all-columns + # - id: check-database-casing-consistency always_run: true - repo: https://github.com/sqlfluff/sqlfluff From a616818d71242a42d349f043f33907a1ae6c3bc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 15:19:11 -0800 Subject: [PATCH 21/29] Move ymllint to root --- transform/.yamllint => .yamllint | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename transform/.yamllint => .yamllint (100%) diff --git a/transform/.yamllint b/.yamllint similarity index 100% rename from transform/.yamllint rename to .yamllint From c3c331669b995130f80bf8d3ceeb3e86ea1d9d4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 15:29:16 -0800 Subject: [PATCH 22/29] Change role --- transform/macros/cicd/grant_access_to_pr_database.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/transform/macros/cicd/grant_access_to_pr_database.sql b/transform/macros/cicd/grant_access_to_pr_database.sql index 2cd4223..039da96 100644 --- a/transform/macros/cicd/grant_access_to_pr_database.sql +++ b/transform/macros/cicd/grant_access_to_pr_database.sql @@ -5,7 +5,7 @@ #} {%- macro grant_access_to_pr_database() -%} - {% set db_role_name = 'z_db_balboa_tst' %} + {% set db_role_name = 'analyst' %} {% set db_name = target.database %} {% set apply_db_grants_sql %} @@ -13,8 +13,8 @@ {% endset %} {% do run_query(apply_db_grants_sql) %} - {% set schemas_list %} + select schema_name from {{ db_name }}.information_schema.schemata where schema_name not in ('INFORMATION_SCHEMA','PUBLIC','DBT_TEST__AUDIT') @@ -24,9 +24,9 @@ {% for schema in schemas %} {% set apply_schema_grants_sql %} - grant usage on schema {{db_name}}.{{ schema[0] }} to z_schema_{{schema[0]}}; - grant select on all tables in schema {{db_name}}.{{ schema[0] }} to role z_tables_views_general; - grant select on all views in schema {{db_name}}.{{ schema[0] }} to role z_tables_views_general; + grant usage on schema {{db_name}}.{{ schema[0] }} to {{schema[0]}}; + grant select on all tables in schema {{db_name}}.{{ schema[0] }} to role analyst; + grant select on all views in schema {{db_name}}.{{ schema[0] }} to role analyst; {% endset %} {% do run_query(apply_schema_grants_sql) %} From 87b701b688c9ccb5edfe1c2d2461f6839a692228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 15:41:41 -0800 Subject: [PATCH 23/29] Change casing --- .../models/L1_staging/country_data/stg_country_populations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transform/models/L1_staging/country_data/stg_country_populations.yml b/transform/models/L1_staging/country_data/stg_country_populations.yml index c366d62..588ea97 100644 --- a/transform/models/L1_staging/country_data/stg_country_populations.yml +++ b/transform/models/L1_staging/country_data/stg_country_populations.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: STG_COUNTRY_POPULATIONS + - name: stg_country_populations description: 'Raw population information from Github Datasets repository' columns: - name: year From 5542fed6b8b700aad7d5d2b9d4a9e839484b0e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 15:45:32 -0800 Subject: [PATCH 24/29] Remove additional if statements --- .github/workflows/pull_request_build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index 04ef306..fd41f62 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -83,7 +83,7 @@ jobs: run: "dbt build --fail-fast --defer --state logs --select state:modified+ --empty" - name: Governance run of dbt with EMPTY models using full run - if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' || contains(github.event.pull_request.labels.*.name, 'full-refresh') }} + if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }} run: "dbt build --fail-fast --empty" - name: Generate Docs Combining Prod and branch catalog.json @@ -91,7 +91,7 @@ jobs: run: "dbt-coves generate docs --merge-deferred --state logs" - name: Generate Docs Combining Prod and branch catalog.json - if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' || contains(github.event.pull_request.labels.*.name, 'full-refresh') }} + if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }} run: "dbt-coves generate docs --merge-deferred --state logs" - name: Run governance checks @@ -103,7 +103,7 @@ jobs: run: "dbt build --fail-fast --defer --state logs --select state:modified+" - name: Run dbt build full run - if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' || contains(github.event.pull_request.labels.*.name, 'full-refresh') }} + if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }} run: "dbt build --fail-fast" - name: Grant access to PR database From c50c4041cb3dd2778468c77f93beaccf329c19f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 18:14:07 -0800 Subject: [PATCH 25/29] Update get_artifacts. fix GA script. --- automate/dbt/get_artifacts.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/automate/dbt/get_artifacts.sh b/automate/dbt/get_artifacts.sh index 485cd01..7182f43 100755 --- a/automate/dbt/get_artifacts.sh +++ b/automate/dbt/get_artifacts.sh @@ -1,9 +1,8 @@ #! /bin/bash -# Cause script to exit on errors except for controlled checks +# Cause script to exit on error set -e -# Navigate to the DBT project directory cd $DATACOVES__DBT_HOME mkdir -p logs @@ -12,24 +11,25 @@ dbt run-operation get_last_artifacts # Check if manifest,son exist, count lines if does or set to 0 if [ -e "logs/manifest.json" ]; then - LINES_IN_MANIFEST="$(grep -c '^' logs/manifest.json)" +LINES_IN_MANIFEST="$(grep -c '^' logs/manifest.json)" else LINES_IN_MANIFEST="0" fi -if [ $LINES_IN_MANIFEST -eq 0 ]; then - echo "Production manifest for dbt was not found in Snowflake." +if [ $LINES_IN_MANIFEST -eq 0 ] +then + echo "Manifest not found in Snowflake stage, contact the Snowflake administrator to load a updated manifest to snowflake." # This is used by github actions - # echo "::set-output name=manifest_found::false" + echo "::set-output name=manifest_found::false" # This is used by Jenkins - echo "false" >> temp_MANIFEST_FOUND.txt + # echo "false" > temp_MANIFEST_FOUND.txt else echo "Updated manifest from production" # This is used by github actions - # echo "::set-output name=manifest_found::true" + echo "::set-output name=manifest_found::true" # This is used by Jenkins - echo "true" >> temp_MANIFEST_FOUND.txt + # echo "true" > temp_MANIFEST_FOUND.txt fi From 52d08e8f70486b72ee6953139beb98a5f32a0339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 18:14:19 -0800 Subject: [PATCH 26/29] Fix github script --- .github/workflows/pull_request_build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index fd41f62..04ef306 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -83,7 +83,7 @@ jobs: run: "dbt build --fail-fast --defer --state logs --select state:modified+ --empty" - name: Governance run of dbt with EMPTY models using full run - if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }} + if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' || contains(github.event.pull_request.labels.*.name, 'full-refresh') }} run: "dbt build --fail-fast --empty" - name: Generate Docs Combining Prod and branch catalog.json @@ -91,7 +91,7 @@ jobs: run: "dbt-coves generate docs --merge-deferred --state logs" - name: Generate Docs Combining Prod and branch catalog.json - if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }} + if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' || contains(github.event.pull_request.labels.*.name, 'full-refresh') }} run: "dbt-coves generate docs --merge-deferred --state logs" - name: Run governance checks @@ -103,7 +103,7 @@ jobs: run: "dbt build --fail-fast --defer --state logs --select state:modified+" - name: Run dbt build full run - if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }} + if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' || contains(github.event.pull_request.labels.*.name, 'full-refresh') }} run: "dbt build --fail-fast" - name: Grant access to PR database From 9851b41bad8250578bc739f1a4eea1eb32962148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 18:17:10 -0800 Subject: [PATCH 27/29] Remove extra generate docs step --- .github/workflows/pull_request_build.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index 04ef306..fad51e9 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -90,10 +90,6 @@ jobs: if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' && contains(github.event.pull_request.labels.*.name, 'full-refresh') != true }} run: "dbt-coves generate docs --merge-deferred --state logs" - - name: Generate Docs Combining Prod and branch catalog.json - if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' || contains(github.event.pull_request.labels.*.name, 'full-refresh') }} - run: "dbt-coves generate docs --merge-deferred --state logs" - - name: Run governance checks run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD" From f441d19da7071dc6ac0d1200e190a7288ef03161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 18:20:01 -0800 Subject: [PATCH 28/29] Change data test --- .../models/L1_staging/country_data/stg_country_populations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transform/models/L1_staging/country_data/stg_country_populations.yml b/transform/models/L1_staging/country_data/stg_country_populations.yml index 588ea97..3bf856f 100644 --- a/transform/models/L1_staging/country_data/stg_country_populations.yml +++ b/transform/models/L1_staging/country_data/stg_country_populations.yml @@ -9,7 +9,7 @@ models: - name: country_name description: The name of the country data_tests: - - unique + - not_null - name: value description: The population value for a particular year and country - name: country_code From 5e8c9002b40c1d4a7fb39a97ac5c0c3110e6d42c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Sun, 10 Nov 2024 18:30:05 -0800 Subject: [PATCH 29/29] debug grant_access_to_pr_database macro --- transform/macros/cicd/grant_access_to_pr_database.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/transform/macros/cicd/grant_access_to_pr_database.sql b/transform/macros/cicd/grant_access_to_pr_database.sql index 039da96..97a08ef 100644 --- a/transform/macros/cicd/grant_access_to_pr_database.sql +++ b/transform/macros/cicd/grant_access_to_pr_database.sql @@ -18,15 +18,16 @@ select schema_name from {{ db_name }}.information_schema.schemata where schema_name not in ('INFORMATION_SCHEMA','PUBLIC','DBT_TEST__AUDIT') + {{print(schema_list)}} {% endset %} {% set schemas = run_query(schemas_list) %} {% for schema in schemas %} {% set apply_schema_grants_sql %} - grant usage on schema {{db_name}}.{{ schema[0] }} to {{schema[0]}}; - grant select on all tables in schema {{db_name}}.{{ schema[0] }} to role analyst; - grant select on all views in schema {{db_name}}.{{ schema[0] }} to role analyst; + grant usage on schema {{db_name}}.{{ schema[0] }} to {{db_role_name}}; + grant select on all tables in schema {{db_name}}.{{ schema[0] }} to role {{db_role_name}}; + grant select on all views in schema {{db_name}}.{{ schema[0] }} to role {{db_role_name}}; {% endset %} {% do run_query(apply_schema_grants_sql) %}