From 8e7f72052540660163f63fd969ed2eb03163d9c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Wed, 13 Nov 2024 12:13:19 -0800 Subject: [PATCH 1/3] Testing --- .pre-commit-config.yaml | 2 +- .../L1_staging/loans/_loans_mayrapena1324.yml | 8 +++ .../stg_mayrapena1324_personal_loans.sql | 71 +++++++++++++++++++ .../stg_mayrapena1324_personal_loans.yml | 61 ++++++++++++++++ 4 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 transform/models/L1_staging/loans/_loans_mayrapena1324.yml create mode 100644 transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql create mode 100644 transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 94e6f43..2dbac27 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ files: ^transform/models/ repos: - repo: https://github.com/dbt-checkpoint/dbt-checkpoint - rev: v2.0.5 + rev: v2.0.6 hooks: - id: check-source-table-has-description diff --git a/transform/models/L1_staging/loans/_loans_mayrapena1324.yml b/transform/models/L1_staging/loans/_loans_mayrapena1324.yml new file mode 100644 index 0000000..b21d575 --- /dev/null +++ b/transform/models/L1_staging/loans/_loans_mayrapena1324.yml @@ -0,0 +1,8 @@ +version: 2 + +sources: + - name: MAYRAPENA1324 + database: RAW + tables: + - name: PERSONAL_LOANS + description: 'Personal Loan table' diff --git a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql new file mode 100644 index 0000000..3e9d375 --- /dev/null +++ b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql @@ -0,0 +1,71 @@ +with raw_source as ( + + select * + from {{ source('MAYRAPENA1324', 'PERSONAL_LOANS') }} + +), + +final as ( + + select + "_AIRBYTE_RAW_ID"::varchar as airbyte_raw_id, + "_AIRBYTE_EXTRACTED_AT"::timestamp_tz as airbyte_extracted_at, + "_AIRBYTE_META"::variant as airbyte_meta, + "TOTAL_ACC"::float as total_acc, + "ANNUAL_INC"::float as annual_inc, + "EMP_LENGTH"::varchar as emp_length, + "DESC"::varchar as desc, + "TOTAL_PYMNT"::float as total_pymnt, + "LAST_PYMNT_D"::varchar as last_pymnt_d, + "ADDR_STATE"::varchar as addr_state, + "NEXT_PYMNT_D"::varchar as next_pymnt_d, + "EMP_TITLE"::varchar as emp_title, + "COLLECTION_RECOVERY_FEE"::float as collection_recovery_fee, + "MTHS_SINCE_LAST_MAJOR_DEROG"::float as mths_since_last_major_derog, + "INQ_LAST_6MTHS"::float as inq_last_6mths, + "SUB_GRADE"::varchar as sub_grade, + "FUNDED_AMNT_INV"::float as funded_amnt_inv, + "DELINQ_2YRS"::float as delinq_2yrs, + "LOAN_ID"::varchar as loan_id, + "FUNDED_AMNT"::float as funded_amnt, + "VERIFICATION_STATUS"::varchar as verification_status, + "DTI"::float as dti, + "TOTAL_REC_PRNCP"::float as total_rec_prncp, + "GRADE"::varchar as grade, + "HOME_OWNERSHIP"::varchar as home_ownership, + "ISSUE_D"::varchar as issue_d, + "MTHS_SINCE_LAST_DELINQ"::float as mths_since_last_delinq, + "OUT_PRNCP"::float as out_prncp, + "PUB_REC"::float as pub_rec, + "INT_RATE"::float as int_rate, + "ZIP_CODE"::varchar as zip_code, + "OPEN_ACC"::float as open_acc, + "TERM"::varchar as term, + "PYMNT_PLAN"::varchar as pymnt_plan, + "URL"::varchar as url, + "REVOL_BAL"::float as revol_bal, + "RECOVERIES"::float as recoveries, + "LAST_PYMNT_AMNT"::float as last_pymnt_amnt, + "LOAN_AMNT"::float as loan_amnt, + "PURPOSE"::varchar as purpose, + "INITIAL_LIST_STATUS"::varchar as initial_list_status, + "TOTAL_REC_INT"::float as total_rec_int, + "TOTAL_PYMNT_INV"::float as total_pymnt_inv, + "MTHS_SINCE_LAST_RECORD"::float as mths_since_last_record, + "LAST_CREDIT_PULL_D"::varchar as last_credit_pull_d, + "TOTAL_REC_LATE_FEE"::float as total_rec_late_fee, + "MEMBER_ID"::float as member_id, + "POLICY_CODE"::float as policy_code, + "TITLE"::varchar as title, + "LOAN_STATUS"::varchar as loan_status, + "INSTALLMENT"::float as installment, + "EARLIEST_CR_LINE"::varchar as earliest_cr_line, + "REVOL_UTIL"::varchar as revol_util, + "OUT_PRNCP_INV"::float as out_prncp_inv, + "COLLECTIONS_12_MTHS_EX_MED"::float as collections_12_mths_ex_med + + from raw_source + +) + +select * from final diff --git a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml new file mode 100644 index 0000000..a4581f0 --- /dev/null +++ b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml @@ -0,0 +1,61 @@ +version: 2 + +models: + - name: PERSONAL_LOANS + description: '' + columns: + - name: airbyte_raw_id + - name: airbyte_extracted_at + - name: airbyte_meta + - name: total_acc + - name: annual_inc + - name: emp_length + - name: desc + - name: total_pymnt + - name: last_pymnt_d + - name: addr_state + - name: next_pymnt_d + - name: emp_title + - name: collection_recovery_fee + - name: mths_since_last_major_derog + - name: inq_last_6mths + - name: sub_grade + - name: funded_amnt_inv + - name: delinq_2yrs + - name: loan_id + - name: funded_amnt + - name: verification_status + - name: dti + - name: total_rec_prncp + - name: grade + - name: home_ownership + - name: issue_d + - name: mths_since_last_delinq + - name: out_prncp + - name: pub_rec + - name: int_rate + - name: zip_code + - name: open_acc + - name: term + - name: pymnt_plan + - name: url + - name: revol_bal + - name: recoveries + - name: last_pymnt_amnt + - name: loan_amnt + - name: purpose + - name: initial_list_status + - name: total_rec_int + - name: total_pymnt_inv + - name: mths_since_last_record + - name: last_credit_pull_d + - name: total_rec_late_fee + - name: member_id + - name: policy_code + - name: title + - name: loan_status + - name: installment + - name: earliest_cr_line + - name: revol_util + - name: out_prncp_inv + - name: collections_12_mths_ex_med From 2f0915c57a3a3120b2eae5acd600b877e0b07ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mayra=20Pe=C3=B1a?= Date: Wed, 13 Nov 2024 12:21:06 -0800 Subject: [PATCH 2/3] Change name of model to match file --- .../L1_staging/loans/stg_mayrapena1324_personal_loans.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml index a4581f0..08ad9e1 100644 --- a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml +++ b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: PERSONAL_LOANS + - name: STG_MAYRAPENA1324_PERSONAL_LOANS description: '' columns: - name: airbyte_raw_id From 5cfce0cce7012d4342f89b9e651a0e936d5f8d7c Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Wed, 13 Nov 2024 15:17:42 -0800 Subject: [PATCH 3/3] update to model name --- .../templates/staging_model_props.yml | 4 +- .../L1_staging/loans/_loans_mayrapena1324.yml | 8 --- .../stg_mayrapena1324_personal_loans.sql | 71 ------------------- .../stg_mayrapena1324_personal_loans.yml | 61 ---------------- 4 files changed, 2 insertions(+), 142 deletions(-) delete mode 100644 transform/models/L1_staging/loans/_loans_mayrapena1324.yml delete mode 100644 transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql delete mode 100644 transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml diff --git a/transform/.dbt_coves/templates/staging_model_props.yml b/transform/.dbt_coves/templates/staging_model_props.yml index 8725bf0..0ca65e8 100644 --- a/transform/.dbt_coves/templates/staging_model_props.yml +++ b/transform/.dbt_coves/templates/staging_model_props.yml @@ -1,8 +1,8 @@ version: 2 models: - - name: {{model}} - description: '' + - name: stg_{{relation.schema | lower }}_{{model | lower }} + description: 'TESTING' columns: {%- for cols in nested.values() %} {%- for col in cols %} diff --git a/transform/models/L1_staging/loans/_loans_mayrapena1324.yml b/transform/models/L1_staging/loans/_loans_mayrapena1324.yml deleted file mode 100644 index b21d575..0000000 --- a/transform/models/L1_staging/loans/_loans_mayrapena1324.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: 2 - -sources: - - name: MAYRAPENA1324 - database: RAW - tables: - - name: PERSONAL_LOANS - description: 'Personal Loan table' diff --git a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql deleted file mode 100644 index 3e9d375..0000000 --- a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql +++ /dev/null @@ -1,71 +0,0 @@ -with raw_source as ( - - select * - from {{ source('MAYRAPENA1324', 'PERSONAL_LOANS') }} - -), - -final as ( - - select - "_AIRBYTE_RAW_ID"::varchar as airbyte_raw_id, - "_AIRBYTE_EXTRACTED_AT"::timestamp_tz as airbyte_extracted_at, - "_AIRBYTE_META"::variant as airbyte_meta, - "TOTAL_ACC"::float as total_acc, - "ANNUAL_INC"::float as annual_inc, - "EMP_LENGTH"::varchar as emp_length, - "DESC"::varchar as desc, - "TOTAL_PYMNT"::float as total_pymnt, - "LAST_PYMNT_D"::varchar as last_pymnt_d, - "ADDR_STATE"::varchar as addr_state, - "NEXT_PYMNT_D"::varchar as next_pymnt_d, - "EMP_TITLE"::varchar as emp_title, - "COLLECTION_RECOVERY_FEE"::float as collection_recovery_fee, - "MTHS_SINCE_LAST_MAJOR_DEROG"::float as mths_since_last_major_derog, - "INQ_LAST_6MTHS"::float as inq_last_6mths, - "SUB_GRADE"::varchar as sub_grade, - "FUNDED_AMNT_INV"::float as funded_amnt_inv, - "DELINQ_2YRS"::float as delinq_2yrs, - "LOAN_ID"::varchar as loan_id, - "FUNDED_AMNT"::float as funded_amnt, - "VERIFICATION_STATUS"::varchar as verification_status, - "DTI"::float as dti, - "TOTAL_REC_PRNCP"::float as total_rec_prncp, - "GRADE"::varchar as grade, - "HOME_OWNERSHIP"::varchar as home_ownership, - "ISSUE_D"::varchar as issue_d, - "MTHS_SINCE_LAST_DELINQ"::float as mths_since_last_delinq, - "OUT_PRNCP"::float as out_prncp, - "PUB_REC"::float as pub_rec, - "INT_RATE"::float as int_rate, - "ZIP_CODE"::varchar as zip_code, - "OPEN_ACC"::float as open_acc, - "TERM"::varchar as term, - "PYMNT_PLAN"::varchar as pymnt_plan, - "URL"::varchar as url, - "REVOL_BAL"::float as revol_bal, - "RECOVERIES"::float as recoveries, - "LAST_PYMNT_AMNT"::float as last_pymnt_amnt, - "LOAN_AMNT"::float as loan_amnt, - "PURPOSE"::varchar as purpose, - "INITIAL_LIST_STATUS"::varchar as initial_list_status, - "TOTAL_REC_INT"::float as total_rec_int, - "TOTAL_PYMNT_INV"::float as total_pymnt_inv, - "MTHS_SINCE_LAST_RECORD"::float as mths_since_last_record, - "LAST_CREDIT_PULL_D"::varchar as last_credit_pull_d, - "TOTAL_REC_LATE_FEE"::float as total_rec_late_fee, - "MEMBER_ID"::float as member_id, - "POLICY_CODE"::float as policy_code, - "TITLE"::varchar as title, - "LOAN_STATUS"::varchar as loan_status, - "INSTALLMENT"::float as installment, - "EARLIEST_CR_LINE"::varchar as earliest_cr_line, - "REVOL_UTIL"::varchar as revol_util, - "OUT_PRNCP_INV"::float as out_prncp_inv, - "COLLECTIONS_12_MTHS_EX_MED"::float as collections_12_mths_ex_med - - from raw_source - -) - -select * from final diff --git a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml b/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml deleted file mode 100644 index 08ad9e1..0000000 --- a/transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml +++ /dev/null @@ -1,61 +0,0 @@ -version: 2 - -models: - - name: STG_MAYRAPENA1324_PERSONAL_LOANS - description: '' - columns: - - name: airbyte_raw_id - - name: airbyte_extracted_at - - name: airbyte_meta - - name: total_acc - - name: annual_inc - - name: emp_length - - name: desc - - name: total_pymnt - - name: last_pymnt_d - - name: addr_state - - name: next_pymnt_d - - name: emp_title - - name: collection_recovery_fee - - name: mths_since_last_major_derog - - name: inq_last_6mths - - name: sub_grade - - name: funded_amnt_inv - - name: delinq_2yrs - - name: loan_id - - name: funded_amnt - - name: verification_status - - name: dti - - name: total_rec_prncp - - name: grade - - name: home_ownership - - name: issue_d - - name: mths_since_last_delinq - - name: out_prncp - - name: pub_rec - - name: int_rate - - name: zip_code - - name: open_acc - - name: term - - name: pymnt_plan - - name: url - - name: revol_bal - - name: recoveries - - name: last_pymnt_amnt - - name: loan_amnt - - name: purpose - - name: initial_list_status - - name: total_rec_int - - name: total_pymnt_inv - - name: mths_since_last_record - - name: last_credit_pull_d - - name: total_rec_late_fee - - name: member_id - - name: policy_code - - name: title - - name: loan_status - - name: installment - - name: earliest_cr_line - - name: revol_util - - name: out_prncp_inv - - name: collections_12_mths_ex_med