-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #2 from datacoves/feature/mayrapena1324
Feature/mayrapena1324
- Loading branch information
Showing
16 changed files
with
230 additions
and
36 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,30 +1,48 @@ | ||
| {# Macro for returning dbt manifest from a snowflake stage. #} | ||
| {# | ||
| dbt run-operation get_last_artifacts | ||
| #} | ||
| #} | ||
| {# Once this is completed, deferral and state modifiers are available using --state logs #} | ||
|
|
||
| {% macro get_last_artifacts(stage = 'RAW.DBT_ARTIFACTS.ARTIFACTS') %} | ||
| {# we will put the manifest.json in the log directory and use the with the --state param in dbt #} | ||
| {% macro get_last_artifacts() %} | ||
| {# Fallback variable used to run/debug macro in vscode #} | ||
| {% set stage_name = 'RAW.DBT_ARTIFACTS.ARTIFACTS' %} | ||
|
|
||
| {# We will put the manifest.json in the log directory and use it with the --state param in dbt #} | ||
| {% set logs_dir = env_var('DATACOVES__DBT_HOME') ~ "/logs/" %} | ||
|
|
||
| {# List only the .json files in the root folder (excludes archive dir) #} | ||
| {% set list_stage_query %} | ||
| LIST @{{ stage }} PATTERN = '^((?!(archive/)).)*.json$'; | ||
| LIST @{{ stage_name }} PATTERN = '^((?!(archive/)).)*.json$'; | ||
| {% endset %} | ||
|
|
||
| {{ print("\nCurrent items in stage " ~ stage) }} | ||
| {{ print("\nCurrent items in stage " ~ stage_name) }} | ||
| {% set results = run_query(list_stage_query) %} | ||
| {{ results.exclude('md5').print_table(max_column_width=40) }} | ||
| {{ print("\n" ~ "="*85) }} | ||
|
|
||
| {% set artifacts_destination = "file://" + logs_dir %} | ||
| {% if results and results.rows %} | ||
|
|
||
| {% set get_query %} | ||
| get @{{ stage }}/manifest.json {{ artifacts_destination }}; | ||
| get @{{ stage }}/catalog.json {{ artifacts_destination }}; | ||
| {% endset %} | ||
| {% set artifacts_destination = "file://" + logs_dir %} | ||
|
|
||
| {# Download and print manifest.json #} | ||
| {% set get_manifest_query %} | ||
| get @{{ stage_name }}/manifest.json {{ artifacts_destination }}; | ||
| {% endset %} | ||
| {% set download_manifest_results = run_query(get_manifest_query) %}x | ||
| {{ print("Manifest Downloaded") }} | ||
| {{ download_manifest_results.print_table(max_column_width=40) }} | ||
|
|
||
| {# Download and print catalog.json #} | ||
| {% set get_catalog_query %} | ||
| get @{{ stage_name }}/catalog.json {{ artifacts_destination }}; | ||
| {% endset %} | ||
| {% set download_catalog_results = run_query(get_catalog_query) %} | ||
| {{ print("Catalog Downloaded") }} | ||
| {{ download_catalog_results.print_table(max_column_width=40) }} | ||
|
|
||
| {% set results = run_query(get_query) %} | ||
| {% else %} | ||
| {{ print("No artifacts found in stage " ~ stage_name ~ ". Skipping file download.") }} | ||
| {% endif %} | ||
|
|
||
| {% endmacro %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
6 changes: 3 additions & 3 deletions
6
transform/models/L1_staging/country_data/stg_country_populations.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| version: 2 | ||
|
|
||
| sources: | ||
| - name: MAYRAPENA1324 | ||
| database: RAW | ||
| tables: | ||
| - name: PERSONAL_LOANS | ||
| description: 'A personal loans source table' |
71 changes: 71 additions & 0 deletions
71
transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| with raw_source as ( | ||
|
|
||
| select * | ||
| from {{ source('MAYRAPENA1324', 'PERSONAL_LOANS') }} | ||
|
|
||
| ), | ||
|
|
||
| final as ( | ||
|
|
||
| select | ||
| "_AIRBYTE_RAW_ID"::varchar as airbyte_raw_id, | ||
| "_AIRBYTE_EXTRACTED_AT"::timestamp_tz as airbyte_extracted_at, | ||
| "_AIRBYTE_META"::variant as airbyte_meta, | ||
| "TOTAL_ACC"::float as total_acc, | ||
| "ANNUAL_INC"::float as annual_inc, | ||
| "EMP_LENGTH"::varchar as emp_length, | ||
| "DESC"::varchar as desc, | ||
| "TOTAL_PYMNT"::float as total_pymnt, | ||
| "LAST_PYMNT_D"::varchar as last_pymnt_d, | ||
| "ADDR_STATE"::varchar as addr_state, | ||
| "NEXT_PYMNT_D"::varchar as next_pymnt_d, | ||
| "EMP_TITLE"::varchar as emp_title, | ||
| "COLLECTION_RECOVERY_FEE"::float as collection_recovery_fee, | ||
| "MTHS_SINCE_LAST_MAJOR_DEROG"::float as mths_since_last_major_derog, | ||
| "INQ_LAST_6MTHS"::float as inq_last_6mths, | ||
| "SUB_GRADE"::varchar as sub_grade, | ||
| "FUNDED_AMNT_INV"::float as funded_amnt_inv, | ||
| "DELINQ_2YRS"::float as delinq_2yrs, | ||
| "LOAN_ID"::varchar as loan_id, | ||
| "FUNDED_AMNT"::float as funded_amnt, | ||
| "VERIFICATION_STATUS"::varchar as verification_status, | ||
| "DTI"::float as dti, | ||
| "TOTAL_REC_PRNCP"::float as total_rec_prncp, | ||
| "GRADE"::varchar as grade, | ||
| "HOME_OWNERSHIP"::varchar as home_ownership, | ||
| "ISSUE_D"::varchar as issue_d, | ||
| "MTHS_SINCE_LAST_DELINQ"::float as mths_since_last_delinq, | ||
| "OUT_PRNCP"::float as out_prncp, | ||
| "PUB_REC"::float as pub_rec, | ||
| "INT_RATE"::float as int_rate, | ||
| "ZIP_CODE"::varchar as zip_code, | ||
| "OPEN_ACC"::float as open_acc, | ||
| "TERM"::varchar as term, | ||
| "PYMNT_PLAN"::varchar as pymnt_plan, | ||
| "URL"::varchar as url, | ||
| "REVOL_BAL"::float as revol_bal, | ||
| "RECOVERIES"::float as recoveries, | ||
| "LAST_PYMNT_AMNT"::float as last_pymnt_amnt, | ||
| "LOAN_AMNT"::float as loan_amnt, | ||
| "PURPOSE"::varchar as purpose, | ||
| "INITIAL_LIST_STATUS"::varchar as initial_list_status, | ||
| "TOTAL_REC_INT"::float as total_rec_int, | ||
| "TOTAL_PYMNT_INV"::float as total_pymnt_inv, | ||
| "MTHS_SINCE_LAST_RECORD"::float as mths_since_last_record, | ||
| "LAST_CREDIT_PULL_D"::varchar as last_credit_pull_d, | ||
| "TOTAL_REC_LATE_FEE"::float as total_rec_late_fee, | ||
| "MEMBER_ID"::float as member_id, | ||
| "POLICY_CODE"::float as policy_code, | ||
| "TITLE"::varchar as title, | ||
| "LOAN_STATUS"::varchar as loan_status, | ||
| "INSTALLMENT"::float as installment, | ||
| "EARLIEST_CR_LINE"::varchar as earliest_cr_line, | ||
| "REVOL_UTIL"::varchar as revol_util, | ||
| "OUT_PRNCP_INV"::float as out_prncp_inv, | ||
| "COLLECTIONS_12_MTHS_EX_MED"::float as collections_12_mths_ex_med | ||
|
|
||
| from raw_source | ||
|
|
||
| ) | ||
|
|
||
| select * from final |
61 changes: 61 additions & 0 deletions
61
transform/models/L1_staging/loans/stg_mayrapena1324_personal_loans.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| version: 2 | ||
|
|
||
| models: | ||
| - name: stg_mayrapena1324_personal_loans | ||
| description: 'A staging model for personal loans' | ||
| columns: | ||
| - name: airbyte_raw_id | ||
| - name: airbyte_extracted_at | ||
| - name: airbyte_meta | ||
| - name: total_acc | ||
| - name: annual_inc | ||
| - name: emp_length | ||
| - name: desc | ||
| - name: total_pymnt | ||
| - name: last_pymnt_d | ||
| - name: addr_state | ||
| - name: next_pymnt_d | ||
| - name: emp_title | ||
| - name: collection_recovery_fee | ||
| - name: mths_since_last_major_derog | ||
| - name: inq_last_6mths | ||
| - name: sub_grade | ||
| - name: funded_amnt_inv | ||
| - name: delinq_2yrs | ||
| - name: loan_id | ||
| - name: funded_amnt | ||
| - name: verification_status | ||
| - name: dti | ||
| - name: total_rec_prncp | ||
| - name: grade | ||
| - name: home_ownership | ||
| - name: issue_d | ||
| - name: mths_since_last_delinq | ||
| - name: out_prncp | ||
| - name: pub_rec | ||
| - name: int_rate | ||
| - name: zip_code | ||
| - name: open_acc | ||
| - name: term | ||
| - name: pymnt_plan | ||
| - name: url | ||
| - name: revol_bal | ||
| - name: recoveries | ||
| - name: last_pymnt_amnt | ||
| - name: loan_amnt | ||
| - name: purpose | ||
| - name: initial_list_status | ||
| - name: total_rec_int | ||
| - name: total_pymnt_inv | ||
| - name: mths_since_last_record | ||
| - name: last_credit_pull_d | ||
| - name: total_rec_late_fee | ||
| - name: member_id | ||
| - name: policy_code | ||
| - name: title | ||
| - name: loan_status | ||
| - name: installment | ||
| - name: earliest_cr_line | ||
| - name: revol_util | ||
| - name: out_prncp_inv | ||
| - name: collections_12_mths_ex_med |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| with raw_source as ( | ||
|
|
||
| select * from {{ ref('stg_mayrapena1324_personal_loans') }} | ||
|
|
||
| ), | ||
|
|
||
| final as ( | ||
|
|
||
| select | ||
| grade, | ||
| avg(loan_amnt) as avg_loan_amount, | ||
| count(*) as total_loans | ||
| from raw_source | ||
| where loan_status = 'Fully Paid' | ||
| group by grade | ||
| order by grade | ||
|
|
||
| ) | ||
|
|
||
| select * from final |
Oops, something went wrong.