From 03f39d2fc3b615d22a43ff73c47bcaf18e4cdcdb Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 28 Aug 2023 16:18:45 -0500 Subject: [PATCH 01/10] MagicBot/add-union-schema updates --- CHANGELOG.md | 11 ++++++++++ README.md | 12 ++++++++++ dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- models/amazon_ads.yml | 22 +++++++++++++++++++ models/amazon_ads__account_report.sql | 7 +++++- models/amazon_ads__ad_group_report.sql | 9 +++++++- models/amazon_ads__ad_report.sql | 14 +++++++++--- models/amazon_ads__campaign_report.sql | 12 +++++++--- models/amazon_ads__keyword_report.sql | 10 ++++++++- models/amazon_ads__portfolio_report.sql | 8 ++++++- models/amazon_ads__search_report.sql | 10 ++++++++- models/docs.md | 3 +++ .../int_amazon_ads__portfolio_history.sql | 2 ++ packages.yml | 4 ++++ 15 files changed, 115 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dbfded..72b4bf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +# dbt_amazon_ads v0.3.0 +[PR #11](https://github.com/fivetran/dbt_amazon_ads/pull/11) includes the following updates: +## Feature update πŸŽ‰ +- Unioning capability! This adds the ability to union source data from multiple amazon_ads connectors. Refer to the [README](https://github.com/fivetran/dbt_amazon_ads/blob/main/README.md) for more details. + +## Under the hood 🚘 +- In the source package, updated tmp models to union source data using the `fivetran_utils.union_data` macro. +- To distinguish which source each field comes from, added `source_relation` column in each staging and downstream model and applied the `fivetran_utils.source_relation` macro. +- Updated tests to account for the new `source_relation` column. + - The `source_relation` column is included in all joins and window function partition clauses in the transform package. + # dbt_amazon_ads v0.2.0 [PR #6](https://github.com/fivetran/dbt_amazon_ads/pull/6) includes the following updates: ## 🚨 Breaking changes diff --git a/README.md b/README.md index 841109c..cfd10c2 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,18 @@ vars: ## (Optional) Step 5: Additional configurations
Expand for configurations +### Union multiple connectors +If you have multiple amazon_ads connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `amazon_ads_union_schemas` OR `amazon_ads_union_databases` variables (cannot do both) in your root `dbt_project.yml` file: + +```yml +vars: + amazon_ads_union_schemas: ['amazon_ads_usa','amazon_ads_canada'] # use this if the data is in different schemas/datasets of the same database/project + amazon_ads_union_databases: ['amazon_ads_usa','amazon_ads_canada'] # use this if the data is in different databases/projects but uses the same schema name +``` +Please be aware that the native `source.yml` connection set up in the package will not function when the union schema/database feature is utilized. Although the data will be correctly combined, you will not observe the sources linked to the package models in the Directed Acyclic Graph (DAG). This happens because the package includes only one defined `source.yml`. + +To connect your multiple schema/database sources to the package models, follow the steps outlined in the [Union Data Defined Sources Configuration](https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source) section of the Fivetran Utils documentation for the union_data macro. This will ensure a proper configuration and correct visualization of connections in the DAG. + ### Passing Through Additional Metrics By default, this package will select `clicks`, `impressions`, and `cost` from the source reporting tables to store into the staging models. If you would like to pass through additional metrics to the staging models, add the following configurations to your `dbt_project.yml` file. These variables allow the pass-through fields to be aliased (`alias`) if desired, but not required. Use the following format for declaring the respective pass-through variables: diff --git a/dbt_project.yml b/dbt_project.yml index 2810ecf..40728d2 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'amazon_ads' -version: '0.2.0' +version: '0.3.0' config-version: 2 require-dbt-version: [">=1.3.0", "<2.0.0"] models: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index bd5a18a..2227c80 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'amazon_ads_integration_tests' -version: '0.2.0' +version: '0.3.0' profile: 'integration_tests' diff --git a/models/amazon_ads.yml b/models/amazon_ads.yml index 8e10c7b..fe86074 100644 --- a/models/amazon_ads.yml +++ b/models/amazon_ads.yml @@ -1,3 +1,4 @@ +UPDATE TESTS AND CHECK SOURCE_RELATION ADDED PROPERLY!!! (delete line when done.) version: 2 models: @@ -6,9 +7,12 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: + - source_relation - date_day - profile_id columns: + - name: source_relation + description: "{{ doc('source_relation') }}" - name: date_day description: "{{ doc('report_date') }}" - name: profile_id @@ -35,9 +39,12 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: + - source_relation - date_day - ad_group_id columns: + - name: source_relation + description: "{{ doc('source_relation') }}" - name: date_day description: "{{ doc('report_date') }}" - name: account_id @@ -82,9 +89,12 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: + - source_relation - date_day - ad_id columns: + - name: source_relation + description: "{{ doc('source_relation') }}" - name: date_day description: "{{ doc('report_date') }}" - name: account_id @@ -137,9 +147,12 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: + - source_relation - date_day - campaign_id columns: + - name: source_relation + description: "{{ doc('source_relation') }}" - name: date_day description: "{{ doc('report_date') }}" - name: account_id @@ -180,9 +193,12 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: + - source_relation - date_day - portfolio_id columns: + - name: source_relation + description: "{{ doc('source_relation') }}" - name: date_day description: "{{ doc('report_date') }}" - name: account_id @@ -227,9 +243,12 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: + - source_relation - date_day - keyword_id columns: + - name: source_relation + description: "{{ doc('source_relation') }}" - name: date_day description: "{{ doc('report_date') }}" - name: account_id @@ -280,10 +299,13 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: + - source_relation - date_day - search_term - keyword_id columns: + - name: source_relation + description: "{{ doc('source_relation') }}" - name: date_day description: "{{ doc('report_date') }}" - name: account_id diff --git a/models/amazon_ads__account_report.sql b/models/amazon_ads__account_report.sql index 2fe34cc..c1da5c0 100644 --- a/models/amazon_ads__account_report.sql +++ b/models/amazon_ads__account_report.sql @@ -1,3 +1,5 @@ +ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) + {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} with report as ( @@ -20,6 +22,7 @@ campaigns as ( fields as ( select + report.source_relation, report.date_day, account_info.account_name, account_info.account_id, @@ -36,11 +39,13 @@ fields as ( left join campaigns on campaigns.campaign_id = report.campaign_id + and campaigns.source_relation = report.source_relation left join account_info on account_info.profile_id = campaigns.profile_id + and account_info.source_relation = campaigns.source_relation - {{ dbt_utils.group_by(5) }} + {{ dbt_utils.group_by(6) }} ) select * diff --git a/models/amazon_ads__ad_group_report.sql b/models/amazon_ads__ad_group_report.sql index b3a330f..1484df3 100644 --- a/models/amazon_ads__ad_group_report.sql +++ b/models/amazon_ads__ad_group_report.sql @@ -1,3 +1,5 @@ +ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) + {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} with report as ( @@ -30,6 +32,7 @@ ad_groups as ( fields as ( select + report.source_relation, report.date_day, account_info.account_name, account_info.account_id, @@ -55,14 +58,18 @@ fields as ( left join ad_groups on ad_groups.ad_group_id = report.ad_group_id + and ad_groups.source_relation = report.source_relation left join campaigns on campaigns.campaign_id = ad_groups.campaign_id + and campaigns.source_relation = ad_groups.source_relation left join portfolios on portfolios.portfolio_id = campaigns.portfolio_id + and portfolios.source_relation = campaigns.source_relation left join account_info on account_info.profile_id = campaigns.profile_id + and account_info.source_relation = campaigns.source_relation - {{ dbt_utils.group_by(15) }} + {{ dbt_utils.group_by(16) }} ) select * diff --git a/models/amazon_ads__ad_report.sql b/models/amazon_ads__ad_report.sql index 8a8da2f..c061e23 100644 --- a/models/amazon_ads__ad_report.sql +++ b/models/amazon_ads__ad_report.sql @@ -1,3 +1,5 @@ +ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) + {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} with report as ( @@ -36,6 +38,7 @@ ads as ( fields as ( select + report.source_relation, report.date_day, account_info.account_name, account_info.account_id, @@ -65,16 +68,21 @@ fields as ( left join ads on ads.ad_id = report.ad_id + and ads.source_relation = report.source_relation left join ad_groups on ad_groups.ad_group_id = report.ad_group_id + and ad_groups.source_relation = report.source_relation left join campaigns on campaigns.campaign_id = report.campaign_id + and campaigns.source_relation = report.source_relation left join portfolios - on portfolios.portfolio_id = campaigns.portfolio_id + on portfolios.portfolio_id = campaigns.portfolio_id + and portfolios.source_relation = campaigns.source_relation left join account_info - on account_info.profile_id = campaigns.profile_id + on account_info.profile_id = campaigns.profile_id + and account_info.source_relation = campaigns.source_relation - {{ dbt_utils.group_by(19) }} + {{ dbt_utils.group_by(20) }} ) select * diff --git a/models/amazon_ads__campaign_report.sql b/models/amazon_ads__campaign_report.sql index 4e406af..a2e0650 100644 --- a/models/amazon_ads__campaign_report.sql +++ b/models/amazon_ads__campaign_report.sql @@ -1,3 +1,5 @@ +ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) + {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} with report as ( @@ -24,6 +26,7 @@ campaigns as ( fields as ( select + report.source_relation, report.date_day, account_info.account_name, account_info.account_id, @@ -47,12 +50,15 @@ fields as ( left join campaigns on campaigns.campaign_id = report.campaign_id + and campaigns.source_relation = report.source_relation left join portfolios - on portfolios.portfolio_id = campaigns.portfolio_id + on portfolios.portfolio_id = campaigns.portfolio_id + and portfolios.source_relation = campaigns.source_relation left join account_info - on account_info.profile_id = campaigns.profile_id + on account_info.profile_id = campaigns.profile_id + and account_info.source_relation = campaigns.source_relation - {{ dbt_utils.group_by(13) }} + {{ dbt_utils.group_by(14) }} ) select * diff --git a/models/amazon_ads__keyword_report.sql b/models/amazon_ads__keyword_report.sql index 0f9299c..b54f7b8 100644 --- a/models/amazon_ads__keyword_report.sql +++ b/models/amazon_ads__keyword_report.sql @@ -1,3 +1,5 @@ +ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) + {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} with report as ( @@ -36,6 +38,7 @@ keywords as ( fields as ( select + report.source_relation, report.date_day, account_info.account_name, account_info.account_id, @@ -64,16 +67,21 @@ fields as ( left join keywords on keywords.keyword_id = report.keyword_id + and keywords.source_relation = report.source_relation left join ad_groups on ad_groups.ad_group_id = report.ad_group_id + and ad_groups.source_relation = report.source_relation left join campaigns on campaigns.campaign_id = report.campaign_id + and campaigns.source_relation = report.source_relation left join portfolios on portfolios.portfolio_id = campaigns.portfolio_id + and portfolios.source_relation = campaigns.source_relation left join account_info on account_info.profile_id = campaigns.profile_id + and account_info.source_relation = campaigns.source_relation - {{ dbt_utils.group_by(18) }} + {{ dbt_utils.group_by(19) }} ) select * diff --git a/models/amazon_ads__portfolio_report.sql b/models/amazon_ads__portfolio_report.sql index e887ebc..1e1622a 100644 --- a/models/amazon_ads__portfolio_report.sql +++ b/models/amazon_ads__portfolio_report.sql @@ -1,3 +1,5 @@ +ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) + {{ config(enabled=fivetran_utils.enabled_vars(['ad_reporting__amazon_ads_enabled','amazon_ads__portfolio_history_enabled'])) }} with report as ( @@ -26,6 +28,7 @@ campaigns as ( fields as ( select + report.source_relation, report.date_day, account_info.account_name, account_info.account_id, @@ -52,12 +55,15 @@ fields as ( left join campaigns on campaigns.portfolio_id = portfolios.portfolio_id + and campaigns.source_relation = portfolios.source_relation left join account_info on account_info.profile_id = campaigns.profile_id + and account_info.source_relation = campaigns.source_relation left join report on report.campaign_id = campaigns.campaign_id + and report.source_relation = campaigns.source_relation - {{ dbt_utils.group_by(15) }} + {{ dbt_utils.group_by(16) }} ) select * diff --git a/models/amazon_ads__search_report.sql b/models/amazon_ads__search_report.sql index 13a367b..91a18df 100644 --- a/models/amazon_ads__search_report.sql +++ b/models/amazon_ads__search_report.sql @@ -1,3 +1,5 @@ +ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) + {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} with report as ( @@ -36,6 +38,7 @@ keywords as ( fields as ( select + report.source_relation, report.date_day, account_info.account_name, account_info.account_id, @@ -64,16 +67,21 @@ fields as ( left join keywords on keywords.keyword_id = report.keyword_id + and keywords.source_relation = report.source_relation left join ad_groups on ad_groups.ad_group_id = report.ad_group_id + and ad_groups.source_relation = report.source_relation left join campaigns on campaigns.campaign_id = report.campaign_id + and campaigns.source_relation = report.source_relation left join portfolios on portfolios.portfolio_id = campaigns.portfolio_id + and portfolios.source_relation = campaigns.source_relation left join account_info on account_info.profile_id = campaigns.profile_id + and account_info.source_relation = campaigns.source_relation - {{ dbt_utils.group_by(18) }} + {{ dbt_utils.group_by(19) }} ) select * diff --git a/models/docs.md b/models/docs.md index 0c9b12f..9485ba5 100644 --- a/models/docs.md +++ b/models/docs.md @@ -176,4 +176,7 @@ The state of the record (enabled, paused, or archived). {% docs targeting %} A string representation of the expression object used in the targeting clause. +{% enddocs %} +{% docs source_relation %} +The source of the record if the unioning functionality is being used. If not this field will be empty. {% enddocs %} \ No newline at end of file diff --git a/models/intermediate/int_amazon_ads__portfolio_history.sql b/models/intermediate/int_amazon_ads__portfolio_history.sql index 6fd8718..972f190 100644 --- a/models/intermediate/int_amazon_ads__portfolio_history.sql +++ b/models/intermediate/int_amazon_ads__portfolio_history.sql @@ -1,3 +1,5 @@ +ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) + {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} {# This intermediate model creates a dummy portfolio table if the user does not use portfolios. diff --git a/packages.yml b/packages.yml index 09e4a43..4bcc0b2 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,7 @@ packages: - package: fivetran/amazon_ads_source version: [">=0.2.0", "<0.3.0"] + +- git: https://github.com/fivetran/amazon_ads_source.git + revision: MagicBot/add-union-schema + warn-unpinned: false \ No newline at end of file From f8682f43a23c4cab60895d7f19793bb610a37096 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 29 Aug 2023 14:41:27 -0500 Subject: [PATCH 02/10] revise magicbot --- models/amazon_ads.yml | 1 - models/amazon_ads__account_report.sql | 2 -- models/amazon_ads__ad_group_report.sql | 2 -- models/amazon_ads__ad_report.sql | 2 -- models/amazon_ads__campaign_report.sql | 2 -- models/amazon_ads__keyword_report.sql | 2 -- models/amazon_ads__portfolio_report.sql | 2 -- models/amazon_ads__search_report.sql | 2 -- models/docs.md | 1 + models/intermediate/int_amazon_ads__portfolio_history.sql | 3 +-- packages.yml | 6 +++--- 11 files changed, 5 insertions(+), 20 deletions(-) diff --git a/models/amazon_ads.yml b/models/amazon_ads.yml index fe86074..f52f93b 100644 --- a/models/amazon_ads.yml +++ b/models/amazon_ads.yml @@ -1,4 +1,3 @@ -UPDATE TESTS AND CHECK SOURCE_RELATION ADDED PROPERLY!!! (delete line when done.) version: 2 models: diff --git a/models/amazon_ads__account_report.sql b/models/amazon_ads__account_report.sql index c1da5c0..dbc2301 100644 --- a/models/amazon_ads__account_report.sql +++ b/models/amazon_ads__account_report.sql @@ -1,5 +1,3 @@ -ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) - {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} with report as ( diff --git a/models/amazon_ads__ad_group_report.sql b/models/amazon_ads__ad_group_report.sql index 1484df3..6f7f927 100644 --- a/models/amazon_ads__ad_group_report.sql +++ b/models/amazon_ads__ad_group_report.sql @@ -1,5 +1,3 @@ -ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) - {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} with report as ( diff --git a/models/amazon_ads__ad_report.sql b/models/amazon_ads__ad_report.sql index c061e23..46da202 100644 --- a/models/amazon_ads__ad_report.sql +++ b/models/amazon_ads__ad_report.sql @@ -1,5 +1,3 @@ -ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) - {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} with report as ( diff --git a/models/amazon_ads__campaign_report.sql b/models/amazon_ads__campaign_report.sql index a2e0650..f5128b1 100644 --- a/models/amazon_ads__campaign_report.sql +++ b/models/amazon_ads__campaign_report.sql @@ -1,5 +1,3 @@ -ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) - {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} with report as ( diff --git a/models/amazon_ads__keyword_report.sql b/models/amazon_ads__keyword_report.sql index b54f7b8..0e882f7 100644 --- a/models/amazon_ads__keyword_report.sql +++ b/models/amazon_ads__keyword_report.sql @@ -1,5 +1,3 @@ -ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) - {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} with report as ( diff --git a/models/amazon_ads__portfolio_report.sql b/models/amazon_ads__portfolio_report.sql index 1e1622a..89563a1 100644 --- a/models/amazon_ads__portfolio_report.sql +++ b/models/amazon_ads__portfolio_report.sql @@ -1,5 +1,3 @@ -ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) - {{ config(enabled=fivetran_utils.enabled_vars(['ad_reporting__amazon_ads_enabled','amazon_ads__portfolio_history_enabled'])) }} with report as ( diff --git a/models/amazon_ads__search_report.sql b/models/amazon_ads__search_report.sql index 91a18df..90adff4 100644 --- a/models/amazon_ads__search_report.sql +++ b/models/amazon_ads__search_report.sql @@ -1,5 +1,3 @@ -ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) - {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} with report as ( diff --git a/models/docs.md b/models/docs.md index 9485ba5..b3d004b 100644 --- a/models/docs.md +++ b/models/docs.md @@ -177,6 +177,7 @@ The state of the record (enabled, paused, or archived). {% docs targeting %} A string representation of the expression object used in the targeting clause. {% enddocs %} + {% docs source_relation %} The source of the record if the unioning functionality is being used. If not this field will be empty. {% enddocs %} \ No newline at end of file diff --git a/models/intermediate/int_amazon_ads__portfolio_history.sql b/models/intermediate/int_amazon_ads__portfolio_history.sql index 972f190..a659880 100644 --- a/models/intermediate/int_amazon_ads__portfolio_history.sql +++ b/models/intermediate/int_amazon_ads__portfolio_history.sql @@ -1,5 +1,3 @@ -ADD source_relation WHERE NEEDED + CHECK JOINS AND WINDOW FUNCTIONS! (Delete this line when done.) - {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} {# This intermediate model creates a dummy portfolio table if the user does not use portfolios. @@ -12,6 +10,7 @@ with portfolios as ( from {{ var('portfolio_history') }} where is_most_recent_record = True {% else %} + cast(null as {{ dbt.type_string() }}) as source_relation, {# uses the columns macro from the source package to populate column names #} {%- set columns = amazon_ads_source.get_portfolio_history_columns() -%} {% for column in columns %} diff --git a/packages.yml b/packages.yml index 4bcc0b2..3a5cb70 100644 --- a/packages.yml +++ b/packages.yml @@ -1,7 +1,7 @@ packages: - - package: fivetran/amazon_ads_source - version: [">=0.2.0", "<0.3.0"] + # - package: fivetran/amazon_ads_source + # version: [">=0.2.0", "<0.3.0"] -- git: https://github.com/fivetran/amazon_ads_source.git +- git: https://github.com/fivetran/dbt_amazon_ads_source.git revision: MagicBot/add-union-schema warn-unpinned: false \ No newline at end of file From b95657d4867ba84b4d21bb52ffca4510c173424f Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 30 Aug 2023 20:25:24 -0500 Subject: [PATCH 03/10] update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cfd10c2..32be13c 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Include the following amazon_ads package version in your `packages.yml` file: ```yaml packages: - package: fivetran/amazon_ads - version: [">=0.2.0", "<0.3.0"] # we recommend using ranges to capture non-breaking changes automatically + version: [">=0.3.0", "<0.4.0"] # we recommend using ranges to capture non-breaking changes automatically ``` Do NOT include the `amazon_ads_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well. @@ -150,7 +150,7 @@ This dbt package is dependent on the following dbt packages. Be aware that these ```yml packages: - package: fivetran/amazon_ads_source - version: [">=0.2.0", "<0.3.0"] + version: [">=0.3.0", "<0.4.0"] - package: fivetran/fivetran_utils version: [">=0.4.0", "<0.5.0"] From 4743f2d08d0dffe074bfb04cc19e919b9c530999 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 1 Sep 2023 13:57:25 -0500 Subject: [PATCH 04/10] Update packages.yml --- packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages.yml b/packages.yml index 3a5cb70..43aff36 100644 --- a/packages.yml +++ b/packages.yml @@ -1,7 +1,7 @@ packages: # - package: fivetran/amazon_ads_source - # version: [">=0.2.0", "<0.3.0"] + # version: [">=0.3.0", "<0.4.0"] - git: https://github.com/fivetran/dbt_amazon_ads_source.git revision: MagicBot/add-union-schema - warn-unpinned: false \ No newline at end of file + warn-unpinned: false From 2932aede2737ee05b5efb4a92d188eadf5e8a870 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 5 Sep 2023 20:39:48 -0500 Subject: [PATCH 05/10] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72b4bf3..84838c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - In the source package, updated tmp models to union source data using the `fivetran_utils.union_data` macro. - To distinguish which source each field comes from, added `source_relation` column in each staging and downstream model and applied the `fivetran_utils.source_relation` macro. - Updated tests to account for the new `source_relation` column. - - The `source_relation` column is included in all joins and window function partition clauses in the transform package. + - The `source_relation` column is included in all joins in the transform package. # dbt_amazon_ads v0.2.0 [PR #6](https://github.com/fivetran/dbt_amazon_ads/pull/6) includes the following updates: From f11a26d436bbb2e28b7a98d00399f46ee6297437 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 5 Sep 2023 21:45:07 -0500 Subject: [PATCH 06/10] update for databricks --- .buildkite/hooks/pre-command | 3 ++- .buildkite/pipeline.yml | 1 + integration_tests/ci/sample.profiles.yml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 04c85c0..81b7b00 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") \ No newline at end of file +export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") +export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6e76ea0..281f1d4 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -69,5 +69,6 @@ steps: - "CI_DATABRICKS_DBT_HOST" - "CI_DATABRICKS_DBT_HTTP_PATH" - "CI_DATABRICKS_DBT_TOKEN" + - "CI_DATABRICKS_DBT_CATALOG" commands: | bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 1353821..4055a75 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -45,7 +45,7 @@ integration_tests: schema: amazon_ads_integration_tests_02 threads: 8 databricks: - catalog: null + catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" schema: amazon_ads_integration_tests_02 From 132539f19209fc26a33f4cfea9c9e41f29690248 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 6 Oct 2023 09:57:04 -0500 Subject: [PATCH 07/10] update changelog and readme --- CHANGELOG.md | 2 +- README.md | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84838c3..31fbb71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # dbt_amazon_ads v0.3.0 [PR #11](https://github.com/fivetran/dbt_amazon_ads/pull/11) includes the following updates: ## Feature update πŸŽ‰ -- Unioning capability! This adds the ability to union source data from multiple amazon_ads connectors. Refer to the [README](https://github.com/fivetran/dbt_amazon_ads/blob/main/README.md) for more details. +- Unioning capability! This adds the ability to union source data from multiple amazon_ads connectors. Refer to the [Union Multiple Connectors README section](https://github.com/fivetran/dbt_amazon_ads/blob/main/README.md#union-multiple-connectors) for more details. ## Under the hood 🚘 - In the source package, updated tmp models to union source data using the `fivetran_utils.union_data` macro. diff --git a/README.md b/README.md index 32be13c..d27bead 100644 --- a/README.md +++ b/README.md @@ -76,8 +76,6 @@ vars: ``` ## (Optional) Step 5: Additional configurations -
Expand for configurations - ### Union multiple connectors If you have multiple amazon_ads connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `amazon_ads_union_schemas` OR `amazon_ads_union_databases` variables (cannot do both) in your root `dbt_project.yml` file: @@ -134,8 +132,6 @@ If an individual source table has a different name than the package expects, add vars: amazon_ads__identifier: your_table_name ``` -
- ## (Optional) Step 6: Orchestrate your models with Fivetran Transformations for dbt Coreβ„’
Expand for more details From cb6e1ea7b06a46eeca18130f5c8d6cde61060f57 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 6 Oct 2023 10:56:36 -0500 Subject: [PATCH 08/10] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31fbb71..3dab272 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,8 @@ ## Under the hood 🚘 - In the source package, updated tmp models to union source data using the `fivetran_utils.union_data` macro. - To distinguish which source each field comes from, added `source_relation` column in each staging and downstream model and applied the `fivetran_utils.source_relation` macro. + - The `source_relation` column is included in all joins in the transform package. - Updated tests to account for the new `source_relation` column. - - The `source_relation` column is included in all joins in the transform package. # dbt_amazon_ads v0.2.0 [PR #6](https://github.com/fivetran/dbt_amazon_ads/pull/6) includes the following updates: From fbac353ce94929296d94bc3161bb89eb68f38a00 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 12 Oct 2023 10:16:38 -0500 Subject: [PATCH 09/10] Update packages.yml --- packages.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages.yml b/packages.yml index 43aff36..89041e5 100644 --- a/packages.yml +++ b/packages.yml @@ -1,7 +1,3 @@ packages: - # - package: fivetran/amazon_ads_source - # version: [">=0.3.0", "<0.4.0"] - -- git: https://github.com/fivetran/dbt_amazon_ads_source.git - revision: MagicBot/add-union-schema - warn-unpinned: false + - package: fivetran/amazon_ads_source + version: [">=0.3.0", "<0.4.0"] From 58ecc1429d12a91da8579971987aa13ae471ce9c Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 12 Oct 2023 14:46:36 -0500 Subject: [PATCH 10/10] Update requirements.txt --- integration_tests/requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 87ae0f9..ca5a642 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -4,4 +4,6 @@ dbt-redshift>=1.3.0,<2.0.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 \ No newline at end of file +dbt-databricks>=1.3.0,<2.0.0 + +oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437