feature/adding the connector_type table to the package as well as other packa…

CHANGELOG.md

@@ -1,3 +1,7 @@
+# dbt_fivetran_log v0.6.4
+## Feature - connector_type table added
+- Added the connector_type table to the package and incorporated some of it's fields to the `fivetran_log__connector_status` model.
+- Also made some changes to the staging connector, user and log tables as per the connector release notes: https://fivetran.com/docs/logs/fivetran-log/changelog
 # dbt_fivetran_log v0.6.3
 ## Fixes
 - Modified the argument used for the identifier in the get_relation macro used in the does_table_exist macro from name to identifier. This avoids issues on snowflake where the name of a table defined in a source yaml may be in lowercase while in snowflake it is uppercased.

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'fivetran_log'
-version: '0.6.3'
+version: '0.6.4'
 
 require-dbt-version: [">=1.0.0", "<2.0.0"]
 
@@ -21,6 +21,7 @@ vars:
     account_membership: "{{ source('fivetran_log', 'account_membership') }}"
     active_volume: "{{ source('fivetran_log', 'active_volume') }}"
     connector: "{{ source('fivetran_log', 'connector') }}"
+    connector_type: "{{ source('fivetran_log', 'connector_type') }}"
     credits_used: "{{ source('fivetran_log', 'credits_used') }}"
     destination: "{{ source('fivetran_log', 'destination') }}"
     destination_membership: "{{ source('fivetran_log', 'destination_membership') }}"

integration_tests/ci/sample.profiles.yml

@@ -16,14 +16,14 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: fivetrans_logs_integration_tests
+      schema: fivetrans_logs_integration_tests_1
       threads: 8
     bigquery:
       type: bigquery
       method: service-account
       keyfile: "{{ env_var('GCLOUD_SERVICE_KEY_PATH') }}"
       project: 'dbt-package-testing'
-      schema: fivetrans_logs_integration_tests
+      schema: fivetrans_logs_integration_tests_1
       threads: 8
     snowflake:
       type: snowflake
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: fivetrans_logs_integration_tests
+      schema: fivetrans_logs_integration_tests_1
       threads: 8
     postgres:
       type: postgres
@@ -42,12 +42,12 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: fivetrans_logs_integration_tests
+      schema: fivetrans_logs_integration_tests_1
       threads: 8
     spark:
       type: spark
       method: http
-      schema: fivetrans_logs_integration_tests
+      schema: fivetrans_logs_integration_tests_1
       host: "{{ env_var('CI_SPARK_DBT_HOST') }}"
       organization: "{{ env_var('CI_SPARK_DBT_ORGANIZATION') }}"
       token: "{{ env_var('CI_SPARK_DBT_TOKEN') }}"

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'fivetran_log_integration_tests'
-version: '0.6.3'
+version: '0.6.4'
 config-version: 2
 profile: 'integration_tests'
 
@@ -10,11 +10,12 @@ dispatch:
 vars:
   fivetran_log:
 
-    fivetran_log_schema: fivetrans_logs_integration_tests
+    fivetran_log_schema: fivetrans_logs_integration_tests_1
     fivetran_log_account_identifier: "account"
     fivetran_log_account_membership_identifier: "account_membership"
     fivetran_log_active_volume_identifier: "active_volume"
     fivetran_log_connector_identifier: "connector"
+    fivetran_log_connector_type_identifier: "connector_type"
     fivetran_log_credits_used_identifier: "credits_used"
     fivetran_log_usage_cost_identifier: "usage_cost"
     fivetran_log_destination_identifier: "destination"

integration_tests/seeds/connector_type.csv

@@ -0,0 +1,11 @@
+_fivetran_synced,availability,broken,created_at,deleted,official_connector_name,public_beta_at,release_at,type,id
+2022-07-05T10:59:37.158,DEVELOPMENT,false,2017-07-07T13:52:12,false,Google Ads (AdWords),,,Marketing,adwords
+2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2018-03-08T23:55:49,false,Google Sheets,2018-03-09T19:09:28,2018-03-09T19:09:28,File,google_sheets
+2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2019-07-31T16:08:02,false,Fivetran Log,2020-03-13T10:44:09,2020-04-06T05:46:33,Free,fivetran_log
+2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2017-02-15T03:06:56,false,Shopify,2017-04-10T17:15:05,2017-12-14T01:37:29,Sales,shopify
+2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2018-12-10T10:54:16,false,Amplitude,2019-01-07T10:04:58,2019-07-02T17:23:16,Events,amplitude
+2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2021-09-29T09:01:54.237,false,Stripe,2021-09-29T09:01:54.237,2021-09-29T09:01:54.237,Finance,stripe
+2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2017-12-07T08:29:42,false,QuickBooks,2017-12-07T08:29:42,2017-12-07T08:29:42,Finance,quickbooks
+2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2019-05-28T09:59:07,true,Recurly,2019-05-28T09:59:07,2019-09-05T23:09:17,Finance,recurly
+2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2020-09-29T19:00:45.287,false,Qualtrics,2022-01-11T18:01:21.651,2022-07-27T19:10:30.269,Support,qualtrics
+2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2017-12-06T21:45:17,false,Zendesk Support,2017-12-06T21:45:17,2018-03-28T02:05:35,Support,zendesk

integration_tests/seeds/log.csv

@@ -1,25 +1,25 @@
-id,time_stamp,_fivetran_synced,connector_id,event,message_data,message_event,transformation_id
-legislative_lazy,2021-02-13 13:34:03.343,2021-02-16 21:11:07.065,legislative_lazy,WARNING,"{""type"":""table_excluded_by_system"",""message"":""salesforce.affectlayer__AffectLayer_User_Setting__ChangeEvent has been Excluded by system. Reason : Not queryable""}",warning,
-paint_wedges,2021-02-12 08:15:05.555,2021-02-16 21:48:51.493,paint_wedges,INFO,"{""method"":""GET"",""uri"":""https://google.com""}",api_call,
-intrinsic_departed,2021-12-09 14:27:00.504,2021-12-09 20:30:53.959,intrinsic_departed,INFO,,sync_end,
-intrinsic_departed,2021-12-09 14:26:52.433,2021-12-09 20:30:53.957,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":4,""operationType"":""REPLACED_OR_INSERTED"",""table"":""fivetran_audit""}",records_modified,
-intrinsic_departed,2021-12-09 14:26:52.242,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:51.703,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:44,2021-12-09 20:30:53.952,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_start,
-intrinsic_departed,2021-12-09 14:26:42.084,2021-12-09 20:30:53.950,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":9,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_history""}",records_modified,
-intrinsic_departed,2021-12-09 14:26:42.029,2021-12-09 20:30:53.949,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":5,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_insights""}",records_modified,
-intrinsic_departed,2021-12-09 14:26:41.886,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:41.875,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:40.880,2021-12-09 20:30:53.946,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:40.737,2021-12-09 20:30:53.944,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:39.469,2021-12-09 20:30:53.935,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":1,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_history""}",records_modified,
-intrinsic_departed,2021-12-09 14:26:39.446,2021-12-09 20:30:53.932,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":3,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_insights""}",records_modified,
-intrinsic_departed,2021-12-09 14:26:39.092,2021-12-09 20:30:53.930,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:39.087,2021-12-09 20:30:53.929,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:37.735,2021-12-09 20:30:53.928,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:37.691,2021-12-09 20:30:53.920,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:29.860,2021-12-09 20:30:53.904,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_start,
-intrinsic_departed,2021-12-09 14:26:29.814,2021-12-09 20:30:53.903,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_start,
-intrinsic_departed,2021-12-09 14:26:29.744,2021-12-09 20:30:53.901,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_start,
-intrinsic_departed,2021-12-09 14:26:29.719,2021-12-09 20:30:53.878,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_start,
-intrinsic_departed,2021-12-09 14:26:05.907,2021-12-09 20:30:53.778,intrinsic_departed,INFO,,sync_start,
+id,time_stamp,_fivetran_synced,connector_id,event,message_data,message_event,transformation_id,sync_id
+legislative_lazy,2021-02-13 13:34:03.343,2021-02-16 21:11:07.065,legislative_lazy,WARNING,"{""type"":""table_excluded_by_system"",""message"":""salesforce.affectlayer__AffectLayer_User_Setting__ChangeEvent has been Excluded by system. Reason : Not queryable""}",warning,,25
+paint_wedges,2021-02-12 08:15:05.555,2021-02-16 21:48:51.493,paint_wedges,INFO,"{""method"":""GET"",""uri"":""https://google.com""}",api_call,,24
+intrinsic_departed,2021-12-09 14:27:00.504,2021-12-09 20:30:53.959,intrinsic_departed,INFO,,sync_end,,23
+intrinsic_departed,2021-12-09 14:26:52.433,2021-12-09 20:30:53.957,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":4,""operationType"":""REPLACED_OR_INSERTED"",""table"":""fivetran_audit""}",records_modified,,22
+intrinsic_departed,2021-12-09 14:26:52.242,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,,21
+intrinsic_departed,2021-12-09 14:26:51.703,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,,20
+intrinsic_departed,2021-12-09 14:26:44,2021-12-09 20:30:53.952,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_start,,19
+intrinsic_departed,2021-12-09 14:26:42.084,2021-12-09 20:30:53.950,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":9,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_history""}",records_modified,,18
+intrinsic_departed,2021-12-09 14:26:42.029,2021-12-09 20:30:53.949,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":5,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_insights""}",records_modified,,17
+intrinsic_departed,2021-12-09 14:26:41.886,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end,,16
+intrinsic_departed,2021-12-09 14:26:41.875,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end,,15
+intrinsic_departed,2021-12-09 14:26:40.880,2021-12-09 20:30:53.946,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end,,14
+intrinsic_departed,2021-12-09 14:26:40.737,2021-12-09 20:30:53.944,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end,,13
+intrinsic_departed,2021-12-09 14:26:39.469,2021-12-09 20:30:53.935,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":1,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_history""}",records_modified,,12
+intrinsic_departed,2021-12-09 14:26:39.446,2021-12-09 20:30:53.932,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":3,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_insights""}",records_modified,,11
+intrinsic_departed,2021-12-09 14:26:39.092,2021-12-09 20:30:53.930,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end,,9
+intrinsic_departed,2021-12-09 14:26:39.087,2021-12-09 20:30:53.929,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end,,8
+intrinsic_departed,2021-12-09 14:26:37.735,2021-12-09 20:30:53.928,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end,,7
+intrinsic_departed,2021-12-09 14:26:37.691,2021-12-09 20:30:53.920,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end,,6
+intrinsic_departed,2021-12-09 14:26:29.860,2021-12-09 20:30:53.904,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_start,,5
+intrinsic_departed,2021-12-09 14:26:29.814,2021-12-09 20:30:53.903,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_start,,4
+intrinsic_departed,2021-12-09 14:26:29.744,2021-12-09 20:30:53.901,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_start,,3
+intrinsic_departed,2021-12-09 14:26:29.719,2021-12-09 20:30:53.878,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_start,,2
+intrinsic_departed,2021-12-09 14:26:05.907,2021-12-09 20:30:53.778,intrinsic_departed,INFO,,sync_start,,1

integration_tests/seeds/user.csv

@@ -1,3 +1,3 @@
-id,_fivetran_synced,created_at,email,email_disabled,family_name,given_name,phone,verified
-barista_fibula,2021-03-12 21:03:46.208,2020-05-19 20:09:23.765024,[email protected],false,Squarepants,Spngebob,,false
-backlog_jack,2021-03-12 21:03:45.888,2020-08-19 15:12:41.254224,[email protected],false,McGee,Knucklehead,,true
+id,_fivetran_synced,created_at,email,email_disabled,family_name,given_name,phone,verified,_fivetran_deleted
+barista_fibula,2021-03-12 21:03:46.208,2020-05-19 20:09:23.765024,[email protected],false,Squarepants,Spngebob,,false,false
+backlog_jack,2021-03-12 21:03:45.888,2020-08-19 15:12:41.254224,[email protected],false,McGee,Knucklehead,,true,false

macros/get_brand_columns.sql

@@ -11,7 +11,8 @@
     {"name": "destination_id", "datatype": dbt_utils.type_string()},
     {"name": "paused", "datatype": "boolean"},
     {"name": "service_version", "datatype": dbt_utils.type_int()},
-    {"name": "signed_up", "datatype": dbt_utils.type_timestamp()}
+    {"name": "signed_up", "datatype": dbt_utils.type_timestamp()},
+    {"name": "sync_frequency", "datatype": dbt_utils.type_int()}
 ] %}
 
 {{ return(columns) }}

models/fivetran_log.yml

@@ -20,8 +20,14 @@ models:
       - name: connector_name
         description: Name of the connector.
 
+      - name: official_connector_name
+        description: The official name of the connector.
+
       - name: connector_type
-        description: The kind of connector (ie google sheets, webhooks).
+        description: The type of connector (e.g. Productivity, Finance).
+
+      - name: availability
+        description: The general availability of the connector.
 
       - name: destination_id
         description: >
@@ -49,6 +55,12 @@ models:
       - name: set_up_at
         description: Timestamp of when the connector was set up.
 
+      - name: is_deleted
+        description: Boolean indicating if the connector has been deleted.
+
+      - name: is_broken
+        description: Boolean indicating if the connector is broken.
+
       - name: number_of_schema_changes_last_month
         description: > 
           The number of schema changes (creating tables or schemas and altering tables) in the past 30 days.

models/fivetran_log__connector_status.sql

@@ -50,6 +50,13 @@ connector as (
 
 ),
 
+connector_type as (
+    select
+        *
+    from
+        {{ ref('stg_fivetran_log__connector_type') }}
+),
+
 destination as (
 
     select * 
@@ -61,10 +68,17 @@ connector_metrics as (
     select
         connector.connector_id,
         connector.connector_name,
-        connector.connector_type,
+        connector_type.connector as official_connector_name,
+        connector_type.connector_type,
+        connector_type.availability as connector_type_availability,
         connector.destination_id,
         connector.is_paused,
         connector.set_up_at,
+        connector_type.is_deleted as is_connector_type_deleted,
+        connector_type.is_broken as is_connector_type_broken,
+        connector_type.created_at as connector_type_created_at,
+        connector_type.public_beta_at as connector_type_public_beta_at,
+        connector_type.release_at as connector_type_released_at,
         max(case when connector_log.event_subtype = 'sync_start' then connector_log.created_at else null end) as last_sync_started_at,
 
         max(case when connector_log.event_subtype = 'sync_end' 
@@ -90,9 +104,11 @@ connector_metrics as (
         max(case when event_type = 'WARNING' then connector_log.created_at else null end) as last_warning_at
 
     from connector 
+    left join connector_type
+        on connector_type.connector_type_id = connector.connector_type
     left join connector_log 
         on connector_log.connector_id = connector.connector_id
-    {{ dbt_utils.group_by(n=6) }}
+    {{ dbt_utils.group_by(n=13) }}
 
 ),
 
@@ -133,13 +149,20 @@ connector_recent_logs as (
     select 
         connector_health.connector_id,
         connector_health.connector_name,
+        connector_health.official_connector_name,
         connector_health.connector_type,
+        connector_health.connector_type_availability,
         connector_health.destination_id,
         connector_health.connector_health,
         connector_health.last_successful_sync_completed_at,
         connector_health.last_sync_started_at,
         connector_health.last_sync_completed_at,
         connector_health.set_up_at,
+        connector_health.is_connector_type_deleted,
+        connector_health.is_connector_type_broken,
+        connector_health.connector_type_created_at,
+        connector_health.connector_type_public_beta_at,
+        connector_health.connector_type_released_at,
         connector_log.event_subtype,
         connector_log.event_type,
         connector_log.message_data
@@ -156,7 +179,7 @@ connector_recent_logs as (
             and {{ fivetran_utils.json_parse(string="connector_log.message_data", string_path=["status"]) }} ='RESCHEDULED'
             and {{ fivetran_utils.json_parse(string="connector_log.message_data", string_path=["reason"]) }} like '%intended behavior%')
 
-    {{ dbt_utils.group_by(n=12) }} -- de-duping error messages
+    {{ dbt_utils.group_by(n=19) }} -- de-duping error messages
 
 
 ),
@@ -166,14 +189,21 @@ final as (
     select
         connector_recent_logs.connector_id,
         connector_recent_logs.connector_name,
+        connector_recent_logs.official_connector_name,
         connector_recent_logs.connector_type,
+        connector_recent_logs.connector_type_availability,
         connector_recent_logs.destination_id,
         destination.destination_name,
         connector_recent_logs.connector_health,
         connector_recent_logs.last_successful_sync_completed_at,
         connector_recent_logs.last_sync_started_at,
         connector_recent_logs.last_sync_completed_at,
         connector_recent_logs.set_up_at,
+        connector_recent_logs.is_connector_type_deleted,
+        connector_recent_logs.is_connector_type_broken,
+        connector_recent_logs.connector_type_created_at,
+        connector_recent_logs.connector_type_public_beta_at,
+        connector_recent_logs.connector_type_released_at,
         coalesce(schema_changes.number_of_schema_changes_last_month, 0) as number_of_schema_changes_last_month
 
         {% if var('fivetran_log_using_sync_alert_messages', true) %}
@@ -186,7 +216,7 @@ final as (
         on connector_recent_logs.connector_id = schema_changes.connector_id 
 
     join destination on destination.destination_id = connector_recent_logs.destination_id
-    {{ dbt_utils.group_by(n=11) }}
+    {{ dbt_utils.group_by(n=18) }}
 )
 
 select * from final