🚨🚨✨Source Klaviyo: Migrate to API version '2023-10-15' (#32128)

airbytehq · Nov 6, 2023 · 6583dc5 · 6583dc5
1 parent af2dba6
commit 6583dc5
Show file tree

Hide file tree

Showing 21 changed files with 859 additions and 952 deletions.
diff --git a/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml
@@ -7,7 +7,7 @@ acceptance_tests:
           extra_records: true
         ignored_fields:
           email_templates:
-            - name: html
+            - name: "attributes/html"
               bypass_reason: unstable data
   connection:
     tests:
@@ -19,8 +19,7 @@ acceptance_tests:
     tests:
       - config_path: secrets/config.json
         backward_compatibility_tests_config:
-          disable_for_version: "1.0.0"
-          previous_connector_version: "0.5.0"
+          disable_for_version: "1.1.0"
   full_refresh:
     tests:
       - config_path: secrets/config.json

diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json
@@ -2,42 +2,52 @@
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "timestamp": 9621295127 },
+      "stream_state": { "datetime": "2120-10-10T00:00:00Z" },
       "stream_descriptor": { "name": "events" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "timestamp": "2120-10-10T00:00:00Z" },
+      "stream_state": { "updated": "2120-10-10T00:00:00Z" },
       "stream_descriptor": { "name": "global_exclusions" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "created": "2120-10-10 00:00:00" },
+      "stream_state": {
+        "updated": "2120-10-10 00:00:00",
+        "archived": {
+          "updated": "2120-10-10 00:00:00"
+        }
+      },
       "stream_descriptor": { "name": "flows" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "created": "2120-10-10 00:00:00" },
+      "stream_state": { "updated": "2120-10-10 00:00:00" },
       "stream_descriptor": { "name": "metrics" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "created": "2120-10-10 00:00:00" },
+      "stream_state": { "updated": "2120-10-10 00:00:00" },
       "stream_descriptor": { "name": "lists" }
     }
   },
   {
     "type": "STREAM",
     "stream": {
-      "stream_state": { "created": "2120-10-10 00:00:00" },
+      "stream_state": {
+        "updated_at": "2120-10-10 00:00:00",
+        "archived": {
+          "updated_at": "2120-10-10 00:00:00"
+        }
+      },
       "stream_descriptor": { "name": "campaigns" }
     }
   },
@@ -47,5 +57,12 @@
       "stream_state": { "updated": "2120-10-10 00:00:00" },
       "stream_descriptor": { "name": "profiles" }
     }
+  },
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": { "updated": "2120-10-10 00:00:00" },
+      "stream_descriptor": { "name": "email_templates" }
+    }
   }
 ]
diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-klaviyo/integration_tests/configured_catalog.json
@@ -41,7 +41,7 @@
         "namespace": null
       },
       "sync_mode": "full_refresh",
-      "cursor_field": ["timestamp"],
+      "cursor_field": ["datetime"],
       "destination_sync_mode": "append",
       "primary_key": [["id"]]
     },
@@ -52,13 +52,13 @@
         "supported_sync_modes": ["full_refresh"],
         "source_defined_cursor": null,
         "default_cursor_field": null,
-        "source_defined_primary_key": [["email"]],
+        "source_defined_primary_key": [["id"]],
         "namespace": null
       },
       "sync_mode": "full_refresh",
       "cursor_field": null,
       "destination_sync_mode": "append",
-      "primary_key": [["email"]]
+      "primary_key": [["id"]]
     },
     {
       "stream": {

diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl
diff --git a/airbyte-integrations/connectors/source-klaviyo/metadata.yaml b/airbyte-integrations/connectors/source-klaviyo/metadata.yaml
@@ -8,7 +8,7 @@ data:
   definitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde
   connectorBuildOptions:
     baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c
-  dockerImageTag: 1.1.0
+  dockerImageTag: 2.0.0
   dockerRepository: airbyte/source-klaviyo
   githubIssueLabel: source-klaviyo
   icon: klaviyo.svg
@@ -20,11 +20,21 @@ data:
     oss:
       enabled: true
   releaseStage: generally_available
+  suggestedStreams:
+    streams:
+      - events
+      - campaigns
+      - lists
+      - metrics
+      - flows
   releases:
     breakingChanges:
       1.0.0:
         message: In this release, for 'events' stream changed type of 'event_properties/items/quantity' field from integer to number. Users will need to refresh the source schema and reset events streams after upgrading.
         upgradeDeadline: "2023-11-30"
+      2.0.0:
+        message: In this release, streams 'campaigns', 'email_templates', 'events', 'flows', 'global_exclusions', 'lists', and 'metrics' are now pulling data using latest API which has a different schema. Users will need to refresh the source schemas and reset these streams after upgrading.
+        upgradeDeadline: "2023-11-30"
   documentationUrl: https://docs.airbyte.com/integrations/sources/klaviyo
   tags:
     - language:python

diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/availability_strategy.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/availability_strategy.py
@@ -11,14 +11,14 @@
 from requests import HTTPError, codes
 
 
-class KlaviyoAvailabilityStrategyLatest(HttpAvailabilityStrategy):
+class KlaviyoAvailabilityStrategy(HttpAvailabilityStrategy):
     def reasons_for_unavailable_status_codes(
         self, stream: Stream, logger: logging.Logger, source: Optional[Source], error: HTTPError
     ) -> Dict[int, str]:
         reasons_for_codes: Dict[int, str] = super().reasons_for_unavailable_status_codes(stream, logger, source, error)
         reasons_for_codes[codes.UNAUTHORIZED] = (
             "This is most likely due to insufficient permissions on the credentials in use. "
-            "Try to grant required permissions/scopes or re-authenticate"
+            f"Try to create and use an API key with read permission for the '{stream.name}' stream granted"
         )
 
         return reasons_for_codes
diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py
@@ -0,0 +1,2 @@
+class KlaviyoBackoffError(Exception):
+    """An exception which is raised when 'retry-after' time is longer than 'max_time' specified"""
diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/campaigns.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/campaigns.json
@@ -1,55 +1,50 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
-    "object": { "type": "string" },
+    "type": { "type": "string" },
     "id": { "type": "string" },
-    "name": { "type": "string" },
-    "created": { "type": ["null", "string"], "format": "date-time" },
-    "updated": { "type": ["null", "string"], "format": "date-time" },
-    "status": { "type": "string" },
-    "status_id": { "type": "integer" },
-    "status_label": { "type": "string" },
-    "from_name": { "type": "string" },
-    "from_email": { "type": "string" },
-    "num_recipients": { "type": "integer" },
-    "lists": {
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "object": { "type": "string" },
-          "id": { "type": "string" },
-          "name": { "type": "string" },
-          "created": { "type": "string", "format": "date-time" },
-          "updated": { "type": "string", "format": "date-time" },
-          "person_count": { "type": "integer" },
-          "list_type": { "type": "string" },
-          "folder": { "type": ["null", "string"] }
-        }
+    "updated_at": { "type": ["null", "string"], "format": "date-time" },
+    "attributes": {
+      "type": ["null", "object"],
+      "additionalProperties": true,
+      "properties": {
+        "name": { "type": "string" },
+        "status": { "type": "string" },
+        "archived": { "type": "boolean" },
+        "channel": { "type": "string" },
+        "audiences": {
+          "type": ["null", "object"],
+          "additionalProperties": true
+        },
+        "send_options": {
+          "type": ["null", "object"],
+          "properties": {
+            "use_smart_sending": { "type": "boolean" }
+          }
+        },
+        "message": { "type": "string" },
+        "tracking_options": {
+          "type": ["null", "object"],
+          "additionalProperties": true
+        },
+        "send_strategy": {
+          "type": ["null", "object"],
+          "additionalProperties": true
+        },
+        "created_at": { "type": ["null", "string"], "format": "date-time" },
+        "scheduled_at": { "type": ["null", "string"], "format": "date-time" },
+        "updated_at": { "type": ["null", "string"], "format": "date-time" },
+        "send_time": { "type": ["null", "string"], "format": "date-time" }
       }
     },
-    "excluded_lists": {
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "object": { "type": "string" },
-          "id": { "type": "string" },
-          "name": { "type": "string" },
-          "created": { "type": "string", "format": "date-time" },
-          "updated": { "type": "string", "format": "date-time" },
-          "person_count": { "type": "integer" },
-          "list_type": { "type": "string" },
-          "folder": { "type": ["null", "string"] }
-        }
-      }
+    "links": {
+      "type": ["null", "object"],
+      "additionalProperties": true
     },
-    "is_segmented": { "type": "boolean" },
-    "send_time": { "type": ["null", "string"], "format": "date-time" },
-    "sent_at": { "type": ["null", "string"], "format": "date-time" },
-    "campaign_type": { "type": "string" },
-    "subject": { "type": ["null", "string"] },
-    "message_type": { "type": "string" },
-    "template_id": { "type": ["null", "string"] }
+    "relationships": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    }
   }
 }
diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json
@@ -2,28 +2,25 @@
   "type": "object",
   "additionalProperties": true,
   "properties": {
-    "object": {
-      "type": "string"
+    "type": { "type": "string" },
+    "id": { "type": "string" },
+    "updated": { "type": ["null", "string"], "format": "date-time" },
+    "attributes": {
+      "type": ["null", "object"],
+      "additionalProperties": true,
+      "properties": {
+        "name": { "type": "string" },
+        "editor_type": { "type": ["null", "string"] },
+        "html": { "type": "string" },
+        "text": { "type": ["null", "string"] },
+        "created": { "type": ["null", "string"], "format": "date-time" },
+        "updated": { "type": ["null", "string"], "format": "date-time" },
+        "company_id": { "type": ["null", "string"] }
+      }
     },
-    "id": {
-      "type": "string"
-    },
-    "name": {
-      "type": ["null", "string"]
-    },
-    "html": {
-      "type": ["null", "string"]
-    },
-    "is_writeable": {
-      "type": ["null", "boolean"]
-    },
-    "created": {
-      "type": "string",
-      "format": "date-time"
-    },
-    "updated": {
-      "type": "string",
-      "format": "date-time"
+    "links": {
+      "type": ["null", "object"],
+      "additionalProperties": true
     }
   }
 }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		class KlaviyoBackoffError(Exception):
		"""An exception which is raised when 'retry-after' time is longer than 'max_time' specified"""