Support incremental load

meltano.yml

@@ -1,29 +1,23 @@
 version: 1
 send_anonymous_usage_stats: true
-project_id: "tap-f1"
+project_id: tap-f1
 default_environment: test
 environments:
 - name: test
 plugins:
   extractors:
-  - name: "tap-f1"
-    namespace: "tap_f1"
+  - name: tap-f1
+    namespace: tap_f1
     pip_url: -e .
     capabilities:
     - state
     - catalog
     - discover
     - about
     - stream-maps
-    config:
-      start_date: '2010-01-01T00:00:00Z'
     settings:
-    # TODO: To configure using Meltano, declare settings and their types here:
-    - name: username
-    - name: password
-      kind: password
     - name: start_date
-      value: '2010-01-01T00:00:00Z'
+      kind: date_iso8601
   loaders:
   - name: target-jsonl
     variant: andyh1203

tap_f1/streams.py

@@ -1,6 +1,8 @@
 """Stream type classes for tap-f1."""
 
 
+from datetime import date
+
 from singer_sdk import typing as th
 
 from tap_f1.client import F1Stream
@@ -23,6 +25,7 @@ class SeasonsStream(F1Stream):
 
     name = "seasons"
     primary_keys = ["season"]
+    replication_key = "season"
     path = "/seasons.json"
     records_jsonpath = "MRData.SeasonTable.Seasons[*]"
 
@@ -32,6 +35,11 @@ class SeasonsStream(F1Stream):
     ).to_dict()
 
     def get_child_context(self, record, context):
+        start_date = date.fromisoformat(self.config["start_date"])
+
+        if start_date.year > int(record["season"]):
+            return None
+
         return {"season": record["season"]}
 
 
@@ -106,6 +114,7 @@ class RacesStream(F1Stream):
     parent_stream_type = SeasonsStream
     name = "races"
     primary_keys = ["season", "round"]
+    replication_key = "date"
     path = "/{season}.json"
     records_jsonpath = "MRData.RaceTable.Races[*]"
 
@@ -171,6 +180,12 @@ class RacesStream(F1Stream):
     ).to_dict()
 
     def get_child_context(self, record, context):
+        value = self.get_starting_replication_key_value(context)
+        start_date = date.fromisoformat(value)
+
+        if start_date > date.fromisoformat(record["date"]):
+            return None
+
         return {
             "season": record["season"],
             "round": record["round"],

tap_f1/tap.py

@@ -1,5 +1,8 @@
 """F1 tap class."""
 
+from datetime import date
+
+import singer_sdk.typing as th
 from singer_sdk import Tap
 
 from tap_f1 import streams
@@ -25,6 +28,14 @@ class TapF1(Tap):
 
     name = "tap-f1"
 
+    config_jsonschema = th.PropertiesList(
+        th.Property(
+            "start_date",
+            th.DateType,
+            default=date(date.today().year, 1, 1).isoformat(),
+        ),
+    ).to_dict()
+
     def discover_streams(self):
         return [stream_type(self) for stream_type in STREAM_TYPES]