From 78068508c68c69c4051cb8fec26b19014c7a3647 Mon Sep 17 00:00:00 2001 From: Reuben Frankel Date: Fri, 9 Feb 2024 21:55:42 +0000 Subject: [PATCH 1/3] Ignore some Ruff rules --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f621fad..7738084 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,8 +29,12 @@ warn_unused_configs = true [tool.ruff] ignore = [ + "ANN001", # missing-type-function-argument "ANN101", # missing-type-self "ANN102", # missing-type-cls + "ANN201", # missing-return-type-undocumented-public-function + "COM812", # missing-trailing-comma + "ISC001", # single-line-implicit-string-concatenation ] select = ["ALL"] src = ["tap_f1"] From 10e0ed1b22835d366f94f2724b48e8088f97452f Mon Sep 17 00:00:00 2001 From: Reuben Frankel Date: Fri, 9 Feb 2024 21:58:33 +0000 Subject: [PATCH 2/3] Lint/formatting fixes --- .pre-commit-config.yaml | 2 ++ tap_f1/client.py | 3 +++ tap_f1/pagination.py | 7 +++++++ tap_f1/streams.py | 37 +++++++++++++++++++++++-------------- tap_f1/tap.py | 6 ++++-- tests/test_core.py | 3 --- 6 files changed, 39 insertions(+), 19 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d2661e3..6083de2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,6 +28,7 @@ repos: hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] + exclude: tests/ - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy @@ -36,3 +37,4 @@ repos: - id: mypy additional_dependencies: - types-requests + exclude: tests/ diff --git a/tap_f1/client.py b/tap_f1/client.py index 3e7fd85..ff33dff 100644 --- a/tap_f1/client.py +++ b/tap_f1/client.py @@ -1,6 +1,7 @@ """REST client handling, including F1Stream base class.""" from singer_sdk.streams import RESTStream +from typing_extensions import override from tap_f1.pagination import F1Paginator @@ -11,9 +12,11 @@ class F1Stream(RESTStream): url_base = "https://ergast.com/api/f1" _limit = 1000 + @override def get_new_paginator(self): return F1Paginator(0, self._limit) + @override def get_url_params(self, context, next_page_token): params = super().get_url_params(context, next_page_token) params["limit"] = self._limit diff --git a/tap_f1/pagination.py b/tap_f1/pagination.py index bb65d31..83208fb 100644 --- a/tap_f1/pagination.py +++ b/tap_f1/pagination.py @@ -1,7 +1,14 @@ +"""Pagination classes for tap-f1.""" + + from singer_sdk.pagination import BaseOffsetPaginator +from typing_extensions import override class F1Paginator(BaseOffsetPaginator): + """Base API paginator.""" + + @override def has_more(self, response): data = response.json()["MRData"] diff --git a/tap_f1/streams.py b/tap_f1/streams.py index 5d35287..f9a878c 100644 --- a/tap_f1/streams.py +++ b/tap_f1/streams.py @@ -4,11 +4,15 @@ from datetime import date from singer_sdk import typing as th +from typing_extensions import override from tap_f1.client import F1Stream class SpeedUnitType(th.StringType): + """Speed unit type.""" + + @override @th.DefaultInstanceProperty def type_dict(self): return { @@ -24,7 +28,7 @@ class SeasonsStream(F1Stream): """Define seasons stream.""" name = "seasons" - primary_keys = ["season"] + primary_keys = ("season",) replication_key = "season" path = "/seasons.json" records_jsonpath = "MRData.SeasonTable.Seasons[*]" @@ -34,6 +38,7 @@ class SeasonsStream(F1Stream): th.Property("url", th.URIType), ).to_dict() + @override def get_child_context(self, record, context): start_date = date.fromisoformat(self.config["start_date"]) @@ -48,7 +53,7 @@ class CircuitsStream(F1Stream): parent_stream_type = SeasonsStream name = "circuits" - primary_keys = ["circuitId"] + primary_keys = ("circuitId",) path = "/{season}/circuits.json" records_jsonpath = "MRData.CircuitTable.Circuits[*]" @@ -74,7 +79,7 @@ class DriversStream(F1Stream): parent_stream_type = SeasonsStream context_key = "Driver" name = "drivers" - primary_keys = ["driverId"] + primary_keys = ("driverId",) path = "/{season}/drivers.json" records_jsonpath = "MRData.DriverTable.Drivers[*]" @@ -96,7 +101,7 @@ class ConstructorsStream(F1Stream): parent_stream_type = SeasonsStream context_key = "Constructor" name = "constructors" - primary_keys = ["constructorId"] + primary_keys = ("constructorId",) path = "/{season}/constructors.json" records_jsonpath = "MRData.ConstructorTable.Constructors[*]" @@ -113,7 +118,7 @@ class RacesStream(F1Stream): parent_stream_type = SeasonsStream name = "races" - primary_keys = ["season", "round"] + primary_keys = ("season", "round") replication_key = "date" path = "/{season}.json" records_jsonpath = "MRData.RaceTable.Races[*]" @@ -179,6 +184,7 @@ class RacesStream(F1Stream): ), ).to_dict() + @override def get_child_context(self, record, context): value = self.get_starting_replication_key_value(context) start_date = date.fromisoformat(value) @@ -197,7 +203,7 @@ class QualifyingResultsStream(F1Stream): parent_stream_type = RacesStream name = "qualifying_results" - primary_keys = ["season", "round", "number"] + primary_keys = ("season", "round", "number") path = "/{season}/{round}/qualifying.json" records_jsonpath = "MRData.RaceTable.Races[*].QualifyingResults[*]" @@ -239,7 +245,7 @@ class SprintResultsStream(F1Stream): parent_stream_type = RacesStream name = "sprints_results" - primary_keys = ["season", "round", "number"] + primary_keys = ("season", "round", "number") path = "/{season}/{round}/sprint.json" records_jsonpath = "MRData.RaceTable.Races[*].SprintResults[*]" @@ -310,7 +316,7 @@ class RaceResultsStream(F1Stream): parent_stream_type = RacesStream name = "race_results" - primary_keys = ["season", "round", "number"] + primary_keys = ("season", "round", "number") path = "/{season}/{round}/results.json" records_jsonpath = "MRData.RaceTable.Races[*].Results[*]" @@ -375,6 +381,7 @@ class RaceResultsStream(F1Stream): ), ).to_dict() + @override def get_child_context(self, record, context): return { **super().get_child_context(record, context), @@ -387,7 +394,7 @@ class LapsStream(F1Stream): parent_stream_type = RaceResultsStream name = "laps" - primary_keys = ["season", "round", "driverId", "number"] + primary_keys = ("season", "round", "driverId", "number") path = "/{season}/{round}/drivers/{driverId}/laps.json" records_jsonpath = "MRData.RaceTable.Races[*].Laps[*]" @@ -403,7 +410,7 @@ class LapsStream(F1Stream): th.Property("driverId", th.StringType), th.Property("position", th.StringType), th.Property("time", th.StringType), - ) + ), ), ), ).to_dict() @@ -414,7 +421,7 @@ class PitStopsStream(F1Stream): parent_stream_type = RacesStream name = "pit_stops" - primary_keys = ["season", "round", "driverId", "stop"] + primary_keys = ("season", "round", "driverId", "stop") path = "/{season}/{round}/pitstops.json" records_jsonpath = "MRData.RaceTable.Races[*].PitStops[*]" @@ -434,7 +441,7 @@ class DriverStandingsStream(F1Stream): parent_stream_type = RacesStream name = "driver_standings" - primary_keys = ["season", "round", "driverId", "position"] + primary_keys = ("season", "round", "driverId", "position") path = "/{season}/{round}/driverStandings.json" records_jsonpath = "MRData.StandingsTable.StandingsLists[*].DriverStandings[*]" @@ -467,11 +474,12 @@ class DriverStandingsStream(F1Stream): th.Property("url", th.URIType), th.Property("name", th.StringType), th.Property("nationality", th.StringType), - ) + ), ), ), ).to_dict() + @override def post_process(self, row, context): # driverId forms part of primary key row["driverId"] = row["Driver"]["driverId"] @@ -484,7 +492,7 @@ class ConstructorStandingsStream(F1Stream): parent_stream_type = RacesStream name = "constructor_standings" - primary_keys = ["season", "round", "constructorId", "position"] + primary_keys = ("season", "round", "constructorId", "position") path = "/{season}/{round}/constructorStandings.json" records_jsonpath = "MRData.StandingsTable.StandingsLists[*].ConstructorStandings[*]" @@ -507,6 +515,7 @@ class ConstructorStandingsStream(F1Stream): ), ).to_dict() + @override def post_process(self, row, context): # constructorId forms part of primary key row["constructorId"] = row["Constructor"]["constructorId"] diff --git a/tap_f1/tap.py b/tap_f1/tap.py index c09bb77..8d50d70 100644 --- a/tap_f1/tap.py +++ b/tap_f1/tap.py @@ -1,9 +1,10 @@ """F1 tap class.""" -from datetime import date +from datetime import date, datetime, timezone import singer_sdk.typing as th from singer_sdk import Tap +from typing_extensions import override from tap_f1 import streams @@ -32,10 +33,11 @@ class TapF1(Tap): th.Property( "start_date", th.DateType, - default=date(date.today().year, 1, 1).isoformat(), + default=date(datetime.now(tz=timezone.utc).year, 1, 1).isoformat(), ), ).to_dict() + @override def discover_streams(self): return [stream_type(self) for stream_type in STREAM_TYPES] diff --git a/tests/test_core.py b/tests/test_core.py index ccdb053..9f309c9 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -17,6 +17,3 @@ tap_class=TapF1, config=SAMPLE_CONFIG, ) - - -# TODO: Create additional tests as appropriate for your tap. From aeee6391956ef0469cb4304f7dfa7e5f25213584 Mon Sep 17 00:00:00 2001 From: Reuben Frankel Date: Wed, 6 Mar 2024 21:25:27 +0000 Subject: [PATCH 3/3] Add Makefile --- Makefile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2620f4b --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +.PHONY: help init lint test + +help: + @echo AVAILABLE COMMANDS + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-23s\033[0m%s\n", $$1, $$2}' + +init: ## Initialise repo for local development + @poetry install --sync --with dev + @poetry run pre-commit install --install-hooks + +lint: ## Lint files + poetry run pre-commit run ruff + +test: ## Run tests + @poetry run pytest