diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f4b1bee..2bbd9e1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,6 +34,13 @@ jobs: run: | poetry env use ${{ matrix.python-version }} poetry install + - name: Cache HTTP responses + uses: actions/cache@v4 + with: + path: ~/.cache/tap-f1.sqlite + key: http-responses-${{ github.run_id }} + restore-keys: | + http-responses- - name: Test with pytest run: | poetry run pytest diff --git a/poetry.lock b/poetry.lock index 8ed94d7..e3d80a8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -122,6 +122,32 @@ urllib3 = [ [package.extras] crt = ["awscrt (==0.19.19)"] +[[package]] +name = "cattrs" +version = "24.1.2" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-24.1.2-py3-none-any.whl", hash = "sha256:67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0"}, + {file = "cattrs-24.1.2.tar.gz", hash = "sha256:8028cfe1ff5382df59dd36474a86e02d817b06eaf8af84555441bac915d2ef85"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +msgspec = ["msgspec (>=0.18.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + [[package]] name = "certifi" version = "2024.7.4" @@ -582,6 +608,22 @@ files = [ {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"}, ] +[[package]] +name = "platformdirs" +version = "4.3.6" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, + {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, +] + +[package.extras] +docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"] +type = ["mypy (>=1.11.2)"] + [[package]] name = "pluggy" version = "1.5.0" @@ -754,6 +796,36 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requests-cache" +version = "1.2.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.1-py3-none-any.whl", hash = "sha256:1285151cddf5331067baa82598afe2d47c7495a1334bfe7a7d329b43e9fd3603"}, + {file = "requests_cache-1.2.1.tar.gz", hash = "sha256:68abc986fdc5b8d0911318fbb5f7c80eebcd4d01bfacc6685ecf8876052511d1"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + [[package]] name = "rpds-py" version = "0.17.1" @@ -1179,6 +1251,20 @@ files = [ {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + [[package]] name = "urllib3" version = "1.26.19" @@ -1216,4 +1302,4 @@ s3 = ["fs-s3fs"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "01c7fe8e8b8f1f36fa96a5aa5ff52311c4532963bf2b730c3daa1453c62c7386" +content-hash = "ecd6e88a09f4d15908bf590cdecc2995b9abbe3a8a49111aa6ac4ff60fd6467d" diff --git a/pyproject.toml b/pyproject.toml index 3c2bd9b..cc27dad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ python = "^3.8" singer-sdk = { version="~=0.42.1" } fs-s3fs = { version = "~=1.1.1", optional = true } requests = "~=2.32.3" +requests-cache = "^1.2.1" [tool.poetry.group.dev.dependencies] pytest = ">=7.4.0" @@ -30,6 +31,8 @@ warn_unused_configs = true [tool.ruff] ignore = [ "ANN001", # missing-type-function-argument + "ANN002", # missing-type-args + "ANN003", # missing-type-kwargs "ANN101", # missing-type-self "ANN102", # missing-type-cls "ANN201", # missing-return-type-undocumented-public-function diff --git a/tap_f1/client.py b/tap_f1/client.py index ff33dff..4cb7fce 100644 --- a/tap_f1/client.py +++ b/tap_f1/client.py @@ -1,5 +1,8 @@ """REST client handling, including F1Stream base class.""" +from datetime import timedelta + +from requests_cache import CachedSession from singer_sdk.streams import RESTStream from typing_extensions import override @@ -12,6 +15,15 @@ class F1Stream(RESTStream): url_base = "https://ergast.com/api/f1" _limit = 1000 + def __init__(self, *args, **kwargs) -> None: + """Initialise the F1 stream.""" + super().__init__(*args, **kwargs) + self._requests_session = CachedSession( + self.tap_name, + use_cache_dir=True, + expire_after=timedelta(days=1), + ) + @override def get_new_paginator(self): return F1Paginator(0, self._limit)