From b1bc4688d8054b22c506f9e3e271fdf427f8e31c Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Thu, 14 Nov 2024 17:36:58 +0000 Subject: [PATCH] feat(repositories): Add MetOffice DataHub repository --- .github/workflows/branch_ci.yml | 2 +- .github/workflows/main_ci.yml | 3 + .github/workflows/tagged_ci.yml | 4 +- src/nwp_consumer/{cmd => cmd.tmp}/__init__.py | 0 src/nwp_consumer/{cmd => cmd.tmp}/main.py | 0 .../internal/entities/parameters.py | 4 +- .../internal/ports/repositories.py | 29 +- .../model_repositories/__init__.py | 4 +- .../{metoffice_global.py => ceda_ftp.py} | 33 +- .../model_repositories/ecmwf_realtime.py | 25 +- .../model_repositories/mo_datahub.py | 298 ++++++++++++++++++ .../{noaa_gfs.py => noaa_s3.py} | 26 +- ...t_metoffice_global.py => test_ceda_ftp.py} | 4 +- .../model_repositories/test_mo_datahub.py | 34 ++ .../{test_noaa_gfs.py => test_noaa_s3.py} | 2 +- 15 files changed, 380 insertions(+), 88 deletions(-) rename src/nwp_consumer/{cmd => cmd.tmp}/__init__.py (100%) rename src/nwp_consumer/{cmd => cmd.tmp}/main.py (100%) rename src/nwp_consumer/internal/repositories/model_repositories/{metoffice_global.py => ceda_ftp.py} (90%) create mode 100644 src/nwp_consumer/internal/repositories/model_repositories/mo_datahub.py rename src/nwp_consumer/internal/repositories/model_repositories/{noaa_gfs.py => noaa_s3.py} (91%) rename src/nwp_consumer/internal/repositories/model_repositories/{test_metoffice_global.py => test_ceda_ftp.py} (96%) create mode 100644 src/nwp_consumer/internal/repositories/model_repositories/test_mo_datahub.py rename src/nwp_consumer/internal/repositories/model_repositories/{test_noaa_gfs.py => test_noaa_s3.py} (98%) diff --git a/.github/workflows/branch_ci.yml b/.github/workflows/branch_ci.yml index 863b00f6..80f50523 100644 --- a/.github/workflows/branch_ci.yml +++ b/.github/workflows/branch_ci.yml @@ -152,7 +152,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 2d0201b6..3ab9fe2a 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -17,6 +17,8 @@ concurrency: jobs: # Define an autotagger job that creates tags on changes to master + # Use #major #minor in merge commit messages to bump version beyond patch + # See https://github.com/RueLaLa/auto-tagger?tab=readme-ov-file#usage tag: runs-on: ubuntu-latest if: | @@ -34,3 +36,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_PR_NUMBER: ${{ github.event.number }} + diff --git a/.github/workflows/tagged_ci.yml b/.github/workflows/tagged_ci.yml index f4cb5a5a..ac0953cf 100644 --- a/.github/workflows/tagged_ci.yml +++ b/.github/workflows/tagged_ci.yml @@ -22,7 +22,6 @@ env: jobs: - # Job to create a container # Job for building container image # * Builds and pushes an OCI Container image to the registry defined in the environment variables build-container: @@ -30,7 +29,6 @@ jobs: permissions: contents: read packages: write - needs: ["lint-typecheck", "test-unit"] steps: # Do a non-shallow clone of the repo to ensure tags are present @@ -44,7 +42,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} diff --git a/src/nwp_consumer/cmd/__init__.py b/src/nwp_consumer/cmd.tmp/__init__.py similarity index 100% rename from src/nwp_consumer/cmd/__init__.py rename to src/nwp_consumer/cmd.tmp/__init__.py diff --git a/src/nwp_consumer/cmd/main.py b/src/nwp_consumer/cmd.tmp/main.py similarity index 100% rename from src/nwp_consumer/cmd/main.py rename to src/nwp_consumer/cmd.tmp/main.py diff --git a/src/nwp_consumer/internal/entities/parameters.py b/src/nwp_consumer/internal/entities/parameters.py index c5c0e398..9284bb12 100644 --- a/src/nwp_consumer/internal/entities/parameters.py +++ b/src/nwp_consumer/internal/entities/parameters.py @@ -136,7 +136,7 @@ def metadata(self) -> ParameterData: "incident on the surface expected over the next hour.", units="W/m^2", limits=ParameterLimits(upper=1500, lower=0), - alternate_shortnames=["swavr", "ssrd", "dswrf"], + alternate_shortnames=["swavr", "ssrd", "dswrf", "sdswrf"], ) case self.DOWNWARD_LONGWAVE_RADIATION_FLUX_GL.name: return ParameterData( @@ -146,7 +146,7 @@ def metadata(self) -> ParameterData: "incident on the surface expected over the next hour.", units="W/m^2", limits=ParameterLimits(upper=500, lower=0), - alternate_shortnames=["strd", "dlwrf"], + alternate_shortnames=["strd", "dlwrf", "sdlwrf"], ) case self.RELATIVE_HUMIDITY_SL.name: return ParameterData( diff --git a/src/nwp_consumer/internal/ports/repositories.py b/src/nwp_consumer/internal/ports/repositories.py index f3e2938b..1a7cacc2 100644 --- a/src/nwp_consumer/internal/ports/repositories.py +++ b/src/nwp_consumer/internal/ports/repositories.py @@ -13,14 +13,17 @@ import abc import datetime as dt +import logging import pathlib from collections.abc import Callable, Iterator import xarray as xr -from returns.result import ResultE +from returns.result import ResultE, Success from nwp_consumer.internal import entities +log = logging.getLogger("nwp-consumer") + class ModelRepository(abc.ABC): """Interface for a repository that produces raw NWP data. @@ -121,6 +124,30 @@ def model() -> entities.ModelMetadata: """Metadata about the model.""" pass + @staticmethod + def _rename_or_drop_vars(ds: xr.Dataset, allowed_parameters: list[entities.Parameter]) \ + -> xr.Dataset: + """Rename variables to match expected names, dropping invalid ones. + + Returns a dataset with all variables in it renamed to a known `entities.Parameter` + name, if a matching parameter exists, and it is an allowed parameter. Otherwise, + the variable is dropped from the dataset. + + Args: + ds: The xarray dataset to rename. + allowed_parameters: The list of parameters allowed in the resultant dataset. + """ + for var in ds.data_vars: + param_result = entities.Parameter.try_from_alternate(str(var)) + match param_result: + case Success(p): + if p in allowed_parameters: + ds = ds.rename_vars({var: p.value}) + continue + log.debug("Dropping invalid parameter '%s' from dataset", var) + ds = ds.drop_vars(str(var)) + return ds + class ZarrRepository(abc.ABC): """Interface for a repository that stores Zarr NWP data.""" diff --git a/src/nwp_consumer/internal/repositories/model_repositories/__init__.py b/src/nwp_consumer/internal/repositories/model_repositories/__init__.py index f7e444d3..e8c4947d 100644 --- a/src/nwp_consumer/internal/repositories/model_repositories/__init__.py +++ b/src/nwp_consumer/internal/repositories/model_repositories/__init__.py @@ -1,6 +1,6 @@ -from .metoffice_global import CEDAFTPModelRepository +from .ceda_ftp import CEDAFTPModelRepository from .ecmwf_realtime import ECMWFRealTimeS3ModelRepository -from .noaa_gfs import NOAAS3ModelRepository +from .noaa_s3 import NOAAS3ModelRepository __all__ = [ "CEDAFTPModelRepository", diff --git a/src/nwp_consumer/internal/repositories/model_repositories/metoffice_global.py b/src/nwp_consumer/internal/repositories/model_repositories/ceda_ftp.py similarity index 90% rename from src/nwp_consumer/internal/repositories/model_repositories/metoffice_global.py rename to src/nwp_consumer/internal/repositories/model_repositories/ceda_ftp.py index f3aa1de8..76c10955 100644 --- a/src/nwp_consumer/internal/repositories/model_repositories/metoffice_global.py +++ b/src/nwp_consumer/internal/repositories/model_repositories/ceda_ftp.py @@ -161,11 +161,6 @@ def model() -> entities.ModelMetadata: @override def fetch_init_data(self, it: dt.datetime) \ -> Iterator[Callable[..., ResultE[list[xr.DataArray]]]]: - # Ensure class is authenticated - authenticate_result = self.authenticate() - if isinstance(authenticate_result, Failure): - yield delayed(Result.from_failure)(authenticate_result.failure()) - return parameter_stubs: list[str] = [ "Total_Downward_Surface_SW_Flux", @@ -252,9 +247,7 @@ def _download(self, url: str) -> ResultE[pathlib.Path]: f.write(chunk) f.flush() log.debug( - "Downloaded %s to %s (%s bytes)", - url, - local_path, + f"Downloaded '{url}' to '{local_path}' (%s bytes)", local_path.stat().st_size, ) except Exception as e: @@ -268,7 +261,7 @@ def _download(self, url: str) -> ResultE[pathlib.Path]: @staticmethod def _convert(path: pathlib.Path) -> ResultE[list[xr.DataArray]]: - """Convert a file to an xarray dataset. + """Convert a local grib file to xarray DataArrays. Args: path: The path to the file to convert. @@ -309,25 +302,3 @@ def _convert(path: pathlib.Path) -> ResultE[list[xr.DataArray]]: ), ) return Success([da]) - - - @staticmethod - def _rename_or_drop_vars(ds: xr.Dataset, allowed_parameters: list[entities.Parameter]) \ - -> xr.Dataset: - """Rename variables to match the expected names, dropping invalid ones. - - Args: - ds: The xarray dataset to rename. - allowed_parameters: The list of parameters allowed in the resultant dataset. - """ - for var in ds.data_vars: - param_result = entities.Parameter.try_from_alternate(str(var)) - match param_result: - case Success(p): - if p in allowed_parameters: - ds = ds.rename_vars({var: p.value}) - continue - log.warning("Dropping invalid parameter '%s' from dataset", var) - ds = ds.drop_vars(str(var)) - return ds - diff --git a/src/nwp_consumer/internal/repositories/model_repositories/ecmwf_realtime.py b/src/nwp_consumer/internal/repositories/model_repositories/ecmwf_realtime.py index 35b3ae38..4708b8b5 100644 --- a/src/nwp_consumer/internal/repositories/model_repositories/ecmwf_realtime.py +++ b/src/nwp_consumer/internal/repositories/model_repositories/ecmwf_realtime.py @@ -143,9 +143,10 @@ def fetch_init_data(self, it: dt.datetime) \ f"in bucket path '{self.bucket}/ecmwf'. Ensure files exist at the given path " "named with the expected pattern, e.g. 'A2S10250000102603001.", )) + return log.debug( - f"Found {len(urls)} files for init time '{it.strftime('%Y-%m-%d %H:%M')}' " + f"Found {len(urls)} file(s) for init time '{it.strftime('%Y-%m-%d %H:%M')}' " f"in bucket path '{self.bucket}/ecmwf'.", ) for url in urls: @@ -309,25 +310,3 @@ def _wanted_file(filename: str, it: dt.datetime, max_step: int) -> bool: "%Y%m%d%H%M%z", ) return tt < it + dt.timedelta(hours=max_step) - - - @staticmethod - def _rename_or_drop_vars(ds: xr.Dataset, allowed_parameters: list[entities.Parameter]) \ - -> xr.Dataset: - """Rename variables to match the expected names, dropping invalid ones. - - Args: - ds: The xarray dataset to rename. - allowed_parameters: The list of parameters allowed in the resultant dataset. - """ - for var in ds.data_vars: - param_result = entities.Parameter.try_from_alternate(str(var)) - match param_result: - case Success(p): - if p in allowed_parameters: - ds = ds.rename_vars({var: p.value}) - continue - log.warning("Dropping invalid parameter '%s' from dataset", var) - ds = ds.drop_vars(str(var)) - return ds - diff --git a/src/nwp_consumer/internal/repositories/model_repositories/mo_datahub.py b/src/nwp_consumer/internal/repositories/model_repositories/mo_datahub.py new file mode 100644 index 00000000..aff21d80 --- /dev/null +++ b/src/nwp_consumer/internal/repositories/model_repositories/mo_datahub.py @@ -0,0 +1,298 @@ +"""Repository implementation for data from MetOffice's DataHub service. + +The API documentation for the MetOffice Weather Datahub can be found at: +https://datahub.metoffice.gov.uk/docs/f/category/atmospheric/type/atmospheric/api-documentation +""" + +import datetime as dt +import json +import logging +import os +import pathlib +import urllib.error +import urllib.request +from collections.abc import Callable, Iterator +from typing import TYPE_CHECKING, ClassVar, override + +import numpy as np +import xarray as xr +from joblib import delayed +from returns.result import Failure, ResultE, Success + +from nwp_consumer.internal import entities, ports + +if TYPE_CHECKING: + import http.client + +log = logging.getLogger("nwp-consumer") + + +class MetOfficeDatahubModelRepository(ports.ModelRepository): + """Repository implementation for data from MetOffice's DataHub service.""" + + base_url: ClassVar[str] = "https://data.hub.api.metoffice.gov.uk/atmospheric-models/1.0.0/orders" + + request_url: str + order_id: str + _headers: dict[str, str] + + def __init__(self, order_id: str, api_key: str) -> None: + """Create a new instance.""" + self._headers = { + "Accept": "application/json", + "apikey": api_key, + } + self.order_id = order_id + self.request_url = f"{self.base_url}/{self.order_id}/latest" + + @staticmethod + @override + def repository() -> entities.ModelRepositoryMetadata: + return entities.ModelRepositoryMetadata( + name="MetOffice-Weather-Datahub", + is_archive=False, + is_order_based=True, + running_hours=[0, 12], + delay_minutes=60, + max_connections=10, + required_env=["METOFFICE_API_KEY", "METOFFICE_ORDER_ID"], + optional_env={}, + postprocess_options=entities.PostProcessOptions(), + ) + + @staticmethod + @override + def model() -> entities.ModelMetadata: + return entities.ModelMetadata( + name="UM-Global", + resolution="17km", + expected_coordinates=entities.NWPDimensionCoordinateMap( + init_time=[], + step=list(range(0, 55)), + variable=sorted( + [ + entities.Parameter.CLOUD_COVER_TOTAL, + entities.Parameter.CLOUD_COVER_HIGH, + entities.Parameter.CLOUD_COVER_MEDIUM, + entities.Parameter.CLOUD_COVER_LOW, + entities.Parameter.VISIBILITY_SL, + entities.Parameter.RELATIVE_HUMIDITY_SL, + entities.Parameter.SNOW_DEPTH_GL, + entities.Parameter.DOWNWARD_SHORTWAVE_RADIATION_FLUX_GL, + entities.Parameter.TEMPERATURE_SL, + entities.Parameter.WIND_U_COMPONENT_10m, + entities.Parameter.WIND_V_COMPONENT_10m, + ], + ), + latitude=[ + float(f"{lat:.4f}") for lat in np.arange(89.856, -89.856 - 0.156, -0.156) + ], + longitude=[ + float(f"{lon:.4f}") for lon in np.concatenate([ + np.arange(-45, 45, 0.234), + np.arange(45, 135, 0.234), + np.arange(135, 225, 0.234), + np.arange(225, 315, 0.234), + ]) + ], + ), + ) + + @classmethod + @override + def authenticate(cls) -> ResultE["MetOfficeDatahubModelRepository"]: + """Authenticate with the MetOffice DataHub service.""" + if all(k not in os.environ for k in cls.repository().required_env): + return Failure(ValueError( + f"Missing required environment variables: {cls.repository().required_env}", + )) + api_key: str = os.environ["METOFFICE_API_KEY"] + order_id: str = os.environ["METOFFICE_ORDER_ID"] + return Success(cls(order_id=order_id, api_key=api_key)) + + @override + def fetch_init_data( + self, it: dt.datetime, + ) -> Iterator[Callable[..., ResultE[list[xr.DataArray]]]]: + """Fetch raw data files for an init time as xarray datasets.""" + req: urllib.request.Request = urllib.request.Request( # noqa: S310 + url=self.request_url + f"?detail=MINIMAL&runfilter={it:%Y%m%d%H}", + headers=self._headers, + method="GET", + ) + + # Request the list of files + try: + response: http.client.HTTPResponse = urllib.request.urlopen(req, timeout=30) # noqa: S310 + except Exception as e: + yield delayed(Failure( + "Unable to list files from MetOffice DataHub for order " + f"{self.order_id} at '{self.request_url}'. " + f"Ensure API key and Order ID are correct. Error context: {e}", + )) + return + try: + data = json.loads( + response.read().decode(response.info().get_param("charset") or "utf-8"), + ) + except Exception as e: + yield delayed(Failure( + "Unable to decode JSON response from MetOffice DataHub. " + "Check the response from the '/latest' endpoint looks as expected. " + f"Error context: {e}", + )) + return + urls: list[str] = [] + if "orderDetails" in data and "files" in data["orderDetails"]: + for filedata in data["orderDetails"]["files"]: + if "fileId" in filedata and "+" not in filedata["fileId"]: + urls.append(f"{self.request_url}/{filedata["fileId"]}/data") + + log.debug( + f"Found {len(urls)} file(s) for init time '{it.strftime('%Y-%m-%d %H:%M')}' " + f"in order '{self.order_id}'.", + ) + + for url in urls: + yield delayed(self._download_and_convert)(url) + + def _download_and_convert(self, url: str) -> ResultE[list[xr.DataArray]]: + """Download and convert a grib file from MetOffice Weather Datahub API. + + Args: + url: The URL of the file of interest. + """ + return self._download(url).bind(self._convert) + + def _download(self, url: str) -> ResultE[pathlib.Path]: + """Download a grib file from MetOffice Weather Datahub API. + + Args: + url: The URL of the file of interest. + """ + local_path: pathlib.Path = ( + pathlib.Path( + os.getenv( + "RAWDIR", + f"~/.local/cache/nwp/{self.repository().name}/{self.model().name}/raw", + ), + ) / f"{url.split("/")[-2]}.grib" + ).expanduser() + + # Only download the file if not already present + if not local_path.exists() or local_path.stat().st_size == 0: + local_path.parent.mkdir(parents=True, exist_ok=True) + log.debug("Requesting file from MetOffice Weather Datahub API at: '%s'", url) + + req: urllib.request.Request = urllib.request.Request( # noqa: S310 + url=url, + headers=self._headers | {"Accept": "application/x-grib"}, + method="GET", + ) + + # Request the file + try: + response: http.client.HTTPResponse = urllib.request.urlopen( # noqa: S310 + req, + timeout=60, + ) + except Exception as e: + return Failure( + "Unable to request file data from MetOffice DataHub at " + f"'{url}': {e}", + ) + + # Download the file + log.debug("Downloading %s to %s", url, local_path) + try: + with local_path.open("wb") as f: + for chunk in iter(lambda: response.read(16 * 1024), b""): + f.write(chunk) + f.flush() + log.debug( + f"Downloaded '{url}' to '{local_path}' (%s bytes)", + local_path.stat().st_size, + ) + except Exception as e: + return Failure( + OSError( + f"Error saving '{url}' to '{local_path}': {e}", + ), + ) + + return Success(local_path) + + @staticmethod + def _convert(path: pathlib.Path) -> ResultE[list[xr.DataArray]]: + """Convert a local grib file to xarray DataArrays. + + Args: + path: The path to the file to convert. + """ + try: + # Read the file as a dataset, also reading the values of the keys in 'read_keys' + ds: xr.Dataset = xr.open_dataset( + path, + engine="cfgrib", + backend_kwargs={"read_keys": ["name", "parameterNumber"], "indexpath": ""}, + chunks={ + "time": 1, + "step": -1, + }, + ) + except Exception as e: + return Failure( + OSError( + f"Error opening '{path}' as xarray Dataset: {e}", + ), + ) + + # Wind parameters are surfaced in the dataset as 'unknown' + # and have to be differentiated via the parameterNumber attribute + # which lines up with the last number in the GRIB2 code specified below + # https://datahub.metoffice.gov.uk/docs/glossary?groups=Wind&sortOrder=GRIB2_CODE + name = next(iter(ds.data_vars)) + parameter_number = ds[name].attrs["GRIB_parameterNumber"] + match name, parameter_number: + case "unknown", 192: + ds = ds.rename({name: "u10"}) + case "unknown", 193: + ds = ds.rename({name: "v10"}) + case "unknown", 194: + ds = ds.rename({name: "wdir"}) + case "unknown", 195: + ds = ds.rename({name: "wdir10"}) + case "unknown", _: + log.warning( + "Encountered unknown parameter with parameterNumber %s", + parameter_number, + ) + + try: + da: xr.DataArray = ( + ds.pipe(MetOfficeDatahubModelRepository._rename_or_drop_vars, ) + .rename(name_dict={"time": "init_time"}) + .expand_dims(dim="init_time") + .expand_dims(dim="step") + .to_dataarray(name=MetOfficeDatahubModelRepository.model().name) + ) + da = ( + da.drop_vars( + names=[ + c for c in ds.coords + if c not in ["init_time", "step", "variable", "latitude", "longitude"] + ], + errors="ignore", + ) + .transpose("init_time", "step", "variable", "latitude", "longitude") + .sortby(variables=["step", "variable", "longitude"]) + .sortby(variables="latitude", ascending=False) + ) + except Exception as e: + return Failure( + ValueError( + f"Error processing {path} to DataArray: {e}", + ), + ) + + return Success([da]) diff --git a/src/nwp_consumer/internal/repositories/model_repositories/noaa_gfs.py b/src/nwp_consumer/internal/repositories/model_repositories/noaa_s3.py similarity index 91% rename from src/nwp_consumer/internal/repositories/model_repositories/noaa_gfs.py rename to src/nwp_consumer/internal/repositories/model_repositories/noaa_s3.py index c0e63df5..e2be7658 100644 --- a/src/nwp_consumer/internal/repositories/model_repositories/noaa_gfs.py +++ b/src/nwp_consumer/internal/repositories/model_repositories/noaa_s3.py @@ -188,7 +188,8 @@ def _download(self, url: str) -> ResultE[pathlib.Path]: return Success(local_path) - def _convert(self, path: pathlib.Path) -> ResultE[list[xr.DataArray]]: + @staticmethod + def _convert(path: pathlib.Path) -> ResultE[list[xr.DataArray]]: """Convert a GFS file to an xarray DataArray collection. Args: @@ -206,7 +207,7 @@ def _convert(self, path: pathlib.Path) -> ResultE[list[xr.DataArray]]: "squeeze": True, "filter_by_keys": { "shortName": [ - x for v in self.model().expected_coordinates.variable + x for v in NOAAS3ModelRepository.model().expected_coordinates.variable for x in v.metadata().alternate_shortnames ], }, @@ -228,7 +229,7 @@ def _convert(self, path: pathlib.Path) -> ResultE[list[xr.DataArray]]: try: ds = NOAAS3ModelRepository._rename_or_drop_vars( ds=ds, - allowed_parameters=self.model().expected_coordinates.variable, + allowed_parameters=NOAAS3ModelRepository.model().expected_coordinates.variable, ) # Ignore datasets with no variables of interest if len(ds.data_vars) == 0: @@ -280,23 +281,4 @@ def _wanted_file(filename: str, it: dt.datetime, max_step: int) -> bool: return False return not int(match.group(2)) > max_step - @staticmethod - def _rename_or_drop_vars(ds: xr.Dataset, allowed_parameters: list[entities.Parameter]) \ - -> xr.Dataset: - """Rename variables to match the expected names, dropping invalid ones. - - Args: - ds: The xarray dataset to rename. - allowed_parameters: The list of parameters allowed in the resultant dataset. - """ - for var in ds.data_vars: - param_result = entities.Parameter.try_from_alternate(str(var)) - match param_result: - case Success(p): - if p in allowed_parameters: - ds = ds.rename_vars({var: p.value}) - continue - log.debug("Dropping invalid parameter '%s' from dataset", var) - ds = ds.drop_vars(str(var)) - return ds diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_metoffice_global.py b/src/nwp_consumer/internal/repositories/model_repositories/test_ceda_ftp.py similarity index 96% rename from src/nwp_consumer/internal/repositories/model_repositories/test_metoffice_global.py rename to src/nwp_consumer/internal/repositories/model_repositories/test_ceda_ftp.py index 65799504..3d372c24 100644 --- a/src/nwp_consumer/internal/repositories/model_repositories/test_metoffice_global.py +++ b/src/nwp_consumer/internal/repositories/model_repositories/test_ceda_ftp.py @@ -8,7 +8,7 @@ from nwp_consumer.internal import entities -from .metoffice_global import CEDAFTPModelRepository +from .ceda_ftp import CEDAFTPModelRepository class TestCEDAFTPModelRepository(unittest.TestCase): @@ -22,7 +22,7 @@ def test__download_and_convert(self) -> None: """Test the _download_and_convert method.""" auth_result = CEDAFTPModelRepository.authenticate() - self.assertTrue(is_successful(auth_result), msg=f"Error: {auth_result.failure}") + self.assertTrue(is_successful(auth_result), msg=f"Error: {auth_result}") c = auth_result.unwrap() test_it: dt.datetime = dt.datetime(2021, 1, 1, 0, tzinfo=dt.UTC) diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_mo_datahub.py b/src/nwp_consumer/internal/repositories/model_repositories/test_mo_datahub.py new file mode 100644 index 00000000..6ea85fe0 --- /dev/null +++ b/src/nwp_consumer/internal/repositories/model_repositories/test_mo_datahub.py @@ -0,0 +1,34 @@ +import dataclasses +import datetime as dt +import os +import unittest + +from returns.pipeline import flow, is_successful +from returns.pointfree import bind + +from nwp_consumer.internal import entities +from .mo_datahub import MetOfficeDatahubModelRepository + +class TestMetOfficeDatahubModelRepository(unittest.TestCase): + """Test the business methods of the MetOfficeDatahubModelRepository class.""" + + @unittest.skipIf( + condition="CI" in os.environ, + reason="Skipping integration test that requires MetOffice DataHub access.", + ) + def test__download(self) -> None: + """Test the _download method.""" + + auth_result = MetOfficeDatahubModelRepository.authenticate() + self.assertTrue(is_successful(auth_result), msg=f"Error: {auth_result}") + c = auth_result.unwrap() + + test_it = c.repository().determine_latest_it_from(dt.datetime.now(tz=dt.UTC)) + + dl_result = c._download( + f"{c.request_url}/agl_u-component-of-wind-surface-adjusted_10.0_{test_it:%Y%m%d%H}_1/data", + ) + + self.assertTrue(is_successful(dl_result), msg=f"Error: {dl_result}") + + diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_noaa_gfs.py b/src/nwp_consumer/internal/repositories/model_repositories/test_noaa_s3.py similarity index 98% rename from src/nwp_consumer/internal/repositories/model_repositories/test_noaa_gfs.py rename to src/nwp_consumer/internal/repositories/model_repositories/test_noaa_s3.py index e6a23f58..2e75452e 100644 --- a/src/nwp_consumer/internal/repositories/model_repositories/test_noaa_gfs.py +++ b/src/nwp_consumer/internal/repositories/model_repositories/test_noaa_s3.py @@ -8,7 +8,7 @@ from returns.pipeline import is_successful from ...entities import NWPDimensionCoordinateMap -from .noaa_gfs import NOAAS3ModelRepository +from .noaa_s3 import NOAAS3ModelRepository if TYPE_CHECKING: import xarray as xr