From 0f45e00854b51f185e366740a8d8261129036970 Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:53:39 +0000 Subject: [PATCH] feat(repositories): More granular grib tests --- .../model_repositories/test_ceda_ftp.py | 28 ++++++++---- .../model_repositories/test_ecmwf_realtime.py | 32 ++++++++----- ...TP_UM-Global_ssrd_20241105T00_S01-03.grib} | Bin ...UM-Global_u_20241105T00_S01-03_AreaC.grib} | Bin ...ealtime_HRES-IFS_10u_20241104T00_S60.grib} | Bin ...altime_HRES-IFS_ssrd_20241104T00_S60.grib} | Bin ...atahub_UM-Global_t2m_20241120T00_S00.grib} | Bin ...atahub_UM-Global_u10_20241120T00_S17.grib} | Bin ..._NOAAS3_HRES-GFS_10u_20210509T06_S00.grib} | Bin ...OAAS3_HRES-GFS_aptmp_20210509T06_S00.grib} | Bin ..._NOAAS3_HRES-GFS_lcc_20210509T06_S00.grib} | Bin ...st_NOAAS3_HRES-GFS_r_20210509T06_S00.grib} | Bin .../model_repositories/test_mo_datahub.py | 29 +++++++----- .../model_repositories/test_noaa_s3.py | 42 ++++++++++++------ 14 files changed, 87 insertions(+), 44 deletions(-) rename src/nwp_consumer/internal/repositories/model_repositories/{test_UM-Global_ssrd_AreaE.grib => test_gribs/test_CEDAFTP_UM-Global_ssrd_20241105T00_S01-03.grib} (100%) rename src/nwp_consumer/internal/repositories/model_repositories/{test_UM-Global_10u-AreaC.grib => test_gribs/test_CEDAFTP_UM-Global_u_20241105T00_S01-03_AreaC.grib} (100%) rename src/nwp_consumer/internal/repositories/model_repositories/{test_HRES-IFS_10u.grib => test_gribs/test_ECMWFRealtime_HRES-IFS_10u_20241104T00_S60.grib} (100%) rename src/nwp_consumer/internal/repositories/model_repositories/{test_HRES-IFS_ssrd.grib => test_gribs/test_ECMWFRealtime_HRES-IFS_ssrd_20241104T00_S60.grib} (100%) rename src/nwp_consumer/internal/repositories/model_repositories/{test_UM-Global_t2m.grib => test_gribs/test_MODatahub_UM-Global_t2m_20241120T00_S00.grib} (100%) rename src/nwp_consumer/internal/repositories/model_repositories/{test_UM-Global_10u.grib => test_gribs/test_MODatahub_UM-Global_u10_20241120T00_S17.grib} (100%) rename src/nwp_consumer/internal/repositories/model_repositories/{test_HRES-GFS_10u.grib => test_gribs/test_NOAAS3_HRES-GFS_10u_20210509T06_S00.grib} (100%) rename src/nwp_consumer/internal/repositories/model_repositories/{test_HRES-GFS_aptmp.grib => test_gribs/test_NOAAS3_HRES-GFS_aptmp_20210509T06_S00.grib} (100%) rename src/nwp_consumer/internal/repositories/model_repositories/{test_HRES-GFS_lcc.grib => test_gribs/test_NOAAS3_HRES-GFS_lcc_20210509T06_S00.grib} (100%) rename src/nwp_consumer/internal/repositories/model_repositories/{test_HRES-GFS_r.grib => test_gribs/test_NOAAS3_HRES-GFS_r_20210509T06_S00.grib} (100%) diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_ceda_ftp.py b/src/nwp_consumer/internal/repositories/model_repositories/test_ceda_ftp.py index abfe81d..5a56026 100644 --- a/src/nwp_consumer/internal/repositories/model_repositories/test_ceda_ftp.py +++ b/src/nwp_consumer/internal/repositories/model_repositories/test_ceda_ftp.py @@ -78,39 +78,49 @@ def test__convert(self) -> None: class TestCase: filename: str should_error: bool + expected_coords: entities.NWPDimensionCoordinateMap tests: list[TestCase] = [ TestCase( - filename="test_UM-Global_10u-AreaC.grib", + filename="test_CEDAFTP_UM-Global_ssrd_20241105T00_S01-03.grib", + expected_coords = dataclasses.replace( + CEDAFTPModelRepository.model().expected_coordinates, + init_time=[dt.datetime(2024, 11, 5, 0, tzinfo=dt.UTC)], + step=[1, 2, 3], + variable=[entities.parameters.Parameter.DOWNWARD_SHORTWAVE_RADIATION_FLUX_GL], + ), should_error=False, ), TestCase( - filename="test_UM-Global_ssrd_AreaE.grib", + filename="test_CEDAFTP_UM-Global_u_20241105T00_S01-03_AreaC.grib", + expected_coords = dataclasses.replace( + CEDAFTPModelRepository.model().expected_coordinates, + init_time=[dt.datetime(2024, 11, 5, 0, tzinfo=dt.UTC)], + step=[1, 2, 3], + variable=[entities.parameters.Parameter.WIND_U_COMPONENT_10m], + ), should_error=False, ), TestCase( - filename="test_HRES-IFS_10u.grib", + filename="test_MODatahub_UM-Global_t2m_20241120T00_S00.grib", + expected_coords = CEDAFTPModelRepository.model().expected_coordinates, should_error=True, ), ] - expected_coords = dataclasses.replace( - CEDAFTPModelRepository.model().expected_coordinates, - init_time=[dt.datetime(2024, 11, 5, 0, tzinfo=dt.UTC)], - ) for t in tests: with self.subTest(name=t.filename): # Attempt to convert the file result = CEDAFTPModelRepository._convert( - path=pathlib.Path(__file__).parent.absolute() / t.filename, + path=pathlib.Path(__file__).parent.absolute() / "test_gribs" / t.filename, ) region_result: ResultE[dict[str, slice]] = result.do( region for das in result for da in das for region in NWPDimensionCoordinateMap.from_xarray(da).bind( - expected_coords.determine_region, + t.expected_coords.determine_region, ) ) if t.should_error: diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_ecmwf_realtime.py b/src/nwp_consumer/internal/repositories/model_repositories/test_ecmwf_realtime.py index 9465cd8..8caaf06 100644 --- a/src/nwp_consumer/internal/repositories/model_repositories/test_ecmwf_realtime.py +++ b/src/nwp_consumer/internal/repositories/model_repositories/test_ecmwf_realtime.py @@ -7,7 +7,7 @@ from returns.result import Failure, ResultE, Success -from ...entities import NWPDimensionCoordinateMap +from ...entities import NWPDimensionCoordinateMap, Parameter from .ecmwf_realtime import ECMWFRealTimeS3ModelRepository if TYPE_CHECKING: @@ -113,44 +113,52 @@ def test__convert(self) -> None: @dataclasses.dataclass class TestCase: filename: str + expected_coords: NWPDimensionCoordinateMap should_error: bool tests: list[TestCase] = [ TestCase( - filename="test_HRES-IFS_ssrd.grib", + filename="test_ECMWFRealtime_HRES-IFS_ssrd_20241104T00_S60.grib", + expected_coords=dataclasses.replace( + ECMWFRealTimeS3ModelRepository.model().expected_coordinates, + init_time=[dt.datetime(2024, 11, 4, 0, tzinfo=dt.UTC)], + variable=[Parameter.DOWNWARD_SHORTWAVE_RADIATION_FLUX_GL], + step=[60], + ), should_error=False, ), TestCase( - filename="test_HRES-IFS_10u.grib", + filename="test_ECMWFRealtime_HRES-IFS_10u_20241104T00_S60.grib", + expected_coords=dataclasses.replace( + ECMWFRealTimeS3ModelRepository.model().expected_coordinates, + init_time=[dt.datetime(2024, 11, 4, 0, tzinfo=dt.UTC)], + variable=[Parameter.WIND_U_COMPONENT_10m], + step=[60], + ), should_error=False, ), TestCase( - filename="test_UM-Global_t2m.grib", + filename="test_NOAAS3_HRES-GFS_10u_20210509T00_S00.grib", + expected_coords=ECMWFRealTimeS3ModelRepository.model().expected_coordinates, should_error=True, ), ] - expected_coords = dataclasses.replace( - ECMWFRealTimeS3ModelRepository.model().expected_coordinates, - init_time=[dt.datetime(2024, 11, 4, 0, tzinfo=dt.UTC)], - ) - for t in tests: with self.subTest(name=t.filename): # Attempt to convert the file result = ECMWFRealTimeS3ModelRepository._convert( - path=pathlib.Path(__file__).parent.absolute() / t.filename, + path=pathlib.Path(__file__).parent.absolute() / "test_gribs" / t.filename, ) region_result: ResultE[dict[str, slice]] = result.do( region for das in result for da in das for region in NWPDimensionCoordinateMap.from_xarray(da).bind( - expected_coords.determine_region, + t.expected_coords.determine_region, ) ) if t.should_error: self.assertIsInstance(region_result, Failure, msg=f"{region_result}") else: self.assertIsInstance(region_result, Success, msg=f"{region_result}") - diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_UM-Global_ssrd_AreaE.grib b/src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_CEDAFTP_UM-Global_ssrd_20241105T00_S01-03.grib similarity index 100% rename from src/nwp_consumer/internal/repositories/model_repositories/test_UM-Global_ssrd_AreaE.grib rename to src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_CEDAFTP_UM-Global_ssrd_20241105T00_S01-03.grib diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_UM-Global_10u-AreaC.grib b/src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_CEDAFTP_UM-Global_u_20241105T00_S01-03_AreaC.grib similarity index 100% rename from src/nwp_consumer/internal/repositories/model_repositories/test_UM-Global_10u-AreaC.grib rename to src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_CEDAFTP_UM-Global_u_20241105T00_S01-03_AreaC.grib diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_HRES-IFS_10u.grib b/src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_ECMWFRealtime_HRES-IFS_10u_20241104T00_S60.grib similarity index 100% rename from src/nwp_consumer/internal/repositories/model_repositories/test_HRES-IFS_10u.grib rename to src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_ECMWFRealtime_HRES-IFS_10u_20241104T00_S60.grib diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_HRES-IFS_ssrd.grib b/src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_ECMWFRealtime_HRES-IFS_ssrd_20241104T00_S60.grib similarity index 100% rename from src/nwp_consumer/internal/repositories/model_repositories/test_HRES-IFS_ssrd.grib rename to src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_ECMWFRealtime_HRES-IFS_ssrd_20241104T00_S60.grib diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_UM-Global_t2m.grib b/src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_MODatahub_UM-Global_t2m_20241120T00_S00.grib similarity index 100% rename from src/nwp_consumer/internal/repositories/model_repositories/test_UM-Global_t2m.grib rename to src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_MODatahub_UM-Global_t2m_20241120T00_S00.grib diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_UM-Global_10u.grib b/src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_MODatahub_UM-Global_u10_20241120T00_S17.grib similarity index 100% rename from src/nwp_consumer/internal/repositories/model_repositories/test_UM-Global_10u.grib rename to src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_MODatahub_UM-Global_u10_20241120T00_S17.grib diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_HRES-GFS_10u.grib b/src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_NOAAS3_HRES-GFS_10u_20210509T06_S00.grib similarity index 100% rename from src/nwp_consumer/internal/repositories/model_repositories/test_HRES-GFS_10u.grib rename to src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_NOAAS3_HRES-GFS_10u_20210509T06_S00.grib diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_HRES-GFS_aptmp.grib b/src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_NOAAS3_HRES-GFS_aptmp_20210509T06_S00.grib similarity index 100% rename from src/nwp_consumer/internal/repositories/model_repositories/test_HRES-GFS_aptmp.grib rename to src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_NOAAS3_HRES-GFS_aptmp_20210509T06_S00.grib diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_HRES-GFS_lcc.grib b/src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_NOAAS3_HRES-GFS_lcc_20210509T06_S00.grib similarity index 100% rename from src/nwp_consumer/internal/repositories/model_repositories/test_HRES-GFS_lcc.grib rename to src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_NOAAS3_HRES-GFS_lcc_20210509T06_S00.grib diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_HRES-GFS_r.grib b/src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_NOAAS3_HRES-GFS_r_20210509T06_S00.grib similarity index 100% rename from src/nwp_consumer/internal/repositories/model_repositories/test_HRES-GFS_r.grib rename to src/nwp_consumer/internal/repositories/model_repositories/test_gribs/test_NOAAS3_HRES-GFS_r_20210509T06_S00.grib diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_mo_datahub.py b/src/nwp_consumer/internal/repositories/model_repositories/test_mo_datahub.py index 9041548..ca63c05 100644 --- a/src/nwp_consumer/internal/repositories/model_repositories/test_mo_datahub.py +++ b/src/nwp_consumer/internal/repositories/model_repositories/test_mo_datahub.py @@ -6,7 +6,7 @@ from returns.result import Failure, ResultE, Success -from ...entities import NWPDimensionCoordinateMap +from ...entities import NWPDimensionCoordinateMap, Parameter from .mo_datahub import MetOfficeDatahubModelRepository @@ -39,40 +39,49 @@ def test__convert(self) -> None: @dataclasses.dataclass class TestCase: filename: str + expected_coords: NWPDimensionCoordinateMap should_error: bool tests: list[TestCase] = [ TestCase( - filename="test_UM-Global_10u.grib", + filename="test_MODatahub_UM-Global_t2m_20241120T00_S00.grib", + expected_coords=dataclasses.replace( + MetOfficeDatahubModelRepository.model().expected_coordinates, + init_time=[dt.datetime(2024, 11, 20, 0, tzinfo=dt.UTC)], + variable=[Parameter.TEMPERATURE_SL], + step=[0], + ), should_error=False, ), TestCase( - filename="test_UM-Global_t2m.grib", + filename="test_MODatahub_UM-Global_u10_20241120T00_S17.grib", + expected_coords=dataclasses.replace( + MetOfficeDatahubModelRepository.model().expected_coordinates, + init_time=[dt.datetime(2024, 11, 20, 0, tzinfo=dt.UTC)], + variable=[Parameter.WIND_U_COMPONENT_10m], + step=[17], + ), should_error=False, ), TestCase( filename="test_HRES-IFS_10u.grib", + expected_coords=MetOfficeDatahubModelRepository.model().expected_coordinates, should_error=True, ), ] - expected_coords = dataclasses.replace( - MetOfficeDatahubModelRepository.model().expected_coordinates, - init_time=[dt.datetime(2024, 11, 20, 0, tzinfo=dt.UTC)], - ) - for t in tests: with self.subTest(name=t.filename): # Attempt to convert the file result = MetOfficeDatahubModelRepository._convert( - path=pathlib.Path(__file__).parent.absolute() / t.filename, + path=pathlib.Path(__file__).parent.absolute() / "test_gribs" / t.filename, ) region_result: ResultE[dict[str, slice]] = result.do( region for das in result for da in das for region in NWPDimensionCoordinateMap.from_xarray(da).bind( - expected_coords.determine_region, + t.expected_coords.determine_region, ) ) if t.should_error: diff --git a/src/nwp_consumer/internal/repositories/model_repositories/test_noaa_s3.py b/src/nwp_consumer/internal/repositories/model_repositories/test_noaa_s3.py index 8da0c1d..944fc80 100644 --- a/src/nwp_consumer/internal/repositories/model_repositories/test_noaa_s3.py +++ b/src/nwp_consumer/internal/repositories/model_repositories/test_noaa_s3.py @@ -8,7 +8,7 @@ import s3fs from returns.result import Failure, ResultE, Success -from ...entities import NWPDimensionCoordinateMap +from ...entities import NWPDimensionCoordinateMap, Parameter from .noaa_s3 import NOAAS3ModelRepository if TYPE_CHECKING: @@ -117,48 +117,64 @@ def test__convert(self) -> None: @dataclasses.dataclass class TestCase: filename: str + expected_coords: NWPDimensionCoordinateMap should_error: bool tests: list[TestCase] = [ TestCase( - filename="test_HRES-GFS_10u.grib", + filename="test_NOAAS3_HRES-GFS_10u_20210509T06_S00.grib", + expected_coords=dataclasses.replace( + NOAAS3ModelRepository.model().expected_coordinates, + init_time=[dt.datetime(2021, 5, 9, 6, tzinfo=dt.UTC)], + variable=[Parameter.WIND_U_COMPONENT_10m], + step=[0], + ), should_error=False, ), TestCase( - filename="test_HRES-GFS_lcc.grib", + filename="test_NOAAS3_HRES-GFS_lcc_20210509T06_S00.grib", + expected_coords=dataclasses.replace( + NOAAS3ModelRepository.model().expected_coordinates, + init_time=[dt.datetime(2021, 5, 9, 6, tzinfo=dt.UTC)], + variable=[Parameter.CLOUD_COVER_LOW], + step=[0], + ), should_error=False, ), TestCase( - filename="test_HRES-GFS_r.grib", + filename="test_NOAAS3_HRES-GFS_r_20210509T06_S00.grib", + expected_coords=dataclasses.replace( + NOAAS3ModelRepository.model().expected_coordinates, + init_time=[dt.datetime(2021, 5, 9, 6, tzinfo=dt.UTC)], + variable=[Parameter.RELATIVE_HUMIDITY_SL], + step=[0], + ), should_error=False, ), TestCase( - filename="test_HRES-GFS_aptmp.grib", + filename="test_NOAAS3_HRES-GFS_aptmp_20210509T06_S00.grib", + expected_coords=NOAAS3ModelRepository.model().expected_coordinates, should_error=True, ), TestCase( - filename="test_UM-Global_t2m.grib", + filename="test_MODatahub_UM-Global_t2m_20241120T00_S00.grib", + expected_coords=NOAAS3ModelRepository.model().expected_coordinates, should_error=True, ), ] - expected_coords = dataclasses.replace( - NOAAS3ModelRepository.model().expected_coordinates, - init_time=[dt.datetime(2021, 5, 9, 6, tzinfo=dt.UTC)], - ) - for t in tests: with self.subTest(name=t.filename): # Attempt to convert the file result = NOAAS3ModelRepository._convert( - path=pathlib.Path(__file__).parent.absolute() / t.filename, + path=pathlib.Path(__file__).parent.absolute() / "test_gribs" / t.filename, ) region_result: ResultE[dict[str, slice]] = result.do( region for das in result for da in das for region in NWPDimensionCoordinateMap.from_xarray(da).bind( - expected_coords.determine_region, + t.expected_coords.determine_region, ) ) if t.should_error: