From 9b5aac88e0a4e4919f64550aadc0cd072f58f059 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Tue, 26 Sep 2023 16:45:34 +0100 Subject: [PATCH 1/9] and national creation utc limit, start and end target times --- src/national.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/national.py b/src/national.py index f7e4613..51b9f1d 100644 --- a/src/national.py +++ b/src/national.py @@ -39,6 +39,9 @@ def get_national_forecast( forecast_horizon_minutes: Optional[int] = None, user: Auth0User = Security(get_user()), include_metadata: bool = False, + start_datetime_utc: Optional[str] = None, + end_datetime_utc: Optional[str] = None, + creation_limit_utc: Optional[str] = None, ) -> Union[NationalForecast, List[NationalForecastValue]]: """Get the National Forecast @@ -55,6 +58,10 @@ def get_national_forecast( #### Parameters - **forecast_horizon_minutes**: optional forecast horizon in minutes (ex. 60 returns the forecast made an hour before the target time) + - **start_datetime_utc**: optional start datetime for the query. + - **end_datetime_utc**: optional end datetime for the query. + - **creation_utc_limit**: optional, only return forecasts made before this datetime. + Note you can only go 7 days back at the moment """ logger.debug("Get national forecasts") @@ -68,7 +75,14 @@ def get_national_forecast( ) forecast = get_latest_forecast_for_gsps( - session=session, gsp_ids=[0], model_name="blend", historic=True, preload_children=True + session=session, + gsp_ids=[0], + model_name="blend", + historic=True, + preload_children=True, + start_target_time=start_datetime_utc, + end_target_time=end_datetime_utc, + start_created_utc=creation_limit_utc, ) forecast = forecast[0] @@ -76,7 +90,12 @@ def get_national_forecast( forecast_values = forecast.forecast_values else: forecast_values = get_latest_forecast_values_for_a_specific_gsp_from_database( - session=session, gsp_id=0, forecast_horizon_minutes=forecast_horizon_minutes + session=session, + gsp_id=0, + forecast_horizon_minutes=forecast_horizon_minutes, + start_datetime_utc=start_datetime_utc, + end_datetime_utc=end_datetime_utc, + creation_utc_limit=creation_limit_utc, ) logger.debug( From 08c0c951dd1dd881a892d161f750e3ed380f1ede Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Tue, 26 Sep 2023 17:13:06 +0100 Subject: [PATCH 2/9] add tests for national --- src/database.py | 3 ++ src/national.py | 6 +++- src/tests/test_national.py | 61 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/database.py b/src/database.py index 7fd8391..7dca3b1 100644 --- a/src/database.py +++ b/src/database.py @@ -229,6 +229,9 @@ def get_latest_forecast_values_for_a_specific_gsp_from_database( created_utc_limit=creation_utc_limit, ) + if len(forecast_values) == 0: + return [] + # convert to pydantic objects if ( isinstance(forecast_values[0], ForecastValueSevenDaysSQL) diff --git a/src/national.py b/src/national.py index 51b9f1d..72a4473 100644 --- a/src/national.py +++ b/src/national.py @@ -16,7 +16,7 @@ get_truth_values_for_a_specific_gsp_from_database, ) from pydantic_models import NationalForecast, NationalForecastValue, NationalYield -from utils import format_plevels +from utils import format_datetime, format_plevels logger = structlog.stdlib.get_logger() @@ -66,6 +66,10 @@ def get_national_forecast( """ logger.debug("Get national forecasts") + start_datetime_utc = format_datetime(start_datetime_utc) + end_datetime_utc = format_datetime(end_datetime_utc) + creation_limit_utc = format_datetime(creation_limit_utc) + logger.debug("Getting forecast.") if include_metadata: if forecast_horizon_minutes is not None: diff --git a/src/tests/test_national.py b/src/tests/test_national.py index 164e321..fb0ebda 100644 --- a/src/tests/test_national.py +++ b/src/tests/test_national.py @@ -6,6 +6,7 @@ from nowcasting_datamodel.fake import make_fake_national_forecast from nowcasting_datamodel.models import GSPYield, Location, LocationSQL from nowcasting_datamodel.read.read import get_model +from nowcasting_datamodel.save.save import save_all_forecast_values_seven_days from nowcasting_datamodel.save.update import update_all_forecast_latest from database import get_session @@ -42,6 +43,66 @@ def test_read_latest_national_values(db_session, api_client): ) +def test_read_latest_national_values_creation_limit(db_session, api_client): + """Check main solar/GB/national/forecast route works""" + + with freeze_time("2023-01-01"): + model = get_model(db_session, name="blend", version="0.0.1") + + forecast = make_fake_national_forecast( + session=db_session, t0_datetime_utc=datetime.now(tz=timezone.utc) + ) + forecast.model = model + db_session.add(forecast) + update_all_forecast_latest(forecasts=[forecast], session=db_session) + save_all_forecast_values_seven_days(forecasts=[forecast], session=db_session) + + with freeze_time("2023-01-02"): + app.dependency_overrides[get_session] = lambda: db_session + + response = api_client.get("/v0/solar/GB/national/forecast?creation_limit_utc=2023-01-02") + assert response.status_code == 200 + + national_forecast_values = [NationalForecastValue(**f) for f in response.json()] + assert len(national_forecast_values) == 16 + + response = api_client.get("/v0/solar/GB/national/forecast?creation_limit_utc=2022-12-31") + assert response.status_code == 200 + + national_forecast_values = [NationalForecastValue(**f) for f in response.json()] + assert len(national_forecast_values) == 0 + + +def test_read_latest_national_values_start_and_end_filters(db_session, api_client): + """Check main solar/GB/national/forecast route works""" + + with freeze_time("2023-01-01"): + model = get_model(db_session, name="blend", version="0.0.1") + + forecast = make_fake_national_forecast( + session=db_session, t0_datetime_utc=datetime.now(tz=timezone.utc) + ) + forecast.model = model + db_session.add(forecast) + update_all_forecast_latest(forecasts=[forecast], session=db_session) + + app.dependency_overrides[get_session] = lambda: db_session + + response = api_client.get("/v0/solar/GB/national/forecast?start_datetime_utc=2023-01-01") + assert response.status_code == 200 + + national_forecast_values = [NationalForecastValue(**f) for f in response.json()] + assert len(national_forecast_values) == 16 + + response = api_client.get( + "/v0/solar/GB/national/forecast?start_datetime_utc=2023-01-01&end_datetime_utc=2023-01-01 04:00" + ) + assert response.status_code == 200 + + national_forecast_values = [NationalForecastValue(**f) for f in response.json()] + assert len(national_forecast_values) == 9 + + def test_get_national_forecast(db_session, api_client): """Check main solar/GB/national/forecast route works""" From 749ea59dde509f136140e349bee8de93cf057825 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Tue, 26 Sep 2023 17:14:44 +0100 Subject: [PATCH 3/9] lint --- src/tests/test_national.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/test_national.py b/src/tests/test_national.py index fb0ebda..5de34ba 100644 --- a/src/tests/test_national.py +++ b/src/tests/test_national.py @@ -95,7 +95,7 @@ def test_read_latest_national_values_start_and_end_filters(db_session, api_clien assert len(national_forecast_values) == 16 response = api_client.get( - "/v0/solar/GB/national/forecast?start_datetime_utc=2023-01-01&end_datetime_utc=2023-01-01 04:00" + "/v0/solar/GB/national/forecast?start_datetime_utc=2023-01-01&end_datetime_utc=2023-01-01 04:00" # noqa ) assert response.status_code == 200 From eab1081c457309bbabff4e791c7d137290209998 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 16:15:03 +0000 Subject: [PATCH 4/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/tests/test_national.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/test_national.py b/src/tests/test_national.py index 5de34ba..96d1b03 100644 --- a/src/tests/test_national.py +++ b/src/tests/test_national.py @@ -95,7 +95,7 @@ def test_read_latest_national_values_start_and_end_filters(db_session, api_clien assert len(national_forecast_values) == 16 response = api_client.get( - "/v0/solar/GB/national/forecast?start_datetime_utc=2023-01-01&end_datetime_utc=2023-01-01 04:00" # noqa + "/v0/solar/GB/national/forecast?start_datetime_utc=2023-01-01&end_datetime_utc=2023-01-01 04:00" # noqa ) assert response.status_code == 200 From fa66f8e7327cae873b194a7af05f49c344e9774b Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Wed, 27 Sep 2023 11:10:53 +0100 Subject: [PATCH 5/9] small fixes --- src/database.py | 1 + src/national.py | 10 ++++++++-- src/tests/test_national.py | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/database.py b/src/database.py index 7dca3b1..d57f24a 100644 --- a/src/database.py +++ b/src/database.py @@ -222,6 +222,7 @@ def get_latest_forecast_values_for_a_specific_gsp_from_database( session=session, gsp_id=gsp_id, start_datetime=start_datetime, + end_datetime=end_datetime_utc, forecast_horizon_minutes=forecast_horizon_minutes, model_name="blend", model=ForecastValueSevenDaysSQL, diff --git a/src/national.py b/src/national.py index 72a4473..068737d 100644 --- a/src/national.py +++ b/src/national.py @@ -16,7 +16,7 @@ get_truth_values_for_a_specific_gsp_from_database, ) from pydantic_models import NationalForecast, NationalForecastValue, NationalYield -from utils import format_datetime, format_plevels +from utils import filter_forecast_values, format_datetime, format_plevels logger = structlog.stdlib.get_logger() @@ -91,7 +91,13 @@ def get_national_forecast( forecast = forecast[0] forecast = NationalForecast.from_orm_latest(forecast) - forecast_values = forecast.forecast_values + forecasts = filter_forecast_values( + forecasts=[forecast], + start_datetime_utc=start_datetime_utc, + end_datetime_utc=end_datetime_utc, + ) + forecast_values = forecasts[0].forecast_values + else: forecast_values = get_latest_forecast_values_for_a_specific_gsp_from_database( session=session, diff --git a/src/tests/test_national.py b/src/tests/test_national.py index 5de34ba..96d1b03 100644 --- a/src/tests/test_national.py +++ b/src/tests/test_national.py @@ -95,7 +95,7 @@ def test_read_latest_national_values_start_and_end_filters(db_session, api_clien assert len(national_forecast_values) == 16 response = api_client.get( - "/v0/solar/GB/national/forecast?start_datetime_utc=2023-01-01&end_datetime_utc=2023-01-01 04:00" # noqa + "/v0/solar/GB/national/forecast?start_datetime_utc=2023-01-01&end_datetime_utc=2023-01-01 04:00" # noqa ) assert response.status_code == 200 From 41e9127f282ce45f3ea06d07104aab68b38593f1 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Wed, 27 Sep 2023 11:13:42 +0100 Subject: [PATCH 6/9] add test --- src/tests/test_national.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/tests/test_national.py b/src/tests/test_national.py index 96d1b03..80f0ffe 100644 --- a/src/tests/test_national.py +++ b/src/tests/test_national.py @@ -131,6 +131,35 @@ def test_get_national_forecast(db_session, api_client): != national_forecast.forecast_values[0].expected_power_generation_megawatts * 0.9 ) +def test_read_latest_national_values_start_and_end_filters_inculde_metadata(db_session, api_client): + """Check main solar/GB/national/forecast route works""" + + with freeze_time("2023-01-01"): + model = get_model(db_session, name="blend", version="0.0.1") + + forecast = make_fake_national_forecast( + session=db_session, t0_datetime_utc=datetime.now(tz=timezone.utc) + ) + forecast.model = model + db_session.add(forecast) + update_all_forecast_latest(forecasts=[forecast], session=db_session) + + app.dependency_overrides[get_session] = lambda: db_session + + response = api_client.get("/v0/solar/GB/national/forecast?start_datetime_utc=2023-01-01&include_metadata=true") # noqa + assert response.status_code == 200 + + national_forecast = NationalForecast(**response.json()) + assert len(national_forecasts.forecast_values) == 16 + + response = api_client.get( + "/v0/solar/GB/national/forecast?start_datetime_utc=2023-01-01&end_datetime_utc=2023-01-01 04:00&include_metadata=true" # noqa + ) + assert response.status_code == 200 + + national_forecast = NationalForecast(**response.json()) + assert len(national_forecasts.forecast_values) == 9 + def test_get_national_forecast_error(db_session, api_client): """Check main solar/GB/national/forecast route works""" From 32242790b0b724fbc319508353a9a6e457087c7d Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Wed, 27 Sep 2023 11:15:03 +0100 Subject: [PATCH 7/9] fix test --- src/tests/test_national.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/test_national.py b/src/tests/test_national.py index 80f0ffe..b39e801 100644 --- a/src/tests/test_national.py +++ b/src/tests/test_national.py @@ -150,7 +150,7 @@ def test_read_latest_national_values_start_and_end_filters_inculde_metadata(db_s assert response.status_code == 200 national_forecast = NationalForecast(**response.json()) - assert len(national_forecasts.forecast_values) == 16 + assert len(national_forecast.forecast_values) == 16 response = api_client.get( "/v0/solar/GB/national/forecast?start_datetime_utc=2023-01-01&end_datetime_utc=2023-01-01 04:00&include_metadata=true" # noqa @@ -158,7 +158,7 @@ def test_read_latest_national_values_start_and_end_filters_inculde_metadata(db_s assert response.status_code == 200 national_forecast = NationalForecast(**response.json()) - assert len(national_forecasts.forecast_values) == 9 + assert len(national_forecast.forecast_values) == 9 def test_get_national_forecast_error(db_session, api_client): From 95fd1a0848986a9e774717d9344000c980a21a4e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 Sep 2023 10:18:38 +0000 Subject: [PATCH 8/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/tests/test_national.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tests/test_national.py b/src/tests/test_national.py index b39e801..be992cf 100644 --- a/src/tests/test_national.py +++ b/src/tests/test_national.py @@ -131,6 +131,7 @@ def test_get_national_forecast(db_session, api_client): != national_forecast.forecast_values[0].expected_power_generation_megawatts * 0.9 ) + def test_read_latest_national_values_start_and_end_filters_inculde_metadata(db_session, api_client): """Check main solar/GB/national/forecast route works""" @@ -146,7 +147,9 @@ def test_read_latest_national_values_start_and_end_filters_inculde_metadata(db_s app.dependency_overrides[get_session] = lambda: db_session - response = api_client.get("/v0/solar/GB/national/forecast?start_datetime_utc=2023-01-01&include_metadata=true") # noqa + response = api_client.get( + "/v0/solar/GB/national/forecast?start_datetime_utc=2023-01-01&include_metadata=true" + ) # noqa assert response.status_code == 200 national_forecast = NationalForecast(**response.json()) From 7883cf780effa44acdf64ffd216b663329fe3abf Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Wed, 27 Sep 2023 16:14:15 +0100 Subject: [PATCH 9/9] fix --- requirements.txt | 2 +- src/national.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 82a2fbf..44608a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ uvicorn[standard] pydantic numpy requests -nowcasting_datamodel==1.5.15 +nowcasting_datamodel==1.5.17 nowcasting_dataset==3.7.12 sqlalchemy psycopg2-binary diff --git a/src/national.py b/src/national.py index 068737d..3d47e74 100644 --- a/src/national.py +++ b/src/national.py @@ -78,19 +78,28 @@ def get_national_forecast( detail="Can not set forecast_horizon_minutes when including metadata", ) + if creation_limit_utc is None: + historic = True + else: + historic = False + forecast = get_latest_forecast_for_gsps( session=session, gsp_ids=[0], model_name="blend", - historic=True, + historic=historic, preload_children=True, start_target_time=start_datetime_utc, end_target_time=end_datetime_utc, - start_created_utc=creation_limit_utc, + end_created_utc=creation_limit_utc, ) forecast = forecast[0] - forecast = NationalForecast.from_orm_latest(forecast) + if historic: + forecast = NationalForecast.from_orm_latest(forecast) + else: + forecast = NationalForecast.from_orm(forecast) + forecasts = filter_forecast_values( forecasts=[forecast], start_datetime_utc=start_datetime_utc,