Fake data Issue #354 (#360)

* Fake data Issue #354

* Fake data Issue #354

* Fake data Issue #354

* Changed relative imports and removed pre-commit from requirements.txt

* Using more flexible python3 version to attempt to fix pre-commit.ci build issue

* Written new Unit Test for fake forecast with specified GSP ID

* Written unit tests for all endpoints/routes and modified gsp.py to support new tests

* Removed duplicated Test Cases

* 1st test case for fake environment

* Fixed accidental activation of fake environment in gsp.py module

* Written and tested remaining test cases for is_fake

* Moved isintance check to prior to is_fake condition

* Added 2 Tests to test_national.py and cleaned up some logic in test_gsp test cases

* Modified gsp.py and national.py modules and accompanying test cases to address feedback

* Fixed incorrect for loop iteration through list (should be singular ForecastSQL object) to forecasts object in the test_national.py test cases

* Modified test cases to use NationalForecastValue, ForecastValue, and the ManyForecasts as the return objects

* Modified test cases to uss pytest.fixture() to yield values from db_session

* Possible fix for test_read_latest_all_gsp_normalized() and test_read_latest_all_gsp()

* 1st experiment for test_read_truth_national_gsp() and test_read_forecast_values_gsp()

* 1st experiment with make_fake_gsp_yields()

* 2nd experiment with make_fake_gsp_yields() - modified test_gsp routes

* 3rd experiment with make_fake_gsp_yields() - modified List Comprehension

* 4th experiment with make_fake_gsp_yields() - hard coded _gsp_id_

* Removed yield and fixture

* Experiment: Create a separate tests/fake/test_gsp_fake.py test case module

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 default_language_version:
-  python: python3.9
+  python: python3
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks

src/gsp.py

@@ -4,9 +4,11 @@
 from typing import List, Optional, Union
 
 import structlog
+from dotenv import load_dotenv
 from fastapi import APIRouter, Depends, Request, Security, status
 from fastapi.responses import Response
 from fastapi_auth0 import Auth0User
+from nowcasting_datamodel.fake import make_fake_forecast, make_fake_forecasts, make_fake_gsp_yields
 from nowcasting_datamodel.models import Forecast, ForecastValue, ManyForecasts
 from sqlalchemy.orm.session import Session
 
@@ -32,6 +34,7 @@
 
 logger = structlog.stdlib.get_logger()
 adjust_limit = float(os.getenv("ADJUST_MW_LIMIT", 0.0))
+load_dotenv()
 
 
 router = APIRouter(
@@ -40,7 +43,12 @@
 NationalYield = GSPYield
 
 
-# corresponds to route /v0/solar/GB/gsp/forecast/all
+def is_fake():
+    """Start FAKE environment"""
+    return int(os.environ.get("FAKE", 0))
+
+
+# corresponds to route /v0/solar/GB/gsp/forecast/all/
 @router.get(
     "/forecast/all/",
     response_model=Union[ManyForecasts, List[OneDatetimeManyForecastValues]],
@@ -80,11 +88,17 @@ def get_all_available_forecasts(
     - **end_datetime_utc**: optional end datetime for the query. e.g '2023-08-12 14:00:00+00:00'
     """
 
-    logger.info(f"Get forecasts for all gsps. The option is {historic=} for user {user}")
-
-    if gsp_ids is not None:
+    if isinstance(gsp_ids, str):
         gsp_ids = [int(gsp_id) for gsp_id in gsp_ids.split(",")]
 
+    if is_fake:
+        if gsp_ids is None:
+            gsp_ids = [int(gsp_id) for gsp_id in range(1, GSP_TOTAL)]
+
+        make_fake_forecasts(gsp_ids=gsp_ids, session=session)
+
+    logger.info(f"Get forecasts for all gsps. The option is {historic=} for user {user}")
+
     start_datetime_utc = format_datetime(start_datetime_utc)
     end_datetime_utc = format_datetime(end_datetime_utc)
     creation_limit_utc = format_datetime(creation_limit_utc)
@@ -137,6 +151,10 @@ def get_forecasts_for_a_specific_gsp_old_route(
     user: Auth0User = Security(get_user()),
 ) -> Union[Forecast, List[ForecastValue]]:
     """Redirects old API route to new route /v0/solar/GB/gsp/{gsp_id}/forecast"""
+
+    if is_fake:
+        make_fake_forecast(gsp_id=gsp_id, session=session)
+
     return get_forecasts_for_a_specific_gsp(
         request=request,
         gsp_id=gsp_id,
@@ -185,6 +203,8 @@ def get_forecasts_for_a_specific_gsp(
     - **creation_utc_limit**: optional, only return forecasts made before this datetime.
     returns the latest forecast made 60 minutes before the target time)
     """
+    if is_fake:
+        make_fake_forecast(gsp_id=gsp_id, session=session)
 
     logger.info(f"Get forecasts for gsp id {gsp_id} forecast of forecast with only values.")
     logger.info(f"This is for user {user}")
@@ -251,11 +271,18 @@ def get_truths_for_all_gsps(
     - **start_datetime_utc**: optional start datetime for the query.
     - **end_datetime_utc**: optional end datetime for the query.
     """
-    logger.info(f"Get PV Live estimates values for all gsp id and regime {regime} for user {user}")
 
-    if gsp_ids is not None:
+    if isinstance(gsp_ids, str):
         gsp_ids = [int(gsp_id) for gsp_id in gsp_ids.split(",")]
 
+    if is_fake:
+        if gsp_ids is None:
+            gsp_ids = [int(gsp_id) for gsp_id in range(1, GSP_TOTAL)]
+
+        make_fake_gsp_yields(gsp_ids=gsp_ids, session=session)
+
+    logger.info(f"Get PV Live estimates values for all gsp id and regime {regime} for user {user}")
+
     start_datetime_utc = format_datetime(start_datetime_utc)
     end_datetime_utc = format_datetime(end_datetime_utc)
 
@@ -286,6 +313,10 @@ def get_truths_for_a_specific_gsp_old_route(
     user: Auth0User = Security(get_user()),
 ) -> List[GSPYield]:
     """Redirects old API route to new route /v0/solar/GB/gsp/{gsp_id}/pvlive"""
+
+    if is_fake:
+        make_fake_gsp_yields(gsp_ids=[gsp_id], session=session)
+
     return get_truths_for_a_specific_gsp(
         request=request,
         gsp_id=gsp_id,
@@ -331,6 +362,9 @@ def get_truths_for_a_specific_gsp(
     If not set, defaults to N_HISTORY_DAYS env var, which if not set defaults to yesterday.
     """
 
+    if is_fake:
+        make_fake_forecast(gsp_id=gsp_id, session=session)
+
     logger.info(
         f"Get PV Live estimates values for gsp id {gsp_id} " f"and regime {regime} for user {user}"
     )

src/national.py

@@ -10,6 +10,7 @@
 from elexonpy.api_client import ApiClient
 from fastapi import APIRouter, Depends, HTTPException, Query, Request, Security
 from fastapi_auth0 import Auth0User
+from nowcasting_datamodel.fake import make_fake_forecast, make_fake_gsp_yields
 from nowcasting_datamodel.read.read import get_latest_forecast_for_gsps
 from sqlalchemy.orm.session import Session
 
@@ -43,6 +44,11 @@
 elexon_forecast_api = GenerationForecastApi(api_client)
 
 
+def is_fake():
+    """Start FAKE environment"""
+    return int(os.environ.get("FAKE", 0))
+
+
 @router.get(
     "/forecast",
     response_model=Union[NationalForecast, List[NationalForecastValue]],
@@ -88,6 +94,9 @@ def get_national_forecast(
     """
     logger.debug("Get national forecasts")
 
+    if is_fake:
+        make_fake_forecast(gsp_id=0, session=session)
+
     start_datetime_utc = format_datetime(start_datetime_utc)
     end_datetime_utc = format_datetime(end_datetime_utc)
     creation_limit_utc = format_datetime(creation_limit_utc)
@@ -204,6 +213,9 @@ def get_national_pvlive(
     """
     logger.info(f"Get national PV Live estimates values " f"for regime {regime} for  {user}")
 
+    if is_fake:
+        make_fake_gsp_yields(gsp_ids=[0], session=session)
+
     return get_truth_values_for_a_specific_gsp_from_database(
         session=session, gsp_id=0, regime=regime
     )

src/tests/fake/test_gsp_fake.py

@@ -0,0 +1,88 @@
+from nowcasting_datamodel.models import ForecastValue, LocationWithGSPYields, ManyForecasts
+
+from gsp import GSP_TOTAL, is_fake
+
+
+def test_is_fake_specific_gsp(monkeypatch, api_client, gsp_id=1):
+    """### Test FAKE environment specific _gsp_id_ routes are populating
+    with fake data.
+
+    #### Parameters
+    - **gsp_id**: Please set to any non-zero integer that is <= GSP_TOTAL
+    """
+
+    monkeypatch.setenv("FAKE", "1")
+    assert is_fake() == 1
+
+    # Specific _gsp_id_ route/endpoint for successful connection
+    response = api_client.get(f"/v0/solar/GB/gsp/{gsp_id}/forecast")
+    assert response.status_code == 200
+
+    forecast_value = [ForecastValue(**f) for f in response.json()]
+    assert forecast_value is not None
+
+    # Disable is_fake environment
+    monkeypatch.setenv("FAKE", "0")
+
+
+def test_is_fake_get_truths_for_a_specific_gsp(monkeypatch, api_client, gsp_id=1):
+    """### Test FAKE environment specific _gsp_id_ routes are populating
+    with fake data.
+
+    #### Parameters
+    - **gsp_id**: Please set to any non-zero integer that is <= GSP_TOTAL
+    """
+
+    monkeypatch.setenv("FAKE", "1")
+    assert is_fake() == 1
+
+    # Specific _gsp_id_ route/endpoint for successful connection
+    response = api_client.get(f"/v0/solar/GB/gsp/{gsp_id}/pvlive")
+    assert response.status_code == 200
+
+    forecast_value = [ForecastValue(**f) for f in response.json()]
+    assert forecast_value is not None
+
+    # Disable is_fake environment
+    monkeypatch.setenv("FAKE", "0")
+
+
+def test_is_fake_all_available_forecasts(monkeypatch, api_client):
+    """Test FAKE environment for all GSPs are populating
+    with fake data.
+    """
+
+    monkeypatch.setenv("FAKE", "1")
+    assert is_fake() == 1
+
+    # Connect to DB endpoint
+    response = api_client.get("/v0/solar/GB/gsp/forecast/all/")
+    assert response.status_code == 200
+
+    all_forecasts = ManyForecasts(**response.json())
+    assert all_forecasts is not None
+
+    # Disable is_fake environment
+    monkeypatch.setenv("FAKE", "0")
+
+
+def test_is_fake_get_truths_for_all_gsps(
+    monkeypatch, api_client, gsp_ids=list(range(1, GSP_TOTAL))
+):
+    """Test FAKE environment for all GSPs for yesterday and today
+    are populating with fake data.
+    """
+
+    monkeypatch.setenv("FAKE", "1")
+    assert is_fake() == 1
+
+    # Connect to DB endpoint
+    gsp_ids_str = ", ".join(map(str, gsp_ids))
+    response = api_client.get(f"/v0/solar/GB/gsp/pvlive/all?gsp_ids={gsp_ids_str}")
+    assert response.status_code == 200
+
+    all_forecasts = [LocationWithGSPYields(**f) for f in response.json()]
+    assert all_forecasts is not None
+
+    # Disable is_fake environment
+    monkeypatch.setenv("FAKE", "0")

src/tests/test_gsp.py

@@ -53,7 +53,8 @@ def test_read_latest_one_gsp_national(db_session, api_client):
 
     app.dependency_overrides[get_session] = lambda: db_session
 
-    response = api_client.get("/v0/solar/GB/gsp/0/forecast")
+    # response = api_client.get("/v0/solar/GB/gsp/0/forecast")
+    response = api_client.get("/v0/solar/GB/gsp/forecast/0")
 
     assert response.status_code == 200
 
@@ -92,13 +93,13 @@ def test_read_latest_one_gsp_filter_creation_utc(db_session, api_client):
         assert f[0].target_time == forecasts[1].forecast_values[0].target_time
 
 
-def test_read_latest_all_gsp(db_session, api_client):
+def test_read_latest_all_gsp(db_session, api_client, gsp_ids=list(range(0, 10))):
     """Check main solar/GB/gsp/forecast/all route works"""
 
     model = get_model(session=db_session, name="blend", version="0.0.1")
 
     forecasts = make_fake_forecasts(
-        gsp_ids=list(range(0, 10)),
+        gsp_ids=gsp_ids,
         session=db_session,
         t0_datetime_utc=datetime.now(tz=timezone.utc),
     )
@@ -108,7 +109,10 @@ def test_read_latest_all_gsp(db_session, api_client):
 
     app.dependency_overrides[get_session] = lambda: db_session
 
-    response = api_client.get("/v0/solar/GB/gsp/forecast/all/?historic=False")
+    gsp_ids_str = ", ".join(map(str, gsp_ids))
+    response = api_client.get(
+        f"/v0/solar/GB/gsp/forecast/all/?historic=False&gsp_ids={gsp_ids_str}"
+    )
 
     assert response.status_code == 200
 
@@ -168,13 +172,13 @@ def test_read_latest_gsp_id_equal_to_total(db_session, api_client):
     _ = [ForecastValue(**f) for f in response.json()]
 
 
-def test_read_latest_all_gsp_normalized(db_session, api_client):
+def test_read_latest_all_gsp_normalized(db_session, api_client, gsp_ids=list(range(0, 10))):
     """Check main solar/GB/gsp/forecast/all normalized route works"""
 
     model = get_model(session=db_session, name="blend", version="0.0.1")
 
     forecasts = make_fake_forecasts(
-        gsp_ids=list(range(0, 10)),
+        gsp_ids=gsp_ids,
         session=db_session,
         t0_datetime_utc=datetime.now(tz=timezone.utc),
     )
@@ -183,7 +187,10 @@ def test_read_latest_all_gsp_normalized(db_session, api_client):
 
     app.dependency_overrides[get_session] = lambda: db_session
 
-    response = api_client.get("/v0/solar/GB/gsp/forecast/all/?historic=False&normalize=True")
+    gsp_ids_str = ", ".join(map(str, gsp_ids))
+    response = api_client.get(
+        f"/v0/solar/GB/gsp/forecast/all/?historic=False&normalize=True&gsp_ids={gsp_ids_str}"
+    )
 
     assert response.status_code == 200
 
@@ -291,7 +298,7 @@ def test_read_pvlive_for_gsp_id_over_total(db_session, api_client):
     """Check solar/GB/gsp/pvlive returns 204 when gsp_id over total"""
 
     gsp_id = 318
-    response = api_client.get(f"/v0/solar/GB/gsp/pvlive/{gsp_id}")
+    response = api_client.get(f"/v0/solar/GB/gsp/{gsp_id}/pvlive")
 
     assert response.status_code == 204
 

src/tests/test_merged_routes.py

@@ -51,7 +51,7 @@ def test_read_forecast_values_gsp(db_session, api_client):
 
     app.dependency_overrides[get_session] = lambda: db_session
 
-    response = api_client.get("/v0/solar/GB/gsp/1/forecast")
+    response = api_client.get("/v0/solar/GB/gsp/forecast/1")
     assert response.status_code == 200
 
     r_json = response.json()

src/tests/test_national.py

@@ -12,6 +12,7 @@
 
 from database import get_session
 from main import app
+from national import is_fake
 from pydantic_models import NationalForecast, NationalForecastValue
 
 
@@ -248,10 +249,47 @@ def test_read_truth_national_gsp(db_session, api_client):
     db_session.add_all([gsp_yield_1_sql, gsp_yield_2_sql, gsp_yield_3_sql, gsp_sql_1])
 
     app.dependency_overrides[get_session] = lambda: db_session
+    yield db_session
 
-    response = api_client.get("/v0/solar/GB/national/pvlive/")
+    response = api_client.get("/v0/solar/GB/national/pvlive/0")
     assert response.status_code == 200
 
     r_json = response.json()
     assert len(r_json) == 3
     _ = [GSPYield(**gsp_yield) for gsp_yield in r_json]
+
+
+def test_is_fake_national_all_available_forecasts(monkeypatch, api_client):
+    """Test FAKE environment for all GSPs are populating
+    with fake data.
+    """
+
+    monkeypatch.setenv("FAKE", "1")
+    assert is_fake() == 1
+    # Connect to DB endpoint
+    response = api_client.get("/v0/solar/GB/national/forecast")
+    assert response.status_code == 200
+
+    national_forecast_values = [NationalForecastValue(**f) for f in response.json()]
+    assert national_forecast_values is not None
+
+    # Disable is_fake environment
+    monkeypatch.setenv("FAKE", "0")
+
+
+def test_is_fake_national_get_truths_for_all_gsps(monkeypatch, api_client):
+    """Test FAKE environment for all GSPs for yesterday and today
+    are populating with fake data.
+    """
+
+    monkeypatch.setenv("FAKE", "1")
+    assert is_fake() == 1
+    # Connect to DB endpoint
+    response = api_client.get("/v0/solar/GB/national/pvlive/")
+    assert response.status_code == 200
+
+    national_forecast_values = [NationalForecastValue(**f) for f in response.json()]
+    assert national_forecast_values is not None
+
+    # Disable is_fake environment
+    monkeypatch.setenv("FAKE", "0")