From c8c292c48c1056f1a1a69e13eba1b86d2ea1b3fd Mon Sep 17 00:00:00 2001 From: Siddharth Date: Sat, 7 Dec 2024 18:39:28 +0530 Subject: [PATCH 01/15] feat: add core forecast processing and saving logic --- neso_solar_consumer/format_forecast.py | 55 ++++++++++++++++++++++++++ neso_solar_consumer/save_forecasts.py | 17 ++++++++ 2 files changed, 72 insertions(+) create mode 100644 neso_solar_consumer/format_forecast.py create mode 100644 neso_solar_consumer/save_forecasts.py diff --git a/neso_solar_consumer/format_forecast.py b/neso_solar_consumer/format_forecast.py new file mode 100644 index 0000000..42129bf --- /dev/null +++ b/neso_solar_consumer/format_forecast.py @@ -0,0 +1,55 @@ +from datetime import datetime, timezone +import pandas as pd +from nowcasting_datamodel.models import ForecastSQL, ForecastValue +from nowcasting_datamodel.read.read import get_latest_input_data_last_updated, get_location +from nowcasting_datamodel.read.read_models import get_model + + +def format_to_forecast_sql(data: pd.DataFrame, model_tag: str, model_version: str, session) -> list: + """ + Convert fetched NESO solar data into ForecastSQL objects. + + Parameters: + data (pd.DataFrame): The input DataFrame with solar forecast data. + model_tag (str): The name/tag of the model. + model_version (str): The version of the model. + session (Session): SQLAlchemy session for database access. + + Returns: + list: A list of ForecastSQL objects. + """ + # Get the model object + model = get_model(name=model_tag, version=model_version, session=session) + + # Fetch the last updated input data timestamp + input_data_last_updated = get_latest_input_data_last_updated(session=session) + + forecasts = [] + for _, row in data.iterrows(): + if pd.isnull(row["start_utc"]) or pd.isnull(row["solar_forecast_kw"]): + continue # Skip rows with missing critical data + + # Convert "start_utc" to a datetime object + target_time = datetime.fromisoformat(row["start_utc"]).replace(tzinfo=timezone.utc) + + forecast_values = [ + ForecastValue( + target_time=target_time, + expected_power_generation_megawatts=row["solar_forecast_kw"] + ).to_orm() + ] + + location = get_location(session=session, gsp_id=row.get("gsp_id", 0)) + + forecast = ForecastSQL( + model=model, + forecast_creation_time=datetime.now(tz=timezone.utc), + location=location, + input_data_last_updated=input_data_last_updated, + forecast_values=forecast_values, + historic=False, + ) + + forecasts.append(forecast) + + return forecasts diff --git a/neso_solar_consumer/save_forecasts.py b/neso_solar_consumer/save_forecasts.py new file mode 100644 index 0000000..6dd9183 --- /dev/null +++ b/neso_solar_consumer/save_forecasts.py @@ -0,0 +1,17 @@ +# save_forecasts.py + +from nowcasting_datamodel.save.save import save + +def save_forecasts_to_db(forecasts: list, session): + """ + Save a list of ForecastSQL objects to the database using the nowcasting_datamodel `save` function. + + Parameters: + forecasts (list): The list of ForecastSQL objects to save. + session (Session): SQLAlchemy session for database access. + """ + save( + forecasts=forecasts, + session=session, + save_to_last_seven_days=True, # Save forecasts to the last seven days table + ) From c870fbfe533db912b710bb5fa2176baa83a1d486 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Sat, 7 Dec 2024 18:39:46 +0530 Subject: [PATCH 02/15] test: add tests for forecast data fetching and processing --- tests/test_format_forecast.py | 94 +++++++++++++++++++++++++++++++++++ tests/test_save_forecasts.py | 72 +++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 tests/test_format_forecast.py create mode 100644 tests/test_save_forecasts.py diff --git a/tests/test_format_forecast.py b/tests/test_format_forecast.py new file mode 100644 index 0000000..60ec9b9 --- /dev/null +++ b/tests/test_format_forecast.py @@ -0,0 +1,94 @@ +""" +Test Suite for `format_to_forecast_sql` Function + +This script validates the functionality of the `format_to_forecast_sql` function. +It checks if the function correctly converts the input data into SQLAlchemy-compatible +ForecastSQL objects. + +### How to Run the Tests: + +You can run the entire suite of tests in this file using `pytest` from the command line: + + pytest tests/test_format_forecast.py + +To run a specific test, you can specify the function name: + + pytest tests/test_format_forecast.py::test_format_to_forecast_sql_real + +For verbose output, use the -v flag: + + pytest tests/test_format_forecast.py -v + +To run tests matching a specific pattern, use the -k option: + + pytest tests/test_format_forecast.py -k "format_to_forecast_sql" + +""" + +import pytest +from typing import Generator +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from nowcasting_datamodel.models.base import Base_Forecast +from nowcasting_datamodel.models import MLModelSQL +from neso_solar_consumer.fetch_data import fetch_data +from neso_solar_consumer.format_forecast import format_to_forecast_sql + +# Test configuration +RESOURCE_ID = "db6c038f-98af-4570-ab60-24d71ebd0ae5" +LIMIT = 5 +COLUMNS = ["DATE_GMT", "TIME_GMT", "EMBEDDED_SOLAR_FORECAST"] +RENAME_COLUMNS = { + "DATE_GMT": "start_utc", + "TIME_GMT": "end_utc", + "EMBEDDED_SOLAR_FORECAST": "solar_forecast_kw", +} +MODEL_NAME = "real_data_model" +MODEL_VERSION = "1.0" + + +@pytest.fixture +def db_session() -> Generator: + """ + Create a PostgreSQL database session for testing. + + Returns: + Generator: A session object to interact with the database. + """ + engine = create_engine("postgresql://postgres:12345@localhost/testdb") + Base_Forecast.metadata.create_all(engine) # Create tables + Session = sessionmaker(bind=engine) + session = Session() + + # Add a dummy model entry for the test + model = MLModelSQL(name=MODEL_NAME, version=MODEL_VERSION) + session.add(model) + session.commit() + + yield session + + session.close() + engine.dispose() + + +def test_format_to_forecast_sql_real(db_session): + """ + Test `format_to_forecast_sql` with real data fetched from NESO API. + + Steps: + 1. Fetch data from the NESO API. + 2. Convert the data to ForecastSQL objects using `format_to_forecast_sql`. + 3. Validate the number of generated forecasts matches the input data. + 4. Verify that the model metadata (name, version) is correctly assigned. + """ + # Fetch data + data = fetch_data(RESOURCE_ID, LIMIT, COLUMNS, RENAME_COLUMNS) + assert not data.empty, "fetch_data returned an empty DataFrame!" + + # Format data to ForecastSQL objects + forecasts = format_to_forecast_sql(data, MODEL_NAME, MODEL_VERSION, db_session) + + # Assertions + assert len(forecasts) == len(data), "Mismatch in number of forecasts and data rows!" + assert forecasts[0].model.name == MODEL_NAME, "Model name does not match!" + assert forecasts[0].model.version == MODEL_VERSION, "Model version does not match!" diff --git a/tests/test_save_forecasts.py b/tests/test_save_forecasts.py new file mode 100644 index 0000000..ccd6867 --- /dev/null +++ b/tests/test_save_forecasts.py @@ -0,0 +1,72 @@ +""" +Integration Test for Fetching, Formatting, and Saving Forecast Data + +This script validates the integration of fetching real data, formatting it into ForecastSQL objects, +and saving it to the database. +""" + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from nowcasting_datamodel.models.base import Base_Forecast +from nowcasting_datamodel.models import ForecastSQL +from neso_solar_consumer.fetch_data import fetch_data +from neso_solar_consumer.format_forecast import format_to_forecast_sql +from neso_solar_consumer.save_forecasts import save_forecasts_to_db + +# Test configuration +RESOURCE_ID = "db6c038f-98af-4570-ab60-24d71ebd0ae5" +LIMIT = 5 +COLUMNS = ["DATE_GMT", "TIME_GMT", "EMBEDDED_SOLAR_FORECAST"] +RENAME_COLUMNS = { + "DATE_GMT": "start_utc", + "TIME_GMT": "end_utc", + "EMBEDDED_SOLAR_FORECAST": "solar_forecast_kw", +} +MODEL_NAME = "real_data_model" +MODEL_VERSION = "1.0" + +@pytest.fixture +def db_session(): + """ + Create a PostgreSQL database session for testing. + + Returns: + Generator: A session object to interact with the database. + """ + engine = create_engine("postgresql://postgres:12345@localhost/testdb") + Base_Forecast.metadata.create_all(engine) # Create tables + Session = sessionmaker(bind=engine) + session = Session() + + yield session + + session.close() + engine.dispose() + +def test_save_real_forecasts(db_session): + """ + Integration test: Fetch real data, format it into forecasts, and save to the database. + + Steps: + 1. Fetch real data from the NESO API. + 2. Format the data into ForecastSQL objects. + 3. Save the forecasts to the database. + 4. Verify that the forecasts are correctly saved. + """ + # Step 1: Fetch real data + df = fetch_data(RESOURCE_ID, LIMIT, COLUMNS, RENAME_COLUMNS) + assert not df.empty, "fetch_data returned an empty DataFrame!" + + # Step 2: Format data into ForecastSQL objects + forecasts = format_to_forecast_sql(df, MODEL_NAME, MODEL_VERSION, db_session) + assert forecasts, "No forecasts were generated from the fetched data!" + + # Step 3: Save forecasts to the database + save_forecasts_to_db(forecasts, db_session) + + # Step 4: Verify forecasts are saved in the database + saved_forecast = db_session.query(ForecastSQL).first() + assert saved_forecast is not None, "No forecast was saved to the database!" + assert saved_forecast.model.name == MODEL_NAME, "Model name does not match!" + assert len(saved_forecast.forecast_values) > 0, "No forecast values were saved!" From 4b9aaa75e9a09af719ced015b95366ce2d35c207 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Sat, 7 Dec 2024 18:46:25 +0530 Subject: [PATCH 03/15] chore: update pyproject.toml for dependencies --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 81bb507..53a2484 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,5 +11,5 @@ dependencies = [ [project.optional-dependencies] dev = [ - "pytest","black","ruff" + "pytest","black","ruff","sqlalchemy" ] From 46d31aab20fe1643d5a680f161bf7c8eb8a2dfef Mon Sep 17 00:00:00 2001 From: Siddharth Date: Sat, 7 Dec 2024 18:59:59 +0530 Subject: [PATCH 04/15] chore: apply Black and Ruff fixes --- neso_solar_consumer/fetch_data.py | 10 +++------- neso_solar_consumer/format_forecast.py | 2 +- neso_solar_consumer/save_forecasts.py | 1 + tests/test_fetch_data.py | 6 ++---- tests/test_save_forecasts.py | 2 ++ 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/neso_solar_consumer/fetch_data.py b/neso_solar_consumer/fetch_data.py index 0267af8..01c21d8 100644 --- a/neso_solar_consumer/fetch_data.py +++ b/neso_solar_consumer/fetch_data.py @@ -8,9 +8,7 @@ import pandas as pd -def fetch_data( - resource_id: str, limit: int, columns: list, rename_columns: dict -) -> pd.DataFrame: +def fetch_data(resource_id: str, limit: int, columns: list, rename_columns: dict) -> pd.DataFrame: """ Fetch data from the NESO API and process it into a Pandas DataFrame. @@ -55,9 +53,7 @@ def fetch_data( return pd.DataFrame() -def fetch_data_using_sql( - sql_query: str, columns: list, rename_columns: dict -) -> pd.DataFrame: +def fetch_data_using_sql(sql_query: str, columns: list, rename_columns: dict) -> pd.DataFrame: """ Fetch data from the NESO API using an SQL query, process it, and return specific columns with renamed headers. @@ -98,4 +94,4 @@ def fetch_data_using_sql( except Exception as e: print(f"An error occurred: {e}") - return pd.DataFrame() \ No newline at end of file + return pd.DataFrame() diff --git a/neso_solar_consumer/format_forecast.py b/neso_solar_consumer/format_forecast.py index 42129bf..69fd29d 100644 --- a/neso_solar_consumer/format_forecast.py +++ b/neso_solar_consumer/format_forecast.py @@ -35,7 +35,7 @@ def format_to_forecast_sql(data: pd.DataFrame, model_tag: str, model_version: st forecast_values = [ ForecastValue( target_time=target_time, - expected_power_generation_megawatts=row["solar_forecast_kw"] + expected_power_generation_megawatts=row["solar_forecast_kw"], ).to_orm() ] diff --git a/neso_solar_consumer/save_forecasts.py b/neso_solar_consumer/save_forecasts.py index 6dd9183..abc5966 100644 --- a/neso_solar_consumer/save_forecasts.py +++ b/neso_solar_consumer/save_forecasts.py @@ -2,6 +2,7 @@ from nowcasting_datamodel.save.save import save + def save_forecasts_to_db(forecasts: list, session): """ Save a list of ForecastSQL objects to the database using the nowcasting_datamodel `save` function. diff --git a/tests/test_fetch_data.py b/tests/test_fetch_data.py index dc7ae79..b020a0b 100644 --- a/tests/test_fetch_data.py +++ b/tests/test_fetch_data.py @@ -26,7 +26,7 @@ """ -import pytest +import pytest # noqa: F401 from neso_solar_consumer.fetch_data import fetch_data, fetch_data_using_sql @@ -69,6 +69,4 @@ def test_data_consistency(): """ df_api = fetch_data(resource_id, limit, columns, rename_columns) df_sql = fetch_data_using_sql(sql_query, columns, rename_columns) - assert df_api.equals( - df_sql - ), "Data from fetch_data and fetch_data_using_sql are inconsistent!" + assert df_api.equals(df_sql), "Data from fetch_data and fetch_data_using_sql are inconsistent!" diff --git a/tests/test_save_forecasts.py b/tests/test_save_forecasts.py index ccd6867..6419b15 100644 --- a/tests/test_save_forecasts.py +++ b/tests/test_save_forecasts.py @@ -26,6 +26,7 @@ MODEL_NAME = "real_data_model" MODEL_VERSION = "1.0" + @pytest.fixture def db_session(): """ @@ -44,6 +45,7 @@ def db_session(): session.close() engine.dispose() + def test_save_real_forecasts(db_session): """ Integration test: Fetch real data, format it into forecasts, and save to the database. From 88732444e7ac9edbf2151ded962832ba4edb3cb3 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Sat, 7 Dec 2024 19:21:35 +0530 Subject: [PATCH 05/15] fix: update CI workflow to install dev dependencies --- .github/workflows/pytest.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index e94ea54..20868fc 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -8,6 +8,7 @@ on: - cron: "0 12 * * 1" pull_request_target: types: [opened, synchronize, reopened, ready_for_review] + jobs: call-run-python-tests: uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all @@ -16,4 +17,9 @@ jobs: pytest_cov_dir: "neso_solar_consumer" os_list: '["ubuntu-latest"]' python-version: "['3.11']" - pytest_numcpus: '1' \ No newline at end of file + pytest_numcpus: '1' + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Install dependencies + run: pip install .[dev] # Install with dev dependencies From e5dbd02b1431c1e47d4c78c4d2859f332c9751a5 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Sat, 7 Dec 2024 19:28:12 +0530 Subject: [PATCH 06/15] Revert "fix: update CI workflow to install dev dependencies" This reverts commit 88732444e7ac9edbf2151ded962832ba4edb3cb3. --- .github/workflows/pytest.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 20868fc..e94ea54 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -8,7 +8,6 @@ on: - cron: "0 12 * * 1" pull_request_target: types: [opened, synchronize, reopened, ready_for_review] - jobs: call-run-python-tests: uses: openclimatefix/.github/.github/workflows/python-test.yml@issue/pip-all @@ -17,9 +16,4 @@ jobs: pytest_cov_dir: "neso_solar_consumer" os_list: '["ubuntu-latest"]' python-version: "['3.11']" - pytest_numcpus: '1' - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Install dependencies - run: pip install .[dev] # Install with dev dependencies + pytest_numcpus: '1' \ No newline at end of file From c2261f26e0dd11b65060f34a0c2d6126e13b148e Mon Sep 17 00:00:00 2001 From: Siddharth Date: Tue, 17 Dec 2024 11:13:02 +0530 Subject: [PATCH 07/15] fix: move sqlalchemy to default dependencies for CI --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 53a2484..64b002d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,10 +6,10 @@ build-backend = "setuptools.build_meta" name = "neso_solar_consumer" version = "0.1" dependencies = [ - "pandas" + "pandas","sqlalchemy" ] [project.optional-dependencies] dev = [ - "pytest","black","ruff","sqlalchemy" -] + "pytest", "black", "ruff" +] \ No newline at end of file From 9c933d2a7f8221324664b0f834106e499f187029 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Tue, 17 Dec 2024 15:02:22 +0530 Subject: [PATCH 08/15] Refactor format_to_forecast_sql: streamline location creation with get_location --- neso_solar_consumer/format_forecast.py | 80 ++++++++++++++++---------- 1 file changed, 50 insertions(+), 30 deletions(-) diff --git a/neso_solar_consumer/format_forecast.py b/neso_solar_consumer/format_forecast.py index 69fd29d..cdea554 100644 --- a/neso_solar_consumer/format_forecast.py +++ b/neso_solar_consumer/format_forecast.py @@ -1,55 +1,75 @@ +import logging from datetime import datetime, timezone import pandas as pd from nowcasting_datamodel.models import ForecastSQL, ForecastValue from nowcasting_datamodel.read.read import get_latest_input_data_last_updated, get_location from nowcasting_datamodel.read.read_models import get_model +# Configure logging (set to INFO for production; use DEBUG during debugging) +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +logger = logging.getLogger(__name__) + def format_to_forecast_sql(data: pd.DataFrame, model_tag: str, model_version: str, session) -> list: """ - Convert fetched NESO solar data into ForecastSQL objects. + Convert NESO solar forecast data into a single ForecastSQL object. - Parameters: - data (pd.DataFrame): The input DataFrame with solar forecast data. - model_tag (str): The name/tag of the model. - model_version (str): The version of the model. - session (Session): SQLAlchemy session for database access. + Args: + data (pd.DataFrame): Input DataFrame with forecast data. + model_tag (str): The model name/tag. + model_version (str): The model version. + session: SQLAlchemy session for database access. Returns: - list: A list of ForecastSQL objects. + list: A list containing one ForecastSQL object. """ - # Get the model object + logger.info("Starting format_to_forecast_sql process...") + + # Step 1: Retrieve model metadata model = get_model(name=model_tag, version=model_version, session=session) + logger.debug(f"Model Retrieved: {model}") - # Fetch the last updated input data timestamp + # Step 2: Fetch input data last updated timestamp input_data_last_updated = get_latest_input_data_last_updated(session=session) + logger.debug(f"Input Data Last Updated: {input_data_last_updated}") - forecasts = [] + # Step 3: Fetch or create the location using get_location + location = get_location(session=session, gsp_id=0) + logger.debug(f"Location Retrieved or Created: {location}") + + # Step 4: Process rows into ForecastValue objects + forecast_values = [] for _, row in data.iterrows(): if pd.isnull(row["start_utc"]) or pd.isnull(row["solar_forecast_kw"]): - continue # Skip rows with missing critical data + logger.debug("Skipping row due to missing data") + continue - # Convert "start_utc" to a datetime object target_time = datetime.fromisoformat(row["start_utc"]).replace(tzinfo=timezone.utc) + forecast_value = ForecastValue( + target_time=target_time, + expected_power_generation_megawatts=row["solar_forecast_kw"], + ).to_orm() + forecast_values.append(forecast_value) + logger.debug(f"Forecast Value Created: {forecast_value}") - forecast_values = [ - ForecastValue( - target_time=target_time, - expected_power_generation_megawatts=row["solar_forecast_kw"], - ).to_orm() - ] - - location = get_location(session=session, gsp_id=row.get("gsp_id", 0)) + if not forecast_values: + logger.warning("No valid forecast values found in the data. Exiting.") + return [] - forecast = ForecastSQL( - model=model, - forecast_creation_time=datetime.now(tz=timezone.utc), - location=location, - input_data_last_updated=input_data_last_updated, - forecast_values=forecast_values, - historic=False, - ) + # Step 5: Create a single ForecastSQL object + forecast = ForecastSQL( + model=model, + forecast_creation_time=datetime.now(tz=timezone.utc), + location=location, # Directly using the location from get_location + input_data_last_updated=input_data_last_updated, + forecast_values=forecast_values, + historic=False, + ) + logger.debug(f"ForecastSQL Object Created: {forecast}") - forecasts.append(forecast) + # Step 6: Add to session and flush + session.add(forecast) + session.flush() + logger.info("ForecastSQL object successfully added to session and flushed.") - return forecasts + return [forecast] From 89ab0bfbac044d2309376d3221d20c0161e98a9a Mon Sep 17 00:00:00 2001 From: Siddharth Date: Tue, 17 Dec 2024 15:13:11 +0530 Subject: [PATCH 09/15] Clean and document test for format_to_forecast_sql function --- tests/test_format_forecast.py | 97 ++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 29 deletions(-) diff --git a/tests/test_format_forecast.py b/tests/test_format_forecast.py index 60ec9b9..3b13036 100644 --- a/tests/test_format_forecast.py +++ b/tests/test_format_forecast.py @@ -2,27 +2,27 @@ Test Suite for `format_to_forecast_sql` Function This script validates the functionality of the `format_to_forecast_sql` function. -It checks if the function correctly converts the input data into SQLAlchemy-compatible -ForecastSQL objects. +It ensures that the function correctly creates a single ForecastSQL object with multiple +ForecastValue entries. ### How to Run the Tests: -You can run the entire suite of tests in this file using `pytest` from the command line: - +Run the entire suite: pytest tests/test_format_forecast.py -To run a specific test, you can specify the function name: - +Run a specific test: pytest tests/test_format_forecast.py::test_format_to_forecast_sql_real -For verbose output, use the -v flag: - +For verbose output: pytest tests/test_format_forecast.py -v -To run tests matching a specific pattern, use the -k option: - - pytest tests/test_format_forecast.py -k "format_to_forecast_sql" +Note: +This test assumes a local PostgreSQL database configured with: + - Username: `postgres` + - Password: `12345` + - Database: `testdb` +This setup is temporary for local testing and may require adjustment for CI/CD environments. """ import pytest @@ -30,11 +30,11 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from nowcasting_datamodel.models.base import Base_Forecast -from nowcasting_datamodel.models import MLModelSQL +from nowcasting_datamodel.models import MLModelSQL, ForecastSQL, ForecastValue from neso_solar_consumer.fetch_data import fetch_data from neso_solar_consumer.format_forecast import format_to_forecast_sql -# Test configuration +# Test configuration constants RESOURCE_ID = "db6c038f-98af-4570-ab60-24d71ebd0ae5" LIMIT = 5 COLUMNS = ["DATE_GMT", "TIME_GMT", "EMBEDDED_SOLAR_FORECAST"] @@ -50,45 +50,84 @@ @pytest.fixture def db_session() -> Generator: """ - Create a PostgreSQL database session for testing. + Fixture to set up and tear down a PostgreSQL database session for testing. - Returns: - Generator: A session object to interact with the database. + Database connection details (modify if required): + - Host: localhost + - User: postgres + - Password: 12345 + - Database: testdb + + Creates fresh tables before each test and cleans up after execution. """ engine = create_engine("postgresql://postgres:12345@localhost/testdb") + Base_Forecast.metadata.drop_all(engine) # Drop tables if they already exist Base_Forecast.metadata.create_all(engine) # Create tables + Session = sessionmaker(bind=engine) session = Session() - # Add a dummy model entry for the test + # Add a dummy ML model for testing + session.query(MLModelSQL).delete() # Ensure no pre-existing data model = MLModelSQL(name=MODEL_NAME, version=MODEL_VERSION) session.add(model) session.commit() - yield session + yield session # Provide session to the test function + # Cleanup session.close() engine.dispose() def test_format_to_forecast_sql_real(db_session): """ - Test `format_to_forecast_sql` with real data fetched from NESO API. + Test `format_to_forecast_sql` with real data fetched from the NESO API. Steps: - 1. Fetch data from the NESO API. - 2. Convert the data to ForecastSQL objects using `format_to_forecast_sql`. - 3. Validate the number of generated forecasts matches the input data. - 4. Verify that the model metadata (name, version) is correctly assigned. + 1. Fetch test data from the NESO API. + 2. Format the data into a ForecastSQL object using the target function. + 3. Validate the creation of ForecastSQL and associated ForecastValue entries. + 4. Verify database state for correctness. + + Expected Outcomes: + - A single ForecastSQL object is created. + - The number of ForecastValue entries matches the input data. + - The model metadata (name, version) matches the expected values. + - No redundant or duplicate objects are added to the database. """ - # Fetch data + # Step 1: Fetch mock data from the API data = fetch_data(RESOURCE_ID, LIMIT, COLUMNS, RENAME_COLUMNS) assert not data.empty, "fetch_data returned an empty DataFrame!" - # Format data to ForecastSQL objects + # Step 2: Format the data into a ForecastSQL object forecasts = format_to_forecast_sql(data, MODEL_NAME, MODEL_VERSION, db_session) - # Assertions - assert len(forecasts) == len(data), "Mismatch in number of forecasts and data rows!" - assert forecasts[0].model.name == MODEL_NAME, "Model name does not match!" - assert forecasts[0].model.version == MODEL_VERSION, "Model version does not match!" + # Step 3: Validate the ForecastSQL object + assert len(forecasts) == 1, "More than one ForecastSQL object was created!" + forecast = forecasts[0] + + # Validate the number of ForecastValue entries + assert len(forecast.forecast_values) == len(data), ( + f"Mismatch in the number of ForecastValue entries. " + f"Expected: {len(data)}, Got: {len(forecast.forecast_values)}" + ) + + # Step 4: Validate model metadata + assert forecast.model.name == MODEL_NAME, f"Model name mismatch. Expected: {MODEL_NAME}" + assert forecast.model.version == MODEL_VERSION, f"Model version mismatch. Expected: {MODEL_VERSION}" + + # Step 5: Validate database state + forecasts_in_db = db_session.query(ForecastSQL).all() + assert len(forecasts_in_db) == 1, "Unexpected number of ForecastSQL objects in the database!" + + total_forecast_values = db_session.query(ForecastValue).count() + assert total_forecast_values == len(data), ( + f"Mismatch in the number of ForecastValue entries in the database. " + f"Expected: {len(data)}, Got: {total_forecast_values}" + ) + + # Debugging information for better visibility + print(f"ForecastSQL object created successfully with {len(forecast.forecast_values)} ForecastValues.") + print(f"Total ForecastSQL objects in database: {len(forecasts_in_db)}") + print(f"Total ForecastValue objects in database: {total_forecast_values}") From 77e17e0753d0283dce6d4270d43dd7df19966fb2 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Tue, 17 Dec 2024 15:21:01 +0530 Subject: [PATCH 10/15] Fix missing dependency --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 64b002d..5bde208 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "neso_solar_consumer" version = "0.1" dependencies = [ - "pandas","sqlalchemy" + "pandas","sqlalchemy","nowcasting_datamodel" ] [project.optional-dependencies] From e3328975ed1ffeb3a6189e4d14bd3c07327651e6 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Tue, 17 Dec 2024 16:38:41 +0530 Subject: [PATCH 11/15] Refactor format_forecast code: remove redundant session operations. --- neso_solar_consumer/format_forecast.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/neso_solar_consumer/format_forecast.py b/neso_solar_consumer/format_forecast.py index cdea554..fed0662 100644 --- a/neso_solar_consumer/format_forecast.py +++ b/neso_solar_consumer/format_forecast.py @@ -67,9 +67,5 @@ def format_to_forecast_sql(data: pd.DataFrame, model_tag: str, model_version: st ) logger.debug(f"ForecastSQL Object Created: {forecast}") - # Step 6: Add to session and flush - session.add(forecast) - session.flush() logger.info("ForecastSQL object successfully added to session and flushed.") - return [forecast] From f2222f128d8f7deb2175c860caee9d2956876f67 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Tue, 17 Dec 2024 16:39:03 +0530 Subject: [PATCH 12/15] Move reusable fixtures and test configs to conftest.py for better reusability. --- tests/conftest.py | 72 ++++++++++++++++++ tests/test_fetch_data.py | 70 ++++++------------ tests/test_format_forecast.py | 134 ++++------------------------------ tests/test_save_forecasts.py | 53 ++++---------- 4 files changed, 122 insertions(+), 207 deletions(-) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..f15ccc4 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,72 @@ +import pytest +from typing import Generator +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from nowcasting_datamodel.models.base import Base_Forecast +from nowcasting_datamodel.models import MLModelSQL + +# Shared Test Configuration Constants +TEST_DB_URL = "postgresql://postgres:12345@localhost/testdb" +RESOURCE_ID = "db6c038f-98af-4570-ab60-24d71ebd0ae5" +LIMIT = 5 +COLUMNS = ["DATE_GMT", "TIME_GMT", "EMBEDDED_SOLAR_FORECAST"] +RENAME_COLUMNS = { + "DATE_GMT": "start_utc", + "TIME_GMT": "end_utc", + "EMBEDDED_SOLAR_FORECAST": "solar_forecast_kw", +} +MODEL_NAME = "real_data_model" +MODEL_VERSION = "1.0" + + +@pytest.fixture(scope="function") +def db_session() -> Generator: + """ + Fixture to set up and tear down a PostgreSQL database session for testing. + + This fixture: + - Creates a fresh database schema before each test. + - Adds a dummy ML model for test purposes. + - Tears down the database session and cleans up resources after each test. + + Returns: + Generator: A SQLAlchemy session object. + """ + # Create database engine and tables + engine = create_engine(TEST_DB_URL) + Base_Forecast.metadata.drop_all(engine) # Drop all tables to ensure a clean slate + Base_Forecast.metadata.create_all(engine) # Recreate the tables + + # Establish session + Session = sessionmaker(bind=engine) + session = Session() + + # Insert a dummy model for testing + session.query(MLModelSQL).delete() # Clean up any pre-existing data + model = MLModelSQL(name=MODEL_NAME, version=MODEL_VERSION) + session.add(model) + session.commit() + + yield session # Provide the session to the test + + # Cleanup: close session and dispose of engine + session.close() + engine.dispose() + + +@pytest.fixture(scope="session") +def test_config(): + """ + Fixture to provide shared test configuration constants. + + Returns: + dict: A dictionary of test configuration values. + """ + return { + "resource_id": RESOURCE_ID, + "limit": LIMIT, + "columns": COLUMNS, + "rename_columns": RENAME_COLUMNS, + "model_name": MODEL_NAME, + "model_version": MODEL_VERSION, + } diff --git a/tests/test_fetch_data.py b/tests/test_fetch_data.py index b020a0b..48285b5 100644 --- a/tests/test_fetch_data.py +++ b/tests/test_fetch_data.py @@ -1,72 +1,44 @@ -""" -Test Suite for `fetch_data` and `fetch_data_using_sql` Functions - -This script contains tests to validate the functionality and consistency of two data-fetching functions: -`fetch_data` (via API) and `fetch_data_using_sql` (via SQL query). It checks that the data returned -by both methods is correctly processed, contains the expected columns, and ensures the consistency -between the two methods. - -### How to Run the Tests: - -You can run the entire suite of tests in this file using `pytest` from the command line: - - pytest tests/test_fetch_data.py - -To run a specific test, you can specify the function name: - - pytest tests/test_fetch_data.py::test_fetch_data_api - -For verbose output, use the -v flag: - - pytest tests/test_fetch_data.py -v - -To run tests matching a specific pattern, use the -k option: - - pytest tests/test_fetch_data.py -k "fetch_data" - -""" - -import pytest # noqa: F401 from neso_solar_consumer.fetch_data import fetch_data, fetch_data_using_sql -resource_id = "db6c038f-98af-4570-ab60-24d71ebd0ae5" -limit = 5 -columns = ["DATE_GMT", "TIME_GMT", "EMBEDDED_SOLAR_FORECAST"] -rename_columns = { - "DATE_GMT": "start_utc", - "TIME_GMT": "end_utc", - "EMBEDDED_SOLAR_FORECAST": "solar_forecast_kw", -} -sql_query = f'SELECT * from "{resource_id}" LIMIT {limit}' - - -def test_fetch_data_api(): +def test_fetch_data_api(test_config): """ Test the fetch_data function to ensure it fetches and processes data correctly via API. """ - df_api = fetch_data(resource_id, limit, columns, rename_columns) + df_api = fetch_data( + test_config["resource_id"], + test_config["limit"], + test_config["columns"], + test_config["rename_columns"], + ) assert not df_api.empty, "fetch_data returned an empty DataFrame!" assert set(df_api.columns) == set( - rename_columns.values() + test_config["rename_columns"].values() ), "Column names do not match after renaming!" -def test_fetch_data_sql(): +def test_fetch_data_sql(test_config): """ Test the fetch_data_using_sql function to ensure it fetches and processes data correctly via SQL. """ - df_sql = fetch_data_using_sql(sql_query, columns, rename_columns) + sql_query = f'SELECT * FROM "{test_config["resource_id"]}" LIMIT {test_config["limit"]}' + df_sql = fetch_data_using_sql(sql_query, test_config["columns"], test_config["rename_columns"]) assert not df_sql.empty, "fetch_data_using_sql returned an empty DataFrame!" assert set(df_sql.columns) == set( - rename_columns.values() + test_config["rename_columns"].values() ), "Column names do not match after renaming!" -def test_data_consistency(): +def test_data_consistency(test_config): """ Validate that the data fetched by fetch_data and fetch_data_using_sql are consistent. """ - df_api = fetch_data(resource_id, limit, columns, rename_columns) - df_sql = fetch_data_using_sql(sql_query, columns, rename_columns) + sql_query = f'SELECT * FROM "{test_config["resource_id"]}" LIMIT {test_config["limit"]}' + df_api = fetch_data( + test_config["resource_id"], + test_config["limit"], + test_config["columns"], + test_config["rename_columns"], + ) + df_sql = fetch_data_using_sql(sql_query, test_config["columns"], test_config["rename_columns"]) assert df_api.equals(df_sql), "Data from fetch_data and fetch_data_using_sql are inconsistent!" diff --git a/tests/test_format_forecast.py b/tests/test_format_forecast.py index 3b13036..22cad0d 100644 --- a/tests/test_format_forecast.py +++ b/tests/test_format_forecast.py @@ -1,133 +1,27 @@ -""" -Test Suite for `format_to_forecast_sql` Function - -This script validates the functionality of the `format_to_forecast_sql` function. -It ensures that the function correctly creates a single ForecastSQL object with multiple -ForecastValue entries. - -### How to Run the Tests: - -Run the entire suite: - pytest tests/test_format_forecast.py - -Run a specific test: - pytest tests/test_format_forecast.py::test_format_to_forecast_sql_real - -For verbose output: - pytest tests/test_format_forecast.py -v - -Note: -This test assumes a local PostgreSQL database configured with: - - Username: `postgres` - - Password: `12345` - - Database: `testdb` - -This setup is temporary for local testing and may require adjustment for CI/CD environments. -""" - -import pytest -from typing import Generator -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from nowcasting_datamodel.models.base import Base_Forecast -from nowcasting_datamodel.models import MLModelSQL, ForecastSQL, ForecastValue from neso_solar_consumer.fetch_data import fetch_data from neso_solar_consumer.format_forecast import format_to_forecast_sql +from nowcasting_datamodel.models import ForecastSQL, ForecastValue -# Test configuration constants -RESOURCE_ID = "db6c038f-98af-4570-ab60-24d71ebd0ae5" -LIMIT = 5 -COLUMNS = ["DATE_GMT", "TIME_GMT", "EMBEDDED_SOLAR_FORECAST"] -RENAME_COLUMNS = { - "DATE_GMT": "start_utc", - "TIME_GMT": "end_utc", - "EMBEDDED_SOLAR_FORECAST": "solar_forecast_kw", -} -MODEL_NAME = "real_data_model" -MODEL_VERSION = "1.0" - - -@pytest.fixture -def db_session() -> Generator: - """ - Fixture to set up and tear down a PostgreSQL database session for testing. - - Database connection details (modify if required): - - Host: localhost - - User: postgres - - Password: 12345 - - Database: testdb - - Creates fresh tables before each test and cleans up after execution. - """ - engine = create_engine("postgresql://postgres:12345@localhost/testdb") - Base_Forecast.metadata.drop_all(engine) # Drop tables if they already exist - Base_Forecast.metadata.create_all(engine) # Create tables - - Session = sessionmaker(bind=engine) - session = Session() - - # Add a dummy ML model for testing - session.query(MLModelSQL).delete() # Ensure no pre-existing data - model = MLModelSQL(name=MODEL_NAME, version=MODEL_VERSION) - session.add(model) - session.commit() - yield session # Provide session to the test function - - # Cleanup - session.close() - engine.dispose() - - -def test_format_to_forecast_sql_real(db_session): +def test_format_to_forecast_sql_real(db_session, test_config): """ Test `format_to_forecast_sql` with real data fetched from the NESO API. - - Steps: - 1. Fetch test data from the NESO API. - 2. Format the data into a ForecastSQL object using the target function. - 3. Validate the creation of ForecastSQL and associated ForecastValue entries. - 4. Verify database state for correctness. - - Expected Outcomes: - - A single ForecastSQL object is created. - - The number of ForecastValue entries matches the input data. - - The model metadata (name, version) matches the expected values. - - No redundant or duplicate objects are added to the database. """ # Step 1: Fetch mock data from the API - data = fetch_data(RESOURCE_ID, LIMIT, COLUMNS, RENAME_COLUMNS) + data = fetch_data( + test_config["resource_id"], + test_config["limit"], + test_config["columns"], + test_config["rename_columns"], + ) assert not data.empty, "fetch_data returned an empty DataFrame!" # Step 2: Format the data into a ForecastSQL object - forecasts = format_to_forecast_sql(data, MODEL_NAME, MODEL_VERSION, db_session) - - # Step 3: Validate the ForecastSQL object - assert len(forecasts) == 1, "More than one ForecastSQL object was created!" - forecast = forecasts[0] - - # Validate the number of ForecastValue entries - assert len(forecast.forecast_values) == len(data), ( - f"Mismatch in the number of ForecastValue entries. " - f"Expected: {len(data)}, Got: {len(forecast.forecast_values)}" - ) - - # Step 4: Validate model metadata - assert forecast.model.name == MODEL_NAME, f"Model name mismatch. Expected: {MODEL_NAME}" - assert forecast.model.version == MODEL_VERSION, f"Model version mismatch. Expected: {MODEL_VERSION}" - - # Step 5: Validate database state - forecasts_in_db = db_session.query(ForecastSQL).all() - assert len(forecasts_in_db) == 1, "Unexpected number of ForecastSQL objects in the database!" - - total_forecast_values = db_session.query(ForecastValue).count() - assert total_forecast_values == len(data), ( - f"Mismatch in the number of ForecastValue entries in the database. " - f"Expected: {len(data)}, Got: {total_forecast_values}" + forecasts = format_to_forecast_sql( + data, test_config["model_name"], test_config["model_version"], db_session ) + assert len(forecasts) == 1, "More than one ForecastSQL object was created!" - # Debugging information for better visibility - print(f"ForecastSQL object created successfully with {len(forecast.forecast_values)} ForecastValues.") - print(f"Total ForecastSQL objects in database: {len(forecasts_in_db)}") - print(f"Total ForecastValue objects in database: {total_forecast_values}") + # Step 3: Validate ForecastSQL content + forecast = forecasts[0] + assert len(forecast.forecast_values) == len(data), "Mismatch in ForecastValue entries!" diff --git a/tests/test_save_forecasts.py b/tests/test_save_forecasts.py index 6419b15..be5b24a 100644 --- a/tests/test_save_forecasts.py +++ b/tests/test_save_forecasts.py @@ -6,47 +6,13 @@ """ import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from nowcasting_datamodel.models.base import Base_Forecast from nowcasting_datamodel.models import ForecastSQL from neso_solar_consumer.fetch_data import fetch_data from neso_solar_consumer.format_forecast import format_to_forecast_sql from neso_solar_consumer.save_forecasts import save_forecasts_to_db -# Test configuration -RESOURCE_ID = "db6c038f-98af-4570-ab60-24d71ebd0ae5" -LIMIT = 5 -COLUMNS = ["DATE_GMT", "TIME_GMT", "EMBEDDED_SOLAR_FORECAST"] -RENAME_COLUMNS = { - "DATE_GMT": "start_utc", - "TIME_GMT": "end_utc", - "EMBEDDED_SOLAR_FORECAST": "solar_forecast_kw", -} -MODEL_NAME = "real_data_model" -MODEL_VERSION = "1.0" - -@pytest.fixture -def db_session(): - """ - Create a PostgreSQL database session for testing. - - Returns: - Generator: A session object to interact with the database. - """ - engine = create_engine("postgresql://postgres:12345@localhost/testdb") - Base_Forecast.metadata.create_all(engine) # Create tables - Session = sessionmaker(bind=engine) - session = Session() - - yield session - - session.close() - engine.dispose() - - -def test_save_real_forecasts(db_session): +def test_save_real_forecasts(db_session, test_config): """ Integration test: Fetch real data, format it into forecasts, and save to the database. @@ -57,11 +23,18 @@ def test_save_real_forecasts(db_session): 4. Verify that the forecasts are correctly saved. """ # Step 1: Fetch real data - df = fetch_data(RESOURCE_ID, LIMIT, COLUMNS, RENAME_COLUMNS) + df = fetch_data( + test_config["resource_id"], + test_config["limit"], + test_config["columns"], + test_config["rename_columns"], + ) assert not df.empty, "fetch_data returned an empty DataFrame!" # Step 2: Format data into ForecastSQL objects - forecasts = format_to_forecast_sql(df, MODEL_NAME, MODEL_VERSION, db_session) + forecasts = format_to_forecast_sql( + df, test_config["model_name"], test_config["model_version"], db_session + ) assert forecasts, "No forecasts were generated from the fetched data!" # Step 3: Save forecasts to the database @@ -70,5 +43,9 @@ def test_save_real_forecasts(db_session): # Step 4: Verify forecasts are saved in the database saved_forecast = db_session.query(ForecastSQL).first() assert saved_forecast is not None, "No forecast was saved to the database!" - assert saved_forecast.model.name == MODEL_NAME, "Model name does not match!" + assert saved_forecast.model.name == test_config["model_name"], "Model name does not match!" assert len(saved_forecast.forecast_values) > 0, "No forecast values were saved!" + + # Debugging Output (Optional) + print("Forecast successfully saved to the database.") + print(f"Number of forecast values: {len(saved_forecast.forecast_values)}") From 1a820b8ea2914b3fcc9d472f03ed41b62ce87cb4 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Tue, 17 Dec 2024 17:34:49 +0530 Subject: [PATCH 13/15] fix(tests): restore missing docstring in test_fetch_data --- tests/test_fetch_data.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_fetch_data.py b/tests/test_fetch_data.py index 48285b5..fc48560 100644 --- a/tests/test_fetch_data.py +++ b/tests/test_fetch_data.py @@ -1,3 +1,30 @@ +""" +Test Suite for `fetch_data` and `fetch_data_using_sql` Functions + +This script contains tests to validate the functionality and consistency of two data-fetching functions: +`fetch_data` (via API) and `fetch_data_using_sql` (via SQL query). It checks that the data returned +by both methods is correctly processed, contains the expected columns, and ensures the consistency +between the two methods. + +### How to Run the Tests: + +You can run the entire suite of tests in this file using `pytest` from the command line: + + pytest tests/test_fetch_data.py + +To run a specific test, you can specify the function name: + + pytest tests/test_fetch_data.py::test_fetch_data_api + +For verbose output, use the -v flag: + + pytest tests/test_fetch_data.py -v + +To run tests matching a specific pattern, use the -k option: + + pytest tests/test_fetch_data.py -k "fetch_data" + +""" from neso_solar_consumer.fetch_data import fetch_data, fetch_data_using_sql From 3e79c3dae06f91a57ac5274726ded53493263ebc Mon Sep 17 00:00:00 2001 From: Siddharth Date: Wed, 18 Dec 2024 09:24:48 +0530 Subject: [PATCH 14/15] Refactor tests to use PostgresContainer --- tests/conftest.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f15ccc4..2757edf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,9 +4,9 @@ from sqlalchemy.orm import sessionmaker from nowcasting_datamodel.models.base import Base_Forecast from nowcasting_datamodel.models import MLModelSQL +from testcontainers.postgres import PostgresContainer # Shared Test Configuration Constants -TEST_DB_URL = "postgresql://postgres:12345@localhost/testdb" RESOURCE_ID = "db6c038f-98af-4570-ab60-24d71ebd0ae5" LIMIT = 5 COLUMNS = ["DATE_GMT", "TIME_GMT", "EMBEDDED_SOLAR_FORECAST"] @@ -19,21 +19,38 @@ MODEL_VERSION = "1.0" +@pytest.fixture(scope="session") +def postgres_container(): + """ + Fixture to spin up a PostgreSQL container for the entire test session. + + This fixture uses `testcontainers` to start a fresh PostgreSQL container and provides + the connection URL dynamically for use in other fixtures. + """ + with PostgresContainer("postgres:15.5") as postgres: + postgres.start() + yield postgres.get_connection_url() + + @pytest.fixture(scope="function") -def db_session() -> Generator: +def db_session(postgres_container) -> Generator: """ Fixture to set up and tear down a PostgreSQL database session for testing. This fixture: + - Connects to the PostgreSQL container provided by `postgres_container`. - Creates a fresh database schema before each test. - Adds a dummy ML model for test purposes. - Tears down the database session and cleans up resources after each test. + Args: + postgres_container (str): The dynamic connection URL provided by PostgresContainer. + Returns: Generator: A SQLAlchemy session object. """ - # Create database engine and tables - engine = create_engine(TEST_DB_URL) + # Use the dynamic connection URL + engine = create_engine(postgres_container) Base_Forecast.metadata.drop_all(engine) # Drop all tables to ensure a clean slate Base_Forecast.metadata.create_all(engine) # Recreate the tables From eeddfbfda1f5841ef4641f075d687672d3f4c764 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Wed, 18 Dec 2024 09:28:38 +0530 Subject: [PATCH 15/15] Fix missing dependency-testcontainers --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5bde208..049d84b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "neso_solar_consumer" version = "0.1" dependencies = [ - "pandas","sqlalchemy","nowcasting_datamodel" + "pandas","sqlalchemy","nowcasting_datamodel","testcontainers" ] [project.optional-dependencies]