diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 7f4165d..b229aaa 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10'] + python-version: ['3.9', '3.10', '3.11'] defaults: run: working-directory: /home/runner/work/pyPrediktorMapClient/pyPrediktorMapClient diff --git a/.readthedocs.yml b/.readthedocs.yml index 21b0814..c8571ff 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -17,7 +17,7 @@ formats: - pdf python: - version: 3.8 + version: 3.9 install: - requirements: docs/requirements.txt - {path: ., method: pip} diff --git a/docs/conf.py b/docs/conf.py index 0c86221..151ecb0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -150,6 +150,8 @@ # If this is True, todo emits a warning for each TODO entries. The default is False. todo_emit_warnings = True +# Since the documentation build process doesn't need to make actual database connections, we can mock the pyodbc module during the documentation build. +autodoc_mock_imports = ["pyodbc"] # -- Options for HTML output ------------------------------------------------- diff --git a/setup.cfg b/setup.cfg index be97793..acd7bee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,7 +41,7 @@ package_dir = =src # Require a min/specific Python version (comma-separated conditions) -python_requires = >=3.8 +python_requires = >=3.9 # Add here dependencies of your project (line-separated), e.g. requests>=2.2,<3.0. # Version specifiers like >=2.2,<3.0 avoid problems due to API changes in @@ -53,7 +53,8 @@ install_requires = aiohttp >= 3.8.1 pydantic >= 2.0 pandas >= 1.4.4 - pyodbc >= 4.0.39 + pyodbc + pyPrediktorUtilities >= 0.3.2 [options.packages.find] where = src @@ -70,6 +71,8 @@ testing = setuptools pytest pytest-cov + pyPrediktorUtilities >= 0.3.2 + pyodbc [options.entry_points] @@ -118,10 +121,10 @@ version = 4.3 package = pyprediktormapclient [tox:tox] -envlist = py38, py39, py310, mypy +envlist = py39, py310, py311, mypy [gh-actions] python = - 3.8: py38 3.9: py39 - 3.10: py310, mypy \ No newline at end of file + 3.10: py310, mypy + 3.11: py311 \ No newline at end of file diff --git a/src/pyprediktormapclient/__init__.py b/src/pyprediktormapclient/__init__.py index b0a9c59..6df284e 100644 --- a/src/pyprediktormapclient/__init__.py +++ b/src/pyprediktormapclient/__init__.py @@ -5,7 +5,7 @@ from .opc_ua import * if sys.version_info[:2] >= (3, 8): - # TODO: Import directly (no need for conditional) when `python_requires = >= 3.8` + # TODO: Import directly (no need for conditional) when `python_requires = >= 3.9` from importlib.metadata import PackageNotFoundError, version # pragma: no cover else: from importlib_metadata import PackageNotFoundError, version # pragma: no cover diff --git a/src/pyprediktormapclient/dwh/context/enercast.py b/src/pyprediktormapclient/dwh/context/enercast.py index 17cfd96..2486bec 100644 --- a/src/pyprediktormapclient/dwh/context/enercast.py +++ b/src/pyprediktormapclient/dwh/context/enercast.py @@ -1,6 +1,6 @@ import json from pydantic import validate_call -from typing import List, Dict, Any, Union +from typing import List, Dict, Union from ..idwh import IDWH @@ -10,19 +10,19 @@ def __init__(self, dwh: IDWH) -> None: self.dwh = dwh @validate_call - def get_plants_to_update(self) -> List[Any]: + def get_plants_to_update(self) -> List: query = "SET NOCOUNT ON; EXEC dwetl.GetEnercastPlantsToUpdate" return self.dwh.fetch(query) @validate_call - def get_live_meter_data(self, asset_name: str) -> List[Any]: + def get_live_meter_data(self, asset_name: str) -> List: query = f"SET NOCOUNT ON; EXEC dwetl.GetEnercastLiveMeterData '{asset_name}'" return self.dwh.fetch(query) @validate_call def upsert_forecast_data( self, enercast_forecast_data: Dict, forecast_type_key: Union[int, None] = None - ) -> List[Any]: + ) -> List: enercast_forecast_data_json = json.dumps({"results": enercast_forecast_data}) query = "EXEC dwetl.UpsertEnercastForecastData ?, ?" diff --git a/src/pyprediktormapclient/dwh/context/plant.py b/src/pyprediktormapclient/dwh/context/plant.py index 28f11c7..4783fa0 100644 --- a/src/pyprediktormapclient/dwh/context/plant.py +++ b/src/pyprediktormapclient/dwh/context/plant.py @@ -1,6 +1,6 @@ import json from pydantic import validate_call -from typing import List, Dict, Any +from typing import List, Dict from ..idwh import IDWH @@ -10,7 +10,7 @@ def __init__(self, dwh: IDWH) -> None: self.dwh = dwh @validate_call - def get_optimal_tracker_angles(self, facility_name: str) -> List[Any]: + def get_optimal_tracker_angles(self, facility_name: str) -> List: query = ( f"SET NOCOUNT ON; EXEC dwetl.GetOptimalTrackerAngleParameters " + f"@FacilityName = N'{facility_name}'" @@ -18,7 +18,7 @@ def get_optimal_tracker_angles(self, facility_name: str) -> List[Any]: return self.dwh.fetch(query) @validate_call - def upsert_optimal_tracker_angles(self, facility_data: Dict) -> List[Any]: + def upsert_optimal_tracker_angles(self, facility_data: Dict) -> List: facility_data_json = json.dumps(facility_data) facility_data_json.replace("'", '"') @@ -33,7 +33,7 @@ def insert_log( data_type: str, has_thrown_error: bool = False, message: str = "", - ) -> List[Any]: + ) -> List: query = "EXEC dwetl.InsertExtDataUpdateLog @plantname = ?, @extkey = ?, @DataType = ?, @Message = ?, @Result = ?" return self.dwh.execute( query, diff --git a/src/pyprediktormapclient/dwh/context/solcast.py b/src/pyprediktormapclient/dwh/context/solcast.py index 07531f1..c6504ac 100644 --- a/src/pyprediktormapclient/dwh/context/solcast.py +++ b/src/pyprediktormapclient/dwh/context/solcast.py @@ -1,6 +1,6 @@ import json from pydantic import validate_call -from typing import List, Dict, Any, Union +from typing import List, Dict, Union from ..idwh import IDWH @@ -10,7 +10,7 @@ def __init__(self, dwh: IDWH) -> None: self.dwh = dwh @validate_call - def get_plants_to_update(self) -> List[Any]: + def get_plants_to_update(self) -> List: query = "SET NOCOUNT ON; EXEC dwetl.GetSolcastPlantsToUpdate" return self.dwh.fetch(query) @@ -20,7 +20,7 @@ def upsert_forecast_data( plantname: str, solcast_forecast_data: Dict, forecast_type_key: Union[int, None] = None, - ) -> List[Any]: + ) -> List: solcast_forecast_data_json = json.dumps( { "results": { diff --git a/src/pyprediktormapclient/dwh/db.py b/src/pyprediktormapclient/dwh/db.py deleted file mode 100644 index 18afee9..0000000 --- a/src/pyprediktormapclient/dwh/db.py +++ /dev/null @@ -1,302 +0,0 @@ -import pyodbc -import logging -import pandas as pd -from typing import List, Any -from pydantic import validate_call - -logger = logging.getLogger(__name__) -logger.addHandler(logging.NullHandler()) - - -class Db: - """Access a PowerView Data Warehouse or other SQL databases. - - Args: - url (str): The URL of the sql server - database (str): The name of the database - username (str): The username - password (str): The password - - Attributes: - connection (pyodbc.Connection): The connection object - cursor (pyodbc.Cursor): The cursor object - """ - - @validate_call - def __init__( - self, - url: str, - database: str, - username: str, - password: str, - driver_index: int = -1, - ) -> None: - """Class initializer. - - Args: - url (str): The URL of the sql server - database (str): The name of the database - username (str): The username - password (str): The password - """ - self.url = url - self.cursor = None - self.database = database - self.username = username - self.password = password - self.connection = None - - self.__set_driver(driver_index) - - self.connection_string = ( - f"UID={self.username};" - + f"PWD={self.password};" - + f"DRIVER={self.driver};" - + f"SERVER={self.url};" - + f"DATABASE={self.database};" - ) - self.connection_attempts = 3 - - self.__connect() - - def __enter__(self): - return self - - @validate_call - def __exit__(self, exc_type, exc_val, exc_tb): - if self.connection is not None: - self.__disconnect() - - """ - Public - """ - - @validate_call - def fetch(self, query: str, to_dataframe: bool = False) -> List[Any]: - """Execute the SQL query to get results from DWH and return the data. - - Use that method for getting data. That means that if you use SELECT or - you'd like to call a stored procedure that returns one or more sets - of data, that is the correct method to use. - - Use that method to GET. - - Args: - query (str): The SQL query to execute. - to_dataframe (bool): If True, return the results as a list - of DataFrames. - - Returns: - List[Any]: The results of the query. If DWH returns multiple - data sets, this method is going to return a list - of result sets (lists). If DWH returns a single data set, - the method is going to return a list representing the single - result set. - - If to_dataframe is True, the data inside each data set - is going to be in DataFrame format. - """ - self.__connect() - self.cursor.execute(query) - - data_sets = [] - while True: - data_set = [] - - columns = [col[0] for col in self.cursor.description] - for row in self.cursor.fetchall(): - data_set.append( - {name: row[index] for index, name in enumerate(columns)} - ) - - data_sets.append(pd.DataFrame(data_set) if to_dataframe else data_set) - - if not self.cursor.nextset(): - break - - return data_sets if len(data_sets) > 1 else data_sets[0] - - @validate_call - def execute(self, query: str, *args, **kwargs) -> List[Any]: - """Execute the SQL query and return the results. - - For instance, if we create a new record in DWH by calling - a stored procedure returning the id of the inserted element or - in our query we use `SELECT SCOPE_IDENTITY() AS LastInsertedId;`, - the DWH is going to return data after executing our write request. - - Please note that here we expect a single result set. Therefore DWH - is obligated to return only one data set and also we're obligated to - construct our query according to this requirement. - - Use that method to CREATE, UPDATE, DELETE or execute business logic. - To NOT use for GET. - - Args: - query (str): The SQL query to execute. - *args: Variable length argument list to pass to cursor.execute(). - **kwargs: Arbitrary keyword arguments to pass to cursor.execute(). - - Returns: - List[Any]: The results of the query. - """ - self.__connect() - self.cursor.execute(query, *args, **kwargs) - - result = [] - try: - result = self.cursor.fetchall() - except Exception: - pass - - self.__commit() - - return result - - """ - Private - Driver - """ - - @validate_call - def __set_driver(self, driver_index: int) -> None: - """Sets the driver to use for the connection to the database. - - Args: - driver (int): The index of the driver to use. If the index is -1 or - in general below 0, pyPrediktorMapClient is going to choose - the driver for you. - """ - if driver_index < 0: - self.driver = self.__get_list_of_available_and_supported_pyodbc_drivers()[0] - return - - if self.__get_number_of_available_pyodbc_drivers() < (driver_index + 1): - raise ValueError( - f"Driver index {driver_index} is out of range. Please use " - + f"the __get_list_of_available_pyodbc_drivers() method " - + f"to list all available drivers." - ) - - self.driver = self.__get_list_of_supported_pyodbc_drivers()[driver_index] - - @validate_call - def __get_number_of_available_pyodbc_drivers(self) -> int: - return len(self.__get_list_of_supported_pyodbc_drivers()) - - @validate_call - def __get_list_of_supported_pyodbc_drivers(self) -> List[Any]: - return pyodbc.drivers() - - @validate_call - def __get_list_of_available_and_supported_pyodbc_drivers(self) -> List[Any]: - available_drivers = [] - for driver in self.__get_list_of_supported_pyodbc_drivers(): - try: - pyodbc.connect( - f"UID={self.username};" - + f"PWD={self.password};" - + f"DRIVER={driver};" - + f"SERVER={self.url};" - + f"DATABASE={self.database};", - timeout=3, - ) - available_drivers.append(driver) - except pyodbc.Error as e: - pass - - return available_drivers - - """ - Private - Connector & Disconnector - """ - - @validate_call - def __connect(self) -> None: - """Establishes a connection to the database.""" - if self.connection: - return - - logging.info("Initiating connection to the database...") - - attempt = 0 - while attempt < self.connection_attempts: - try: - self.connection = pyodbc.connect(self.connection_string) - self.cursor = self.connection.cursor() - logging.info("Connection successfull!") - break - - # Exceptions once thrown there is no point attempting - except pyodbc.DataError as err: - logger.error(f"Data Error {err.args[0]}: {err.args[1]}") - raise - except pyodbc.IntegrityError as err: - logger.error(f"Integrity Error {err.args[0]}: {err.args[1]}") - raise - except pyodbc.ProgrammingError as err: - logger.error(f"Programming Error {err.args[0]}: {err.args[1]}") - logger.warning( - f"There seems to be a problem with your code. Please " - + f"check your code and try again." - ) - raise - except pyodbc.NotSupportedError as err: - logger.error(f"Not supported {err.args[0]}: {err.args[1]}") - raise - - # Exceptions when thrown we can continue attempting - except pyodbc.OperationalError as err: - logger.error(f"Operational Error {err.args[0]}: {err.args[1]}") - logger.warning( - f"Pyodbc is having issues with the connection. This " - + f"could be due to the wrong driver being used. Please " - + f"check your driver with " - + f"the __get_list_of_available_and_supported_pyodbc_drivers() method " - + f"and try again." - ) - - attempt += 1 - if self.__are_connection_attempts_reached(attempt): - raise - except pyodbc.DatabaseError as err: - logger.error(f"Database Error {err.args[0]}: {err.args[1]}") - - attempt += 1 - if self.__are_connection_attempts_reached(attempt): - raise - except pyodbc.Error as err: - logger.error(f"Generic Error {err.args[0]}: {err.args[1]}") - - attempt += 1 - if self.__are_connection_attempts_reached(attempt): - raise - - @validate_call - def __are_connection_attempts_reached(self, attempt) -> bool: - if attempt != self.connection_attempts: - logger.warning("Retrying connection...") - return False - - logger.error( - f"Failed to connect to the DataWarehouse after " - + f"{self.connection_attempts} attempts." - ) - return True - - @validate_call - def __disconnect(self) -> None: - """Closes the connection to the database.""" - if self.connection: - self.connection.close() - - self.cursor = None - self.connection = None - - """ - Private - Low level database operations - """ - - @validate_call - def __commit(self) -> None: - """Commits any changes to the database.""" - self.connection.commit() diff --git a/src/pyprediktormapclient/dwh/dwh.py b/src/pyprediktormapclient/dwh/dwh.py index 845b7fa..90b8459 100644 --- a/src/pyprediktormapclient/dwh/dwh.py +++ b/src/pyprediktormapclient/dwh/dwh.py @@ -3,8 +3,8 @@ import importlib from typing import Dict from pydantic import validate_call +from pyprediktorutilities import Dwh as Db -from .db import Db from . import context from .idwh import IDWH @@ -65,17 +65,11 @@ def __init__( @validate_call def version(self) -> Dict: - """Get the DWH version. + """ + Get the DWH version. Returns: - Dict: A dictionary with the following keys (or similar): - DWHVersion, - UpdateDate, - ImplementedDate, - Comment, - MajorVersionNo, - MinorVersionNo, - InterimVersionNo + Dict: A dictionary with the following keys (or similar): DWHVersion, UpdateDate, ImplementedDate, Comment, MajorVersionNo, MinorVersionNo, InterimVersionNo """ query = "SET NOCOUNT ON; EXEC [dbo].[GetVersion]" results = self.fetch(query) diff --git a/src/pyprediktormapclient/dwh/idwh.py b/src/pyprediktormapclient/dwh/idwh.py index 530dddc..3129be3 100644 --- a/src/pyprediktormapclient/dwh/idwh.py +++ b/src/pyprediktormapclient/dwh/idwh.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Any +from typing import Dict, List from abc import ABC, abstractmethod @@ -8,9 +8,9 @@ def version(self) -> Dict: pass @abstractmethod - def fetch(self, query: str, to_dataframe: bool = False) -> List[Any]: + def fetch(self, query: str, to_dataframe: bool = False) -> List: pass @abstractmethod - def execute(self, query: str, *args, **kwargs) -> List[Any]: + def execute(self, query: str, *args, **kwargs) -> List: pass diff --git a/tests/dwh/test_db.py b/tests/dwh/test_db.py deleted file mode 100644 index 50fffd0..0000000 --- a/tests/dwh/test_db.py +++ /dev/null @@ -1,788 +0,0 @@ -import pytest -import random -import string -import pyodbc -import logging -import pandas as pd -from unittest.mock import Mock -from pyprediktormapclient.dwh.db import Db -from pandas.testing import assert_frame_equal - -""" -Helpers -""" - - -class mock_pyodbc_connection: - def __init__(self, connection_string): - pass - - def cursor(self): - return - - -def mock_pyodbc_connection_throws_error_not_tolerant_to_attempts(connection_string): - raise pyodbc.DataError("Error code", "Error message") - - -def mock_pyodbc_connection_throws_error_tolerant_to_attempts(connection_string): - raise pyodbc.DatabaseError("Error code", "Error message") - - -def grs(): - """Generate a random string.""" - return "".join(random.choices(string.ascii_uppercase + string.digits, k=10)) - - -""" -__init__ -""" - - -def test_init_when_instantiate_db_then_instance_is_created(monkeypatch): - driver_index = 0 - - # Mock the database connection - monkeypatch.setattr( - "pyprediktormapclient.dwh.db.pyodbc.connect", mock_pyodbc_connection - ) - - db = Db(grs(), grs(), grs(), grs(), driver_index) - assert db is not None - - -def test_init_when_instantiate_db_but_no_pyodbc_drivers_available_then_throw_exception( - monkeypatch, -): - driver_index = 0 - - # Mock the absence of ODBC drivers - monkeypatch.setattr("pyprediktormapclient.dwh.db.pyodbc.drivers", lambda: []) - - with pytest.raises(ValueError) as excinfo: - Db(grs(), grs(), grs(), grs(), driver_index) - assert "Driver index 0 is out of range." in str(excinfo.value) - - -def test_init_when_instantiate_db_but_pyodbc_throws_error_with_tolerance_to_attempts_then_throw_exception( - monkeypatch, -): - driver_index = 0 - - # Mock the database connection - monkeypatch.setattr( - "pyprediktormapclient.dwh.db.pyodbc.connect", - mock_pyodbc_connection_throws_error_not_tolerant_to_attempts, - ) - - with pytest.raises(pyodbc.DataError): - Db(grs(), grs(), grs(), grs(), driver_index) - - -def test_init_when_instantiate_db_but_pyodbc_throws_error_tolerant_to_attempts_then_retry_connecting_and_throw_exception( - caplog, monkeypatch -): - driver_index = 0 - - # Mock the database connection - monkeypatch.setattr( - "pyprediktormapclient.dwh.db.pyodbc.connect", - mock_pyodbc_connection_throws_error_tolerant_to_attempts, - ) - - with caplog.at_level(logging.ERROR): - with pytest.raises(pyodbc.DatabaseError): - Db(grs(), grs(), grs(), grs(), driver_index) - - assert any( - "Failed to connect to the DataWarehouse after 3 attempts." in message - for message in caplog.messages - ) - - -def test_init_when_instantiate_dwh_but_driver_index_is_not_passed_then_instance_is_created( - monkeypatch, -): - # Mock the connection method to return a mock connection with a mock cursor - mock_cursor = Mock() - mock_connection = Mock() - mock_connection.cursor.return_value = mock_cursor - monkeypatch.setattr("pyodbc.connect", lambda *args, **kwargs: mock_connection) - monkeypatch.setattr("pyodbc.drivers", lambda: ["Driver1", "Driver2"]) - - db = Db(grs(), grs(), grs(), grs()) - assert db is not None - assert db.driver == "Driver1" - - -""" -fetch -""" - - -def test_fetch_when_init_db_connection_is_successfull_but_fails_when_calling_fetch_then_throw_exception( - monkeypatch, -): - query = "SELECT * FROM mytable" - driver_index = 0 - - # Mock the cursor - mock_cursor = Mock() - - # Mock the connection method to return a mock connection with a mock cursor - mock_connection_success = Mock() - mock_connection_success.cursor.return_value = mock_cursor - - mock_connection_fail = Mock() - mock_connection_fail.cursor.side_effect = pyodbc.DataError( - "Error code", "Database data error" - ) - - monkeypatch.setattr( - "pyodbc.connect", - Mock(side_effect=[mock_connection_success, mock_connection_fail]), - ) - - with pytest.raises(pyodbc.DataError): - db = Db(grs(), grs(), grs(), grs(), driver_index) - db.connection = False - db.fetch(query) - - -def test_fetch_when_to_dataframe_is_false_and_no_data_is_returned_then_return_empty_list( - monkeypatch, -): - query = "SELECT * FROM mytable" - driver_index = 2 - - expected_result = [] - - # Mock the cursor's fetchall methods - mock_cursor = Mock() - mock_cursor.fetchall.return_value = [] - mock_cursor.nextset.return_value = False - mock_cursor.description = [ - ("plantname", None), - ("resource_id", None), - ("api_key", None), - ("ExtForecastTypeKey", None), - ("hours", None), - ("output_parameters", None), - ("period", None), - ] - - # Mock the connection method to return a mock connection with a mock cursor - mock_connection = Mock() - mock_connection.cursor.return_value = mock_cursor - monkeypatch.setattr("pyodbc.connect", lambda *args, **kwargs: mock_connection) - monkeypatch.setattr("pyodbc.drivers", lambda: ["Driver1", "Driver2", "Driver3"]) - - db = Db(grs(), grs(), grs(), grs(), driver_index) - actual_result = db.fetch(query) - - mock_cursor.execute.assert_called_once_with(query) - assert actual_result == expected_result - - -def test_fetch_when_to_dataframe_is_false_and_single_data_set_is_returned_then_return_list_representing_single_data_set( - monkeypatch, -): - query = "SELECT * FROM mytable" - driver_index = 2 - data_returned_by_db = [ - ( - "XY-ZK", - "1234-abcd-efgh-5678", - "SOME_KEY", - 13, - 168, - "pv_power_advanced", - "PT15M", - ), - ( - "XY-ZK", - "1234-abcd-efgh-5678", - "SOME_KEY", - 14, - 168, - "pv_power_advanced", - "PT15M", - ), - ( - "KL-MN", - "1234-abcd-efgh-5678", - "SOME_KEY", - 13, - 168, - "pv_power_advanced", - "PT15M", - ), - ] - - expected_result = [ - { - "plantname": "XY-ZK", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 13, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - { - "plantname": "XY-ZK", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 14, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - { - "plantname": "KL-MN", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 13, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - ] - - # Mock the cursor's fetchall methods - mock_cursor = Mock() - mock_cursor.fetchall.return_value = data_returned_by_db - mock_cursor.nextset.return_value = False - mock_cursor.description = [ - ("plantname", None), - ("resource_id", None), - ("api_key", None), - ("ExtForecastTypeKey", None), - ("hours", None), - ("output_parameters", None), - ("period", None), - ] - - # Mock the connection method to return a mock connection with a mock cursor - mock_connection = Mock() - mock_connection.cursor.return_value = mock_cursor - monkeypatch.setattr("pyodbc.connect", lambda *args, **kwargs: mock_connection) - monkeypatch.setattr("pyodbc.drivers", lambda: ["Driver1", "Driver2", "Driver3"]) - - db = Db(grs(), grs(), grs(), grs(), driver_index) - actual_result = db.fetch(query) - - mock_cursor.execute.assert_called_once_with(query) - assert actual_result == expected_result - - -def test_fetch_when_to_dataframe_is_false_and_multiple_data_sets_are_returned_then_return_list_of_lists_representing_multiple_data_sets( - monkeypatch, -): - query = "SELECT * FROM mytable" - driver_index = 2 - data_returned_by_db_set_one = [ - ( - "XY-ZK", - "1234-abcd-efgh-5678", - "SOME_KEY", - 13, - 168, - "pv_power_advanced", - "PT15M", - ), - ( - "XY-ZK", - "1234-abcd-efgh-5678", - "SOME_KEY", - 14, - 168, - "pv_power_advanced", - "PT15M", - ), - ( - "KL-MN", - "1234-abcd-efgh-5678", - "SOME_KEY", - 13, - 168, - "pv_power_advanced", - "PT15M", - ), - ] - data_returned_by_db_set_two = [ - ( - "ALPHA", - "1234-abcd-efgh-5678", - "SOME_KEY", - 13, - 168, - "pv_power_advanced", - "PT15M", - ), - ( - "BETA", - "1234-abcd-efgh-5678", - "SOME_KEY", - 14, - 168, - "pv_power_advanced", - "PT15M", - ), - ] - - expected_result = [ - [ - { - "plantname": "XY-ZK", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 13, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - { - "plantname": "XY-ZK", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 14, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - { - "plantname": "KL-MN", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 13, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - ], - [ - { - "plantname": "ALPHA", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 13, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - { - "plantname": "BETA", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 14, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - ], - ] - - # Mock the cursor's fetchall methods - mock_cursor = Mock() - mock_cursor.fetchall.side_effect = [ - data_returned_by_db_set_one, - data_returned_by_db_set_two, - ] - mock_cursor.nextset.side_effect = [True, False] - mock_cursor.description = [ - ("plantname", None), - ("resource_id", None), - ("api_key", None), - ("ExtForecastTypeKey", None), - ("hours", None), - ("output_parameters", None), - ("period", None), - ] - - # Mock the connection method to return a mock connection with a mock cursor - mock_connection = Mock() - mock_connection.cursor.return_value = mock_cursor - monkeypatch.setattr("pyodbc.connect", lambda *args, **kwargs: mock_connection) - monkeypatch.setattr("pyodbc.drivers", lambda: ["Driver1", "Driver2", "Driver3"]) - - db = Db(grs(), grs(), grs(), grs(), driver_index) - actual_result = db.fetch(query) - - mock_cursor.execute.assert_called_once_with(query) - assert actual_result == expected_result - - -def test_fetch_when_to_dataframe_is_true_and_no_data_is_returned_then_return_empty_dataframe( - monkeypatch, -): - query = "SELECT * FROM mytable" - driver_index = 2 - - # Mock the cursor's fetchall methods - mock_cursor = Mock() - mock_cursor.fetchall.return_value = [] - mock_cursor.nextset.return_value = False - mock_cursor.description = [ - ("plantname", None), - ("resource_id", None), - ("api_key", None), - ("ExtForecastTypeKey", None), - ("hours", None), - ("output_parameters", None), - ("period", None), - ] - - # Mock the connection method to return a mock connection with a mock cursor - mock_connection = Mock() - mock_connection.cursor.return_value = mock_cursor - monkeypatch.setattr("pyodbc.connect", lambda *args, **kwargs: mock_connection) - monkeypatch.setattr("pyodbc.drivers", lambda: ["Driver1", "Driver2", "Driver3"]) - - db = Db(grs(), grs(), grs(), grs(), driver_index) - actual_result = db.fetch(query, True) - - mock_cursor.execute.assert_called_once_with(query) - assert actual_result.empty - - -def test_fetch_when_to_dataframe_is_true_and_single_data_set_is_returned_then_return_dataframe( - monkeypatch, -): - query = "SELECT * FROM mytable" - driver_index = 2 - data_returned_by_db = [ - ( - "XY-ZK", - "1234-abcd-efgh-5678", - "SOME_KEY", - 13, - 168, - "pv_power_advanced", - "PT15M", - ), - ( - "XY-ZK", - "1234-abcd-efgh-5678", - "SOME_KEY", - 14, - 168, - "pv_power_advanced", - "PT15M", - ), - ( - "KL-MN", - "1234-abcd-efgh-5678", - "SOME_KEY", - 13, - 168, - "pv_power_advanced", - "PT15M", - ), - ] - - expected_result = [ - { - "plantname": "XY-ZK", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 13, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - { - "plantname": "XY-ZK", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 14, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - { - "plantname": "KL-MN", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 13, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - ] - expected_df = pd.DataFrame(expected_result) - - # Mock the cursor's fetchall methods - mock_cursor = Mock() - mock_cursor.fetchall.return_value = data_returned_by_db - mock_cursor.nextset.return_value = False - mock_cursor.description = [ - ("plantname", None), - ("resource_id", None), - ("api_key", None), - ("ExtForecastTypeKey", None), - ("hours", None), - ("output_parameters", None), - ("period", None), - ] - - # Mock the connection method to return a mock connection with a mock cursor - mock_connection = Mock() - mock_connection.cursor.return_value = mock_cursor - monkeypatch.setattr("pyodbc.connect", lambda *args, **kwargs: mock_connection) - monkeypatch.setattr("pyodbc.drivers", lambda: ["Driver1", "Driver2", "Driver3"]) - - db = Db(grs(), grs(), grs(), grs(), driver_index) - actual_result = db.fetch(query, True) - - mock_cursor.execute.assert_called_once_with(query) - assert_frame_equal( - actual_result.reset_index(drop=True), - expected_df.reset_index(drop=True), - check_dtype=False, - ) - - -def test_fetch_when_to_dataframe_is_true_and_multiple_data_sets_are_returned_then_return_list_of_dataframes_representing_multiple_data_sets( - monkeypatch, -): - query = "SELECT * FROM mytable" - driver_index = 2 - data_returned_by_db_set_one = [ - ( - "XY-ZK", - "1234-abcd-efgh-5678", - "SOME_KEY", - 13, - 168, - "pv_power_advanced", - "PT15M", - ), - ( - "XY-ZK", - "1234-abcd-efgh-5678", - "SOME_KEY", - 14, - 168, - "pv_power_advanced", - "PT15M", - ), - ( - "KL-MN", - "1234-abcd-efgh-5678", - "SOME_KEY", - 13, - 168, - "pv_power_advanced", - "PT15M", - ), - ] - data_returned_by_db_set_two = [ - ( - "ALPHA", - "1234-abcd-efgh-5678", - "SOME_KEY", - 13, - 168, - "pv_power_advanced", - "PT15M", - ), - ( - "BETA", - "1234-abcd-efgh-5678", - "SOME_KEY", - 14, - 168, - "pv_power_advanced", - "PT15M", - ), - ] - - expected_result_set_one = [ - { - "plantname": "XY-ZK", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 13, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - { - "plantname": "XY-ZK", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 14, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - { - "plantname": "KL-MN", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 13, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - ] - expected_result_set_two = [ - { - "plantname": "ALPHA", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 13, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - { - "plantname": "BETA", - "resource_id": "1234-abcd-efgh-5678", - "api_key": "SOME_KEY", - "ExtForecastTypeKey": 14, - "hours": 168, - "output_parameters": "pv_power_advanced", - "period": "PT15M", - }, - ] - expected_df_set_one = pd.DataFrame(expected_result_set_one) - expected_df_set_two = pd.DataFrame(expected_result_set_two) - - # Mock the cursor's fetchall methods - mock_cursor = Mock() - mock_cursor.fetchall.side_effect = [ - data_returned_by_db_set_one, - data_returned_by_db_set_two, - ] - mock_cursor.nextset.side_effect = [True, False] - mock_cursor.description = [ - ("plantname", None), - ("resource_id", None), - ("api_key", None), - ("ExtForecastTypeKey", None), - ("hours", None), - ("output_parameters", None), - ("period", None), - ] - - # Mock the connection method to return a mock connection with a mock cursor - mock_connection = Mock() - mock_connection.cursor.return_value = mock_cursor - monkeypatch.setattr("pyodbc.connect", lambda *args, **kwargs: mock_connection) - monkeypatch.setattr("pyodbc.drivers", lambda: ["Driver1", "Driver2", "Driver3"]) - - db = Db(grs(), grs(), grs(), grs(), driver_index) - actual_result = db.fetch(query, True) - - mock_cursor.execute.assert_called_once_with(query) - assert_frame_equal( - actual_result[0].reset_index(drop=True), - expected_df_set_one, - check_dtype=False, - ) - assert_frame_equal( - actual_result[1].reset_index(drop=True), - expected_df_set_two, - check_dtype=False, - ) - - -""" -execute -""" - - -def test_execute_when_init_db_connection_is_successfull_but_fails_when_calling_execute_then_throw_exception( - monkeypatch, -): - query = "INSERT INTO mytable VALUES (1, 'test')" - driver_index = 0 - - # Mock the cursor - mock_cursor = Mock() - - # Mock the connection method to return a mock connection with a mock cursor - mock_connection_success = Mock() - mock_connection_success.cursor.return_value = mock_cursor - - mock_connection_fail = Mock() - mock_connection_fail.cursor.side_effect = pyodbc.DataError( - "Error code", "Database data error" - ) - - monkeypatch.setattr( - "pyodbc.connect", - Mock(side_effect=[mock_connection_success, mock_connection_fail]), - ) - - with pytest.raises(pyodbc.DataError): - db = Db(grs(), grs(), grs(), grs(), driver_index) - db.connection = False - db.execute(query) - - -def test_execute_when_parameter_passed_then_fetch_results_and_return_data(monkeypatch): - query = "INSERT INTO mytable VALUES (?, ?)" - param_one = "John" - param_two = "Smith" - driver_index = 0 - expected_result = [{"id": 13}] - - # Mock the cursor and execute - mock_cursor = Mock() - mock_execute = Mock() - - # Mock the connection method to return a mock connection with a mock cursor - mock_connection = Mock() - mock_connection.cursor.return_value = mock_cursor - - monkeypatch.setattr( - "pyodbc.connect", - Mock(return_value=mock_connection), - ) - - # Mock the fetch method - mock_fetch = Mock(return_value=expected_result) - mock_cursor.execute = mock_execute - mock_cursor.fetchall = mock_fetch - - db = Db(grs(), grs(), grs(), grs(), driver_index) - actual_result = db.execute(query, param_one, param_two) - - mock_execute.assert_called_once_with(query, param_one, param_two) - mock_fetch.assert_called_once() - assert actual_result == expected_result - - -def test_execute_when_fetchall_throws_error_then_return_empty_list(monkeypatch): - query = "INSERT INTO mytable VALUES (?, ?)" - param_one = "John" - param_two = "Smith" - driver_index = 0 - - # Mock the cursor and execute - mock_cursor = Mock() - mock_execute = Mock() - mock_fetchall = Mock(side_effect=Exception("Error occurred")) - - # Mock the connection method to return a mock connection with a mock cursor - mock_connection = Mock() - mock_connection.cursor.return_value = mock_cursor - - monkeypatch.setattr( - "pyodbc.connect", - Mock(return_value=mock_connection), - ) - - # Mock the fetchall method - mock_cursor.execute = mock_execute - mock_cursor.fetchall = mock_fetchall - - db = Db(grs(), grs(), grs(), grs(), driver_index) - actual_result = db.execute(query, param_one, param_two) - - mock_execute.assert_called_once_with(query, param_one, param_two) - mock_fetchall.assert_called_once() - assert actual_result == [] diff --git a/tests/dwh/test_dwh.py b/tests/dwh/test_dwh.py index a3e86a7..a98c477 100644 --- a/tests/dwh/test_dwh.py +++ b/tests/dwh/test_dwh.py @@ -43,7 +43,7 @@ def test_init_when_instantiate_dwh_then_instance_is_created(monkeypatch): # Mock the database connection monkeypatch.setattr( - "pyprediktormapclient.dwh.db.pyodbc.connect", mock_pyodbc_connection + "pyprediktorutilities.dwh.pyodbc.connect", mock_pyodbc_connection ) dwh = DWH(grs(), grs(), grs(), grs(), driver_index) @@ -59,7 +59,7 @@ def test_init_when_instantiate_dwh_but_no_pyodbc_drivers_available_then_throw_ex driver_index = 0 # Mock the absence of ODBC drivers - monkeypatch.setattr("pyprediktormapclient.dwh.db.pyodbc.drivers", lambda: []) + monkeypatch.setattr("pyprediktorutilities.dwh.pyodbc.drivers", lambda: []) with pytest.raises(ValueError) as excinfo: DWH(grs(), grs(), grs(), grs(), driver_index) @@ -73,7 +73,7 @@ def test_init_when_instantiate_dwh_but_pyodbc_throws_error_with_tolerance_to_att # Mock the database connection monkeypatch.setattr( - "pyprediktormapclient.dwh.db.pyodbc.connect", + "pyprediktorutilities.dwh.pyodbc.connect", mock_pyodbc_connection_throws_error_not_tolerant_to_attempts, ) @@ -88,7 +88,7 @@ def test_init_when_instantiate_dwh_but_pyodbc_throws_error_tolerant_to_attempts_ # Mock the database connection monkeypatch.setattr( - "pyprediktormapclient.dwh.db.pyodbc.connect", + "pyprediktorutilities.dwh.pyodbc.connect", mock_pyodbc_connection_throws_error_tolerant_to_attempts, )