From 03065c92140de62a5a9f455572902cc916d2964f Mon Sep 17 00:00:00 2001 From: MeenBna Date: Tue, 26 Mar 2024 14:18:22 +0100 Subject: [PATCH 1/2] Fix: addressed a bug in the opc ua values api request code. --- ...ploring_API_Functions_Authentication.ipynb | 41 ++- .../Test_API_Functions_New_Features.ipynb | 307 ------------------ requirements.txt | 1 + src/pyprediktormapclient/opc_ua.py | 2 +- 4 files changed, 41 insertions(+), 310 deletions(-) delete mode 100644 notebooks/Test_API_Functions_New_Features.ipynb diff --git a/notebooks/Exploring_API_Functions_Authentication.ipynb b/notebooks/Exploring_API_Functions_Authentication.ipynb index ad1693e..8640f81 100644 --- a/notebooks/Exploring_API_Functions_Authentication.ipynb +++ b/notebooks/Exploring_API_Functions_Authentication.ipynb @@ -203,6 +203,19 @@ "string_sets_for_first_park.dataframe" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# All inverter data for the site\n", + "inverter_json = model_data.get_object_descendants(\n", + " \"InverterType\", [site_id], \"PV_Assets\"\n", + ")\n", + "inverters = AnalyticsHelper(inverter_json)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -237,6 +250,23 @@ "opc_data = OPC_UA(rest_url=opcua_rest_url, opcua_url=opcua_server_url, namespaces=namespace_list, auth_client=auth_client)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# History data\n", + "inverters_hist_df = opc_data.get_historical_aggregated_values(\n", + " start_time=datetime.datetime.now() - datetime.timedelta(2),\n", + " end_time=datetime.datetime.now() - datetime.timedelta(1),\n", + " pro_interval=600000,\n", + " agg_name=\"Average\",\n", + " variable_list=inverters.variables_as_list([\"DCPower\"]),\n", + ")\n", + "inverters_hist_df" + ] + }, { "cell_type": "code", "execution_count": null, @@ -245,7 +275,7 @@ "source": [ "# Live value data of trackers\n", "live_value = opc_data.get_values(\n", - " trackers.variables_as_list([\"AngleMeasured\", \"AngleSetpoint\"])\n", + " trackers.variables_as_list([\"AngleMeasured\"])\n", ")\n", "live_value" ] @@ -266,6 +296,13 @@ ")\n", "one_day_historic_tracker_data" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -284,7 +321,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.8" + "version": "3.12.2" } }, "nbformat": 4, diff --git a/notebooks/Test_API_Functions_New_Features.ipynb b/notebooks/Test_API_Functions_New_Features.ipynb deleted file mode 100644 index bfd1e89..0000000 --- a/notebooks/Test_API_Functions_New_Features.ipynb +++ /dev/null @@ -1,307 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Import Libraries" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Import the required packages and libraries\n", - "import datetime\n", - "import os\n", - "from dotenv import load_dotenv \n", - "from pathlib import Path" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Import Scripts" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pyprediktormapclient.opc_ua import OPC_UA\n", - "from pyprediktormapclient.model_index import ModelIndex\n", - "from pyprediktormapclient.auth_client import AUTH_CLIENT\n", - "from pyprediktormapclient.analytics_helper import AnalyticsHelper\n", - "from pyprediktormapclient.shared import *" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Import Envrionment Variables" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Consider obtaining the envrionment variables from .env file if you are running this locally from source.\n", - "dotenv_path = Path(\"../.env\")\n", - "load_dotenv(dotenv_path=dotenv_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "username = os.environ[\"USERNAME\"]\n", - "password = os.environ[\"PASSWORD\"]\n", - "opcua_rest_url = os.environ[\"OPC_UA_REST_URL\"]\n", - "opcua_server_url = os.environ[\"OPC_UA_SERVER_URL\"]\n", - "model_index_url = os.environ[\"MODEL_INDEX_URL\"]\n", - "ory_url = os.environ[\"ORY_URL\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Getting ory bearer token\n", - "auth_client = AUTH_CLIENT(rest_url=ory_url, username=username, password=password)\n", - "auth_client.request_new_ory_token()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Download data from modelindex api" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Connecting to ModelIndex APIs \n", - "model_data = ModelIndex(url=model_index_url)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Listed sites on the model index api server\n", - "namespaces = model_data.get_namespace_array()\n", - "namespaces" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Types of Objects\n", - "object_types_json = model_data.get_object_types()\n", - "object_types = AnalyticsHelper(object_types_json)\n", - "object_types.dataframe" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Unique types of Objects\n", - "object_types_unique = object_types.dataframe[[\"Id\", \"Name\"]].drop_duplicates()\n", - "object_types_unique" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# To get typeId by type name of an object\n", - "object_type_id = model_data.get_object_type_id_from_name(\"SiteType\")\n", - "object_type_id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# To get the objects of a type\n", - "sites_json = model_data.get_objects_of_type(\"SiteType\")\n", - "\n", - "# Send the returned JSON into a normalizer to get Id, Type, Name, Props and Vars as columns\n", - "sites = AnalyticsHelper(sites_json)\n", - "sites.list_of_names()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Analytics helper\n", - "sites.variables_as_dataframe()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sites.list_of_ids()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Selecting the single site\n", - "site_id = sites.list_of_ids()[0]\n", - "site_id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Get all stringsets for one park\n", - "string_sets_for_first_park_as_json = model_data.get_object_descendants(\n", - " \"StringSetType\", [site_id], \"PV_Assets\"\n", - ")\n", - "string_sets_for_first_park = AnalyticsHelper(string_sets_for_first_park_as_json)\n", - "string_sets_for_first_park.dataframe" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Ancestors of an object type, get all trackers that are ancestor of the parks string sets\n", - "\n", - "trackers_as_json = model_data.get_object_ancestors(\n", - " \"TrackerType\", string_sets_for_first_park.list_of_ids(), \"PV_Serves\"\n", - ")\n", - "trackers = AnalyticsHelper(trackers_as_json)\n", - "trackers.variables_as_dataframe()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Download data from the opc ua api" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "namespace_list = object_types.namespaces_as_list(namespaces)\n", - "\n", - "# Initating the OPC UA API with a fixed namespace list\n", - "opc_data = OPC_UA(rest_url=opcua_rest_url, opcua_url=opcua_server_url, namespaces=namespace_list)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Live value data of trackers\n", - "live_value = opc_data.get_values(\n", - " trackers.variables_as_list([\"AngleMeasured\", \"AngleSetpoint\"])\n", - ")\n", - "live_value" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Raw historic value data of trackers\n", - "one_day_raw_historic_tracker_data = opc_data.get_historical_raw_values(\n", - " start_time=(datetime.datetime.now() - datetime.timedelta(30)),\n", - " end_time=(datetime.datetime.now() - datetime.timedelta(29)),\n", - " variable_list=trackers.variables_as_list([\"AngleSetpoint\"]),\n", - ")\n", - "one_day_raw_historic_tracker_data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Aggregated historic value data of trackers, 1 days worth of data 30 days ago\n", - "one_day_historic_tracker_data = opc_data.get_historical_aggregated_values(\n", - " start_time=(datetime.datetime.now() - datetime.timedelta(30)),\n", - " end_time=(datetime.datetime.now() - datetime.timedelta(29)),\n", - " pro_interval=3600000,\n", - " agg_name=\"Average\",\n", - " variable_list=trackers.variables_as_list([\"AngleMeasured\"]),\n", - ")\n", - "one_day_historic_tracker_data" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv_auth", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/requirements.txt b/requirements.txt index 5cd2a4d..664a1fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ sphinx_rtd_theme tox +ipykernel -e . python-dotenv \ No newline at end of file diff --git a/src/pyprediktormapclient/opc_ua.py b/src/pyprediktormapclient/opc_ua.py index 17409f5..d40e618 100644 --- a/src/pyprediktormapclient/opc_ua.py +++ b/src/pyprediktormapclient/opc_ua.py @@ -224,7 +224,7 @@ def _get_variable_list_as_list(self, variable_list: list) -> list: else: raise TypeError("Unsupported type in variable_list") - return new_vars + return new_vars def get_values(self, variable_list: List[Variables]) -> List: From 633991503b742aea0e1c568671d6dba667bd097c Mon Sep 17 00:00:00 2001 From: MeenBna Date: Thu, 4 Apr 2024 18:44:33 +0200 Subject: [PATCH 2/2] Updated tests for handling pyodbc errors in DWH initialization. --- src/pyprediktormapclient/dwh/dwh.py | 2 +- tests/dwh/test_dwh.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pyprediktormapclient/dwh/dwh.py b/src/pyprediktormapclient/dwh/dwh.py index ec63d7d..ed38754 100644 --- a/src/pyprediktormapclient/dwh/dwh.py +++ b/src/pyprediktormapclient/dwh/dwh.py @@ -3,7 +3,7 @@ import importlib from typing import Dict from pydantic import validate_call -from pyprediktorutilities import Dwh as Db +from pyprediktorutilities.dwh.dwh import Dwh as Db from . import context from .idwh import IDWH diff --git a/tests/dwh/test_dwh.py b/tests/dwh/test_dwh.py index 534ee38..7860780 100644 --- a/tests/dwh/test_dwh.py +++ b/tests/dwh/test_dwh.py @@ -5,7 +5,7 @@ import logging import datetime from unittest.mock import Mock -from pyprediktormapclient.dwh import DWH +from pyprediktormapclient.dwh.dwh import DWH """ Mock Functions @@ -44,7 +44,7 @@ def test_init_when_instantiate_dwh_but_pyodbc_throws_error_with_tolerance_to_att # Mock the database connection monkeypatch.setattr( - "pyprediktorutilities.dwh.pyodbc.connect", + "pyprediktorutilities.dwh.dwh.pyodbc.connect", mock_pyodbc_connection_throws_error_not_tolerant_to_attempts, ) @@ -59,7 +59,7 @@ def test_init_when_instantiate_dwh_but_pyodbc_throws_error_tolerant_to_attempts_ # Mock the database connection monkeypatch.setattr( - "pyprediktorutilities.dwh.pyodbc.connect", + "pyprediktorutilities.dwh.dwh.pyodbc.connect", mock_pyodbc_connection_throws_error_tolerant_to_attempts, ) @@ -99,7 +99,7 @@ def test_version_when_version_data_is_returned_then_return_version_data(monkeypa ( "2.3.1", datetime.datetime(2023, 11, 14, 7, 5, 19, 830000), - "Updated DWH from procs", + "Updated Dwh from procs", 2, 3, 1, @@ -110,7 +110,7 @@ def test_version_when_version_data_is_returned_then_return_version_data(monkeypa expected_result = { "DWHVersion": "2.3.1", "UpdateDate": datetime.datetime(2023, 11, 14, 7, 5, 19, 830000), - "Comment": "Updated DWH from procs", + "Comment": "Updated Dwh from procs", "MajorVersionNo": 2, "MinorVersionNo": 3, "InterimVersionNo": 1,