From 90d6f25ce85f71f35a1625a258cb4ef52daf2f96 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Fri, 10 May 2024 10:14:10 -0700 Subject: [PATCH 01/12] Releases/9.0.1 (#3171) (#3181) * Update readthedocs requirements.txt * Update conf.py * Update requirements_demo.txt Add missing pandas requirement for demo * work around for issue #3154 * Fix for security issue #3168 (#3169) * Fix for security issue #3168 * handling clean up errors in test * testing group commands in different test module * moved group and role test to different module * moved group and role test to different module * Added a cache for agent names since platform start * Fixes process overload from file events * fixed issue with variable definition. * Remove PersistentDict from web-user.json file. * Update admin_endpoints.py Handle behavior of removing PersistentDict * Update version to 9.0.1 --------- Co-authored-by: Chandrika Sivaramakrishnan Co-authored-by: Chandrika Co-authored-by: Andrew Rodgers --- docs/source/conf.py | 4 ++-- volttron/platform/__init__.py | 2 +- volttron/platform/vip/agent/subsystems/auth.py | 6 ++---- volttron/platform/web/admin_endpoints.py | 15 +++++++++++---- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index dc75f60eb2..10c31ff659 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -90,9 +90,9 @@ def __getattr__(cls, name): author = 'The VOLTTRON Community' # The short X.Y version -version = '9.0' +version = '9.0.1' # The full version, including alpha/beta/rc tags -release = '9.0' +release = '9.0.1' # -- General configuration --------------------------------------------------- diff --git a/volttron/platform/__init__.py b/volttron/platform/__init__.py index d6ce14cdd2..5a7525b81e 100644 --- a/volttron/platform/__init__.py +++ b/volttron/platform/__init__.py @@ -35,7 +35,7 @@ from urllib.parse import urlparse from ..utils.frozendict import FrozenDict -__version__ = '9.0rc0' +__version__ = '9.0.1' _log = logging.getLogger(__name__) diff --git a/volttron/platform/vip/agent/subsystems/auth.py b/volttron/platform/vip/agent/subsystems/auth.py index 8ba4e2dcba..9337d0e513 100644 --- a/volttron/platform/vip/agent/subsystems/auth.py +++ b/volttron/platform/vip/agent/subsystems/auth.py @@ -265,6 +265,7 @@ def update_rpc_method_capabilities(self): """ rpc_method_authorizations = {} rpc_methods = self.get_rpc_exports() + updated_rpc_authorizations = None for method in rpc_methods: if len(method.split(".")) > 1: pass @@ -295,9 +296,7 @@ def update_rpc_method_capabilities(self): _log.info( f"Skipping updating rpc auth capabilities for agent " f"{self._core().identity} connecting to remote address: {self._core().address} ") - updated_rpc_authorizations = None except gevent.timeout.Timeout: - updated_rpc_authorizations = None _log.warning(f"update_id_rpc_authorization rpc call timed out for {self._core().identity} {rpc_method_authorizations}") except MethodNotFound: _log.warning("update_id_rpc_authorization method is missing from " @@ -306,7 +305,6 @@ def update_rpc_method_capabilities(self): "dynamic RPC authorizations.") return except Exception as e: - updated_rpc_authorizations = None _log.exception(f"Exception when calling rpc method update_id_rpc_authorizations for identity: " f"{self._core().identity} Exception:{e}") if updated_rpc_authorizations is None: @@ -318,7 +316,7 @@ def update_rpc_method_capabilities(self): f"the identity of the agent" ) return - if rpc_method_authorizations != updated_rpc_authorizations: + if rpc_method_authorizations != updated_rpc_authorizations and updated_rpc_authorizations is not None: for method in updated_rpc_authorizations: self.set_rpc_authorizations( method, updated_rpc_authorizations[method] diff --git a/volttron/platform/web/admin_endpoints.py b/volttron/platform/web/admin_endpoints.py index 9de8d05a5c..d307b2f353 100644 --- a/volttron/platform/web/admin_endpoints.py +++ b/volttron/platform/web/admin_endpoints.py @@ -46,7 +46,6 @@ from volttron.platform import get_home from volttron.platform import jsonapi from volttron.utils import VolttronHomeFileReloader -from volttron.utils.persistance import PersistentDict _log = logging.getLogger(__name__) @@ -84,7 +83,7 @@ def __init__(self, rmq_mgmt=None, ssl_public_key: bytes = None, rpc_caller=None) else: self._ssl_public_key = None - self._userdict = None + self._userdict = {} self.reload_userdict() self._observer = Observer() @@ -96,7 +95,14 @@ def __init__(self, rmq_mgmt=None, ssl_public_key: bytes = None, rpc_caller=None) def reload_userdict(self): webuserpath = os.path.join(get_home(), 'web-users.json') - self._userdict = PersistentDict(webuserpath, format="json") + if os.path.exists(webuserpath): + with open(webuserpath) as fp: + try: + self._userdict = jsonapi.loads(fp.read()) + except json.decoder.JSONDecodeError: + self._userdict = {} + # Keep same behavior as with PersistentDict + raise ValueError("File not in a supported format") def get_routes(self): """ @@ -339,4 +345,5 @@ def add_user(self, username, unencrypted_pw, groups=None, overwrite=False): groups=groups ) - self._userdict.sync() + with open(os.path.join(get_home(), 'web-users.json'), 'w') as fp: + fp.write(jsonapi.dumps(self._userdict, indent=2)) From 02533d6910afc425aa6012d30400ef68294414f0 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 1 May 2024 07:48:52 -0700 Subject: [PATCH 02/12] Modbustk error handler doesn't include .message --- .../platform_driver/interfaces/modbus_tk/client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py index beea69b023..6581e3cff2 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py @@ -669,9 +669,10 @@ def read_request(self, request): ) self._data.update(request.parse_values(results)) except (AttributeError, ModbusError) as err: - if "Exception code" in err.message: - raise Exception("{0}: {1}".format(err.message, - helpers.TABLE_EXCEPTION_CODE.get(err.message[-1], "UNDEFINED"))) + if err is ModbusError: + code = err.get_exception_code() + raise Exception(f'{err.args[0]}, {helpers.TABLE_EXCEPTION_CODE.get(code, "UNDEFINED")}') + logger.warning("modbus read_all() failure on request: %s\tError: %s", request, err) def read_all(self): From 10fcabf80025d8e571352e0a2399182c1451ec95 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Wed, 10 Apr 2024 14:21:58 -0700 Subject: [PATCH 03/12] Fixed saving of state. --- services/core/ActuatorAgent/actuator/scheduler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/core/ActuatorAgent/actuator/scheduler.py b/services/core/ActuatorAgent/actuator/scheduler.py index c1190e5491..8bb8b4bf2a 100644 --- a/services/core/ActuatorAgent/actuator/scheduler.py +++ b/services/core/ActuatorAgent/actuator/scheduler.py @@ -21,14 +21,14 @@ # # ===----------------------------------------------------------------------=== # }}} - - import bisect import logging -from pickle import dumps, loads + +from base64 import b64encode from collections import defaultdict, namedtuple from copy import deepcopy from datetime import timedelta +from pickle import dumps, loads from volttron.platform.agent import utils @@ -340,7 +340,7 @@ def save_state(self, now): try: self._cleanup(now) - self.save_state_callback(dumps(self.tasks)) + self.save_state_callback(b64encode(dumps(self.tasks)).decode("utf-8")) except Exception: _log.error('Failed to save scheduler state!') From 8605a4ae8a560faf6921bd3b00837f75d917641f Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Wed, 10 Apr 2024 14:24:52 -0700 Subject: [PATCH 04/12] Fixed missing preempted task information when scheduling. --- services/core/ActuatorAgent/actuator/scheduler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/core/ActuatorAgent/actuator/scheduler.py b/services/core/ActuatorAgent/actuator/scheduler.py index 8bb8b4bf2a..b2fcc4ae52 100644 --- a/services/core/ActuatorAgent/actuator/scheduler.py +++ b/services/core/ActuatorAgent/actuator/scheduler.py @@ -411,7 +411,10 @@ def request_slots(self, agent_id, id_, requests, priority, now=None): self.save_state(now) - return RequestResult(True, preempted_tasks, '') + if preempted_tasks: + return RequestResult(True, list(preempted_tasks), 'TASK_WERE_PREEMPTED') + else: + return RequestResult(True, {}, '') def cancel_task(self, agent_id, task_id, now): if task_id not in self.tasks: From c24f01152b0bbc4b23cd608845bc0818ec2fc752 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Tue, 16 Apr 2024 18:08:12 -0700 Subject: [PATCH 05/12] Allow preempted device information to reach user. --- services/core/ActuatorAgent/actuator/agent.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/services/core/ActuatorAgent/actuator/agent.py b/services/core/ActuatorAgent/actuator/agent.py index 369489ed75..82f666c9ca 100644 --- a/services/core/ActuatorAgent/actuator/agent.py +++ b/services/core/ActuatorAgent/actuator/agent.py @@ -1382,11 +1382,8 @@ def _request_new_schedule(self, sender, task_id, priority, requests, publish_res 'data': {'agentID': sender, 'taskID': task_id}}) - # If we are successful we do something else with the real result data - data = result.data if not result.success else {} - results = {'result': success, - 'data': data, + 'data': result.data, 'info': result.info_string} if publish_result: From 99f8eda86a99efd671b6115a65ad9d68d66ac928 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Tue, 16 Apr 2024 18:08:56 -0700 Subject: [PATCH 06/12] Fixes to driver exception handling related to bacnet pings and heartbeats. --- services/core/PlatformDriverAgent/platform_driver/agent.py | 5 ++++- .../platform_driver/interfaces/bacnet.py | 7 +++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/agent.py b/services/core/PlatformDriverAgent/platform_driver/agent.py index 967e2c81f1..5c3d097433 100644 --- a/services/core/PlatformDriverAgent/platform_driver/agent.py +++ b/services/core/PlatformDriverAgent/platform_driver/agent.py @@ -487,7 +487,10 @@ def heart_beat(self): """ _log.debug("sending heartbeat") for device in self.instances.values(): - device.heart_beat() + try: + device.heart_beat() + except (Exception, gevent.Timeout) as e: + _log.warning(f'Failed to set heart_beat point on device: {device.device_name} -- {e}.') @RPC.export def revert_point(self, path, point_name, **kwargs): diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py index b2fc973e10..d7f7c606e1 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py @@ -22,7 +22,7 @@ # ===----------------------------------------------------------------------=== # }}} - +import gevent import logging from datetime import datetime, timedelta @@ -98,9 +98,8 @@ def ping_target(self): pinged = True except errors.Unreachable: _log.warning("Unable to reach BACnet proxy.") - - except errors.VIPError: - _log.warning("Error trying to ping device.") + except (Exception, gevent.Timeout) as e: + _log.warning(f"Error trying to ping device: {e}") self.scheduled_ping = None From 8fbfc051e6ffbc70ac4ed7d533e1094532df6523 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Fri, 10 May 2024 12:59:00 -0700 Subject: [PATCH 07/12] Added device information to error messages. --- .../PlatformDriverAgent/platform_driver/interfaces/bacnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py index d7f7c606e1..d591bc8b45 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py @@ -99,7 +99,7 @@ def ping_target(self): except errors.Unreachable: _log.warning("Unable to reach BACnet proxy.") except (Exception, gevent.Timeout) as e: - _log.warning(f"Error trying to ping device: {e}") + _log.warning(f"Error trying to ping device with device_id '{self.device_id}' at {self.target_address}: {e}") self.scheduled_ping = None From 84ca12b2ff98bd40dfa9521f888138d6098bf2c6 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Fri, 28 Jun 2024 13:24:47 -0700 Subject: [PATCH 08/12] Added on_property method to get_point and set_point. Replaced and extended broken functionality to get_priortity_array on get_point. --- .../PlatformDriverAgent/platform_driver/interfaces/bacnet.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py index d591bc8b45..6ae4a4c8fa 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py @@ -97,9 +97,10 @@ def ping_target(self): self.vip.rpc.call(self.proxy_address, 'ping_device', self.target_address, self.device_id).get(timeout=self.timeout) pinged = True except errors.Unreachable: - _log.warning("Unable to reach BACnet proxy.") + _log.warning(f"Unable to reach BACnet proxy at: {self.proxy_address}.") except (Exception, gevent.Timeout) as e: - _log.warning(f"Error trying to ping device with device_id '{self.device_id}' at {self.target_address}: {e}") + _log.warning(f"Error trying to ping device with device_id '{self.device_id}' at {self.target_address}" + f"through proxy {self.proxy_address}: {e}") self.scheduled_ping = None From 94c908017c5ca729294c6e4abf614fd15382dcc6 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Fri, 28 Jun 2024 11:21:15 -0700 Subject: [PATCH 09/12] Added on_property method to get_point and set_point. Replaced and extended broken functionality to get_priortity_array on get_point. --- .../platform_driver/interfaces/bacnet.py | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py index 6ae4a4c8fa..370a3d2d9d 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py @@ -108,16 +108,25 @@ def ping_target(self): if not pinged: self.schedule_ping() - def get_point(self, point_name, get_priority_array=False): + def get_point(self, point_name, on_property=None): register = self.get_register_by_name(point_name) - property_name = "priorityArray" if get_priority_array else register.property - register_index = None if get_priority_array else register.index - result = self.vip.rpc.call(self.proxy_address, 'read_property', - self.target_address, register.object_type, - register.instance_number, property_name, register_index).get(timeout=self.timeout) + if on_property is None: + result = self.vip.rpc.call(self.proxy_address, 'read_property', + self.target_address, register.object_type, + register.instance_number, register.property, register.index).get(timeout=self.timeout) + else: + point_map = {} + point_map[register.point_name] = [register.object_type, + register.instance_number, + on_property, + register.index] + result = self.vip.rpc.call(self.proxy_address, 'read_properties', + self.target_address, point_map, + self.max_per_request, True).get(timeout=self.timeout) + result = list(result.values())[0] return result - def set_point(self, point_name, value, priority=None): + def set_point(self, point_name, value, priority=None, on_property=None): # TODO: support writing from an array. register = self.get_register_by_name(point_name) if register.read_only: @@ -130,7 +139,7 @@ def set_point(self, point_name, value, priority=None): args = [self.target_address, value, register.object_type, register.instance_number, - register.property, + on_property if on_property is not None else register.property, priority if priority is not None else register.priority, register.index] result = self.vip.rpc.call(self.proxy_address, 'write_property', *args).get(timeout=self.timeout) From adf79e10b10558e9fb202eb67bd60b7cb4596b9c Mon Sep 17 00:00:00 2001 From: jiangyilin123 Date: Mon, 26 Feb 2024 13:06:37 -0800 Subject: [PATCH 10/12] modified chargepoint interface for platform driver --- .../interfaces/chargepoint/__init__.py | 31 ++++++++++--------- .../interfaces/chargepoint/async_service.py | 10 +++--- .../chargepoint/credential_check.py | 6 ++-- .../interfaces/chargepoint/service.py | 25 ++++++++------- update_scripts/update.driver | 2 ++ 5 files changed, 42 insertions(+), 32 deletions(-) create mode 100644 update_scripts/update.driver diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py index 0f8d26e5bb..0e915905c8 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py @@ -27,12 +27,11 @@ import logging import abc import sys - from . import service as cps -from . import async_service as async - +from . import async_service as async_service from .. import BaseInterface, BaseRegister, BasicRevert, DriverInterfaceError -from suds.sudsobject import asdict +#from suds.sudsobject import asdict +from zeep.helpers import serialize_object _log = logging.getLogger(__name__) @@ -54,7 +53,7 @@ point_name_mapping = {"Status.TimeStamp": "TimeStamp"} service = {} -gevent.spawn(async.web_service) +gevent.spawn(async_service.web_service) def recursive_asdict(d): @@ -64,7 +63,7 @@ def recursive_asdict(d): http://stackoverflow.com/questions/2412486/serializing-a-suds-object-in-python """ out = {} - for k, v in asdict(d).items(): + for k, v in serialize_object(d, dict).items(): if hasattr(v, '__keylist__'): out[k] = recursive_asdict(v) elif isinstance(v, list): @@ -196,7 +195,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getStations - result = async.CPRequest.request(method, self.timeout, stationID=self.station_id) + result = async_service.CPRequest.request(method, self.timeout, stationID=self.station_id) result.wait() return self.get_register(result.value, method) @@ -237,7 +236,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getLoad - result = async.CPRequest.request(method, self.timeout, stationID=self.station_id) + result = async_service.CPRequest.request(method, self.timeout, stationID=self.station_id) result.wait() return self.get_register(result.value, method) @@ -263,7 +262,7 @@ def value(self, x): kwargs = {'stationID': self.station_id} if self.attribute_name == 'shedState' and not value: method = service[self.username].clearShedState - result = async.CPRequest.request(method, 0, stationID=self.station_id) + result = async_service.CPRequest.request(method, 0, stationID=self.station_id) elif self.attribute_name == 'shedState': _log.error('shedState may only be written with value 0. If you want to shedLoad, write to ' 'allowedLoad or percentShed') @@ -273,7 +272,7 @@ def value(self, x): kwargs[self.attribute_name] = value if self.port: kwargs['portNumber'] = self.port - result = async.CPRequest.request(method, 0, **kwargs) + result = async_service.CPRequest.request(method, 0, **kwargs) result.wait() if result.value.responseCode != "100": @@ -322,7 +321,7 @@ def value(self): if self.port: kwargs['portNumber'] = self.port - result = async.CPRequest.request(method, self.timeout, **kwargs) + result = async_service.CPRequest.request(method, self.timeout, **kwargs) result.wait() return self.get_register(result.value, method, False) @@ -348,7 +347,7 @@ def value(self, x): if self.attribute_name == 'clearAlarms' and value: kwargs = {'stationID': self.station_id} method = service[self.username].clearAlarms - result = async.CPRequest.request(method, 0, **kwargs) + result = async_service.CPRequest.request(method, 0, **kwargs) result.wait() if result.value.responseCode not in ['100', '153']: @@ -383,7 +382,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getChargingSessionData - result = async.CPRequest.request(method, self.timeout, stationID=self.station_id) + result = async_service.CPRequest.request(method, self.timeout, stationID=self.station_id) result.wait() # Of Note, due to API limitations, port number is ignored for these calls @@ -418,7 +417,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getStationStatus - result = async.CPRequest.request(method, self.timeout, self.station_id) + result = async_service.CPRequest.request(method, self.timeout, self.station_id) result.wait() return self.get_register(result.value, method) @@ -455,7 +454,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getStationRights - result = async.CPRequest.request(method, self.timeout, stationID=self.station_id) + result = async_service.CPRequest.request(method, self.timeout, stationID=self.station_id) result.wait() # Note: this does not go through get_register, as it is of a unique type, 'dictionary.' @@ -523,6 +522,8 @@ def parse_config(self, config_dict, registry_config_str): return for regDef in registry_config_str: + print(regDef) + _log.debug(f'RegDef is {regDef}') # Skip lines that have no address yet. if not regDef['Attribute Name']: continue diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py index ca98c8a753..4a273cbc80 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py @@ -47,15 +47,16 @@ import gevent.event import gevent.queue import logging -import suds +#import suds +import zeep from gevent import monkey from .service import CPAPIException from datetime import datetime, timedelta monkey.patch_all() _log = logging.getLogger(__name__) -SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.0.wsdl" - +#SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.0.wsdl" +SERVICE_WSDL_URL = "http://localhost:8080/cp_api_5.1.wsdl" # Queue for Web API requests and responses. It is managed by the long running # web_service() greenlet. web_service_queue = gevent.queue.Queue() @@ -253,7 +254,8 @@ def web_service(): web_cache[item_key] = cache_item if not client_set: - client_set.add(suds.client.Client(SERVICE_WSDL_URL)) + #client_set.add(suds.client.Client(SERVICE_WSDL_URL)) + client_set.add(zeep.Client(SERVICE_WSDL_URL)) client = client_set.pop() gevent.spawn(web_call, item, client) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py index 9687d08d63..9f8a8f1586 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py @@ -1,5 +1,6 @@ from . import service as cps -import suds +#import suds +import zeep import io station_csv = { @@ -176,5 +177,6 @@ else: print("Some other error happened") - except suds.WebFault as a: + #except suds.WebFault as a: + except zeep.exception.Fault as e: print("Sorry, your API credentials are invalid. Please contact Chargepoint for assistance.") diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py index da845c676e..0ef7d381a1 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py @@ -22,8 +22,10 @@ # ===----------------------------------------------------------------------=== # }}} -import suds.client -import suds.wsse +#import suds.client +#import suds.wsse +import zeep +from zeep.wsse.username import UsernameToken import logging logger = logging.getLogger('chargepoint') @@ -769,18 +771,18 @@ def __init__(self, username=None, password=None): """ self._username = username self._password = password - self._suds_client = None + self._zeep_client = None @property def _client(self): """Initialize the SUDS client if necessary.""" - if self._suds_client is None: - self._suds_client = suds.client.Client(SERVICE_WSDL_URL) + if self._zeep_client is None: + self._zeep_client = zeep.Client(SERVICE_WSDL_URL) # Add SOAP Security tokens self.set_security_token() - return self._suds_client + return self._zeep_client @property def _soap_service(self): @@ -788,13 +790,14 @@ def _soap_service(self): def set_security_token(self): # Add SOAP Security tokens - security = suds.wsse.Security() - token = suds.wsse.UsernameToken(self._username, self._password) - security.tokens.append(token) - self._suds_client.set_options(wsse=security) + #security = suds.wsse.Security() + #token = suds.wsse.UsernameToken(self._username, self._password) + #security.tokens.append(token) + #self._zeep_client.set_options(wsse=security) + self._zeep_client = zeep.Client(SERVICE_WSDL_URL, wsse=UsernameToken(self._username, self._password)) def set_client(self, client): - self._suds_client = client + self._zeep_client = client self.set_security_token() def clearAlarms(self, **kwargs): diff --git a/update_scripts/update.driver b/update_scripts/update.driver new file mode 100644 index 0000000000..9f9c3346f8 --- /dev/null +++ b/update_scripts/update.driver @@ -0,0 +1,2 @@ +#!/bin/bash +python ~/volttron/scripts/install-agent.py -s /home/vboxuser/chargepoint/volttron_chargepoint/services/core/PlatformDriverAgent -c /home/vboxuser/chargepoint/volttron_chargepoint/config/driver.config -i platform.driver -t driver --force --start --priority 60 From 6b77994787ae7b360ed7e2d3418ddf46cca32317 Mon Sep 17 00:00:00 2001 From: jiangyilin123 Date: Tue, 5 Mar 2024 16:27:11 -0800 Subject: [PATCH 11/12] replace suds to zeep for chargepoint interface in platform.driver --- .../interfaces/chargepoint/README.rst | 2 +- .../interfaces/chargepoint/__init__.py | 4 +- .../interfaces/chargepoint/async_service.py | 7 +-- .../chargepoint/credential_check.py | 2 - .../interfaces/chargepoint/requirements.txt | 2 +- .../interfaces/chargepoint/service.py | 52 +++++++++---------- update_scripts/update.driver | 2 - 7 files changed, 31 insertions(+), 40 deletions(-) delete mode 100644 update_scripts/update.driver diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/README.rst b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/README.rst index 218b1dba8d..4da6ce77a3 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/README.rst +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/README.rst @@ -12,7 +12,7 @@ activated environment: :: - pip install suds-jurko + pip install zeep Alternatively requirements can be installed from requirements.txt using: diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py index 0e915905c8..de3c7786ac 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py @@ -522,8 +522,6 @@ def parse_config(self, config_dict, registry_config_str): return for regDef in registry_config_str: - print(regDef) - _log.debug(f'RegDef is {regDef}') # Skip lines that have no address yet. if not regDef['Attribute Name']: continue @@ -559,7 +557,7 @@ def parse_config(self, config_dict, registry_config_str): description=description, port_number=port_num, username=config_dict['username'], - timeout=config_dict['cacheExpiration'] + timeout=config_dict.get('cacheExpiration',0) ) self.insert_register(register) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py index 4a273cbc80..e142a30d86 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py @@ -47,16 +47,14 @@ import gevent.event import gevent.queue import logging -#import suds import zeep from gevent import monkey from .service import CPAPIException from datetime import datetime, timedelta -monkey.patch_all() _log = logging.getLogger(__name__) -#SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.0.wsdl" -SERVICE_WSDL_URL = "http://localhost:8080/cp_api_5.1.wsdl" + +SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.1.wsdl" # Queue for Web API requests and responses. It is managed by the long running # web_service() greenlet. web_service_queue = gevent.queue.Queue() @@ -254,7 +252,6 @@ def web_service(): web_cache[item_key] = cache_item if not client_set: - #client_set.add(suds.client.Client(SERVICE_WSDL_URL)) client_set.add(zeep.Client(SERVICE_WSDL_URL)) client = client_set.pop() gevent.spawn(web_call, item, client) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py index 9f8a8f1586..e06c51b25d 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py @@ -1,5 +1,4 @@ from . import service as cps -#import suds import zeep import io @@ -177,6 +176,5 @@ else: print("Some other error happened") - #except suds.WebFault as a: except zeep.exception.Fault as e: print("Sorry, your API credentials are invalid. Please contact Chargepoint for assistance.") diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/requirements.txt b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/requirements.txt index 274ad9e069..f61b24a644 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/requirements.txt +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/requirements.txt @@ -1 +1 @@ -suds-jurko==0.6 \ No newline at end of file +zeep==4.2.1 \ No newline at end of file diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py index 0ef7d381a1..559f78e5ca 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py @@ -22,16 +22,14 @@ # ===----------------------------------------------------------------------=== # }}} -#import suds.client -#import suds.wsse import zeep from zeep.wsse.username import UsernameToken +from zeep import Settings import logging logger = logging.getLogger('chargepoint') -SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.0.wsdl" - +SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.1.wsdl" CPAPI_SUCCESS = '100' XMPP_EVENTS = [ @@ -158,13 +156,13 @@ class CPStation: """Wrapper around the getStations() return by Chargepoint API. Data surrounding a Chargepoint Station can generally be categorized as static or dynamic. Chargepoint API has two - basic calls, getLoad and getStation, that each return station data. getLoad returns the stationLoadData SUDS - object, and getStation returns the stationDataExtended SUDS object. These are each kept as separate meta-data + basic calls, getLoad and getStation, that each return station data. getLoad returns the stationLoadData object, + and getStation returns the stationDataExtended object. These are each kept as separate meta-data parameters. :param cps: Chargepoint Service object. - :param sld: stationLoadData SUDS object. - :param sde: stationDataExtended SUDS object. + :param sld: stationLoadData object. + :param sde: stationDataExtended object. (stationDataExtended){ stationID = "1:00001" @@ -760,8 +758,8 @@ class CPService: """ Python wrapper around the Chargepoint WebServices API. - Current Version: 5.0 - Docs: ChargePoint_Web_Services_API_Guide_Ver4.1_Rev5.pdf + Current Version: 5.1 + Docs: ChargePoint_Web_Services_API_Guide_Ver5.1_Rev1.13.pdf """ def __init__(self, username=None, password=None): @@ -775,7 +773,7 @@ def __init__(self, username=None, password=None): @property def _client(self): - """Initialize the SUDS client if necessary.""" + """Initialize the ZEEP client if necessary.""" if self._zeep_client is None: self._zeep_client = zeep.Client(SERVICE_WSDL_URL) @@ -790,11 +788,10 @@ def _soap_service(self): def set_security_token(self): # Add SOAP Security tokens - #security = suds.wsse.Security() - #token = suds.wsse.UsernameToken(self._username, self._password) - #security.tokens.append(token) - #self._zeep_client.set_options(wsse=security) - self._zeep_client = zeep.Client(SERVICE_WSDL_URL, wsse=UsernameToken(self._username, self._password)) + #TODO:might need to put this in config + #NOTE: wihtout this setting, zeep will not get result + settins = Settings(strict=False, xml_huge_tree=True, xsd_ignore_sequence_order=True) + self._zeep_client = zeep.Client(SERVICE_WSDL_URL, wsse=UsernameToken(self._username, self._password),settings=settins) def set_client(self, client): self._zeep_client = client @@ -823,7 +820,7 @@ def clearAlarms(self, **kwargs): :returns SOAP reply object. If successful, there will be a responseCode of '100'. """ - searchQuery = self._client.factory.create('clearAlarmsSearchQuery') + searchQuery = self._client.get_type('ns0:clearAlarmsSearchQuery')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.clearAlarms(searchQuery) @@ -839,7 +836,7 @@ def clearShedState(self, **kwargs): :returns SOAP reply object. If successful, there will be a responseCode of '100'. """ - searchQuery = self._client.factory.create('shedQueryInputData') + searchQuery = self._client.get_type('ns0:shedQueryInputData')() if 'stationID' in kwargs.keys(): setattr(searchQuery, 'shedStation', {'stationID': kwargs['stationID']}) elif 'sgID' in kwargs.keys(): @@ -893,7 +890,7 @@ def getAlarms(self, **kwargs): } """ - searchQuery = self._client.factory.create('getAlarmsSearchQuery') + searchQuery = self._client.get_type('ns0:getAlarmsSearchQuery')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getAlarms(searchQuery) @@ -968,7 +965,7 @@ def getChargingSessionData(self, **kwargs): } """ - searchQuery = self._client.factory.create('sessionSearchdata') + searchQuery = self._client.get_type('ns0:sessionSearchdata')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getChargingSessionData(searchQuery) @@ -1021,7 +1018,8 @@ def getLoad(self, **kwargs): """ # @ToDo: Figure out what type of request searchQuery should be here. - searchQuery = self._client.factory.create('stationSearchRequestExtended') + # @Note: Looks like it should be {sgID: xsd:int, stationID: xsd:string, sessionID: xsd:long} + searchQuery = {} for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getLoad(searchQuery) @@ -1062,7 +1060,7 @@ def getOrgsAndStationGroups(self, **kwargs): } """ - searchQuery = self._client.factory.create('getOrgsAndStationGroupsSearchQuery') + searchQuery = self._client.get_type('ns0:getOrgsAndStationGroupsSearchQuery')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getOrgsAndStationGroups(searchQuery) @@ -1217,7 +1215,7 @@ def getStationRights(self, **kwargs): } """ - searchQuery = self._client.factory.create('stationRightsSearchRequest') + searchQuery = self._client.get_type('ns0:stationRightsSearchRequest')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getStationRights(searchQuery) @@ -1362,7 +1360,8 @@ def getStations(self, **kwargs): moreFlag = 0 } """ - searchQuery = self._client.factory.create('stationSearchRequestExtended') + + searchQuery = self._client.get_type('ns0:stationSearchRequestExtended')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getStations(searchQuery) @@ -1449,7 +1448,7 @@ def getUsers(self, **kwargs): } """ - searchQuery = self._client.factory.create('getUsersSearchRequest') + searchQuery = self._client.get_type('ns0:getUsersSearchRequest')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getUsers(searchQuery) @@ -1487,7 +1486,8 @@ def shedLoad(self, **kwargs): :returns SOAP reply object. If successful, there will be a responseCode of '100'. """ - searchQuery = self._client.factory.create('shedLoadQueryInputData') + + searchQuery = self._client.get_type('ns0:shedLoadQueryInputData')() port = kwargs.pop('portNumber', None) query_params = {'stationID': kwargs['stationID']} if port: diff --git a/update_scripts/update.driver b/update_scripts/update.driver deleted file mode 100644 index 9f9c3346f8..0000000000 --- a/update_scripts/update.driver +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -python ~/volttron/scripts/install-agent.py -s /home/vboxuser/chargepoint/volttron_chargepoint/services/core/PlatformDriverAgent -c /home/vboxuser/chargepoint/volttron_chargepoint/config/driver.config -i platform.driver -t driver --force --start --priority 60 From 87650e1fbd11c5ce7ec87a872e0262e237ddfc24 Mon Sep 17 00:00:00 2001 From: jiangyilin123 Date: Fri, 15 Mar 2024 16:07:59 -0700 Subject: [PATCH 12/12] correct ChargingSessionData published format --- .../interfaces/chargepoint/__init__.py | 18 +++++++++++++++--- .../interfaces/chargepoint/service.py | 16 ++++++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py index de3c7786ac..a106540650 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py @@ -138,6 +138,16 @@ def read_only_check(self): raise IOError("Trying to write to a point configured read only: {0}".format(self.attribute_name)) return True + def get_last_non_none_value(self,lst): + """ + Depends on port number, the result could be a list with None value + get last non-None value as result + """ + for item in reversed(lst): + if item is not None: + return item + return None + def get_register(self, result, method, port_flag=True): """Gets correct register from API response. @@ -150,9 +160,10 @@ def get_register(self, result, method, port_flag=True): :return: Correct register value cast to appropriate python type. Returns None if there is an error. """ try: - value = getattr(result, self.attribute_name)(self.port)[0] \ + _log.debug(f'In get_register, to get {self.attribute_name}, the port_flag is {port_flag}') + value = self.get_last_non_none_value(getattr(result, self.attribute_name)(self.port)) \ if port_flag \ - else getattr(result, self.attribute_name)(None)[0] + else self.get_last_non_none_value(getattr(result, self.attribute_name)(None)) return self.sanitize_output(self.data_type, value) except cps.CPAPIException as exception: if exception._responseCode not in ['153']: @@ -386,7 +397,8 @@ def value(self): result.wait() # Of Note, due to API limitations, port number is ignored for these calls - return self.get_register(result.value, method, False) + # NOTE: Change this port number for Chargingsession data. + return self.get_register(result.value, method) @value.setter def value(self, x): diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py index 559f78e5ca..f9b404a723 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py @@ -416,8 +416,17 @@ def get_port_value(port_number, data, attribute): if flag: logger.warning("Station does not have a definition for port {0}".format(port_number)) else: - logger.warning("Response does not have Ports defined") - return None + if (attribute in ['sessionID', 'startTime', 'endTime', 'Energy', 'rfidSerialNumber', 'driverAccountNumber', + 'driverName']) and int(data['portNumber']) == port_number: + try: + data_attribute = data[attribute] + return data_attribute + except: + logger.warning(f'Response does not have {attribute} field') + return None + else: + logger.warning("Response does not have Ports defined") + return None @staticmethod def check_output(attribute, parent_dict): @@ -441,6 +450,7 @@ def get_attr_from_response(name_string, response, portNum=None): else CPAPIResponse.is_not_found(name_string)) else: list.append(CPAPIResponse.get_port_value(portNum, item, name_string)) + logger.debug(f'{name_string} list for {portNum} is {list}') return list @@ -632,12 +642,14 @@ def Type(self, port=None): def startTime(self, port=None): if port: + logger.debug(f'startTime port is {port}') return CPAPIResponse.get_attr_from_response('startTime', self.stations, port) else: return [self.pricing_helper('startTime', station) for station in self.stations] def endTime(self, port=None): if port: + logger.debug(f'endTime port is {port}') return CPAPIResponse.get_attr_from_response('endTime', self.stations, port) else: return [self.pricing_helper('endTime', station) for station in self.stations]