From bd43cf8294bd608ddcd6619fdab872288ccde12e Mon Sep 17 00:00:00 2001 From: Jun Jiang Date: Tue, 29 Oct 2024 09:45:59 +0100 Subject: [PATCH 1/6] Update Readme Signed-off-by: Jun Jiang --- README.md | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 5b42231..057da53 100644 --- a/README.md +++ b/README.md @@ -6,22 +6,36 @@ Data logging is the process of acquiring data over time from various sources, typically using sensors or instruments, and storing them in one or multiple outputs, such as files or databases. -This Python package provides easy understandable interfaces for various data sources and outputs, facilitating a quick +This Python package provides easily understandable interfaces for various data sources and outputs, facilitating a quick and easy configuration for data logging and data transfer. Potential use cases include field measurements, test bench monitoring, and Hardware-in-the-Loop (HiL) development. With its versatile capabilities, this toolbox aims to enhance the efficiency of data acquisition processes across different applications. +## Data logger + +As the key component in the data logging process, the data logger in this toolbox ensures high flexibility in the +logging procedure, featuring the following capabilities: + +- Read and write data from and to multiple systems simultaneously +- Rename each variable in data sources for each output individually +- Perform data type conversion for each variable in data sources for each data output individually + +The following types of data loggers are available in the toolbox: + +- Periodic trigger (time trigger) +- MQTT on-message trigger + ## Currently supported systems -The toolbox currently supports the following platforms and protocols: +The toolbox currently supports the following platforms and protocols, as shown in the table: + +| System | Read from system
(data source) | Write to system
(data output) | Note | +|:-------------------------------------------------------:|:---------------------------------:|:--------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Beckhoff PLC](https://www.beckhoff.com/) | Yes | Yes | - | +| [ICP DAS](https://www.icpdas.com/) | Yes | Yes (not tested) | Currently, the package only supports the [DCON Based I/O Expansion Unit](https://www.icpdas.com/en/product/guide+Remote__I_O__Module__and__Unit+Ethernet__I_O__Modules+IO__Expansion__Unit) with the I-87K series. | +| [MQTT protocol](https://mqtt.org/) | Yes | Yes | - | +| [The Things Network](https://www.thethingsnetwork.org/) | Yes | Yes (not tested) | Communication is via MQTT Server supported by The Things Stack. | +| [Sensor Electronic](http://sensor-electronic.pl/) | Yes | No | The package supports the Air Distribution Measuring System ([AirDistSys 5000](http://sensor-electronic.pl/pdf/KAT_AirDistSys5000.pdf)) and the Thermal Condition Monitoring System ([ThermCondSys 5500](http://sensor-electronic.pl/pdf/KAT_ThermCondSys5500.pdf)). Device configuration is possible, but it is not directly accessible via the data source API. | -- [Beckhoff PLC](https://www.beckhoff.com/) -- [ICP DAS](https://www.icpdas.com/) (Currently, the package only supports the -[DCON Based I/O Expansion Unit](https://www.icpdas.com/en/product/guide+Remote__I_O__Module__and__Unit+Ethernet__I_O__Modules+IO__Expansion__Unit) -with the I-87K series.) -- [MQTT protocol](https://mqtt.org/) -- [Sensor Electronic](http://sensor-electronic.pl/), which includes the air distribution measuring system -[AirDistSys 5000](http://sensor-electronic.pl/pdf/KAT_AirDistSys5000.pdf), and thermal condition monitoring system -[ThermCondSys 5500](http://sensor-electronic.pl/pdf/KAT_ThermCondSys5500.pdf) From 23fc9cea8a8993d6321a7f14674eb96af6a264cf Mon Sep 17 00:00:00 2001 From: Jun Jiang Date: Tue, 29 Oct 2024 12:50:22 +0100 Subject: [PATCH 2/6] Add data type conversion, tested Signed-off-by: Jun Jiang --- ebcmeasurements/Base/DataLogger.py | 142 +++++++++++++++++-- ebcmeasurements/Base/UML_Class.puml | 42 +++++- ebcmeasurements/Mqtt/MqttDataSourceOutput.py | 27 +++- examples/e02_multiple_sources_outputs.py | 39 ++++- examples/e05_mqtt.py | 5 + setup.py | 2 +- 6 files changed, 233 insertions(+), 24 deletions(-) diff --git a/ebcmeasurements/Base/DataLogger.py b/ebcmeasurements/Base/DataLogger.py index 340e203..9ff546d 100644 --- a/ebcmeasurements/Base/DataLogger.py +++ b/ebcmeasurements/Base/DataLogger.py @@ -11,10 +11,14 @@ class DataLoggerBase(ABC): + # Class attribute: supported types by data type conversions + _types_of_data_type_conversion = ('str', 'int', 'float', 'bool', 'bytes') + def __init__( self, data_sources_mapping: dict[str, DataSource.DataSourceBase | DataSourceOutput.DataSourceOutputBase], data_outputs_mapping: dict[str, DataOutput.DataOutputBase | DataSourceOutput.DataSourceOutputBase], + data_type_conversion_mapping: dict[str, dict[str, dict[str, str]]] | None = None, data_rename_mapping: dict[str, dict[str, dict[str, str]]] | None = None, **kwargs ): @@ -35,6 +39,31 @@ def __init__( ... } + The format of data_type_conversion_mapping is as follows: + { + '': { + <'output1_name'>: { + : , + ... + }, + <'output2_name'>: { + : , + ... + }, + }, + '': { + <'output1_name'>: { + : , + ... + }, + <'output2_name'>: { + : , + ... + }, + }, + ... + } + The format of data_rename_mapping is as follows: { '': { @@ -62,6 +91,8 @@ def __init__( :param data_sources_mapping: Mapping of multiple data sources :param data_outputs_mapping: Mapping of multiple data outputs + :param data_type_conversion_mapping: Mapping of multiple data type conversions, None to use default data types + provided by data sources, supported types are 'str', 'int', 'float', 'bool', 'bytes' :param data_rename_mapping: Mapping of rename for data sources and data outputs, None to use default names provided by data sources :param kwargs: @@ -81,6 +112,16 @@ def __init__( for k, do in data_outputs_mapping.items() } + # Data type conversion mapping of data sources and outputs + if data_type_conversion_mapping is not None: + # Check data type conversion mapping of data sources and outputs + self._check_data_type_conversion_mapping_input(data_type_conversion_mapping=data_type_conversion_mapping) + # Init the data type conversion mapping (full mapping) + self._data_type_conversion_mapping = self._init_data_type_conversion_mapping( + data_type_conversion_mapping=data_type_conversion_mapping) + else: + self._data_type_conversion_mapping = None + # Check rename mapping of data sources and outputs if data_rename_mapping is not None: self._check_data_rename_mapping_input( @@ -88,7 +129,7 @@ def __init__( explicit=kwargs.get('data_rename_mapping_explicit', False) ) - # Init the data rename mapping + # Init the data rename mapping (full mapping) self._data_rename_mapping = self._init_data_rename_mapping( data_rename_mapping=data_rename_mapping if data_rename_mapping is not None else {}, ) @@ -148,18 +189,45 @@ def __init__( # Count of logging self.log_count = 0 - def _check_data_rename_mapping_input(self, data_rename_mapping: dict, explicit: bool): - """Check input dict of data rename mapping""" - def _check_data_source_name(data_source_name): - """Check if data source name available in data sources""" - if data_source_name not in self._data_sources_mapping.keys(): - raise ValueError(f"Invalid data source name '{data_source_name}' for rename mapping") + def _check_data_source_name(self, data_source_name: str): + """Check if data source name available in data sources""" + if data_source_name not in self._data_sources_mapping.keys(): + raise ValueError(f"Invalid data source name '{data_source_name}' for rename mapping") + + def _check_data_output_name(self, data_output_name: str): + """Check if data output name available in data outputs""" + if data_output_name not in self._data_outputs_mapping.keys(): + raise ValueError(f"Invalid data output name '{data_output_name}' for rename mapping") - def _check_data_output_name(data_output_name): - """Check if data output name available in data outputs""" - if data_output_name not in self._data_outputs_mapping.keys(): - raise ValueError(f"Invalid data output name '{data_output_name}' for rename mapping") + def _check_data_type_conversion_mapping_input(self, data_type_conversion_mapping: dict): + """Check input dict of data type conversion mapping""" + # Check data source, data output, and mapping + for ds_name, output_dict in data_type_conversion_mapping.items(): + self._check_data_source_name(ds_name) + for do_name, mapping in output_dict.items(): + self._check_data_output_name(do_name) + for typ in mapping.values(): + if typ not in self._types_of_data_type_conversion: + raise ValueError(f"Invalid data type '{typ}' for data type conversion mapping, it must be one " + f"of '{self._types_of_data_type_conversion}'") + + def _init_data_type_conversion_mapping( + self, data_type_conversion_mapping: dict) -> dict[str, dict[str, dict[str, str | None]]]: + """Init data type conversion mapping for all data sources to all data outputs, if the conversion mapping for a + variable name is unavailable, return None""" + return { + ds_name: { + do_name: { + var: data_type_conversion_mapping.get(ds_name, {}).get(do_name, {}).get(var, None) + for var in ds.all_variable_names + } + for do_name in self._data_outputs_mapping.keys() + } + for ds_name, ds in self._data_sources_mapping.items() + } + def _check_data_rename_mapping_input(self, data_rename_mapping: dict, explicit: bool): + """Check input dict of data rename mapping""" def _explicit_check_rename_mapping(data_source_name, rename_mapping): """Explicit check if all keys in the rename mapping are available in data source""" for key in rename_mapping.keys(): @@ -171,9 +239,9 @@ def _explicit_check_rename_mapping(data_source_name, rename_mapping): # Check data source, data output, and mapping for ds_name, output_dict in data_rename_mapping.items(): - _check_data_source_name(ds_name) + self._check_data_source_name(ds_name) for do_name, mapping in output_dict.items(): - _check_data_output_name(do_name) + self._check_data_output_name(do_name) if explicit: _explicit_check_rename_mapping(ds_name, mapping) @@ -238,11 +306,25 @@ def read_data_all_sources(self) -> dict[str, dict]: } def log_data_all_outputs(self, data: dict[str, dict], timestamp: str = None): + def process_variable_name(data_source_name: str, data_output_name: str, variable_name: str) -> str: + # Rename the variable based on rename mapping + return self._data_rename_mapping[data_source_name][data_output_name][variable_name] + + def process_variable_value(data_source_name: str, data_output_name: str, variable_name: str, value): + if self._data_type_conversion_mapping is None: + # No data type conversion + return value + else: + return self.convert_data_type( + value=value, + data_type=self._data_type_conversion_mapping[data_source_name][data_output_name][variable_name] + ) + """Log data to all data outputs""" for do_name, do in self._data_outputs_mapping.items(): # Unzip and rename key for the current output unzipped_data = { - self._data_rename_mapping[ds_name][do_name][var]: value + process_variable_name(ds_name, do_name, var): process_variable_value(ds_name, do_name, var, value) for ds_name, ds_data in data.items() for var, value in ds_data.items() } @@ -276,18 +358,48 @@ def get_timestamp_now() -> str: """Get the timestamp by now""" return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) + @staticmethod + def convert_data_type(value, data_type: str | None) -> bool | str | int | float | bytes | None: + """Convert a single value to defined type""" + if value is None: + return None + + if data_type is None: + return value + + try: + if data_type == 'str': + return str(value) + elif data_type == 'int': + return int(value) + elif data_type == 'float': + return float(value) + elif data_type == 'bool': + # Converts any non-zero or non-empty string to True, otherwise False + return bool(value) and value not in (0, '', None) + elif data_type == 'bytes': + # Convert to bytes using UTF-8 encoding by default + return bytes(str(value), 'utf-8') + else: + raise ValueError(f"Unsupported data type '{data_type}'") + except ValueError: + logger.warning(f"Could not convert value '{value}' to type '{data_type}'") + return value + class DataLoggerTimeTrigger(DataLoggerBase): def __init__( self, data_sources_mapping: dict[str, DataSource.DataSourceBase | DataSourceOutput.DataSourceOutputBase], data_outputs_mapping: dict[str, DataOutput.DataOutputBase | DataSourceOutput.DataSourceOutputBase], + data_type_conversion_mapping: dict[str, dict[str, dict[str, str]]] | None = None, data_rename_mapping: dict[str, dict[str, dict[str, str]]] | None = None, **kwargs ): """Time triggerd data logger""" logger.info("Initializing DataLoggerTimeTrigger ...") - super().__init__(data_sources_mapping, data_outputs_mapping, data_rename_mapping, **kwargs) + super().__init__( + data_sources_mapping, data_outputs_mapping, data_type_conversion_mapping, data_rename_mapping, **kwargs) def run_data_logging(self, interval: int | float, duration: int | float | None): """ diff --git a/ebcmeasurements/Base/UML_Class.puml b/ebcmeasurements/Base/UML_Class.puml index 87d1588..1c7b620 100644 --- a/ebcmeasurements/Base/UML_Class.puml +++ b/ebcmeasurements/Base/UML_Class.puml @@ -75,13 +75,19 @@ package Base.DataOutput{ package Base.DataLogger{ abstract class DataLoggerBase { + # _types_of_data_type_conversion: tuple[str, ...] {static} # _data_sources_mapping: dict[str, DataSourceBase | DataSourceOutput.DataSourceOutputBase] # _data_outputs_mapping: dict[str, DataOutputBase | DataSourceOutput.DataSourceOutputBase] + # _data_type_conversion_mapping: dict[str, dict[str, dict[str, str]]] | None # _data_rename_mapping: dict[str, dict[str, dict[str, str]]] | None # _all_duplicates: dict[str, dict[str, list[str]]] # _all_variable_names_dict: dict[str, dict[str, tuple[str, ...]]] + log_count: int - + __init__(\n data_sources_mapping: dict[str, DataSource.DataSourceBase],\n data_outputs_mapping: dict[str, DataOutput.DataOutputBase],\n data_rename_mapping: dict[str, dict[str, dict[str, str]]],\n **kwargs\n) + + __init__(\n data_sources_mapping: dict[str, DataSource.DataSourceBase],\n data_outputs_mapping: dict[str, DataOutput.DataOutputBase],\n data_type_conversion_mapping: dict[str, dict[str, dict[str, str]]],\n data_rename_mapping: dict[str, dict[str, dict[str, str]]],\n **kwargs\n) + # _check_data_source_name(data_source_name: str) + # _check_data_output_name(data_output_name: str) + # _check_data_type_conversion_mapping_input(data_type_conversion_mapping: dict) + # _init_data_type_conversion_mapping(data_type_conversion_mapping: dict): dict[str, dict[str, dict[str, str | None]]] # _check_data_rename_mapping_input(data_rename_mapping: dict, explicit: bool) # _init_data_rename_mapping(data_rename_mapping: dict): dict[str, dict[str, dict[str, str]]] # _get_duplicates_in_data_rename_mapping(data_rename_mapping: dict): dict[str, dict[str, list[str]]] @@ -92,6 +98,7 @@ package Base.DataLogger{ + data_sources_mapping: dict {readOnly} + data_outputs_mapping: dict {readOnly} + {static} get_timestamp_now(): str + + {static} convert_data_type(value, data_type: str | None): bool | str | int | float | bytes | None } note right of DataLoggerBase::_data_sources_mapping { @@ -107,6 +114,37 @@ package Base.DataLogger{ ... } end note + note right of DataLoggerBase::_data_type_conversion_mapping + { + : { + : { + : , + : , + ... + }, + : { + : , + : , + ... + }, + ... + }, + : { + : { + : , + : , + ... + }, + : { + : , + : , + ... + }, + ... + }, + ... + } + end note note right of DataLoggerBase::_data_rename_mapping { : { @@ -198,7 +236,7 @@ package Base.DataLogger{ end note class DataLoggerTimeTrigger implements DataLoggerBase { - + __init__(\n data_sources_mapping: dict[str, DataSource.DataSourceBase | DataSourceOutput.DataSourceOutputBase],\n data_outputs_mapping: dict[str, DataOutput.DataOutputBase | DataSourceOutput.DataSourceOutputBase],\n data_rename_mapping: dict[str, dict[str, dict[str, str]]],\n **kwargs\n) + + __init__(\n data_sources_mapping: dict[str, DataSource.DataSourceBase | DataSourceOutput.DataSourceOutputBase],\n data_outputs_mapping: dict[str, DataOutput.DataOutputBase | DataSourceOutput.DataSourceOutputBase],\n data_type_conversion_mapping: dict[str, dict[str, dict[str, str]]],\n data_rename_mapping: dict[str, dict[str, dict[str, str]]],\n **kwargs\n) + run_data_logging(interval: int | float, duration: int | float | None) } diff --git a/ebcmeasurements/Mqtt/MqttDataSourceOutput.py b/ebcmeasurements/Mqtt/MqttDataSourceOutput.py index 7d9b493..0853f5c 100644 --- a/ebcmeasurements/Mqtt/MqttDataSourceOutput.py +++ b/ebcmeasurements/Mqtt/MqttDataSourceOutput.py @@ -7,6 +7,7 @@ import time import sys import logging.config + # Load logging configuration from file logger = logging.getLogger(__name__) @@ -14,6 +15,7 @@ class MqttDataSourceOutput(DataSourceOutput.DataSourceOutputBase): class MqttDataSource(DataSourceOutput.DataSourceOutputBase.SystemDataSource): """MQTT implementation of nested class SystemDataSource""" + def __init__( self, system: mqtt.Client, @@ -50,6 +52,7 @@ def read_data(self) -> dict: class MqttDataOutput(DataSourceOutput.DataSourceOutputBase.SystemDataOutput): """MQTT implementation of nested class SystemDataOutput""" + def __init__( self, system: mqtt.Client, @@ -92,7 +95,8 @@ def __init__( self, data_source: object, data_outputs_mapping: dict[str: DataOutput.DataOutputBase], - data_rename_mapping: dict[str: dict[str: str]] | None = None, + data_type_conversion_mapping: dict[str, dict[str, str]] | None = None, + data_rename_mapping: dict[str, dict[str, str]] | None = None, ): """MQTT 'on message' triggerd data logger""" logger.info("Initializing MqttDataOnMsgLogger ...") @@ -100,6 +104,8 @@ def __init__( super().__init__( data_sources_mapping={self.data_source_name: data_source}, data_outputs_mapping=data_outputs_mapping, + data_type_conversion_mapping={self.data_source_name: data_type_conversion_mapping} + if data_type_conversion_mapping is not None else None, data_rename_mapping= {self.data_source_name: data_rename_mapping} if data_rename_mapping is not None else None ) @@ -268,6 +274,7 @@ def on_disconnect(self, client, userdata, rc): def activate_on_msg_data_logger( self, data_outputs_mapping: dict[str: DataOutput.DataOutputBase], + data_type_conversion_mapping: dict[str, dict[str, str]] | None = None, data_rename_mapping: dict[str: dict[str: str]] | None = None ): """ @@ -280,14 +287,27 @@ def activate_on_msg_data_logger( ... } + The format of data_type_conversion_mapping is as follows: + { + <'output1_name'>: { + : , + ... + }, + <'output2_name'>: { + : , + ... + }, + ... + } + The format of data_rename_mapping is as follows: { <'output1_name'>: { - : , + : , ... }, <'output2_name'>: { - : , + : , ... }, ... @@ -297,6 +317,7 @@ def activate_on_msg_data_logger( self._on_msg_data_logger = self.MqttDataOnMsgLogger( data_source=self._data_source, data_outputs_mapping=data_outputs_mapping, + data_type_conversion_mapping=data_type_conversion_mapping, data_rename_mapping=data_rename_mapping ) logger.info("The MQTT on-message data logger is activated") diff --git a/examples/e02_multiple_sources_outputs.py b/examples/e02_multiple_sources_outputs.py index c03c682..6062bd0 100644 --- a/examples/e02_multiple_sources_outputs.py +++ b/examples/e02_multiple_sources_outputs.py @@ -32,7 +32,7 @@ def e02_multiple_sources_outputs(): # To avoid ambiguity, the data logger has automatically prefixed data source names to the variable names in the # file header. - input("Please check both files of logged data. Press Enter to continue...") + input("Please check both files of logged data of 'data_logger'. Press Enter to continue...") # It is possible to customize the rename mapping for variable names from each data source to each data output. data_logger_with_rename_1 = DataLogger.DataLoggerTimeTrigger( @@ -58,7 +58,7 @@ def e02_multiple_sources_outputs(): # As a result, the prefix function was automatically deactivated for 'OutA', as the variables have been renamed to # eliminate any ambiguity. However, for 'OutB', the prefix remains because the variables were not renamed for this # output and duplicates still exist. - input("Please check both files of logged data. Press Enter to continue...") + input("Please check both files of logged data of 'data_logger_with_rename_1'. Press Enter to continue...") # It should be noted that the rename mapping must eliminate all duplicates for a data output. If it does not, a # prefix will still be added to all variables logging to this output. @@ -90,7 +90,40 @@ def e02_multiple_sources_outputs(): # As a result, in 'OutB', data source names were prefixed to all variable names due to duplicate data source name # 'RandData1'. - print("Please check both files of logged data.") + input("Please check both files of logged data of 'data_logger_with_rename_2'. Press Enter to continue...") + + # As a new feature in v1.2.0, it is now possible to convert data types during the logging process. To implement + # this, the parameter 'data_type_conversion_mapping' must be set. Its structure is the same as that of the rename + # mapping. + data_logger_with_rename_and_type_conversion = DataLogger.DataLoggerTimeTrigger( + data_sources_mapping={'Sou1': data_source_1, 'Sou2': data_source_2}, + data_outputs_mapping={'OutA': data_output_a, 'OutB': data_output_b}, + data_type_conversion_mapping={ + 'Sou1': { # Type conversion for variables in data source 'Sou1' + 'OutA': { # Type conversion for variables when log the data from 'Sou1' to 'OutA' + 'RandData0': 'int', # Convert 'RandData0' to 'int' when log from 'Sou1' to 'OutA' + } + } + }, + data_rename_mapping={ + 'Sou1': { # Rename variables in data source 'Sou1' + 'OutA': { # Rename variables when log the data from 'Sou1' to 'OutA' + 'RandData0': 'RandData0_S1', # Rename 'RandData0' from Sou1 to 'RandData0_S1' when log to 'OutA' + 'RandData1': 'RandData1_S1', # Rename 'RandData1' from Sou1 to 'RandData1_S1' when log to 'OutA' + }, + }, + 'Sou2': { # Rename variables in data source 'Sou2' + 'OutA': { # Rename variables when log the data from 'Sou1' to 'OutA' + 'RandData0': 'RandData0_S2', # Rename 'RandData0' from Sou2 to 'RandData0_S2' when log to 'OutA' + 'RandData1': 'RandData1_S2', # Rename 'RandData1' from Sou2 to 'RandData1_S2' when log to 'OutA' + }, + }, + } + ) + data_logger_with_rename_and_type_conversion.run_data_logging(interval=1, duration=5) + + # As a result, in 'OutA', the variable 'RandData0' from 'Sou1' was converted to int and renamed to 'RandData0_S1' + print("Please check both files of logged data of 'data_logger_with_rename_and_type_conversion'.") if __name__ == '__main__': diff --git a/examples/e05_mqtt.py b/examples/e05_mqtt.py index 336d414..cc3cf98 100644 --- a/examples/e05_mqtt.py +++ b/examples/e05_mqtt.py @@ -60,6 +60,11 @@ def e05_mqtt(): # Activate on-messeage-logger mqtt_source_on_msg.activate_on_msg_data_logger( data_outputs_mapping={'csv_on_msg': output_on_msg}, + data_type_conversion_mapping={ + 'csv_on_msg': { + 'ebc_measurements/valA': 'int', # Data type conversion is possible + } + }, data_rename_mapping={ 'csv_on_msg': { 'ebc_measurements/valA': 'renamed_valA', # Renaming is possible diff --git a/setup.py b/setup.py index 0e8e4e9..989fedd 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='EBC-Measurements', - version='1.1.0', + version='1.2.0', author='RWTH Aachen University, E.ON Energy Research Center, ' 'Institute for Energy Efficient Buildings and Indoor Climate', author_email='ebc-abos@eonerc.rwth-aachen.de', From 083e476140d8c58b9c94892984a03c8de4e590fd Mon Sep 17 00:00:00 2001 From: Jun Jiang Date: Tue, 29 Oct 2024 13:35:26 +0100 Subject: [PATCH 3/6] Updata Readme Signed-off-by: Jun Jiang --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 057da53..0e408e7 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ logging procedure, featuring the following capabilities: - Read and write data from and to multiple systems simultaneously - Rename each variable in data sources for each output individually +- Automatically prefix variable names to avoid duplicates in data outputs - Perform data type conversion for each variable in data sources for each data output individually The following types of data loggers are available in the toolbox: From 9eb07f896d78ff55e37d78a019152306ec835501 Mon Sep 17 00:00:00 2001 From: Jun Jiang Date: Tue, 29 Oct 2024 14:22:13 +0100 Subject: [PATCH 4/6] Add DocStr Signed-off-by: Jun Jiang --- ebcmeasurements/Base/DataSource.py | 6 ++++++ ebcmeasurements/Beckhoff/AdsDataSourceOutput.py | 2 ++ ebcmeasurements/Icpdas/IcpdasDataSourceOutput.py | 2 ++ ebcmeasurements/Mqtt/MqttDataSourceOutput.py | 2 ++ ebcmeasurements/Mqtt/MqttTheThingsNetwork.py | 2 ++ ebcmeasurements/Sensor_Electronic/SensoSysDataSource.py | 2 ++ 6 files changed, 16 insertions(+) diff --git a/ebcmeasurements/Base/DataSource.py b/ebcmeasurements/Base/DataSource.py index 5114f5f..21dcd86 100644 --- a/ebcmeasurements/Base/DataSource.py +++ b/ebcmeasurements/Base/DataSource.py @@ -43,6 +43,8 @@ def __init__(self, size: int = 10, key_missing_rate: float = 0.5, value_missing_ :param size: Number of variables to generate :param key_missing_rate: Probability of a key being excluded from the final dictionary :param value_missing_rate: Probability of assigning None to a value instead of a random float + + Default variable names are formatted as 'RandData'. """ super().__init__() if not (0.0 <= key_missing_rate <= 1.0): @@ -73,6 +75,8 @@ def __init__( :param str_length: Length of each random string :param key_missing_rate: Probability of a key being excluded from the final dictionary :param value_missing_rate: Probability of assigning None to a value instead of a random float + + Default variable names are formatted as 'RandStr'. """ super().__init__(size, key_missing_rate, value_missing_rate) self.str_length = str_length @@ -99,6 +103,8 @@ def __init__( :param size: Number of variables to generate :param key_missing_rate: Probability of a key being excluded from the final dictionary :param value_missing_rate: Probability of assigning None to a value instead of a random float + + Default variable names are formatted as 'RandBool'. """ super().__init__(size, key_missing_rate, value_missing_rate) self._all_variable_names = tuple(f'RandBool{n}' for n in range(self.size)) # Re-define all data names diff --git a/ebcmeasurements/Beckhoff/AdsDataSourceOutput.py b/ebcmeasurements/Beckhoff/AdsDataSourceOutput.py index 47eef0b..f1c1f64 100644 --- a/ebcmeasurements/Beckhoff/AdsDataSourceOutput.py +++ b/ebcmeasurements/Beckhoff/AdsDataSourceOutput.py @@ -117,6 +117,8 @@ def __init__( :param ams_net_port: See package pyads.Connection.ams_net_port :param source_data_names: List of source names to be read from PLC, None to deactivate read function :param output_data_names: List of output names to be logged to PLC, None to deactivate write function + + Default variable names are the same as in TwinCAT, formatted as '....'. """ logger.info("Initializing AdsDataSourceOutput ...") self.ams_net_id = ams_net_id diff --git a/ebcmeasurements/Icpdas/IcpdasDataSourceOutput.py b/ebcmeasurements/Icpdas/IcpdasDataSourceOutput.py index e0f9efa..ff33b0e 100644 --- a/ebcmeasurements/Icpdas/IcpdasDataSourceOutput.py +++ b/ebcmeasurements/Icpdas/IcpdasDataSourceOutput.py @@ -82,6 +82,8 @@ def __init__( :param io_series: I/O series name, the current version only supports 'I-87K' :param output_dir: Output directory to save initialization information :param ignore_slots_idx: List of slot indices to be ignored by reading or writing data + + Default variable names are formatted as 'MoCh', with both indices starting from 0. """ logger.info(f"Initializing IcpdasDataSourceOutput ...") self.host = host diff --git a/ebcmeasurements/Mqtt/MqttDataSourceOutput.py b/ebcmeasurements/Mqtt/MqttDataSourceOutput.py index 0853f5c..2856d33 100644 --- a/ebcmeasurements/Mqtt/MqttDataSourceOutput.py +++ b/ebcmeasurements/Mqtt/MqttDataSourceOutput.py @@ -145,6 +145,8 @@ def __init__( :param kwargs: 'data_source_all_variable_names': List of all variable names for data source by subscribed topics 'data_output_all_variable_names': List of all variable names for data output by published topics + + Default variable names are the same as topic names, formatted as '//.../'. """ logger.info("Initializing MqttDataSourceOutput ...") self.broker = broker diff --git a/ebcmeasurements/Mqtt/MqttTheThingsNetwork.py b/ebcmeasurements/Mqtt/MqttTheThingsNetwork.py index 313d991..68fa59a 100644 --- a/ebcmeasurements/Mqtt/MqttTheThingsNetwork.py +++ b/ebcmeasurements/Mqtt/MqttTheThingsNetwork.py @@ -77,6 +77,8 @@ def __init__( each device, only these variables will be logged, formatted as {: [, ...]} :param device_downlink_payload_variable_names: Dict containing all variable names in downlink payload for each device, only these variables will be allowed to be set to devices, formatted as {: [, ...]} + + Default variable names are formatted as ':'. """ logger.info("Initializing MqttTheThingsNetwork ...") diff --git a/ebcmeasurements/Sensor_Electronic/SensoSysDataSource.py b/ebcmeasurements/Sensor_Electronic/SensoSysDataSource.py index 7f505ad..675f1f1 100644 --- a/ebcmeasurements/Sensor_Electronic/SensoSysDataSource.py +++ b/ebcmeasurements/Sensor_Electronic/SensoSysDataSource.py @@ -26,6 +26,8 @@ def __init__( :param output_dir: Output dir to save information of found devices, if None, they will not be saved :param all_devices_ids: All possible device's IDs to scan, if None, scan ID from 0 to 255 :param time_out: Timeout in seconds for serial communication + + Default variable names are formatted as '_
'. """ logger.info("Initializing SensoSysDataSource ...") From ee2a034a5d3a1cdbd2cfd1f19dacaff2a032aa7a Mon Sep 17 00:00:00 2001 From: Jun Jiang Date: Tue, 29 Oct 2024 15:24:03 +0100 Subject: [PATCH 5/6] Add set_logging_level for entire package Signed-off-by: Jun Jiang --- ebcmeasurements/__init__.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/ebcmeasurements/__init__.py b/ebcmeasurements/__init__.py index 4a1128e..63b9a8c 100644 --- a/ebcmeasurements/__init__.py +++ b/ebcmeasurements/__init__.py @@ -1,10 +1,35 @@ from .Base import Auxiliary, DataLogger, DataOutput, DataSource from .Beckhoff import AdsDataSourceOutput +from .Icpdas import IcpdasDataSourceOutput +from .Mqtt import MqttDataSourceOutput, MqttTheThingsNetwork from .Sensor_Electronic import SensoSysDataSource import logging -# Configure the root logger +# Configure the root logger with a default leve and format logging.basicConfig( - level=logging.INFO, # Set the logging level + level=logging.INFO, # Set the default logging level format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) + + +def set_logging_level(level: str): + """ + Set the logging level for all loggers in this package. + :param level: The desired logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + """ + level = level.upper() + try: + numeric_level = getattr(logging, level) + # Set the logging level for the root logger and all child loggers + logging.getLogger().setLevel(numeric_level) + for logger_name in [ + 'ebcmeasurements.Base', + 'ebcmeasurements.Beckhoff', + 'ebcmeasurements.Icpdas', + 'ebcmeasurements.Mqtt', + 'ebcmeasurements.Sensor_Electronic', + ]: + logging.getLogger(logger_name).setLevel(numeric_level) + print(f"Logging level set to {level} for all modules.") + except AttributeError: + print(f"Invalid logging level: {level}. Use DEBUG, INFO, WARNING, ERROR, or CRITICAL.") From 99a69d06c9351d7aabaf970b48d0f5e754d49dca Mon Sep 17 00:00:00 2001 From: Jun Jiang Date: Tue, 29 Oct 2024 15:33:46 +0100 Subject: [PATCH 6/6] Remove some empty lines Signed-off-by: Jun Jiang --- ebcmeasurements/Mqtt/MqttDataSourceOutput.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ebcmeasurements/Mqtt/MqttDataSourceOutput.py b/ebcmeasurements/Mqtt/MqttDataSourceOutput.py index 2856d33..9b18b1d 100644 --- a/ebcmeasurements/Mqtt/MqttDataSourceOutput.py +++ b/ebcmeasurements/Mqtt/MqttDataSourceOutput.py @@ -7,7 +7,6 @@ import time import sys import logging.config - # Load logging configuration from file logger = logging.getLogger(__name__) @@ -15,7 +14,6 @@ class MqttDataSourceOutput(DataSourceOutput.DataSourceOutputBase): class MqttDataSource(DataSourceOutput.DataSourceOutputBase.SystemDataSource): """MQTT implementation of nested class SystemDataSource""" - def __init__( self, system: mqtt.Client, @@ -52,7 +50,6 @@ def read_data(self) -> dict: class MqttDataOutput(DataSourceOutput.DataSourceOutputBase.SystemDataOutput): """MQTT implementation of nested class SystemDataOutput""" - def __init__( self, system: mqtt.Client,