diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1c90c7f4d..0be0a3bc6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -394,4 +394,4 @@ jobs: done ls -l var/tmp/pod_ready.txt 2>&1 - name: Run Tests - run: misc/discoverynode/testing/e2e/test_local //mqtt/localhost || true + run: misc/discoverynode/testing/e2e/test_local site_model || true diff --git a/misc/discoverynode/.gitignore b/misc/discoverynode/.gitignore index 0ced3ff6c..eba74f4cd 100644 --- a/misc/discoverynode/.gitignore +++ b/misc/discoverynode/.gitignore @@ -1 +1 @@ -venv/* \ No newline at end of file +venv/ \ No newline at end of file diff --git a/misc/discoverynode/README.md b/misc/discoverynode/README.md index 1de0569fc..e42aafee7 100644 --- a/misc/discoverynode/README.md +++ b/misc/discoverynode/README.md @@ -2,11 +2,53 @@ ## Notes -`vendor` discovery family is a counter which increments every second +`vendor` discovery family is actually sequential number generator at one second increments. + +## Running + +**NOTE** Below commands are run from within the `discoverynode` directory + +1. Setup + +``` +bin/setup +``` + +2. Run + +``` +bin/run SITE_MODEL TARGET DEVICE_ID +``` + +- SITE_MODEL - Path to site model +- TARGET - e.g. `//mqtt/localhost` or `//gbos/bos-platform-testing` +- DEVICE_ID - device ID from site model + +3. (Optional) Running with UDMIS Locally + +**NOTE** This can be destructive to the site model, and file permissions may change +``` +bin/container build +IMAGE_TAG=udmis:latest udmis/bin/actualize ../sites/udmi_site_model +sudo bin/keygen CERT sites/udmi_site_model/devices/AHU-1 +bin/registrar sites/udmi_site_model //mqtt/localhost +``` + +### Troubleshooting + + +#### `ssl.SSLError: [SSL] PEM lib (_ssl.c:3874)` + +The device certificate was not signed by the CA certificate. This occurs if UDMIS is restarted without regenerating certificates, +because the UDMIS script (bin/setup_ca) recreates the CA certificate. + +To fix, run `sudo bin/keygen CERT sites/udmi_site_model/devices/GAT-1` + +Run the script as root + +## Standalone (advanced) -## Standalone -**TODO** - Create `bin/setup` script 1. Setup a python virtual environment and install the required dependencies @@ -125,3 +167,5 @@ TODO - Unit tests - `~/venv/bin/python3 -m pytest tests/` - Integration tests - TODO + +## \ No newline at end of file diff --git a/misc/discoverynode/bin/run b/misc/discoverynode/bin/run index 9c6f8e43b..50e00a158 100755 --- a/misc/discoverynode/bin/run +++ b/misc/discoverynode/bin/run @@ -1,7 +1,93 @@ #!/bin/bash -e +set -x -ROOT_DIR=$(realpath $(dirname $0)/../..) +function error { + echo $* + false +} -cd $ROOT_DIR +ROOT_DIR=$(realpath $(dirname $0)/..) -sudo venv/bin/python3 discoverynode/src/main.py +BASE_CONFIG=$(realpath $ROOT_DIR/etc/base_config.json) +TMP_CONFIG="/tmp/discoverynode_config.json" + +if [[ $# -ne 3 ]]; then + error Usage: $0 SITE_MODEL TARGET DEVICE_ID +fi + +site_model=$1 +target=$2 +device_id=$3 + +if ! [[ -f $site_model/cloud_iot_config.json ]]; then + error "$site_model/cloud_iot_config.json does not exist" +fi + +if ! [[ -f $site_model/devices/$device_id/rsa_private.pem ]]; then + echo $site_model/devices/$device_id/rsa_private.pem not found + error note - only RS256 keys supported +fi + +cp $BASE_CONFIG $TMP_CONFIG + +if [[ -z $SUDO_USER ]]; then + # reset owner of tmp config to the user to avoid file permission issues when next running not as root + chown $SUDO_USER:$SUDO_USER $TMP_CONFIG +fi + +registry_id=$(cat $site_model/cloud_iot_config.json | jq -r '.registry_id') +region=$(cat $site_model/cloud_iot_config.json | jq -r '.cloud_region') + +provider=$(cut -d'/' -f3 <<< $target) +project=$(cut -d'/' -f4 <<< $target) +namespace=$(cut -d'/' -f5 <<< $target) +substitutions= + +if [[ $provider == gbos ]]; then + + if [[ -n $namespace ]]; then + actual_registry=$registry_id~$namespace + else + actual_registry=$registry_id + fi + + substitutions=$(cat < None: + """ Registers a function to execute before state is published. + + The main use case is to update values only when state is published. + + Args: + hook: Function to execute + """ + self.state_hooks.append(hook) + + def execute_state_hooks(self) -> None: + """ Execute registered hooks before state is published. """ + for hook in self.state_hooks: + hook() + def state_monitor(self): self._last_state_hash = None while True: current_hash = self.state.get_hash() if self._last_state_hash != current_hash: + self.execute_state_hooks() self.publish_state() - self._last_state_hash = current_hash + + # Regenerate hash because the "state" may been modified by the hooks + self._last_state_hash = self.state.get_hash() time.sleep(self._STATE_MONITOR_INTERVAL) def publish_state(self): @@ -109,7 +138,6 @@ def publish_state(self): self.publisher.publish_message(self.topic_state, state) def publish_discovery(self, payload): - logging.warning("published discovery for %s:%s", payload.scan_family, payload.scan_addr) self.publisher.publish_message(self.topic_discovery_event, payload.to_json()) def enable_discovery(self,*,bacnet=True,vendor=True,ipv4=True,ether=True): @@ -124,6 +152,8 @@ def enable_discovery(self,*,bacnet=True,vendor=True,ipv4=True,ether=True): number_discovery, ) + self.register_state_hook(number_discovery.on_state_update_hook) + self.components["number_discovery"] = number_discovery if bacnet: @@ -138,6 +168,8 @@ def enable_discovery(self,*,bacnet=True,vendor=True,ipv4=True,ether=True): bacnet_discovery, ) + self.register_state_hook(bacnet_discovery.on_state_update_hook) + self.components["bacnet_discovery"] = bacnet_discovery if ipv4: @@ -150,6 +182,8 @@ def enable_discovery(self,*,bacnet=True,vendor=True,ipv4=True,ether=True): passive_discovery, ) + self.register_state_hook(passive_discovery.on_state_update_hook) + self.components["passive_discovery"] = passive_discovery if ether: @@ -164,5 +198,7 @@ def enable_discovery(self,*,bacnet=True,vendor=True,ipv4=True,ether=True): nmap_banner_scan, ) + self.register_state_hook(nmap_banner_scan.on_state_update_hook) + self.components["nmap_banner_scan"] = nmap_banner_scan diff --git a/misc/discoverynode/src/udmi/discovery/bacnet.py b/misc/discoverynode/src/udmi/discovery/bacnet.py index 7e6769dc6..5a6f53ca0 100644 --- a/misc/discoverynode/src/udmi/discovery/bacnet.py +++ b/misc/discoverynode/src/udmi/discovery/bacnet.py @@ -12,18 +12,41 @@ from typing import Any, Callable import xml.etree.ElementTree import BAC0 +import BAC0.core.io.IOExceptions import udmi.discovery.discovery as discovery import udmi.schema.discovery_event from udmi.schema.discovery_event import DiscoveryEvent +from udmi.schema.discovery_event import DiscoveryPoint import udmi.schema.state - +import ipaddress +import enum +import copy +import dataclasses BAC0.log_level(log_file=None, stdout=None, stderr=None) BAC0.log_level("silence") +class BacnetObjectAcronyms(enum.StrEnum): + """ Mapping of object names to accepted aronyms""" + analogInput = "AI" + analogOutput = "AO" + analogValue = "AV" + binaryInput = "BI" + binaryOutput = "BO" + binaryValue = "BV" + loop = "LP" + multiStateInput = "MSI" + multiStateOutput = "MSO" + multiStateValue = "MSV" + characterstringValue = "CSV" + +class CowardlyQuit(Exception): + pass + + class GlobalBacnetDiscovery(discovery.DiscoveryController): - """Passive Network Discovery.""" + """Bacnet discovery.""" scan_family = "bacnet" @@ -57,8 +80,75 @@ def stop_discovery(self): self.cancelled = True self.result_producer_thread.join() - @discovery.catch_exceptions_to_state + def discover_device(self, device_address, device_id) -> DiscoveryEvent: + + ### Existence of a device + ################################################################### + event = udmi.schema.discovery_event.DiscoveryEvent( + generation=self.config.generation, + scan_family=self.scan_family, + scan_addr=str(device_id), + ) + + try: + ipaddress.ip_address(device_address) + except ValueError: + pass + else: + event.families["ipv4"] = udmi.schema.discovery_event.DiscoveryFamily( + addr=device_address + ) + + ### Basic Properties + ################################################################### + try: + object_name, vendor_name, firmware_version, model_name, serial_number = ( + self.bacnet.readMultiple( + f"{device_address} device {device_id} objectName vendorName" + " firmwareRevision modelName serialNumber" + ) + ) + + logging.info("object_name: %s vendor_name: %s firmware: %s model: %s serial: %s", object_name, vendor_name, firmware_version, model_name, serial_number) + + event.system.serial_no = serial_number + event.system.hardware.make = vendor_name + event.system.hardware.model = model_name + + event.system.ancillary["firmware"] = firmware_version + event.system.ancillary["name"] = object_name + + except (BAC0.core.io.IOExceptions.SegmentationNotSupported, Exception) as err: + logging.exception(f"error reading from {device_address}/{device_id}") + return event + + ### Points + ################################################################### + try: + device = BAC0.device(device_address, device_id, self.bacnet, poll=0) + + for point in device.points: + ref = DiscoveryPoint() + ref.name = point.properties.name + ref.description = point.properties.description + ref.ancillary["present_value"] = point.lastValue + ref.type = point.properties.type + if isinstance(point.properties.units_state, list): + ref.possible_values = point.properties.units_state + elif isinstance(point.properties.units_state, str): + ref.units = point.properties.units_state + point_id = BacnetObjectAcronyms[point.properties.type].value + ":" + point.properties.address + event.refs[point_id] = ref + + except Exception as err: + event.status = udmi.schema.discovery_event.Status("discovery.error", 500, str(err)) + logging.exception(f"error reading from {device_address}/{device_id}") + return event + + return event + @discovery.main_task + @discovery.catch_exceptions_to_state def result_producer(self): while not self.cancelled: try: @@ -68,37 +158,18 @@ def result_producer(self): set(self.bacnet.discoveredDevices.keys()) - self.devices_published ) for device in new_devices: - (address, id) = device - - # if depths ... - # Get make and model - try: - object_name, vendor_name, firmware_version, model_name, serial_number = ( - self.bacnet.readMultiple( - f"{address} device {id} objectName vendorName" - " firmwareRevision modelName serialNumber" - ) - ) - except ValueError: - logging.exception(f"error reading from {address}/{id}") - continue + # Check that it is not cancelled in the inner loop too because this + # can take a long time to enumerate through all found devices. + if self.cancelled: + break - logging.info("object_name %s vendor_name %s firmware %s model %s serial %s", object_name, vendor_name, firmware_version, model_name, serial_number) - event = udmi.schema.discovery_event.DiscoveryEvent( - generation=self.config.generation, - scan_family=self.scan_family, - scan_addr=str(id), - ) - - event.families["ipv4"] = udmi.schema.discovery_event.DiscoveryFamily( - addr=address - ) - event.system.serial_no = serial_number - event.system.hardware.make = vendor_name - event.system.hardware.model = model_name - event.system.software.firmware = firmware_version - - self.publisher(event) + address, id = device + start = time.monotonic() + event = self.discover_device(address, id) + end = time.monotonic() + logging.info(f"discovery for {device} in {end - start} seconds") + + self.publish(event) self.devices_published.add(device) if self.cancelled: diff --git a/misc/discoverynode/src/udmi/discovery/discovery.py b/misc/discoverynode/src/udmi/discovery/discovery.py index 8e226e1b1..a5599221c 100644 --- a/misc/discoverynode/src/udmi/discovery/discovery.py +++ b/misc/discoverynode/src/udmi/discovery/discovery.py @@ -4,7 +4,7 @@ import dataclasses import atexit import enum -from functools import wraps +import functools import logging import sched import threading @@ -18,8 +18,12 @@ import copy import datetime -MAX_THRESHOLD_GENERATION = -10 # [5s] +# TODO: Make this not negative +MAX_THRESHOLD_GENERATION = -10 # [seconds], should be negative + RUNNER_LOOP_INTERVAL = 0.1 + +# TODO: Move into an enum for readability ACTION_START = 1 ACTION_STOP = -1 @@ -30,27 +34,27 @@ def catch_exceptions_to_state(method: Callable): method: method to wrap """ - @wraps(method) + @functools.wraps(method) def _impl(self, *args, **kwargs): try: return method(self, *args, **kwargs) except Exception as err: - self.mark_error_from_exception(err) + self._handle_exception(err) return _impl def main_task(method: Callable): - """Decorator which marks discovery as `done` when wrapped functoin returns. + """Decorator which marks discovery as `done` when wrapped function returns. Args: method: method to wrap """ - @wraps(method) + @functools.wraps(method) def _impl(self, *args, **kwargs): method(self, *args, **kwargs) - self.set_internal_state(states.FINISHED) - self.state.phase = udmi.schema.state.Phase.done + self._set_internal_state(states.FINISHED) + self.state.phase = udmi.schema.state.Phase.stopped return _impl @@ -84,7 +88,7 @@ class states(enum.StrEnum): class DiscoveryController(abc.ABC): - + """""" @property @abc.abstractmethod def scan_family(self): @@ -136,33 +140,46 @@ def __init__(self, state: udmi.schema.state.State, publisher: Callable): self.scheduler_thread = None self.scheduler_thread_stopped = threading.Event() self.generation = None + self.count_events = None + self.publisher_mutex = threading.Lock() atexit.register(self._stop) - def _status_from_exception(self, err: Exception) -> None: - """Create state in status""" + def _increment_event_counter_and_get(self) -> int: + """ Incremenets the internal event counter and returns the new value. + + Returns: + New event count + """ + with self.publisher_mutex: + self.count_events = self.count_events + 1 + return self.count_events + + def _handle_exception(self, err: Exception) -> None: + """Helper function which updates the status when an exception is caught.""" + self._set_internal_state(states.ERROR) self.state.status = udmi.schema.state.Status( category="discovery.error", level=500, message=str(err) ) - - def mark_error_from_exception(self, err: Exception) -> None: - """Helper function to mark""" - self.set_internal_state(states.ERROR) - self._status_from_exception(err) logging.exception(err) def _start(self): + """ """ logging.info("Starting discovery for %s", type(self).__name__) self.state.status = None - self.set_internal_state(states.STARTING) + self._set_internal_state(states.STARTING) + self.count_events = 0 + self.generation = datetime.datetime.now() self.state.generation = self.generation + try: self.start_discovery() - self.set_internal_state(states.STARTED) - self.state.phase = udmi.schema.state.Phase.active except Exception as err: - self.mark_error_from_exception(err) - logging.info("Started... %s", type(self).__name__) + self._handle_exception(err) + else: + self._set_internal_state(states.STARTED) + self.state.phase = udmi.schema.state.Phase.active + logging.info("Started... %s", type(self).__name__) def _stop(self): logging.info("Stopping discovery for %s", type(self).__name__) @@ -172,15 +189,28 @@ def _stop(self): logging.info("Stopped %s", type(self).__name__) self.state.status = None - self.set_internal_state(states.CANCELLING) + self._set_internal_state(states.CANCELLING) try: self.stop_discovery() self.state.phase = udmi.schema.state.Phase.stopped - self.set_internal_state(states.CANCELLED) + self._set_internal_state(states.CANCELLED) except Exception as err: - self.mark_error_from_exception(err) + self._handle_exception(err) - def validate_config(config: udmi.schema.config.DiscoveryFamily): + def publish(self, event: udmi.schema.discovery_event.DiscoveryEvent): + """ Publishes the provided Discovery Event, setting event counts.""" + event_number = self._increment_event_counter_and_get() + event.event_no = event_number + logging.warning("published discovery for %s:%s #%d", event.scan_family, event.scan_addr, event_number) + self.publisher(event) + + def _validate_config(config: udmi.schema.config.DiscoveryFamily): + """ Validates that the + + + Throws: + RuntimeError: If the provided config is invalid + """ if config.scan_duration_sec > config.scan_interval_sec: raise RuntimeError("scan duration cannot be greater than interval") @@ -188,7 +218,7 @@ def validate_config(config: udmi.schema.config.DiscoveryFamily): raise RuntimeError("scan duration or interval cannot be negative") @catch_exceptions_to_state - def scheduler(self, start_time:int, config: udmi.schema.config.DiscoveryFamily): + def _scheduler(self, start_time:int, config: udmi.schema.config.DiscoveryFamily): """ The scheduler starts """ @@ -197,7 +227,7 @@ def scheduler(self, start_time:int, config: udmi.schema.config.DiscoveryFamily): # Initial execution of the scheduler is always to start a discovery next_action = ACTION_START - self.set_internal_state(states.SCHEDULED) + self._set_internal_state(states.SCHEDULED) self.state.phase = udmi.schema.state.Phase.pending # initial generation is from self.state.generation = config.generation @@ -240,7 +270,7 @@ def scheduler(self, start_time:int, config: udmi.schema.config.DiscoveryFamily): sleep_interval = scan_interval_sec - scan_duration_sec next_action_time = time.monotonic() + sleep_interval logging.info("scheduled discovery start for %s in %d seconds", type(self).__name__, scan_duration_sec) - self.set_internal_state(states.SCHEDULED) + self._set_internal_state(states.SCHEDULED) self.state.phase = udmi.schema.state.Phase.pending else: # The scan is not repetitive, exit the scheduler @@ -256,15 +286,17 @@ def controller(self, config_dict: None | dict[str:Any]): Args: config_dict: Complete UDMI configuration as a dictionary """ - logging.info("received config") + + logging.debug("received config %s", config_dict) with self.mutex: try: discovery_config_dict = config_dict["discovery"]["families"][self.scan_family] - except KeyError: + except KeyError as err: + # `self.scan_family`` is not in the config message + # Stop discovery and clear state self._stop() - self.set_internal_state(None) self.config = None - self.reset_udmi_state() + self._reset_udmi_state() return # Create a new config dict (always new) @@ -276,6 +308,7 @@ def controller(self, config_dict: None | dict[str:Any]): # Identical config, do nothing if config == self.config: + logging.debug("identical config, doing nothing") return # config has changed @@ -290,11 +323,12 @@ def controller(self, config_dict: None | dict[str:Any]): # Discovery config empty if not config: + logging.debug("config is now empty, do nothing") return - self.reset_udmi_state() - generation = datetime.datetime.strptime(f"{config.generation}+0000", "%Y-%m-%dT%H:%M:%SZ%z") - time_delta_from_now = generation - datetime.datetime.now(tz=datetime.timezone.utc) + self._reset_udmi_state() + generation_from_config = datetime.datetime.strptime(f"{config.generation}+0000", "%Y-%m-%dT%H:%M:%SZ%z") + time_delta_from_now = generation_from_config - datetime.datetime.now(tz=datetime.timezone.utc) seconds_from_now = time_delta_from_now.total_seconds() if seconds_from_now < MAX_THRESHOLD_GENERATION: @@ -303,17 +337,27 @@ def controller(self, config_dict: None | dict[str:Any]): self.scheduler_thread_stopped.clear() self.scheduler_thread = threading.Thread( - target=self.scheduler, args=[time.monotonic() + seconds_from_now, copy.copy(config)], daemon=True + target=self._scheduler, args=[time.monotonic() + seconds_from_now, copy.copy(config)], daemon=True ) self.scheduler_thread.start() - def done(self): - self.set_internal_state(states.FINISHED) - def set_internal_state(self, new_state: states): + def _set_internal_state(self, new_state: states): + """ Sets the internal state to the given state. + + Arguments: + new_state + + """ logging.info("state now %s, was %s", new_state, self.internal_state) self.internal_state = new_state - def reset_udmi_state(self): + def _reset_udmi_state(self): + """ Resets the UDMI state for this family by nulling all keys. """ for field in dataclasses.fields(self.state): - setattr(self.state, field.name, None) \ No newline at end of file + setattr(self.state, field.name, None) + + def on_state_update_hook(self): + # Check that the state is not reset before setting the active count + if self.state.phase is not None: + self.state.active_count = self.count_events diff --git a/misc/discoverynode/src/udmi/discovery/nmap.py b/misc/discoverynode/src/udmi/discovery/nmap.py index 0b8d4d614..cc068705d 100644 --- a/misc/discoverynode/src/udmi/discovery/nmap.py +++ b/misc/discoverynode/src/udmi/discovery/nmap.py @@ -75,6 +75,4 @@ def nmap_runner(self): "port": {p.port_number: {"banner": p.banner} for p in host.ports} }, ) - self.publisher(event.to_json()) - - self.done() + self.publish(event.to_json()) diff --git a/misc/discoverynode/src/udmi/discovery/numbers.py b/misc/discoverynode/src/udmi/discovery/numbers.py index 9fc388995..91764f388 100644 --- a/misc/discoverynode/src/udmi/discovery/numbers.py +++ b/misc/discoverynode/src/udmi/discovery/numbers.py @@ -30,9 +30,9 @@ def discoverer(self): if self.cancelled: return result = DiscoveryEvent( - generation=self.generation, scan_family=self.scan_family, scan_addr=i + generation=self.generation, scan_family=self.scan_family, scan_addr=str(i) ) - self.publisher(result) + self.publish(result) time.sleep(1) def stop_discovery(self): diff --git a/misc/discoverynode/src/udmi/discovery/passive.py b/misc/discoverynode/src/udmi/discovery/passive.py index f1360c977..5daf4a1e8 100644 --- a/misc/discoverynode/src/udmi/discovery/passive.py +++ b/misc/discoverynode/src/udmi/discovery/passive.py @@ -20,7 +20,8 @@ "ip and " "(" "src net 10.0.0.0/8 or src net 172.16.0.0/12 or src net 192.168.0.0/16 or " - "dst net 10.0.0.0/8 or dst net 172.16.0.0/12 or dst net 192.168.0.0/16" + "dst net 10.0.0.0/8 or dst net 172.16.0.0/12 or dst net 192.168.0.0/16 or " + "dst net 100.64.0.0/10 or src net 100.64.0.0/10" ")" ) @@ -121,7 +122,7 @@ def discovery_service(self): new_device_records = self.device_records - self.devices_records_published for device_record in new_device_records: - self.publisher( + self.publish( udmi.schema.discovery_event.DiscoveryEvent( generation=self.config.generation, scan_addr=device_record.addr, diff --git a/misc/discoverynode/src/udmi/schema/discovery_event.py b/misc/discoverynode/src/udmi/schema/discovery_event.py index 8b390e28c..199a19ec7 100644 --- a/misc/discoverynode/src/udmi/schema/discovery_event.py +++ b/misc/discoverynode/src/udmi/schema/discovery_event.py @@ -5,6 +5,17 @@ import enum import json import udmi.schema.util +from typing import Any, List + + +@dataclasses.dataclass +class Status: + category: str + level: int + message: str + timestamp: datetime.datetime = dataclasses.field( + default_factory=datetime.datetime.now + ) @dataclasses.dataclass @@ -27,6 +38,7 @@ class DiscoverySystem: default_factory=DiscoverySystemSoftware ) serial_no: str | None = None + ancillary: dict[str, Any] = dataclasses.field(default_factory=dict) @dataclasses.dataclass @@ -34,12 +46,22 @@ class DiscoveryFamily: addr: str +@dataclasses.dataclass +class DiscoveryPoint: + ref: str | None = None + name: str | None = None + description: str | None = None + type: str | None = None + units: str | None = None + possible_values: List[Any] | None = None + ancillary: dict[str, Any] = dataclasses.field(default_factory=dict) + + @dataclasses.dataclass class DiscoveryEvent: generation: str scan_family: str scan_addr: str - timestamp = None version: str = "1.5.1" timestamp: datetime.datetime = dataclasses.field( @@ -47,6 +69,10 @@ class DiscoveryEvent: ) families: dict[str, DiscoveryFamily] = dataclasses.field(default_factory=dict) system: DiscoverySystem = dataclasses.field(default_factory=DiscoverySystem) + refs: dict[str, DiscoveryPoint] = dataclasses.field(default_factory=dict) + event_no: int | None = None + status: Status | None = None + def to_json(self) -> str: as_dict = dataclasses.asdict(self) @@ -57,5 +83,5 @@ def to_json(self) -> str: copy.deepcopy(as_dict), None, [{}, None] ) return json.dumps( - as_dict, default=udmi.schema.util.json_serializer, indent=4 + as_dict, default=udmi.schema.util.json_serializer, indent=2 ) diff --git a/misc/discoverynode/src/udmi/schema/state.py b/misc/discoverynode/src/udmi/schema/state.py index e7aa0a113..9b757dec8 100644 --- a/misc/discoverynode/src/udmi/schema/state.py +++ b/misc/discoverynode/src/udmi/schema/state.py @@ -20,7 +20,6 @@ class Phase(enum.StrEnum): pending = 'pending' active = 'active' stopped = 'stopped' - done = 'done' @dataclasses.dataclass @@ -28,6 +27,7 @@ class Discovery: generation: str | None = None phase: Phase | None = None status: Status | None = None + active_count: int | None = None @dataclasses.dataclass @@ -39,6 +39,14 @@ class StateDiscovery: class StateSystemSoftware: version: str | None = None +@dataclasses.dataclass +class StateSystemOperation: + operational: bool | None = None + +@dataclasses.dataclass +class StateSystemHardware: + make: str | None = None + model: str | None = None @dataclasses.dataclass class StateSystem: @@ -46,7 +54,13 @@ class StateSystem: software: StateSystemSoftware = dataclasses.field( default_factory=StateSystemSoftware ) - + hardware: StateSystemHardware = dataclasses.field( + default_factory=StateSystemHardware + ) + operation: StateSystemOperation = dataclasses.field( + default_factory=StateSystemOperation + ) + serial_no: str | None = None @dataclasses.dataclass class LocalnetFamily: diff --git a/misc/discoverynode/src/udmi/schema/util.py b/misc/discoverynode/src/udmi/schema/util.py index 816a9c1d9..3a4750c31 100644 --- a/misc/discoverynode/src/udmi/schema/util.py +++ b/misc/discoverynode/src/udmi/schema/util.py @@ -1,6 +1,7 @@ import dataclasses import datetime import json +import numpy from typing import Any @@ -18,6 +19,13 @@ def datetime_serializer(timestamp: datetime.datetime): def json_serializer(obj: Any): if isinstance(obj, datetime.datetime): return datetime_serializer(obj) + + if isinstance(obj, numpy.integer): + return int(obj) + + if isinstance(obj, numpy.floating): + return float(obj) + raise TypeError(f"Type {type(obj)} is not serializable") @@ -41,8 +49,6 @@ def deep_remove( continue elif isinstance(v, dict): new_dict[k] = deep_remove(v, keys, values) - elif isinstance(v, list): - new_dict[k] = deep_remove(v, keys, values) else: new_dict[k] = v return new_dict diff --git a/misc/discoverynode/testing/docker/bacnet_device/Dockerfile b/misc/discoverynode/testing/docker/bacnet_device/Dockerfile index 5b32dd826..3195689dd 100644 --- a/misc/discoverynode/testing/docker/bacnet_device/Dockerfile +++ b/misc/discoverynode/testing/docker/bacnet_device/Dockerfile @@ -10,7 +10,7 @@ COPY requirements.txt /requirements.txt RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.txt # Copy the virtualenv into a distroless image -FROM gcr.io/distroless/python3-debian12 +FROM gcr.io/distroless/python3-debian12:debug COPY --from=build-venv /venv /venv COPY . /app WORKDIR /app diff --git a/misc/discoverynode/testing/docker/bacnet_device/bacnet_server.py b/misc/discoverynode/testing/docker/bacnet_device/bacnet_server.py index 055dcd017..e88f3ddcf 100644 --- a/misc/discoverynode/testing/docker/bacnet_device/bacnet_server.py +++ b/misc/discoverynode/testing/docker/bacnet_device/bacnet_server.py @@ -2,8 +2,62 @@ import BAC0 import os +from BAC0.core.devices.local.models import ( + analog_input, + analog_output, + analog_value, + binary_input, + binary_output, + binary_value, + character_string, + date_value, + datetime_value, + humidity_input, + humidity_value, + make_state_text, + multistate_input, + multistate_output, + multistate_value, + temperature_input, + temperature_value, +) +from BAC0.core.devices.local.object import ObjectFactory + bacnet = BAC0.lite(port=47808, deviceId=int(os.environ['BACNET_ID'])) +ObjectFactory.clear_objects() + +_new_objects = analog_input( + name="Return Air Temperature", + properties={"units": "degreesCelsius"}, + description="FCU-123", + presentValue=20, +) + +_new_objects = binary_input( + name="Fan On", + description="Main fan", + presentValue=True, +) + +_new_objects = analog_value( + name="Room Temperature", + properties={"units": "degreesCelsius"}, + description="Test Analogue Setpoint", + is_commandable=True, + presentValue=16, +) + +_new_objects = multistate_value( + description="L1 Fire Damper", + properties={"stateText": make_state_text(["Open", "Closed"])}, + name="test_multistate_setpoint", + is_commandable=True, + presentValue=23, +) + +_new_objects.add_objects_to_application(bacnet) + while True: bacnet.discover(global_broadcast=True) - time.sleep(5) \ No newline at end of file + time.sleep(5) diff --git a/misc/discoverynode/testing/docker/discovery_node/Dockerfile b/misc/discoverynode/testing/docker/discovery_node/Dockerfile index a392c155a..cbae3bf28 100644 --- a/misc/discoverynode/testing/docker/discovery_node/Dockerfile +++ b/misc/discoverynode/testing/docker/discovery_node/Dockerfile @@ -10,7 +10,7 @@ COPY requirements.txt /requirements.txt RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.txt # Copy the virtualenv into a distroless image -FROM gcr.io/distroless/python3-debian12 +FROM gcr.io/distroless/python3-debian12:debug COPY --from=build-venv /venv /venv COPY . /app WORKDIR /app diff --git a/misc/discoverynode/testing/e2e/test_local b/misc/discoverynode/testing/e2e/test_local index 5d74ed2c6..a3a10e735 100755 --- a/misc/discoverynode/testing/e2e/test_local +++ b/misc/discoverynode/testing/e2e/test_local @@ -10,7 +10,7 @@ if [[ -z $DN_SITE_PATH ]]; then exit 1 fi -if [[ $2 -ne "" ]]; then +if [[ $2 != "" ]]; then PYTEST_TARGET="-k $2" fi diff --git a/misc/discoverynode/testing/integration/integration_test.sh b/misc/discoverynode/testing/integration/integration_test.sh new file mode 100755 index 000000000..15f6daa12 --- /dev/null +++ b/misc/discoverynode/testing/integration/integration_test.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# No -e so test can be cleaned up once it's complete +set -x +ROOT_DIR=$(dirname $(realpath $0)) + +export DN_SITE_PATH=$(realpath $1) +export DN_TARGET=//mqtt/localhost + +bash $ROOT_DIR/../docker/bacnet_device/build.sh +bash $ROOT_DIR/../docker/discovery_node/build.sh + +docker network create --subnet=192.168.12.0/24 --ip-range=192.168.12.0/24 --gateway=192.168.12.254 discovery-integration || true + +for i in $(seq 1); do + docker run --rm -d \ + --name=discovery-integration-$i \ + --network=discovery-integration\ + --ip=192.168.12.$i \ + -e BACNET_ID=$i \ + test-bacnet-device +done + +docker run -it --rm \ + --entrypoint=/venv/bin/python3 \ + --network=discovery-integration \ + -e I_AM_INTEGRATION_TEST=1 \ + test-discovery_node \ + -m pytest --capture=no --log-cli-level=INFO tests/test_integration.py + +EXIT_CODE=$? + +docker ps -a | grep 'discovery-integration' | awk '{print $1}' | xargs docker stop + +exit $EXIT_CODE \ No newline at end of file