diff --git a/CHANGELOG.md b/CHANGELOG.md index abbc4b72..f0b59444 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 # Unreleased ### Fixed +- Error *may* be inaccurate or wrong when I issue an invalid configuration. (Issue [#304](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/304)) - Get Device Name and corresponding deprecated function (Issue[#341](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/341)) - Failures when adding interfaces to a network (Issue[#329](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/329)) - Add Facility Port to allow adding multiple interfaces (Issue [#289](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/289)) @@ -20,6 +21,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Missing docstrings in interface module (Issue [#313](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/313)) - Missing docstrings in facility_port module (Issue [#312](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/312)) - Missing docstrings in node module (Issue [#318](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/318)) +- Sub Interface Support (Issue [#350](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/350)) +- Advanced reservations (Issue [#345](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/345)) +- Port Mirroring with Basic NICs (Issue [#343](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/343)) +- P4 support (Issue [#340](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/340)) +- ERO Support (Issue [#338](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/338)) +- List hosts (Issue [#331](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/331)) +- AL2S Support (Issue [#325](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/325)) +- Deny infeasible slices (Issue [#326](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/326)) +- Add display of switch port name to network service table listing (Issue [#152](https://github.com/fabric-testbed/fabrictestbed-extensions/issues/152)) ## [1.6.4] - 2024-03-05 diff --git a/README.md b/README.md index bb4346fc..e5877eb2 100644 --- a/README.md +++ b/README.md @@ -89,3 +89,5 @@ help FABlib, please review the [guidelines] first. [configuration]: https://fabric-fablib.readthedocs.io/en/latest/#configuring-fablib [guidelines]: ./CONTRIBUTING.md + + diff --git a/fabrictestbed_extensions/fablib/component.py b/fabrictestbed_extensions/fablib/component.py index cd8c033f..c9eea51b 100644 --- a/fabrictestbed_extensions/fablib/component.py +++ b/fabrictestbed_extensions/fablib/component.py @@ -44,6 +44,8 @@ import jinja2 +from fabrictestbed_extensions.fablib.constants import Constants + if TYPE_CHECKING: from fabrictestbed_extensions.fablib.slice import Slice from fabrictestbed_extensions.fablib.node import Node @@ -60,16 +62,17 @@ class Component: component_model_map = { - "NIC_Basic": ComponentModelType.SharedNIC_ConnectX_6, - "NIC_ConnectX_6": ComponentModelType.SmartNIC_ConnectX_6, - "NIC_ConnectX_5": ComponentModelType.SmartNIC_ConnectX_5, - "NVME_P4510": ComponentModelType.NVME_P4510, - "GPU_TeslaT4": ComponentModelType.GPU_Tesla_T4, - "GPU_RTX6000": ComponentModelType.GPU_RTX6000, - "GPU_A40": ComponentModelType.GPU_A40, - "GPU_A30": ComponentModelType.GPU_A30, - "NIC_OpenStack": ComponentModelType.SharedNIC_OpenStack_vNIC, - "FPGA_Xilinx_U280": ComponentModelType.FPGA_Xilinx_U280, + Constants.CMP_NIC_Basic: ComponentModelType.SharedNIC_ConnectX_6, + Constants.CMP_NIC_ConnectX_6: ComponentModelType.SmartNIC_ConnectX_6, + Constants.CMP_NIC_ConnectX_5: ComponentModelType.SmartNIC_ConnectX_5, + Constants.CMP_NIC_P4: Constants.P4_DedicatedPort, + Constants.CMP_NVME_P4510: ComponentModelType.NVME_P4510, + Constants.CMP_GPU_TeslaT4: ComponentModelType.GPU_Tesla_T4, + Constants.CMP_GPU_RTX6000: ComponentModelType.GPU_RTX6000, + Constants.CMP_GPU_A40: ComponentModelType.GPU_A40, + Constants.CMP_GPU_A30: ComponentModelType.GPU_A30, + Constants.CMP_NIC_OpenStack: ComponentModelType.SharedNIC_OpenStack_vNIC, + Constants.CMP_FPGA_Xilinx_U280: ComponentModelType.FPGA_Xilinx_U280, } def __str__(self): @@ -259,32 +262,6 @@ def list_interfaces( fields=fields, output=output, quiet=quiet, filter_function=filter_function ) - # def list_interfaces(self) -> List[str]: - # """ - # Creates a tabulated string describing all components in the slice. - # - # Intended for printing a list of all components. - # - # :return: Tabulated srting of all components information - # :rtype: String - # """ - # table = [] - # for iface in self.get_interfaces(): - # network_name = "" - # if iface.get_network(): - # network_name = iface.get_network().get_name() - # - # table.append( [ iface.get_name(), - # network_name, - # iface.get_bandwidth(), - # iface.get_vlan(), - # iface.get_mac(), - # iface.get_physical_os_interface_name(), - # iface.get_os_interface(), - # ] ) - # - # return tabulate(table, headers=["Name", "Network", "Bandwidth", "VLAN", "MAC", "Physical OS Interface", "OS Interface" ]) - @staticmethod def calculate_name(node: Node = None, name: str = None) -> str: """ @@ -342,10 +319,13 @@ def __init__(self, node: Node = None, fim_component: FimComponent = None): self.node = node self.interfaces = None - def get_interfaces(self) -> List[Interface]: + def get_interfaces(self, include_subs: bool = True) -> List[Interface]: """ Gets the interfaces attached to this fablib component's FABRIC component. + :param include_subs: Flag indicating if sub interfaces should be included + :type include_subs: bool + :return: a list of the interfaces on this component. :rtype: List[Interface] """ @@ -355,9 +335,12 @@ def get_interfaces(self) -> List[Interface]: if not self.interfaces: self.interfaces = [] for fim_interface in self.get_fim_component().interface_list: - self.interfaces.append( - Interface(component=self, fim_interface=fim_interface) - ) + iface = Interface(component=self, fim_interface=fim_interface) + self.interfaces.append(iface) + if include_subs: + child_interfaces = iface.get_interfaces() + if child_interfaces and len(child_interfaces): + self.interfaces.extend(child_interfaces) return self.interfaces @@ -473,25 +456,23 @@ def get_model(self) -> str: str(self.get_type()) == "SmartNIC" and str(self.get_fim_model()) == "ConnectX-6" ): - return "NIC_ConnectX_6" + return Constants.CMP_NIC_ConnectX_6 elif ( str(self.get_type()) == "SmartNIC" and str(self.get_fim_model()) == "ConnectX-5" ): - return "NIC_ConnectX_5" + return Constants.CMP_NIC_ConnectX_5 elif str(self.get_type()) == "NVME" and str(self.get_fim_model()) == "P4510": - return "NVME_P4510" + return Constants.CMP_NVME_P4510 elif str(self.get_type()) == "GPU" and str(self.get_fim_model()) == "Tesla T4": - return "GPU_TeslaT4" + return Constants.CMP_GPU_TeslaT4 elif str(self.get_type()) == "GPU" and str(self.get_fim_model()) == "RTX6000": - return "GPU_RTX6000" + return Constants.CMP_GPU_RTX6000 elif ( str(self.get_type()) == "SharedNIC" and str(self.get_fim_model()) == "ConnectX-6" ): - return "NIC_Basic" - else: - return None + return Constants.CMP_NIC_Basic def get_reservation_id(self) -> str or None: """ @@ -669,8 +650,9 @@ def get_user_data(self): return {} def delete(self): - for interface in self.get_interfaces(): - interface.delete() + if self.get_interfaces(): + for interface in self.get_interfaces(): + interface.delete() self.get_slice().get_fim_topology().nodes[ self.get_node().get_name() diff --git a/fabrictestbed_extensions/fablib/constants.py b/fabrictestbed_extensions/fablib/constants.py index a9e55279..7d8df3eb 100644 --- a/fabrictestbed_extensions/fablib/constants.py +++ b/fabrictestbed_extensions/fablib/constants.py @@ -158,3 +158,41 @@ class Constants: EMAIL = "email" SSH_KEYS = "sshkeys" EXPIRES_ON = "expires_on" + LEASE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S %z" + + NON_PRETTY_NAME = "non_pretty_name" + PRETTY_NAME = "pretty_name" + HEADER_NAME = "header_name" + AVAILABLE = "Available" + CAPACITY = "Capacity" + ALLOCATED = "Allocated" + VALUE = "value" + + NIC_SHARED_CONNECTX_6 = "SharedNIC-ConnectX-6" + SMART_NIC_CONNECTX_6 = "SmartNIC-ConnectX-6" + SMART_NIC_CONNECTX_5 = "SmartNIC-ConnectX-5" + NVME_P4510 = "NVME-P4510" + GPU_TESLA_T4 = "GPU-Tesla T4" + GPU_RTX6000 = "GPU-RTX6000" + GPU_A30 = "GPU-A30" + GPU_A40 = "GPU-A40" + FPGA_XILINX_U280 = "FPGA-Xilinx-U280" + CORES = "Cores" + RAM = "Ram" + DISK = "Disk" + CPUS = "CPUs" + HOSTS = "Hosts" + P4_SWITCH = "P4-Switch" + + CMP_NIC_Basic = "NIC_Basic" + CMP_NIC_ConnectX_6 = "NIC_ConnectX_6" + CMP_NIC_ConnectX_5 = "NIC_ConnectX_5" + CMP_NIC_P4 = "NIC_P4" + CMP_NVME_P4510 = "NVME_P4510" + CMP_GPU_TeslaT4 = "GPU_TeslaT4" + CMP_GPU_RTX6000 = "GPU_RTX6000" + CMP_GPU_A40 = "GPU_A40" + CMP_GPU_A30 = "GPU_A30" + CMP_NIC_OpenStack = "NIC_OpenStack" + CMP_FPGA_Xilinx_U280 = "FPGA_Xilinx_U280" + P4_DedicatedPort = "P4_DedicatedPort" diff --git a/fabrictestbed_extensions/fablib/fablib.py b/fabrictestbed_extensions/fablib/fablib.py index 175394c7..74735bd8 100644 --- a/fabrictestbed_extensions/fablib/fablib.py +++ b/fabrictestbed_extensions/fablib/fablib.py @@ -68,14 +68,16 @@ import logging import os import random -import time +import traceback import warnings +from fabrictestbed_extensions.fablib.site import Host, Site + warnings.filterwarnings("always", category=DeprecationWarning) from concurrent.futures import ThreadPoolExecutor from ipaddress import IPv4Network, IPv6Network -from typing import TYPE_CHECKING, Dict, List +from typing import TYPE_CHECKING, Dict, List, Tuple import pandas as pd import paramiko @@ -89,6 +91,7 @@ if TYPE_CHECKING: from fabric_cf.orchestrator.swagger_client import Slice as OrchestratorSlice + from fabrictestbed_extensions.fablib.node import Node from fabrictestbed.slice_manager import SliceManager, SliceState, Status from fim.user import Node as FimNode @@ -141,6 +144,16 @@ def list_sites(latlon: bool = True) -> object: """ return fablib.get_default_fablib_manager().list_sites(latlon=latlon) + @staticmethod + def list_hosts() -> object: + """ + Get a string used to print a tabular list of sites with state + + :return: tabulated string of site state + :rtype: str + """ + return fablib.get_default_fablib_manager().list_hosts() + @staticmethod def list_links() -> object: """ @@ -667,6 +680,7 @@ def __init__( self.links = None self.facility_ports = None self.auto_token_refresh = auto_token_refresh + self.last_resources_filtered_by_time = False if not offline: self.ssh_thread_pool_executor = ThreadPoolExecutor(execute_thread_pool_size) @@ -754,7 +768,7 @@ def verify_and_configure(self): logging.info("Project is not specified") raise Exception("Bastion User name is not specified") - self.create_ssh_config() + self.create_ssh_config(overwrite=True) print("Configuration is valid and please save the config!") @@ -800,12 +814,16 @@ def create_ssh_config(self, overwrite: bool = False): dir_path = os.path.dirname(bastion_ssh_config_file) if not os.path.exists(dir_path): - msg = ( - f"Directory {dir_path} does not exist, can not create ssh_config file!" - ) - print(msg) - logging.error(msg) - raise Exception(msg) + try: + os.makedirs(dir_path) + except OSError as e: + msg = ( + f"Directory {dir_path} does not exist, Failed to create directory {dir_path}: {e}, " + f"can not create ssh_config file!" + ) + print(msg) + logging.error(msg) + raise Exception(msg) with open(bastion_ssh_config_file, "w") as f: f.write( @@ -978,12 +996,17 @@ def __create_and_save_key( """ dir_path = os.path.dirname(private_file_path) if not os.path.exists(dir_path): - msg = ( - f"Directory {dir_path} does not exist, can not create {key_type} keys!" - ) - print(msg) - logging.error(msg) - raise Exception(msg) + try: + os.makedirs(dir_path) + except OSError as e: + msg = ( + f"Directory {dir_path} does not exist! Failed to create directory {dir_path}: {e}, " + f"cannot create {key_type} keys!" + ) + print(msg) + logging.error(msg) + raise Exception(msg) + comment = os.path.basename(private_file_path) ssh_keys = self.get_slice_manager().create_ssh_keys( key_type=key_type, @@ -1072,6 +1095,10 @@ def list_sites( pretty_names: bool = True, force_refresh: bool = False, latlon: bool = True, + start: datetime = None, + end: datetime = None, + avoid: List[str] = None, + includes: List[str] = None, ) -> object: """ Lists all the sites and their attributes. @@ -1107,10 +1134,24 @@ def list_sites( :param force_refresh: :type force_refresh: bool :param latlon: convert address to latlon, makes online call to openstreetmaps.org - :rtype: Object + :type: Object + :param start: start time in UTC format: %Y-%m-%d %H:%M:%S %z + :type: datetime + :param end: end time in UTC format: %Y-%m-%d %H:%M:%S %z + :type: datetime + :param avoid: list of sites to avoid + :type: list of string + :param includes: list of sites to include + :type: list of string + """ return self.get_resources( - update=update, force_refresh=force_refresh + update=update, + force_refresh=force_refresh, + start=start, + end=end, + avoid=avoid, + includes=includes, ).list_sites( output=output, fields=fields, @@ -1120,6 +1161,78 @@ def list_sites( latlon=latlon, ) + def list_hosts( + self, + output: str = None, + fields: str = None, + quiet: bool = False, + filter_function=None, + update: bool = True, + pretty_names: bool = True, + force_refresh: bool = False, + start: datetime = None, + end: datetime = None, + avoid: List[str] = None, + includes: List[str] = None, + ) -> object: + """ + Lists all the hosts and their attributes. + + There are several output options: "text", "pandas", and "json" that determine the format of the + output that is returned and (optionally) displayed/printed. + + output: 'text': string formatted with tabular + 'pandas': pandas dataframe + 'json': string in json format + + fields: json output will include all available fields/columns. + + Example: fields=['Name','ConnectX-5 Available', 'NVMe Total'] + + filter_function: A lambda function to filter data by field values. + + Example: filter_function=lambda s: s['ConnectX-5 Available'] > 3 and s['NVMe Available'] <= 10 + + :param output: output format + :type output: str + :param fields: list of fields (table columns) to show + :type fields: List[str] + :param quiet: True to specify printing/display + :type quiet: bool + :param filter_function: lambda function + :type filter_function: lambda + :return: table in format specified by output parameter + :param update: + :type update: bool + :param pretty_names: + :type pretty_names: bool + :param force_refresh: + :type force_refresh: bool + :param start: start time in UTC format: %Y-%m-%d %H:%M:%S %z + :type: datetime + :param end: end time in UTC format: %Y-%m-%d %H:%M:%S %z + :type: datetime + :param avoid: list of sites to avoid + :type: list of string + :param includes: list of sites to include + :type: list of string + + """ + return self.get_resources( + update=update, + force_refresh=force_refresh, + start=start, + end=end, + avoid=avoid, + includes=includes, + ).list_hosts( + output=output, + fields=fields, + quiet=quiet, + filter_function=filter_function, + pretty_names=pretty_names, + ) + def list_links( self, output: str = None, @@ -1352,19 +1465,55 @@ def get_facility_ports(self, update: bool = True) -> FacilityPorts: return self.facility_ports def get_resources( - self, update: bool = True, force_refresh: bool = False + self, + update: bool = True, + force_refresh: bool = False, + start: datetime = None, + end: datetime = None, + avoid: List[str] = None, + includes: List[str] = None, ) -> Resources: """ Get a reference to the resources object. The resources object is used to query for available resources and capacities. + :param update: + :type update: bool + + :param force_refresh: + :type force_refresh: bool + + :param start: start time in UTC format: %Y-%m-%d %H:%M:%S %z + :type: datetime + + :param end: end time in UTC format: %Y-%m-%d %H:%M:%S %z + :type: datetime + + :param avoid: list of sites to avoid + :type: list of string + + :param includes: list of sites to include + :type: list of string + :return: the resources object :rtype: Resources """ - if not self.resources: - self.get_available_resources(update=update, force_refresh=force_refresh) + if not update: + if start or end: + update = True + self.last_resources_filtered_by_time = True + elif self.last_resources_filtered_by_time: + update = True + self.last_resources_filtered_by_time = False - return self.resources + return self.get_available_resources( + update=update, + force_refresh=force_refresh, + start=start, + end=end, + avoid=avoid, + includes=includes, + ) def get_random_site( self, avoid: List[str] = [], filter_function=None, update: bool = True @@ -1567,7 +1716,13 @@ def get_site_advertisement(self, site: str) -> FimNode: return topology.sites[site] def get_available_resources( - self, update: bool = False, force_refresh: bool = False + self, + update: bool = False, + force_refresh: bool = False, + start: datetime = None, + end: datetime = None, + avoid: List[str] = None, + includes: List[str] = None, ) -> Resources: """ Get the available resources. @@ -1577,15 +1732,44 @@ def get_available_resources( information. :param update: + :type update: bool + :param force_refresh: + :type force_refresh: bool + + :param start: start time in UTC format: %Y-%m-%d %H:%M:%S %z + :type: datetime + + :param end: end time in UTC format: %Y-%m-%d %H:%M:%S %z + :type: datetime + + :param avoid: list of sites to avoid + :type: list of string + + :param includes: list of sites to include + :type: list of string + :return: Available Resources object """ from fabrictestbed_extensions.fablib.resources import Resources if self.resources is None: - self.resources = Resources(self, force_refresh=force_refresh) + self.resources = Resources( + self, + force_refresh=force_refresh, + start=start, + end=end, + avoid=avoid, + includes=includes, + ) elif update: - self.resources.update(force_refresh=force_refresh) + self.resources.update( + force_refresh=force_refresh, + start=start, + end=end, + avoid=avoid, + includes=includes, + ) return self.resources @@ -2181,17 +2365,6 @@ def create_list_table(data, fields=None): table.append(row) return table - @staticmethod - def create_list_tableXXX(data, fields=None): - table = [] - for entry in data: - row = [] - for field in fields: - row.append(entry[field]) - - table.append(row) - return table - @staticmethod def create_show_table(data, fields=None, pretty_names_dict={}): table = [] @@ -2212,12 +2385,136 @@ def create_show_table(data, fields=None, pretty_names_dict={}): return table @staticmethod - def create_show_tableXXX(data, fields=None): - table = [] - if fields is None: - for key, value in data.items(): - table.append([key, value]) - else: - for field in fields: - table.append([field, data[field]]) - return table + def __can_allocate_node_in_host( + host: Host, node: Node, allocated: dict, site: Site + ) -> Tuple[bool, str]: + """ + Check if a node can be provisioned on a host node on a site w.r.t available resources on that site + + :return: Tuple indicating status for validation and error message in case of failure + :rtype: Tuple[bool, str] + """ + if host is None or site is None: + return ( + True, + f"Ignoring validation: Host: {host}, Site: {site} not available.", + ) + + msg = f"Node can be allocated on the host: {host.get_name()}." + + if host.get_state() != "Active": + msg = f"Node cannot be allocated on {host.get_name()}, {host.get_name()} is in {host.get_state()}!" + return False, msg + + allocated_core = allocated.setdefault("core", 0) + allocated_ram = allocated.setdefault("ram", 0) + allocated_disk = allocated.setdefault("disk", 0) + available_cores = host.get_core_available() + available_ram = host.get_ram_available() + available_disk = host.get_disk_available() + + if ( + node.get_requested_cores() > available_cores + or node.get_requested_disk() > available_disk + or node.get_requested_ram() > available_ram + ): + msg = f"Insufficient Resources: Host: {host.get_name()} does not meet core/ram/disk requirements." + return False, msg + + # Check if there are enough components available + for c in node.get_components(): + comp_model_type = f"{c.get_type()}-{c.get_fim_model()}" + substrate_component = host.get_component(comp_model_type=comp_model_type) + if not substrate_component: + msg = f"Invalid Request: Host: {host.get_name()} does not have the requested component: {comp_model_type}." + return False, msg + + allocated_comp_count = allocated.setdefault(comp_model_type, 0) + available_comps = ( + substrate_component.capacities.unit + - ( + substrate_component.capacity_allocations.unit + if substrate_component.capacity_allocations + else 0 + ) + - allocated_comp_count + ) + if available_comps <= 0: + msg = f"Insufficient Resources: Host: {host.get_name()} has reached the limit for component: {comp_model_type}." + return False, msg + + allocated[comp_model_type] += 1 + + allocated["core"] += node.get_requested_cores() + allocated["ram"] += node.get_requested_ram() + allocated["disk"] += node.get_requested_disk() + + return True, msg + + def validate_node(self, node: Node, allocated: dict = None) -> Tuple[bool, str]: + """ + Validate a node w.r.t available resources on a site before submission + + :return: Tuple indicating status for validation and error message in case of failure + :rtype: Tuple[bool, str] + """ + try: + error = None + if allocated is None: + allocated = {} + site = self.get_resources().get_site(site_name=node.get_site()) + + if not site: + logging.warning( + f"Ignoring validation: Site: {node.get_site()} not available in resources." + ) + return ( + True, + f"Ignoring validation: Site: {node.get_site()} not available in resources.", + ) + + site_state = site.get_state() + if site_state != "Active": + msg = f"Node cannot be allocated on {node.get_site()}, {node.get_site()} is in {site_state}." + logging.error(msg) + return False, msg + hosts = site.get_hosts() + if not hosts: + msg = f"Node cannot be validated, host information not available for {site}." + logging.error(msg) + return False, msg + + if node.get_host(): + if node.get_host() not in hosts: + msg = f"Invalid Request: Requested Host {node.get_host()} does not exist on site: {node.get_site()}." + logging.error(msg) + return False, msg + + host = hosts.get(node.get_host()) + + allocated_comps = allocated.setdefault(node.get_host(), {}) + status, error = self.__can_allocate_node_in_host( + host=host, node=node, allocated=allocated_comps, site=site + ) + + if not status: + logging.error(error) + return status, error + + for host in hosts.values(): + allocated_comps = allocated.setdefault(host.get_name(), {}) + status, error = self.__can_allocate_node_in_host( + host=host, node=node, allocated=allocated_comps, site=site + ) + if status: + return status, error + + msg = f"Invalid Request: Requested Node cannot be accommodated by any of the hosts on site: {site.get_name()}." + if error: + msg += f" Details: {error}" + logging.error(msg) + return False, msg + except Exception as e: + logging.error(e) + logging.error(traceback.format_exc()) + return False, str(e) diff --git a/fabrictestbed_extensions/fablib/facility_port.py b/fabrictestbed_extensions/fablib/facility_port.py index bbed7fe3..6c50bef8 100644 --- a/fabrictestbed_extensions/fablib/facility_port.py +++ b/fabrictestbed_extensions/fablib/facility_port.py @@ -114,7 +114,7 @@ def show( """ Get a human-readable representation of the facility port. """ - data = self.toDict(pretty_names=True) + data = self.toDict() # fields = ["Name", # ] @@ -171,43 +171,58 @@ def new_facility_port( slice: Slice = None, name: str = None, site: str = None, - vlan: Union[str, list] = None, + vlan: Union[List, str] = None, bandwidth: int = 10, - ): + mtu: int = None, + labels: Labels = None, + peer_labels: Labels = None, + ) -> FacilityPort: """ - Create a new facility port. + Create a new facility port in the given slice. You might want to :py:meth:`Slice.add_facility_port()`, in most cases. - :param Slice: Slice associated with the facility port. - :param name: name of the facility port. - :param site: site associated with the facility port. - :param vlan: VLAN. - :param bandwidth: bandwidth to be used, in Gbps. + :param slice: The slice in which the facility port will be created. + :param name: The name of the facility port. + :param site: The site where the facility port will be located. + :param vlan: A list or single string representing the VLANs for the facility port. + :param bandwidth: The bandwidth capacity for the facility port, default is 10. + :param mtu: MTU size + :param labels: Labels associated with the facility port. + :param peer_labels: Peer labels associated with the facility port. + :return: A FacilityPort object representing the created facility port. """ - if isinstance(vlan, list): - interfaces = [] + if not bandwidth: + bandwidth = 10 + capacities = Capacities(bw=bandwidth) + if mtu: + capacities.mtu = mtu + + interfaces = None + + if vlan: index = 1 + interfaces = [] + if isinstance(vlan, str): + vlan = [vlan] + for v in vlan: iface_tuple = ( f"iface-{index}", Labels(vlan=v), - Capacities(bw=bandwidth), + capacities, ) interfaces.append(iface_tuple) - fim_facility_port = slice.get_fim_topology().add_facility( - name=name, - site=site, - interfaces=interfaces, - ) - else: - fim_facility_port = slice.get_fim_topology().add_facility( - name=name, - site=site, - capacities=Capacities(bw=bandwidth), - labels=Labels(vlan=vlan), - ) + + fim_facility_port = slice.get_fim_topology().add_facility( + name=name, + site=site, + capacities=capacities, + labels=labels, + peer_labels=peer_labels, + interfaces=interfaces, + ) return FacilityPort(slice, fim_facility_port) @staticmethod diff --git a/fabrictestbed_extensions/fablib/interface.py b/fabrictestbed_extensions/fablib/interface.py index aae3950e..7e1abf63 100644 --- a/fabrictestbed_extensions/fablib/interface.py +++ b/fabrictestbed_extensions/fablib/interface.py @@ -33,18 +33,22 @@ import json import logging from ipaddress import IPv4Address -from typing import TYPE_CHECKING, Any, Union +from typing import TYPE_CHECKING, Any, List, Union import jinja2 from fabrictestbed.slice_editor import Flags +from fim.user import Capacities, InterfaceType, Labels from tabulate import tabulate +from fabrictestbed_extensions.fablib.constants import Constants + if TYPE_CHECKING: from fabrictestbed_extensions.fablib.slice import Slice from fabrictestbed_extensions.fablib.node import Node from fabrictestbed_extensions.fablib.network_service import NetworkService from fabrictestbed_extensions.fablib.component import Component from fabrictestbed_extensions.fablib.facility_port import FacilityPort + from fabrictestbed_extensions.fablib.switch import Switch from fabrictestbed.slice_editor import UserData from fim.user.interface import Interface as FimInterface @@ -62,7 +66,9 @@ def __init__( self, component: Component = None, fim_interface: FimInterface = None, - node: FacilityPort = None, + node: Union[Switch, FacilityPort] = None, + model: str = None, + parent: Interface = None, ): """ .. note:: @@ -85,6 +91,9 @@ def __init__( self.network = None self.dev = None self.node = node + self.model = model + self.interfaces = None + self.parent = parent def get_fablib_manager(self): """ @@ -117,8 +126,28 @@ def __str__(self): ["Device", self.get_device_name()], ["Address", self.get_ip_addr()], ["Numa Node", self.get_numa_node()], + ["Switch Port", self.get_switch_port()], ] + subnet = self.get_subnet() + if subnet: + table.append(["Subnet", subnet]) + peer_subnet = self.get_peer_subnet() + if peer_subnet: + table.append(["Peer Subnet", peer_subnet]) + + peer_asn = self.get_peer_asn() + if peer_asn: + table.append(["Peer ASN", peer_asn]) + + peer_bgp = self.get_peer_bgp_key() + if peer_bgp: + table.append(["Peer BGP Key", peer_bgp]) + + peer_account_id = self.get_peer_account_id() + if peer_account_id: + table.append(["Peer Account Id", peer_account_id]) + return tabulate(table) def toJson(self): @@ -155,6 +184,7 @@ def get_pretty_name_dict(): "mode": "Mode", "ip_addr": "IP Address", "numa": "Numa Node", + "switch_port": "Switch Port", } def toDict(self, skip=[]): @@ -206,11 +236,32 @@ def toDict(self, skip=[]): "dev": dev, "ip_addr": ip_addr, "numa": str(self.get_numa_node()), + "switch_port": str(self.get_switch_port()), } + def get_switch_port(self) -> str: + """ + Get the name of the port on the switch corresponding to this interface + + :return: name of the port on switch + :rtype: String + """ + network = self.get_network() + if network and network.get_fim(): + ifs = None + for ifs_name in network.get_fim().interfaces.keys(): + if self.get_name() in ifs_name: + ifs = network.get_fim().interfaces[ifs_name] + break + if ifs and ifs.labels and ifs.labels.local_name: + return ifs.labels.local_name + def get_numa_node(self) -> str: """ - Get NUMA node assoicated with the interface. + Retrieve the NUMA node of the component linked to the interface. + + :return: NUMA node of the linked component. + :rtype: str """ if self.get_component() is not None: return self.get_component().get_numa_node() @@ -232,7 +283,12 @@ def render_template(self, input_string): return output_string def show( - self, fields=None, output=None, quiet=False, colors=False, pretty_names=True + self, + fields=None, + output: str = None, + quiet: bool = False, + colors: bool = False, + pretty_names: bool = True, ): """ Show a table containing the current interface attributes. @@ -256,16 +312,14 @@ def show( :type quiet: bool :param colors: True to specify state colors for pandas output :type colors: bool + :param pretty_names: Display pretty names + :type pretty_names: bool :return: table in format specified by output parameter :rtype: Object """ data = self.toDict() - # fields = ["Name", "Node", "Network", "Bandwidth", "VLAN", - # "MAC", "Device" - # ] - if pretty_names: pretty_names_dict = self.get_pretty_name_dict() else: @@ -284,25 +338,28 @@ def show( def set_auto_config(self): """ - Set interface to auto-configure. - """ - fim_iface = self.get_fim_interface() - fim_iface.flags = Flags(auto_config=True) - # fim_iface.labels = Labels.update(fim_iface.labels, ipv4.... ) + Enable auto-configuration for the interface. - # labels = Labels() - # labels.instance_parent = host_name - # self.get_fim_node().set_properties(labels=labels) + This method sets the `auto_config` flag to `True` for the interface + associated with the current instance. The `auto_config` flag enables + automatic configuration of the interface by Control Framework. - # if_labels = Labels.update(if_labels, ipv4=str(next(ips)), ipv4_subnet=str(network)) - # if_labels = Labels.update(if_labels, vlan="200", ipv4=str(next(ips))) - # fim_iface.set_properties(labels=if_labels) + :return: None + """ + fim_iface = self.get_fim() + fim_iface.flags = Flags(auto_config=True) def unset_auto_config(self): """ - Unset auto-configuration flag on the interface. + Disable auto-configuration for the interface. + + This method sets the `auto_config` flag to `False` for the interface + associated with the current instance. The `auto_config` flag disables + automatic configuration of the interface by Control Framework. + + :return: None """ - fim_iface = self.get_fim_interface() + fim_iface = self.get_fim() fim_iface.flags = Flags(auto_config=False) def get_peer_port_name(self) -> str or None: @@ -319,6 +376,20 @@ def get_peer_port_name(self) -> str or None: else: return None + def get_peer_port_vlan(self) -> str: + """ + Returns the VLAN associated with the interface. + For shared NICs extracts it from label_allocations. + + :return: VLAN to be used for Port Mirroring + :rtype: String + """ + vlan = self.get_vlan() + if not vlan: + label_allocations = self.get_fim().get_property(pname="label_allocations") + if label_allocations: + return label_allocations.vlan + def get_device_name(self) -> str: """ Gets a name of the device name on the node @@ -386,9 +457,10 @@ def get_mac(self) -> str: :rtype: String """ try: - # os_iface = self.get_physical_os_interface() - # mac = os_iface['mac'] - mac = self.get_fim_interface().get_property(pname="label_allocations").mac + if self.parent: + mac = self.parent.get_mac() + else: + mac = self.get_fim().get_property(pname="label_allocations").mac except: mac = None @@ -401,7 +473,6 @@ def get_os_dev(self): :return: device description :rtype: Dict """ - if not self.dev: ip_addr_list_json = self.get_node().ip_addr_list(output="json") @@ -415,22 +486,6 @@ def get_os_dev(self): return None - def get_physical_os_interface(self): - """ - Not intended for API use - """ - - if self.get_network() is None: - return None - - network_name = self.get_network().get_name() - node_name = self.get_node().get_name() - - try: - return self.get_slice().get_interface_map()[network_name][node_name] - except: - return None - def get_physical_os_interface_name(self) -> str: """ Gets a name of the physical interface the operating system uses for this @@ -449,7 +504,9 @@ def get_physical_os_interface_name(self) -> str: def config_vlan_iface(self): """ - Not intended for API use + Configure vlan interface + + NOTE: Not intended for API use """ if self.get_vlan() is not None: self.get_node().add_vlan_os_interface( @@ -460,7 +517,19 @@ def config_vlan_iface(self): def set_ip(self, ip=None, cidr=None, mtu=None): """ - Depricated + Configures IP address for the interface inside the VM + + :param ip: IP address + :type ip: String + + :param cidr: CIDR + :type cidr: String + + :param mtu: MTU + :type mtu: String + + .. deprecated:: 1.7.0 + Use `set_ip_os_interface()` instead. """ if cidr: cidr = str(cidr) @@ -500,7 +569,6 @@ def ip_addr_del(self, addr, subnet): def ip_link_up(self): """ Bring up the link on the interface. - """ if self.get_network(): self.get_node().ip_link_up(None, self) @@ -535,15 +603,31 @@ def set_vlan(self, vlan: Any = None): """ Set the VLAN on the FABRIC request. - :param addr: vlan - :type addr: String or int + :param vlan: vlan + :type vlan: String or int """ if vlan: vlan = str(vlan) - if_labels = self.get_fim_interface().get_property(pname="labels") + if_labels = self.get_fim().get_property(pname="labels") if_labels.vlan = str(vlan) - self.get_fim_interface().set_properties(labels=if_labels) + self.get_fim().set_properties(labels=if_labels) + + return self + + def set_bandwidth(self, bw: int): + """ + Set the Bandwidths on the FABRIC request. + + :param addr: bw + :type addr: int + """ + if not bw: + return + + if_capacities = self.get_fim().get_property(pname="capacities") + if_capacities.bw = int(bw) + self.get_fim().set_properties(capacities=if_capacities) return self @@ -556,8 +640,11 @@ def get_fim_interface(self) -> FimInterface: :return: the FABRIC model node :rtype: fim interface + + .. deprecated:: 1.7.0 + Use `get_fim()` instead. """ - return self.fim_interface + return self.get_fim() def get_bandwidth(self) -> int: """ @@ -569,8 +656,8 @@ def get_bandwidth(self) -> int: """ if self.get_component() and self.get_component().get_model() == "NIC_Basic": return 100 - else: - return self.get_fim_interface().capacities.bw + elif self.get_fim() and self.get_fim().capacities: + return self.get_fim().capacities.bw def get_vlan(self) -> str: """ @@ -580,23 +667,20 @@ def get_vlan(self) -> str: :rtype: String """ try: - vlan = self.get_fim_interface().get_property(pname="labels").vlan + vlan = self.get_fim().get_property(pname="labels").vlan except: vlan = None return vlan def get_reservation_id(self) -> str or None: """ - Get reservation ID for the interface. + Gets the reservation id + + :return: reservation id + :rtype: String """ try: - # TODO THIS DOESNT WORK. - # print(f"{self.get_fim_interface()}") - return ( - self.get_fim_interface() - .get_property(pname="reservation_info") - .reservation_id - ) + return self.get_fim().get_property(pname="reservation_info").reservation_id except: return None @@ -604,14 +688,12 @@ def get_reservation_state(self) -> str or None: """ Gets the reservation state - :return: VLAN + :return: reservation state :rtype: String """ try: return ( - self.get_fim_interface() - .get_property(pname="reservation_info") - .reservation_state + self.get_fim().get_property(pname="reservation_info").reservation_state ) except: return None @@ -624,24 +706,29 @@ def get_error_message(self) -> str: :rtype: String """ try: - return ( - self.get_fim_interface() - .get_property(pname="reservation_info") - .error_message - ) + return self.get_fim().get_property(pname="reservation_info").error_message except: return "" def get_short_name(self): """ - Get short name for the interface. + Retrieve the shortened name of the interface. + + This method strips off the extra parts of the name added by the FIM. Specifically, it removes the + prefix formed by concatenating the node name and the component's short name + followed by a hyphen. + + :return: Shortened name of the interface. + :rtype: str """ - # strip of the extra parts of the name added by fim - return self.get_name()[ - len( - f"{self.get_node().get_name()}-{self.get_component().get_short_name()}-" - ) : - ] + if self.parent: + return self.get_name() + + # Strip off the extra parts of the name added by FIM + prefix_length = len( + f"{self.get_node().get_name()}-{self.get_component().get_short_name()}-" + ) + return self.get_name()[prefix_length:] def get_name(self) -> str: """ @@ -650,7 +737,7 @@ def get_name(self) -> str: :return: the name of this interface :rtype: String """ - return self.get_fim_interface().name + return self.get_fim().name def get_component(self) -> Component: """ @@ -668,7 +755,9 @@ def get_model(self) -> str: :return: the model of this interface's component :rtype: str """ - if self.node: + if self.model: + return self.model + elif self.node: return self.node.get_model() else: return self.get_component().get_model() @@ -755,7 +844,18 @@ def get_ip_link(self): def get_ip_addr_show(self, dev=None): """ - Get the result of running `ip -j addr show` on the interface. + Retrieve the IP address information for a specified network device. + + This method executes the `ip -j addr show` command on the node to get + the IP address information in JSON format for the specified device. + If no device is specified, it defaults to the device name associated + with the current instance. + + :param dev: The name of the network device (optional). + :type dev: str, optional + :return: The JSON output of the `ip -j addr show` command. + :rtype: str + :raises: Logs an error message if the command execution fails. """ try: if not dev: @@ -767,7 +867,7 @@ def get_ip_addr_show(self, dev=None): return stdout except Exception as e: logging.error( - f"Failed to get ip addr show info for interface {self.get_name()} Exception: {e}" + f"Failed to get IP address show info for interface {self.get_name()}. Exception: {e}" ) # fablib.Interface.get_ip_addr() @@ -825,19 +925,24 @@ def get_ips(self, family=None): def get_fim(self): """ - .. warning:: - - Not recommended for most users. + Gets the node's FABRIC Information Model (fim) object. This method + is used to access data at a lower level than FABlib. - Get FABRIC Information Model (fim) object for the interface. + :return: the FABRIC model node + :rtype: fim interface """ - return self.get_fim_interface() + return self.fim_interface def set_user_data(self, user_data: dict): """ - Set user data on the interface. + Set the user data for the interface. + + This method stores the given user data dictionary as a JSON string + in the FIM object associated with the interface. - :param user_data: a `dict`. + :param user_data: The user data to be set. + :type user_data: dict + :return: None """ self.get_fim().set_property( pname="user_data", pval=UserData(json.dumps(user_data)) @@ -845,7 +950,14 @@ def set_user_data(self, user_data: dict): def get_user_data(self): """ - Get user data on the interface. + Retrieve the user data for the interface. + + This method fetches the user data stored in the FIM object associated + with the interface and returns it as a dictionary. If an error occurs, + it returns an empty dictionary. + + :return: The user data dictionary. + :rtype: dict """ try: return json.loads(str(self.get_fim().get_property(pname="user_data"))) @@ -854,7 +966,14 @@ def get_user_data(self): def get_fablib_data(self): """ - Get value associated with `fablib_data` key of user data. + Retrieve the 'fablib_data' from the user data. + + This method extracts and returns the 'fablib_data' field from the + user data dictionary. If an error occurs or the field is not present, + it returns an empty dictionary. + + :return: The 'fablib_data' dictionary. + :rtype: dict """ try: return self.get_user_data()["fablib_data"] @@ -863,7 +982,14 @@ def get_fablib_data(self): def set_fablib_data(self, fablib_data: dict): """ - Set value associated with `fablib_data` key of user data. + Set the 'fablib_data' in the user data. + + This method updates the 'fablib_data' field in the user data dictionary + and stores the updated user data back in the FIM. + + :param fablib_data: The 'fablib_data' to be set. + :type fablib_data: dict + :return: None """ user_data = self.get_user_data() user_data["fablib_data"] = fablib_data @@ -871,11 +997,16 @@ def set_fablib_data(self, fablib_data: dict): def set_network(self, network: NetworkService): """ - Associate a network with the interface. + Set the network for the interface. - Any existing network will be replaced by the new one. + This method assigns the interface to the specified network. If the + interface is already part of another network, it will be removed from + the current network before being added to the new one. - :param network: a :py:class:`.NetworkService` object. + :param network: The network service to assign the interface to. + :type network: NetworkService + :return: The current instance with the updated network. + :rtype: self """ current_network = self.get_network() if current_network: @@ -887,11 +1018,19 @@ def set_network(self, network: NetworkService): def set_ip_addr(self, addr: ipaddress = None, mode: str = None): """ - Set IP address. + Set the IP address for the interface. - :param addr: address to be set, as `IPv4Address` or an - `IPv6Address`. - :param mode: `"auto"`, `"manual"`, or `"config"`. + This method assigns an IP address to the interface based on the provided + address or allocation mode. If an address is provided, it will be allocated + to the interface. If the mode is set to 'AUTO' and no address is provided, + an IP address will be automatically allocated by the network. + + :param addr: The IP address to assign to the interface (optional). + :type addr: ipaddress.IPv4Address or ipaddress.IPv6Address, optional + :param mode: The mode for IP address allocation, e.g., `"auto"`, `"manual"`, or `"config"`. + :type mode: str, optional + :return: The current instance with the updated IP address. + :rtype: self """ fablib_data = self.get_fablib_data() if mode: @@ -903,13 +1042,21 @@ def set_ip_addr(self, addr: ipaddress = None, mode: str = None): elif mode == self.AUTO: if self.get_network(): fablib_data[self.ADDR] = str(self.get_network().allocate_ip()) + self.set_fablib_data(fablib_data) return self def get_ip_addr(self): """ - Get IP address for the interface. + Retrieve the IP address assigned to the interface. + + This method returns the IP address assigned to the interface, either + from the 'fablib_data' or by fetching it via SSH if not available in + the stored data. If the MAC address is not available, it returns None. + + :return: The IP address assigned to the interface. + :rtype: ipaddress.IPv4Address or ipaddress.IPv6Address or str or None """ fablib_data = self.get_fablib_data() if self.ADDR in fablib_data: @@ -926,9 +1073,16 @@ def get_ip_addr(self): def set_mode(self, mode: str = "config"): """ - Set the interface's configuration mode. + Set the mode for the interface. + + This method sets the mode for the interface in the 'fablib_data' + dictionary. The mode determines the configuration behavior of the + interface. - :param mode: `"auto"`, `"manual"`, or `"config"`. + :param mode: The mode to set for the interface (default is "config"). Allowed values: `"auto"`, `"manual"`, or `"config"`.. + :type mode: str + :return: The current instance with the updated mode. + :rtype: self """ fablib_data = self.get_fablib_data() fablib_data[self.MODE] = mode @@ -938,7 +1092,14 @@ def set_mode(self, mode: str = "config"): def get_mode(self): """ - Get the interface's configuration mode. + Retrieve the mode of the interface. + + This method returns the current mode of the interface from the 'fablib_data' + dictionary. If the mode is not set, it defaults to "config" and updates the + 'fablib_data' accordingly. + + :return: The mode of the interface. + :rtype: str """ fablib_data = self.get_fablib_data() if self.MODE not in fablib_data: @@ -949,25 +1110,38 @@ def get_mode(self): def is_configured(self): """ - Return `True` if the interface is configured. + Check if the interface is configured. + + This method checks the 'fablib_data' dictionary to determine if the + interface is marked as configured. + + :return: True if the interface is configured, False otherwise. + :rtype: bool """ fablib_data = self.get_fablib_data() - is_configured = fablib_data.get(self.CONFIGURED) - if is_configured is None or not bool(is_configured): - return False + if fablib_data: + is_configured = fablib_data.get(self.CONFIGURED) + if is_configured is None or not bool(is_configured): + return False return True def config(self): """ - Configure the interface. + Configure the interface based on its mode and network settings. Called when a `.Node` is configured. - Called when a `.Node` is configured. + This method configures the interface by setting its IP address and + bringing it up. It checks the configuration mode and acts accordingly: + - If the mode is 'AUTO' and no address is set, it automatically allocates an IP address. + - If the mode is 'CONFIG' or 'AUTO', it configures the interface with the assigned IP address and subnet. + - If the mode is 'MANUAL', it does not perform any automatic configuration. + + :return: None """ network = self.get_network() if not network: logging.info( - f"interface {self.get_name()} not connected to network, skipping config." + f"Interface {self.get_name()} not connected to a network, skipping configuration." ) return @@ -979,16 +1153,10 @@ def config(self): fablib_data[self.CONFIGURED] = str(True) self.set_fablib_data(fablib_data) - if self.MODE in fablib_data: - mode = fablib_data[self.MODE] - else: - mode = self.MANUAL + mode = fablib_data.get(self.MODE, self.MANUAL) if mode == self.AUTO and addr is None: fablib_data[self.ADDR] = str(self.get_network().allocate_ip()) - # addr = fablib_data[self.ADDR] - # print(f"auto allocated addr: {addr}") - self.set_fablib_data(fablib_data) self.ip_link_up() @@ -1001,26 +1169,197 @@ def config(self): self.ip_link_up() self.ip_addr_add(addr=addr, subnet=ipaddress.ip_network(subnet)) else: - # manual mode... do nothing + # Manual mode; do nothing. pass - def add_mirror(self, port_name: str, name: str = "mirror"): + def add_mirror(self, port_name: str, name: str = "mirror", vlan: str = None): """ - Add port mirroring service to the interface. + Add Port Mirror Service - :param port_name: Name of the port being mirrored. - :param name: Name of the mirror. Default is `"mirror"`. + :param port_name: Mirror Port Name + :type port_name: String + :param vlan: Mirror Port vlan + :type vlan: String + :param name: Name of the Port Mirror service + :type name: String """ self.get_slice().get_fim_topology().add_port_mirror_service( name=name, from_interface_name=port_name, - to_interface=self.get_fim_interface(), + from_interface_vlan=vlan, + to_interface=self.get_fim(), ) def delete(self): """ - Delete the interface. + Delete the interface by removing it from the corresponding network service """ net = self.get_network() + if net: + net.remove_interface(self) + if self.parent and self.parent.get_fim(): + self.parent.get_fim().remove_child_interface(name=self.get_name()) + + def set_subnet(self, ipv4_subnet: str = None, ipv6_subnet: str = None): + """ + Set subnet for the interface. + Used only for interfaces connected to L3VPN service where each interface could be connected to multiple subnets + + :param ipv4_subnet: ipv4 subnet + :type ipv4_subnet: str + + :param ipv6_subnet: ipv6 subnet + :type ipv6_subnet: str + + :raises Exception in case invalid subnet string is specified. + """ + try: + labels = self.get_fim().labels + if not labels: + labels = Labels() + if ipv4_subnet: + ipaddress.ip_network(ipv4_subnet, strict=False) + labels = Labels.update(labels, ipv4_subnet=ipv4_subnet) + elif ipv6_subnet: + ipaddress.ip_network(ipv6_subnet, strict=False) + labels = Labels.update(labels, ipv6_subnet=ipv6_subnet) + + self.get_fim().set_property("labels", labels) + except Exception as e: + logging.error(f"Failed to set the ip subnet e: {e}") + raise e + + def get_subnet(self): + """ + Get Subnet associated with the interface + + :return: ipv4/ipv6 subnet associated with the interface + :rtype: String + """ + if self.get_fim() and self.get_fim().labels: + if self.get_fim().labels.ipv4_subnet: + return self.get_fim().labels.ipv4_subnet + if self.get_fim().labels.ipv6_subnet: + return self.get_fim().labels.ipv6_subnet + + def get_peer_subnet(self): + """ + Get Peer Subnet associated with the interface + + :return: peer ipv4/ipv6 subnet associated with the interface + :rtype: String + """ + if self.get_fim() and self.get_fim().peer_labels: + if self.get_fim().peer_labels.ipv4_subnet: + return self.get_fim().peer_labels.ipv4_subnet + if self.get_fim().peer_labels.ipv6_subnet: + return self.get_fim().peer_labels.ipv6_subnet + + def get_peer_asn(self): + """ + Get Peer ASN; Set only for Peered Interface using L3Peering via AL2S + + :return: peer asn + :rtype: String + """ + if self.get_fim() and self.get_fim().peer_labels: + return self.get_fim().peer_labels.asn + + def get_peer_bgp_key(self): + """ + Get Peer BGP Key; Set only for Peered Interface using L3Peering via AL2S + + :return: peer BGP Key + :rtype: String + """ + if self.get_fim() and self.get_fim().peer_labels: + return self.get_fim().peer_labels.bgp_key + + def get_peer_account_id(self): + """ + Get Peer Account Id associated with the interface + + :return: peer account id associated with the interface (Used when interface is peered to AWS via AL2S) + :rtype: String + """ + if self.get_fim() and self.get_fim().peer_labels: + return self.get_fim().peer_labels.account_id - net.remove_interface(self) + def get_interfaces(self) -> List[Interface]: + """ + Gets the interfaces attached to this fablib component's FABRIC component. + + :return: a list of the interfaces on this component. + :rtype: List[Interface] + """ + + if not self.interfaces: + self.interfaces = [] + for fim_interface in self.get_fim().interface_list: + self.interfaces.append( + Interface( + component=self.get_component(), + fim_interface=fim_interface, + model=str(InterfaceType.SubInterface), + parent=self, + ) + ) + + return self.interfaces + + def add_sub_interface(self, name: str, vlan: str, bw: int = 10): + """ + Add a sub-interface to a dedicated NIC. + + This method adds a sub-interface to a NIC (Network Interface Card) with the specified + name, VLAN (Virtual Local Area Network) ID, and bandwidth. It supports only specific + NIC models. + + :param name: The name of the sub-interface. + :type name: str + + :param vlan: The VLAN ID for the sub-interface. + :type vlan: str + + :param bw: The bandwidth allocated to the sub-interface, in Gbps. Default is 10 Gbps. + :type bw: int + + :raises Exception: If the NIC model does not support sub-interfaces. + """ + if self.get_model() not in [ + Constants.CMP_NIC_ConnectX_5, + Constants.CMP_NIC_ConnectX_6, + ]: + raise Exception( + f"Sub interfaces are only supported for the following NIC models: " + f"{Constants.CMP_NIC_ConnectX_5}, {Constants.CMP_NIC_ConnectX_6}" + ) + + if self.get_fim(): + child_interface = self.get_fim().add_child_interface( + name=name, labels=Labels(vlan=vlan) + ) + child_if_capacities = child_interface.get_property(pname="capacities") + if not child_if_capacities: + child_if_capacities = Capacities() + child_if_capacities.bw = int(bw) + child_interface.set_properties(capacities=child_if_capacities) + if not self.interfaces: + self.interfaces = [] + + ch_iface = Interface( + component=self.get_component(), + fim_interface=child_interface, + model=str(InterfaceType.SubInterface), + ) + self.interfaces.append(ch_iface) + return ch_iface + + def get_type(self) -> str: + """ + Get Interface type + :return: get interface type + :rtype: String + """ + if self.get_fim(): + return self.get_fim().type diff --git a/fabrictestbed_extensions/fablib/network_service.py b/fabrictestbed_extensions/fablib/network_service.py index f1b812da..ce061bd2 100644 --- a/fabrictestbed_extensions/fablib/network_service.py +++ b/fabrictestbed_extensions/fablib/network_service.py @@ -32,8 +32,11 @@ from __future__ import annotations import logging +import threading from typing import TYPE_CHECKING, List, Union +from fim.slivers.path_info import Path +from fim.user import ERO from tabulate import tabulate if TYPE_CHECKING: @@ -46,7 +49,7 @@ from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network import jinja2 -from fabrictestbed.slice_editor import Labels +from fabrictestbed.slice_editor import Capacities, Labels from fabrictestbed.slice_editor import NetworkService as FimNetworkService from fabrictestbed.slice_editor import ServiceType, UserData from fim.slivers.network_service import NSLayer, ServiceType @@ -79,39 +82,53 @@ class NetworkService: fim_special_service_types = ["PortMirror"] @staticmethod - def get_fim_l2network_service_types() -> List[str]: + def __get_fim_l2network_service_types() -> List[str]: """ - Not inteded for API use + Not intended for API use. Returns a list of FIM L2 network service types. + + :return: List of FIM L2 network service types. + :rtype: List[str] """ return NetworkService.fim_l2network_service_types @staticmethod - def get_fim_l3network_service_types() -> List[str]: + def __get_fim_l3network_service_types() -> List[str]: """ - Not inteded for API use + Not intended for API use. Returns a list of FIM L3 network service types. + + :return: List of FIM L3 network service types. + :rtype: List[str] """ return NetworkService.fim_l3network_service_types @staticmethod - def get_fim_special_service_types() -> List[str]: + def __get_fim_special_service_types() -> List[str]: """ - Not intended for API use + Not intended for API use. Returns a list of FIM special service types. + + :return: List of FIM special service types. + :rtype: List[str] """ return NetworkService.fim_special_service_types @staticmethod def get_fim_network_service_types() -> List[str]: """ - Not inteded for API use + Not intended for API use. Returns a list of all FIM network service types. + + :return: List of all FIM network service types. + :rtype: List[str] """ return ( - NetworkService.get_fim_l2network_service_types() - + NetworkService.get_fim_l3network_service_types() - + NetworkService.get_fim_special_service_types() + NetworkService.__get_fim_l2network_service_types() + + NetworkService.__get_fim_l3network_service_types() + + NetworkService.__get_fim_special_service_types() ) @staticmethod - def calculate_l2_nstype(interfaces: List[Interface] = None) -> ServiceType: + def __calculate_l2_nstype( + interfaces: List[Interface] = None, ero_enabled: bool = False + ) -> ServiceType: """ Not inteded for API use @@ -119,6 +136,10 @@ def calculate_l2_nstype(interfaces: List[Interface] = None) -> ServiceType: :param interfaces: a list of interfaces :type interfaces: list[Interface] + + :param ero_enabled: Flag indicating if ERO is specified + :type ero_enabled: bool + :raises Exception: if no network service type is not appropriate for the number of interfaces :return: the network service type :rtype: ServiceType @@ -152,10 +173,8 @@ def calculate_l2_nstype(interfaces: List[Interface] = None) -> ServiceType: # basically the layer-2 point-to-point server template applied is not popping # vlan tags over the MPLS tunnel between two facility ports. if ( - includes_facility_port - and facility_port_interfaces < 2 - and not basic_nic_count - ): + (includes_facility_port and facility_port_interfaces < 2) or ero_enabled + ) and not basic_nic_count: # For now WAN FacilityPorts require L2PTP rtn_nstype = NetworkService.network_service_map["L2PTP"] elif len(interfaces) >= 2: @@ -168,7 +187,7 @@ def calculate_l2_nstype(interfaces: List[Interface] = None) -> ServiceType: return rtn_nstype @staticmethod - def validate_nstype(type, interfaces): + def __validate_nstype(type, interfaces): """ Not intended for API use @@ -183,6 +202,9 @@ def validate_nstype(type, interfaces): :return: true if the network service type is valid based on the number of interfaces :rtype: bool """ + # Just an empty network created; NS type would be validated when add_interface is invoked. + if not len(interfaces): + return True from fabrictestbed_extensions.fablib.facility_port import FacilityPort @@ -268,6 +290,7 @@ def new_portmirror_service( slice: Slice = None, name: str = None, mirror_interface_name: str = None, + mirror_interface_vlan: str = None, receive_interface: Interface or None = None, mirror_direction: str = "both", ) -> NetworkService: @@ -322,6 +345,7 @@ def new_portmirror_service( fim_network_service = slice.topology.add_port_mirror_service( name=name, from_interface_name=mirror_interface_name, + from_interface_vlan=mirror_interface_vlan, to_interface=receive_interface.fim_interface, direction=direction, ) @@ -339,6 +363,7 @@ def new_l3network( interfaces: List[Interface] = [], type: str = None, user_data={}, + technology: str = None, ): """ Not inteded for API use. See slice.add_l3network @@ -362,12 +387,13 @@ def new_l3network( # validate nstype and interface List # NetworkService.validate_nstype(nstype, interfaces) - return NetworkService.new_network_service( + return NetworkService.__new_network_service( slice=slice, name=name, nstype=nstype, interfaces=interfaces, user_data=user_data, + technology=technology, ) @staticmethod @@ -395,21 +421,23 @@ def new_l2network( :rtype: NetworkService """ if type is None: - nstype = NetworkService.calculate_l2_nstype(interfaces=interfaces) + nstype = NetworkService.__calculate_l2_nstype(interfaces=interfaces) else: - if type in NetworkService.get_fim_l2network_service_types(): + if type in NetworkService.__get_fim_l2network_service_types(): nstype = NetworkService.network_service_map[type] else: raise Exception( f"Invalid l2 network type: {type}. Please choose from " - f"{NetworkService.get_fim_l2network_service_types()} or None for automatic selection" + f"{NetworkService.__get_fim_l2network_service_types()} or None for automatic selection" ) # validate nstype and interface List - NetworkService.validate_nstype(nstype, interfaces) + NetworkService.__validate_nstype(nstype, interfaces) # Set default VLANs for P2P networks that did not pass in VLANs - if nstype == ServiceType.L2PTP: # or nstype == ServiceType.L2STS: + if nstype == ServiceType.L2PTP and len( + interfaces + ): # or nstype == ServiceType.L2STS: vlan1 = interfaces[0].get_vlan() vlan2 = interfaces[1].get_vlan() @@ -429,7 +457,7 @@ def new_l2network( # if interface.get_model() != 'NIC_Basic' and not interface.get_vlan(): # # interface.set_vlan("100") - network_service = NetworkService.new_network_service( + network_service = NetworkService.__new_network_service( slice=slice, name=name, nstype=nstype, @@ -439,12 +467,13 @@ def new_l2network( return network_service @staticmethod - def new_network_service( + def __new_network_service( slice: Slice = None, name: str = None, nstype: ServiceType = None, interfaces: List[Interface] = [], user_data: dict = {}, + technology: str = None, ): """ Not intended for API use. See slice.add_l2network @@ -459,6 +488,9 @@ def new_network_service( :param nstype: the type of network service to create :type nstype: ServiceType :param interfaces: a list of interfaces to + :type interfaces: List + :param technology: Specify the technology used should be set to AL2S when using for AL2S peering; otherwise None + :type technology: str :return: the new fablib network service :rtype: NetworkService """ @@ -470,7 +502,7 @@ def new_network_service( f"Create Network Service: Slice: {slice.get_name()}, Network Name: {name}, Type: {nstype}" ) fim_network_service = slice.topology.add_network_service( - name=name, nstype=nstype, interfaces=fim_interfaces + name=name, nstype=nstype, interfaces=fim_interfaces, technology=technology ) network_service = NetworkService( @@ -484,21 +516,24 @@ def new_network_service( @staticmethod def get_l3network_services(slice: Slice = None) -> list: """ - Not intended for API use. + Gets all L3 networks services in this slice + + :return: List of all network services in this slice + :rtype: List[NetworkService] """ topology = slice.get_fim_topology() rtn_network_services = [] fim_network_service = None logging.debug( - f"NetworkService.get_fim_l3network_service_types(): {NetworkService.get_fim_l3network_service_types()}" + f"NetworkService.get_fim_l3network_service_types(): {NetworkService.__get_fim_l3network_service_types()}" ) for net_name, net in topology.network_services.items(): logging.debug(f"scanning network: {net_name}, net: {net}") if ( str(net.get_property("type")) - in NetworkService.get_fim_l3network_service_types() + in NetworkService.__get_fim_l3network_service_types() ): logging.debug(f"returning network: {net_name}, net: {net}") rtn_network_services.append( @@ -510,7 +545,13 @@ def get_l3network_services(slice: Slice = None) -> list: @staticmethod def get_l3network_service(slice: Slice = None, name: str = None): """ - Not inteded for API use. + Gets a particular L3 network service from this slice. + + + :param name: Name network + :type name: String + :return: network services on this slice + :rtype: list[NetworkService] """ for net in NetworkService.get_l3network_services(slice=slice): if net.get_name() == name: @@ -537,7 +578,7 @@ def get_l2network_services(slice: Slice = None) -> list: for net_name, net in topology.network_services.items(): if ( str(net.get_property("type")) - in NetworkService.get_fim_l2network_service_types() + in NetworkService.__get_fim_l2network_service_types() ): rtn_network_services.append( NetworkService(slice=slice, fim_network_service=net) @@ -569,7 +610,10 @@ def get_l2network_service(slice: Slice = None, name: str = None): @staticmethod def get_network_services(slice: Slice = None) -> list: """ - Not inteded for API use. + Gets all network services (L2 and L3) in this slice + + :return: List of all network services in this slice + :rtype: List[NetworkService] """ topology = slice.get_fim_topology() @@ -590,7 +634,12 @@ def get_network_services(slice: Slice = None) -> list: @staticmethod def get_network_service(slice: Slice = None, name: str = None): """ - Not inteded for API use. + Gest a particular network service from this slice. + + :param name: the name of the network service to search for + :type name: str + :return: a particular network service + :rtype: NetworkService """ for net in NetworkService.get_network_services(slice=slice): if net.get_name() == name: @@ -628,6 +677,7 @@ def __init__( pass self.sliver = None + self.lock = threading.Lock() def __str__(self): """ @@ -1015,6 +1065,11 @@ def get_interfaces(self) -> List[Interface]: ) except: logging.warning(f"interface not found: {interface.name}") + from fabrictestbed_extensions.fablib.interface import Interface + + self.interfaces.append( + Interface(fim_interface=interface, node=self) + ) return self.interfaces @@ -1091,7 +1146,10 @@ def add_interface(self, interface: Interface): curr_nstype = self.get_type() if self.get_layer() == NSLayer.L2: - new_nstype = NetworkService.calculate_l2_nstype(interfaces=new_interfaces) + ero_enabled = True if self.get_fim().ero else False + new_nstype = NetworkService.__calculate_l2_nstype( + interfaces=new_interfaces, ero_enabled=ero_enabled + ) if curr_nstype != new_nstype: self.__replace_network_service(new_nstype) else: @@ -1133,7 +1191,10 @@ def remove_interface(self, interface: Interface): curr_nstype = self.get_type() if self.get_layer() == NSLayer.L2: - new_nstype = NetworkService.calculate_l2_nstype(interfaces=interfaces) + ero_enabled = True if self.get_fim().ero else False + new_nstype = NetworkService.__calculate_l2_nstype( + interfaces=interfaces, ero_enabled=ero_enabled + ) if curr_nstype != new_nstype: self.__replace_network_service(new_nstype) @@ -1189,24 +1250,28 @@ def set_allocated_ip(self, addr: IPv4Address or IPv6Address = None): self.set_fablib_data(fablib_data) def allocate_ip(self, addr: IPv4Address or IPv6Address = None): - subnet = self.get_subnet() - allocated_ips = self.get_allocated_ips() - - if addr: - # if addr != subnet.network_address and addr not in allocated_ips: - if addr not in allocated_ips: - self.set_allocated_ip(addr) - return addr - elif ( - type(subnet) == ipaddress.IPv4Network - or type(subnet) == ipaddress.IPv6Network - ): - for host in subnet: - if host != subnet.network_address and host not in allocated_ips: - self.set_allocated_ip(host) - - return host - return None + try: + self.lock.acquire() + subnet = self.get_subnet() + allocated_ips = self.get_allocated_ips() + + if addr: + # if addr != subnet.network_address and addr not in allocated_ips: + if addr not in allocated_ips: + self.set_allocated_ip(addr) + return addr + elif ( + type(subnet) == ipaddress.IPv4Network + or type(subnet) == ipaddress.IPv6Network + ): + for host in subnet: + if host != subnet.network_address and host not in allocated_ips: + self.set_allocated_ip(host) + + return host + return None + finally: + self.lock.release() def set_allocated_ips(self, allocated_ips: list[IPv4Address or IPv6Address]): fablib_data = self.get_fablib_data() @@ -1221,10 +1286,14 @@ def set_allocated_ips(self, allocated_ips: list[IPv4Address or IPv6Address]): self.set_fablib_data(fablib_data) def free_ip(self, addr: IPv4Address or IPv6Address): - allocated_ips = self.get_allocated_ips() - if addr in allocated_ips: - allocated_ips.remove(addr) - self.set_allocated_ips(allocated_ips) + try: + self.lock.acquire() + allocated_ips = self.get_allocated_ips() + if addr in allocated_ips: + allocated_ips.remove(addr) + self.set_allocated_ips(allocated_ips) + finally: + self.lock.release() def make_ip_publicly_routable(self, ipv6: list[str] = None, ipv4: list[str] = None): labels = self.fim_network_service.labels @@ -1250,11 +1319,22 @@ def is_instantiated(self): return False def set_instantiated(self, instantiated: bool = True): + """ + Set instantiated flag in the fablib_data saved in UserData blob in the FIM model + :param instantiated: flag indicating if the service has been instantiated or not + :type instantiated: bool + """ fablib_data = self.get_fablib_data() fablib_data["instantiated"] = str(instantiated) self.set_fablib_data(fablib_data) def config(self): + """ + Sets up the meta data for the Network Service + - For layer3 services, Subnet, gateway and allocated IPs are updated/maintained fablib_data saved in + UserData blob in the FIM model + - For layer2 services, no action is taken + """ if not self.is_instantiated(): self.set_instantiated(True) @@ -1269,3 +1349,74 @@ def config(self): if self.get_gateway() not in allocated_ips: allocated_ips.append(self.get_gateway()) self.set_allocated_ip(self.get_gateway()) + + def peer( + self, + other: NetworkService, + labels: Labels, + peer_labels: Labels, + capacities: Capacities, + ): + """ + Peer a network service; used for AL2S peering between FABRIC Networks and Cloud Networks + Peer this network service to another. A few constraints are enforced like services being + of the same type. Both services will have ServicePort interfaces facing each other over a link. + It typically requires labels and capacities to put on the interface facing the other service + + :param other: network service to be peered + :type other: NetworkService + :param labels: labels + :type labels: Labels + :param peer_labels: peer labels + :type peer_labels: Labels + :param capacities: capacities + :type capacities: Capacities + + """ + # Peer Cloud L3VPN with FABRIC L3VPN + self.get_fim().peer( + other.get_fim(), + labels=labels, + peer_labels=peer_labels, + capacities=capacities, + ) + + def set_l2_route_hops(self, hops: List[str]): + """ + Explicitly define the sequence of sites or hops to be used for a layer 2 connection. + + Users provide a list of site names, which are then mapped by the ControlFramework to the corresponding + layer 2 loopback IP addresses utilized by the Explicit Route Options in the Network Service configuration + on the switch. + + :param hops: A list of site names to be used as hops. + :type hops: List[str] + """ + # Do nothing if hops is None or empty list + if not hops or not len(hops): + return + + interfaces = self.get_interfaces() + + if len(interfaces) != 2 or self.get_type() not in [ + ServiceType.L2STS, + ServiceType.L2PTP, + ]: + raise Exception( + "Network path can only be specified for a Point to Point Layer2 connection!" + ) + + ifs_sites = [] + for ifs in interfaces: + ifs_sites.append(ifs.get_site()) + + resources = self.get_fablib_manager().get_resources() + resources.validate_requested_ero_path( + source=ifs_sites[0], end=ifs_sites[1], hops=hops + ) + p = Path() + p.set_symmetric(hops) + e = ERO() + e.set(payload=p) + ns_type = self.__calculate_l2_nstype(interfaces=interfaces, ero_enabled=True) + self.get_fim().set_properties(type=ns_type, ero=e) diff --git a/fabrictestbed_extensions/fablib/node.py b/fabrictestbed_extensions/fablib/node.py index 5e5ece68..b7f3ff3b 100644 --- a/fabrictestbed_extensions/fablib/node.py +++ b/fabrictestbed_extensions/fablib/node.py @@ -45,27 +45,24 @@ from __future__ import annotations -import concurrent.futures import ipaddress import json import logging -import os import re import select import threading import time import traceback -from concurrent.futures import ThreadPoolExecutor from typing import TYPE_CHECKING, Dict, List, Tuple, Union import jinja2 import paramiko from fabric_cf.orchestrator.orchestrator_proxy import Status +from fim.user import NodeType from IPython.core.display_functions import display from tabulate import tabulate from fabrictestbed_extensions.fablib.network_service import NetworkService -from fabrictestbed_extensions.utils.utils import Utils if TYPE_CHECKING: from fabrictestbed_extensions.fablib.slice import Slice @@ -92,7 +89,13 @@ class Node: default_disk = 10 default_image = "default_rocky_8" - def __init__(self, slice: Slice, node: FimNode): + def __init__( + self, + slice: Slice, + node: FimNode, + validate: bool = False, + raise_exception: bool = False, + ): """ Node constructor, usually invoked by ``Slice.add_node()``. @@ -101,12 +104,22 @@ def __init__(self, slice: Slice, node: FimNode): :param node: the FIM node that this Node represents :type node: Node + + :param validate: Validate node can be allocated w.r.t available resources + :type validate: bool + + :param raise_exception: Raise exception in case validation failes + :type raise_exception: bool + """ super().__init__() self.fim_node = node self.slice = slice self.host = None self.ip_addr_list_json = None + self.validate = validate + self.raise_exception = raise_exception + self.node_type = NodeType.VM # Try to set the username. try: @@ -171,7 +184,12 @@ def get_sliver(self) -> OrchestratorSliver: @staticmethod def new_node( - slice: Slice = None, name: str = None, site: str = None, avoid: List[str] = [] + slice: Slice = None, + name: str = None, + site: str = None, + avoid: List[str] = [], + validate: bool = False, + raise_exception: bool = False, ): """ Not intended for API call. See: Slice.add_node() @@ -191,14 +209,27 @@ def new_node( :param avoid: a list of node names to avoid :type avoid: List[str] + :param validate: Validate node can be allocated w.r.t available resources + :type validate: bool + + :param raise_exception: Raise exception in case of failure + :type raise_exception: bool + :return: a new fablib node :rtype: Node """ if site is None: - [site] = slice.get_fablib_manager().get_random_sites(avoid=avoid) + [site] = slice.get_fablib_manager().get_random_sites( + avoid=avoid, + ) logging.info(f"Adding node: {name}, slice: {slice.get_name()}, site: {site}") - node = Node(slice, slice.topology.add_node(name=name, site=site)) + node = Node( + slice, + slice.topology.add_node(name=name, site=site), + validate=validate, + raise_exception=raise_exception, + ) node.set_capacities( cores=Node.default_cores, ram=Node.default_ram, disk=Node.default_disk ) @@ -833,6 +864,18 @@ def get_cores(self) -> int or None: except: return None + def get_requested_cores(self) -> int or None: + """ + Gets the requested number of cores on the FABRIC node. + + :return: the requested number of cores on the node + :rtype: int + """ + try: + return self.get_fim_node().get_property(pname="capacities").core + except: + return 0 + def get_ram(self) -> int or None: """ Gets the amount of RAM on the FABRIC node. @@ -845,6 +888,18 @@ def get_ram(self) -> int or None: except: return None + def get_requested_ram(self) -> int or None: + """ + Gets the requested amount of RAM on the FABRIC node. + + :return: the requested amount of RAM on the node + :rtype: int + """ + try: + return self.get_fim_node().get_property(pname="capacities").ram + except: + return 0 + def get_disk(self) -> int or None: """ Gets the amount of disk space on the FABRIC node. @@ -857,6 +912,18 @@ def get_disk(self) -> int or None: except: return None + def get_requested_disk(self) -> int or None: + """ + Gets the amount of disk space on the FABRIC node. + + :return: the amount of disk space on the node + :rtype: int + """ + try: + return self.get_fim_node().get_property(pname="capacities").disk + except: + return 0 + def get_image(self) -> str or None: """ Gets the image reference on the FABRIC node. @@ -891,11 +958,14 @@ def get_host(self) -> str or None: try: if self.host is not None: return self.host - return ( - self.get_fim_node() - .get_property(pname="label_allocations") - .instance_parent + label_allocations = self.get_fim_node().get_property( + pname="label_allocations" ) + labels = self.get_fim_node().get_property(pname="labels") + if label_allocations: + return label_allocations.instance_parent + if labels: + return labels.instance_parent except: return None @@ -969,16 +1039,19 @@ def get_error_message(self) -> str or None: except: return "" - def get_interfaces(self) -> List[Interface] or None: + def get_interfaces(self, include_subs: bool = True) -> List[Interface] or None: """ Gets a list of the interfaces associated with the FABRIC node. + :param include_subs: Flag indicating if sub interfaces should be included + :type include_subs: bool + :return: a list of interfaces on the node :rtype: List[Interface] """ interfaces = [] for component in self.get_components(): - for interface in component.get_interfaces(): + for interface in component.get_interfaces(include_subs=include_subs): interfaces.append(interface) return interfaces @@ -1126,9 +1199,18 @@ def add_component( :return: the new component :rtype: Component """ - return Component.new_component( + component = Component.new_component( node=self, model=model, name=name, user_data=user_data ) + if self.validate: + status, error = self.get_fablib_manager().validate_node(node=self) + if not status: + component.delete() + component = None + logging.warning(error) + if self.raise_exception: + raise ValueError(error) + return component def get_components(self) -> List[Component]: """ @@ -2577,9 +2659,24 @@ def set_ip_os_interface( mtu: str = None, ): """ - .. deprecated:: 1.1.3. + Configure IP Address on network interface as seen inside the VM + :param os_iface: Interface name as seen by the OS such as eth1 etc. + :type os_iface: String + + :param vlan: Vlan tag + :type vlan: String + + :param ip: IP address to be assigned to the tagged interface + :type ip: String + + :param cidr: CIDR associated with IP address + :type ip: String + + :param mtu: MTU size + :type mtu: String + + NOTE: This does not add the IP information in the fablib_data """ - # TODO: Add docstring after doc networking classes if cidr: cidr = str(cidr) if mtu: @@ -2674,13 +2771,29 @@ def add_vlan_os_interface( ip: str = None, cidr: str = None, mtu: str = None, - interface: str = None, + interface: Interface = None, ): """ - .. deprecated:: 1.1.3. - """ - # TODO: Add docstring after doc networking classes + Add VLAN tagged interface for a given interface and set IP address on it + + :param os_iface: Interface name as seen by the OS such as eth1 etc. + :type os_iface: String + :param vlan: Vlan tag + :type vlan: String + + :param ip: IP address to be assigned to the tagged interface + :type ip: String + + :param cidr: CIDR associated with IP address + :type ip: String + + :param mtu: MTU size + :type mtu: String + + :param interface: Interface for which tagged interface has to be added + :type interface: Interface + """ if vlan: vlan = str(vlan) if cidr: @@ -2688,8 +2801,8 @@ def add_vlan_os_interface( if mtu: mtu = str(mtu) + ip_command = "sudo ip" try: - gateway = None if interface.get_network().get_layer() == NSLayer.L3: if interface.get_network().get_type() in [ ServiceType.FABNetv6, @@ -2796,7 +2909,8 @@ def get_user_data(self): def delete(self): """ - Remove the node, including components connected to it. + Remove the node from the slice. All components and interfaces associated with + the Node are removed from the Slice. """ for component in self.get_components(): component.delete() @@ -3268,7 +3382,7 @@ def get_cpu_info(self) -> dict: VM INFO looks like: In this example below, no CPU pinning has been applied so CPU Affinity lists all the CPUs After the pinning has been applied, CPU Affinity would show only the pinned CPU - [{'CPU': '116', 'CPU Affinity': '0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127', 'CPU time': '20.2s', 'State': 'running', 'VCPU': '0'}, + [{'CPU': '116', 'CPU Affinity': '0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127', 'CPU time': '20.2s', 'State': 'running', 'VCPU': '0'}, {'CPU': '118', 'CPU Affinity': '0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127', 'CPU time': '9.0s', 'State': 'running', 'VCPU': '1'}, {'CPU': '117', 'CPU Affinity': '0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127', 'CPU time': '8.9s', 'State': 'running', 'VCPU': '2'}, {'CPU': '119', 'CPU Affinity': '0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127', 'CPU time': '8.8s', 'State': 'running', 'VCPU': '3'}, @@ -3421,7 +3535,9 @@ def pin_cpu(self, component_name: str, cpu_range_to_pin: str = None): def os_reboot(self): """ - Reboot the node. + Request Openstack to reboot the VM. + NOTE: This is not same as rebooting the VM via reboot or init 6 command. + Instead this is like openstack server reboot. """ status = self.poa(operation="reboot") if status == "Failed": diff --git a/fabrictestbed_extensions/fablib/resources.py b/fabrictestbed_extensions/fablib/resources.py index 4b0605da..d18973d5 100644 --- a/fabrictestbed_extensions/fablib/resources.py +++ b/fabrictestbed_extensions/fablib/resources.py @@ -33,63 +33,27 @@ import json import logging +from datetime import datetime from typing import List, Tuple -from fabrictestbed.slice_editor import AdvertisedTopology, Capacities +from fabrictestbed.slice_editor import AdvertisedTopology from fabrictestbed.slice_manager import Status -from fim.slivers import maintenance_mode, network_node -from fim.user import composite_node, interface, link, node +from fim.user import interface, link, node from tabulate import tabulate +from fabrictestbed_extensions.fablib.site import ResourceConstants, Site -class Resources: - site_pretty_names = { - "name": "Name", - "state": "State", - "address": "Address", - "location": "Location", - "ptp_capable": "PTP Capable", - "hosts": "Hosts", - "cpus": "CPUs", - "cores_available": "Cores Available", - "cores_capacity": "Cores Capacity", - "cores_allocated": "Cores Allocated", - "ram_available": "RAM Available", - "ram_capacity": "RAM Capacity", - "ram_allocated": "RAM Allocated", - "disk_available": "Disk Available", - "disk_capacity": "Disk Capacity", - "disk_allocated": "Disk Allocated", - "nic_basic_available": "Basic NIC Available", - "nic_basic_capacity": "Basic NIC Capacity", - "nic_basic_allocated": "Basic NIC Allocated", - "nic_connectx_6_available": "ConnectX-6 Available", - "nic_connectx_6_capacity": "ConnectX-6 Capacity", - "nic_connectx_6_allocated": "ConnectX-6 Allocated", - "nic_connectx_5_available": "ConnectX-5 Available", - "nic_connectx_5_capacity": "ConnectX-5 Capacity", - "nic_connectx_5_allocated": "ConnectX-5 Allocated", - "nvme_available": "NVMe Available", - "nvme_capacity": "NVMe Capacity", - "nvme_allocated": "NVMe Allocated", - "tesla_t4_available": "Tesla T4 Available", - "tesla_t4_capacity": "Tesla T4 Capacity", - "tesla_t4_allocated": "Tesla T4 Allocated", - "rtx6000_available": "RTX6000 Available", - "rtx6000_capacity": "RTX6000 Capacity", - "rtx6000_allocated": "RTX6000 Allocated", - "a30_available": "A30 Available", - "a30_capacity": "A30 Capacity", - "a30_allocated": "A30 Allocated", - "a40_available": "A40 Available", - "a40_capacity": "A40 Capacity", - "a40_allocated": "A40 Allocated", - "fpga_u280_available": "U280 Available", - "fpga_u280_capacity": "U280 Capacity", - "fpga_u280_allocated": "U280 Allocated", - } - def __init__(self, fablib_manager, force_refresh: bool = False): +class Resources: + def __init__( + self, + fablib_manager, + force_refresh: bool = False, + start: datetime = None, + end: datetime = None, + avoid: List[str] = None, + includes: List[str] = None, + ): """ :param fablib_manager: a :class:`FablibManager` instance. :type fablib_manager: fablib.FablibManager @@ -97,6 +61,19 @@ def __init__(self, fablib_manager, force_refresh: bool = False): :param force_refresh: force a refresh of available testbed resources. :type force_refresh: bool + + :param start: start time in UTC format: %Y-%m-%d %H:%M:%S %z + :type: datetime + + :param end: end time in UTC format: %Y-%m-%d %H:%M:%S %z + :type: datetime + + :param avoid: list of sites to avoid + :type: list of string + + :param includes: list of sites to include + :type: list of string + """ super().__init__() @@ -104,7 +81,15 @@ def __init__(self, fablib_manager, force_refresh: bool = False): self.topology = None - self.update(force_refresh=force_refresh) + self.sites = {} + + self.update( + force_refresh=force_refresh, + start=start, + end=end, + includes=includes, + avoid=avoid, + ) def __str__(self) -> str: """ @@ -116,54 +101,14 @@ def __str__(self) -> str: :rtype: String """ table = [] - for site_name, site in self.topology.sites.items(): - # logging.debug(f"site -- {site}") - site_sliver = site.get_sliver() - table.append( - [ - site.name, - f"{self.get_ptp_capable()}", - self.get_cpu_capacity(site_sliver), - f"{self.get_core_available(site_sliver)}/{self.get_core_capacity(site_sliver)}", - f"{self.get_ram_available(site_sliver)}/{self.get_ram_capacity(site_sliver)}", - f"{self.get_disk_available(site_sliver)}/{self.get_disk_capacity(site_sliver)}", - # self.get_host_capacity(site), - # self.get_location_postal(site), - # self.get_location_lat_long(site), - f"{self.get_component_available(site_sliver,'SharedNIC-ConnectX-6')}/{self.get_component_capacity(site_sliver,'SharedNIC-ConnectX-6')}", - f"{self.get_component_available(site_sliver,'SmartNIC-ConnectX-6')}/{self.get_component_capacity(site_sliver,'SmartNIC-ConnectX-6')}", - f"{self.get_component_available(site_sliver,'SmartNIC-ConnectX-5')}/{self.get_component_capacity(site_sliver,'SmartNIC-ConnectX-5')}", - f"{self.get_component_available(site_sliver,'NVME-P4510')}/{self.get_component_capacity(site_sliver,'NVME-P4510')}", - f"{self.get_component_available(site_sliver,'GPU-Tesla T4')}/{self.get_component_capacity(site_sliver,'GPU-Tesla T4')}", - f"{self.get_component_available(site_sliver,'GPU-RTX6000')}/{self.get_component_capacity(site_sliver,'GPU-RTX6000')}", - f"{self.get_component_available(site_sliver, 'GPU-A30')}/{self.get_component_capacity(site_sliver, 'GPU-A30')}", - f"{self.get_component_available(site_sliver, 'GPU-A40')}/{self.get_component_capacity(site_sliver, 'GPU-A40')}", - f"{self.get_component_available(site_sliver, 'FPGA-Xilinx-U280')}/{self.get_component_capacity(site_sliver, 'FPGA-Xilinx-U280')}", - ] - ) + headers = [] + for site_name, site in self.sites.items(): + headers, row = site.to_row() + table.append(row) return tabulate( table, - headers=[ - "Name", - "PTP Capable", - "CPUs", - "Cores", - f"RAM ({Capacities.UNITS['ram']})", - f"Disk ({Capacities.UNITS['disk']})", - # "Workers" - # "Physical Address", - # "Location Coordinates" - "Basic (100 Gbps NIC)", - "ConnectX-6 (100 Gbps x2 NIC)", - "ConnectX-5 (25 Gbps x2 NIC)", - "P4510 (NVMe 1TB)", - "Tesla T4 (GPU)", - "RTX6000 (GPU)", - "A30 (GPU)", - "A40 (GPU)", - "FPGA-Xilinx-U280", - ], + headers=headers, ) def show_site( @@ -182,29 +127,25 @@ def show_site( :param site_name: site name :type site_name: String + :param output: Output type + :type output: str + :param fields: List of fields to include + :type fields: List + :param quiet: flag indicating verbose or quiet display + :type quiet: bool + :param pretty_names: flag indicating if pretty names for the fields to be used or not + :type pretty_names: bool + :param latlon: Flag indicating if lat lon to be included or not + :type latlon: bool + :return: Tabulated string of available resources :rtype: String """ - site = self.topology.sites[site_name] - - data = self.site_to_dict(site.get_sliver(), latlon=latlon) - - if pretty_names: - pretty_names_dict = self.site_pretty_names - else: - pretty_names_dict = {} - - site_table = self.get_fablib_manager().show_table( - data, - fields=fields, - title="Site", - output=output, - quiet=quiet, - pretty_names_dict=pretty_names_dict, + site = self.sites.get(site_name) + return site.show( + output=output, fields=fields, quiet=quiet, pretty_names=pretty_names ) - return site_table - def get_site_names(self) -> List[str]: """ Gets a list of all currently available site names @@ -212,23 +153,39 @@ def get_site_names(self) -> List[str]: :return: list of site names :rtype: List[String] """ - site_name_list = [] - for site_name in self.topology.sites.keys(): - site_name_list.append(str(site_name)) + return list(self.sites.keys()) - return site_name_list + def get_site(self, site_name: str) -> Site: + """ + Get a specific site by name. + + :param site_name: The name of the site to retrieve. + :type site_name: str - def get_topology_site(self, site_name: str) -> node.Node: + :return: The specified site. + :rtype: Site """ - Not recommended for most users. + try: + return self.sites.get(site_name) + except Exception as e: + logging.warning(f"Failed to get site {site_name}") + + def __get_topology_site(self, site_name: str) -> node.Node: + """ + Get a specific site from the topology. + + :param site_name: The name of the site to retrieve from the topology. + :type site_name: str + + :return: The node representing the specified site from the topology. + :rtype: node.Node """ try: - return self.topology.sites[site_name] + return self.topology.sites.get(site_name) except Exception as e: logging.warning(f"Failed to get site {site_name}") - return None - def get_state(self, site: str or node.Node or network_node.NodeSliver) -> str: + def get_state(self, site: str or node.Node) -> str: """ Gets the maintenance state of the node @@ -236,20 +193,19 @@ def get_state(self, site: str or node.Node or network_node.NodeSliver) -> str: :type site: String or Node or NodeSliver :return: str(MaintenanceState) """ - site_name = "" try: - if isinstance(site, network_node.NodeSliver): - return str(site.maintenance_info.get(site.get_name()).state) - if isinstance(site, node.Node): - return str(site.maintenance_info.get(site.name).state) - return str(self.get_topology_site(site).maintenance_info.get(site).state) + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_name() except Exception as e: # logging.warning(f"Failed to get site state {site_name}") return "" def get_component_capacity( self, - site: str or node.Node or network_node.NodeSliver, + site: str or node.Node, component_model_name: str, ) -> int: """ @@ -262,25 +218,23 @@ def get_component_capacity( :return: total component capacity :rtype: int """ + component_capacity = 0 try: - if isinstance(site, network_node.NodeSliver): - return site.attached_components_info.get_device( - component_model_name - ).capacities.unit - if isinstance(site, node.Node): - return site.components[component_model_name].capacities.unit - return ( - self.get_topology_site(site) - .components[component_model_name] - .capacities.unit + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_component_capacity( + component_model_name=component_model_name ) + except Exception as e: - # logging.debug(f"Failed to get {component_model_name} capacity {site}") - return 0 + # logging.error(f"Failed to get {component_model_name} capacity {site}: {e}") + return component_capacity def get_component_allocated( self, - site: str or node.Node or network_node.NodeSliver, + site: str or node.Node, component_model_name: str, ) -> int: """ @@ -294,25 +248,22 @@ def get_component_allocated( :return: currently allocated component of this model :rtype: int """ + component_allocated = 0 try: - if isinstance(site, network_node.NodeSliver): - return site.attached_components_info.get_device( - component_model_name - ).capacity_allocations.unit - if isinstance(site, node.Node): - return site.components[component_model_name].capacity_allocations.unit - return ( - self.get_topology_site(site) - .components[component_model_name] - .capacity_allocations.unit + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_component_allocated( + component_model_name=component_model_name ) except Exception as e: - # logging.debug(f"Failed to get {component_model_name} allocated {site}") - return 0 + # logging.error(f"Failed to get {component_model_name} allocated {site}: {e}") + return component_allocated def get_component_available( self, - site: str or node.Node or network_node.NodeSliver, + site: str or node.Node, component_model_name: str, ) -> int: """ @@ -327,16 +278,18 @@ def get_component_available( :rtype: int """ try: - return self.get_component_capacity( - site, component_model_name - ) - self.get_component_allocated(site, component_model_name) + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_component_available( + component_model_name=component_model_name + ) except Exception as e: # logging.debug(f"Failed to get {component_model_name} available {site}") return self.get_component_capacity(site, component_model_name) - def get_location_lat_long( - self, site: str or node.Node or network_node.NodeSliver - ) -> Tuple[float, float]: + def get_location_lat_long(self, site: str or node.Node) -> Tuple[float, float]: """ Gets gets location of a site in latitude and longitude @@ -346,18 +299,16 @@ def get_location_lat_long( :rtype: Tuple(float,float) """ try: - if isinstance(site, network_node.NodeSliver): - return site.get_location().to_latlon() - if isinstance(site, node.Node): - return site.location.to_latlon() - return self.get_topology_site(site).location.to_latlon() + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_location_lat_long() except Exception as e: # logging.warning(f"Failed to get location postal {site}") return 0, 0 - def get_location_postal( - self, site: str or node.Node or network_node.NodeSliver - ) -> str: + def get_location_postal(self, site: str or node.Node) -> str: """ Gets the location of a site by postal address @@ -367,20 +318,18 @@ def get_location_postal( :rtype: String """ try: - if isinstance(site, network_node.NodeSliver): - return site.get_location().postal - if isinstance(site, node.Node): - return site.location.postal - return self.get_topology_site(site).location.postal + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_location_postal() except Exception as e: # logging.debug(f"Failed to get location postal {site}") return "" - def get_host_capacity( - self, site: str or node.Node or network_node.NodeSliver - ) -> int: + def get_host_capacity(self, site: str or node.Node) -> int: """ - Gets the number of worker hosts at the site + Gets the number of hosts at the site :param site: site name or site object :type site: String or Node or NodeSliver @@ -388,18 +337,16 @@ def get_host_capacity( :rtype: int """ try: - if isinstance(site, network_node.NodeSliver): - return site.get_capacities().unit - if isinstance(site, node.Node): - return site.capacities.unit - return self.get_topology_site(site).capacities.unit + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_host_capacity() except Exception as e: # logging.debug(f"Failed to get host count {site}") return 0 - def get_cpu_capacity( - self, site: str or node.Node or network_node.NodeSliver - ) -> int: + def get_cpu_capacity(self, site: str or node.Node) -> int: """ Gets the total number of cpus at the site @@ -409,18 +356,16 @@ def get_cpu_capacity( :rtype: int """ try: - if isinstance(site, network_node.NodeSliver): - return site.get_capacities().cpu - if isinstance(site, node.Node): - return site.capacities.cpu - return self.get_topology_site(site).capacities.cpu + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_cpu_capacity() except Exception as e: # logging.debug(f"Failed to get cpu capacity {site}") return 0 - def get_core_capacity( - self, site: str or node.Node or network_node.NodeSliver - ) -> int: + def get_core_capacity(self, site: str or node.Node) -> int: """ Gets the total number of cores at the site @@ -430,18 +375,16 @@ def get_core_capacity( :rtype: int """ try: - if isinstance(site, network_node.NodeSliver): - return site.get_capacities().core - if isinstance(site, node.Node): - return site.capacities.core - return self.get_topology_site(site).capacities.core + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_core_capacity() except Exception as e: # logging.debug(f"Failed to get core capacity {site}") return 0 - def get_core_allocated( - self, site: str or node.Node or network_node.NodeSliver - ) -> int: + def get_core_allocated(self, site: str or node.Node) -> int: """ Gets the number of currently allocated cores at the site @@ -451,18 +394,16 @@ def get_core_allocated( :rtype: int """ try: - if isinstance(site, network_node.NodeSliver): - return site.get_capacity_allocations().core - if isinstance(site, node.Node): - return site.capacity_allocations.core - return self.get_topology_site(site).capacity_allocations.core + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_core_allocated() except Exception as e: # logging.debug(f"Failed to get cores allocated {site}") return 0 - def get_core_available( - self, site: str or node.Node or network_node.NodeSliver - ) -> int: + def get_core_available(self, site: str or node.Node) -> int: """ Gets the number of currently available cores at the site @@ -472,14 +413,16 @@ def get_core_available( :rtype: int """ try: - return self.get_core_capacity(site) - self.get_core_allocated(site) + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_core_available() except Exception as e: # logging.debug(f"Failed to get cores available {site}") return self.get_core_capacity(site) - def get_ram_capacity( - self, site: str or node.Node or network_node.NodeSliver - ) -> int: + def get_ram_capacity(self, site: str or node.Node) -> int: """ Gets the total amount of memory at the site in GB @@ -489,18 +432,16 @@ def get_ram_capacity( :rtype: int """ try: - if isinstance(site, network_node.NodeSliver): - return site.get_capacities().ram - if isinstance(site, node.Node): - return site.capacities.ram - return self.get_topology_site(site).capacities.ram + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_ram_capacity() except Exception as e: # logging.debug(f"Failed to get ram capacity {site}") return 0 - def get_ram_allocated( - self, site: str or node.Node or network_node.NodeSliver - ) -> int: + def get_ram_allocated(self, site: str or node.Node) -> int: """ Gets the amount of memory currently allocated the site in GB @@ -510,18 +451,16 @@ def get_ram_allocated( :rtype: int """ try: - if isinstance(site, network_node.NodeSliver): - return site.get_capacity_allocations().ram - if isinstance(site, node.Node): - return site.capacity_allocations.ram - return self.get_topology_site(site).capacity_allocations.ram + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_ram_allocated() except Exception as e: # logging.debug(f"Failed to get ram allocated {site}") return 0 - def get_ram_available( - self, site: str or node.Node or network_node.NodeSliver - ) -> int: + def get_ram_available(self, site: str or node.Node) -> int: """ Gets the amount of memory currently available the site in GB @@ -531,14 +470,16 @@ def get_ram_available( :rtype: int """ try: - return self.get_ram_capacity(site) - self.get_ram_allocated(site) + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_ram_available() except Exception as e: # logging.debug(f"Failed to get ram available {site_name}") return self.get_ram_capacity(site) - def get_disk_capacity( - self, site: str or node.Node or network_node.NodeSliver - ) -> int: + def get_disk_capacity(self, site: str or node.Node) -> int: """ Gets the total amount of disk available the site in GB @@ -548,18 +489,16 @@ def get_disk_capacity( :rtype: int """ try: - if isinstance(site, network_node.NodeSliver): - return site.get_capacities().disk - if isinstance(site, node.Node): - return site.capacities.disk - return self.get_topology_site(site).capacities.disk + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_disk_capacity() except Exception as e: # logging.debug(f"Failed to get disk capacity {site}") return 0 - def get_disk_allocated( - self, site: str or node.Node or network_node.NodeSliver - ) -> int: + def get_disk_allocated(self, site: str or node.Node) -> int: """ Gets the amount of disk allocated the site in GB @@ -569,18 +508,16 @@ def get_disk_allocated( :rtype: int """ try: - if isinstance(site, network_node.NodeSliver): - return site.get_capacity_allocations().disk - if isinstance(site, node.Node): - return site.capacity_allocations.disk - return self.get_topology_site(site).capacity_allocations.disk + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_disk_allocated() except Exception as e: # logging.debug(f"Failed to get disk allocated {site}") return 0 - def get_disk_available( - self, site: str or node.Node or network_node.NodeSliver - ) -> int: + def get_disk_available(self, site: str or node.Node) -> int: """ Gets the amount of disk available the site in GB @@ -590,14 +527,16 @@ def get_disk_available( :rtype: int """ try: - return self.get_disk_capacity(site) - self.get_disk_allocated(site) + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_disk_available() except Exception as e: # logging.debug(f"Failed to get disk available {site_name}") return self.get_disk_capacity(site) - def get_ptp_capable( - self, site: str or node.Node or network_node.NodeSliver - ) -> bool: + def get_ptp_capable(self, site: str or node.Node) -> bool: """ Gets the PTP flag of the site - if it has a native PTP capability :param site: site name or object @@ -606,28 +545,63 @@ def get_ptp_capable( :rtype: bool """ try: - if isinstance(site, network_node.NodeSliver): - return site.flags.ptp - if isinstance(site, node.Node): - return site.flags.ptp - return self.get_topology_site(site).flags.ptp + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.get_ptp_capable() except Exception as e: # logging.debug(f"Failed to get PTP status for {site}") return False def get_fablib_manager(self): + """ + Get the Fabric library manager associated with the resources. + + :return: The Fabric library manager. + :rtype: Any + """ return self.fablib_manager - def update(self, force_refresh: bool = False): + def update( + self, + force_refresh: bool = False, + start: datetime = None, + end: datetime = None, + avoid: List[str] = None, + includes: List[str] = None, + ): """ Update the available resources by querying the FABRIC services + :param force_refresh: force a refresh of available testbed + resources. + :type force_refresh: bool + + :param start: start time in UTC format: %Y-%m-%d %H:%M:%S %z + :type: datetime + + :param end: end time in UTC format: %Y-%m-%d %H:%M:%S %z + :type: datetime + + :param avoid: list of sites to avoid + :type: list of string + + :param includes: list of sites to include + :type: list of string """ logging.info(f"Updating available resources") return_status, topology = ( self.get_fablib_manager() .get_slice_manager() - .resources(force_refresh=force_refresh) + .resources( + force_refresh=force_refresh, + level=2, + start=start, + end=end, + excludes=avoid, + includes=includes, + ) ) if return_status != Status.OK: raise Exception( @@ -638,32 +612,30 @@ def update(self, force_refresh: bool = False): self.topology = topology + for site_name, site in self.topology.sites.items(): + s = Site(site=site, fablib_manager=self.get_fablib_manager()) + self.sites[site_name] = s + def get_topology(self, update: bool = False) -> AdvertisedTopology: """ - Not intended for API use - """ - if update or self.topology is None: - self.update() + Get the FIM object of the Resources. - return self.topology + :return: The FIM of the resources. + :rtype: AdvertisedTopology + """ + return self.get_fim(update=update) - def get_site_list(self, update: bool = False) -> List[str]: + def get_fim(self, update: bool = False) -> AdvertisedTopology: """ - Gets a list of all sites by name + Get the FIM object of the Resources. - :param update: (optional) set to True update available resources - :type update: bool - :return: list of site names - :rtype: List[String] + :return: The FIM of the resources. + :rtype: AdvertisedTopology """ if update or self.topology is None: self.update() - rtn_sites = [] - for site_name, site in self.topology.sites.items(): - rtn_sites.append(site_name) - - return rtn_sites + return self.topology def get_link_list(self, update: bool = False) -> List[str]: """ @@ -684,272 +656,38 @@ def get_link_list(self, update: bool = False) -> List[str]: return rtn_links def site_to_json(self, site, latlon=True): + """ + Convert site information into a JSON string. + + :param site: Name of the site or site object. + :type site: str or node.Node + + :param latlon: Flag indicating whether to convert address to latitude and longitude. + :type latlon: bool + + :return: JSON string representation of the site information. + :rtype: str + """ return json.dumps(self.site_to_dict(site, latlon=latlon), indent=4) - def site_to_dict( - self, site: str or node.Node or network_node.NodeSliver, latlon=True - ): + def site_to_dict(self, site: str or node.Node, latlon=True): """ - Convert site information into a dictionary + Convert site information into a dictionary. - :param site: site name or site object - :param latlon: convert address to latlon (makes online call to openstreetmaps.org) - """ - core_a = self.get_core_available(site) - core_c = self.get_core_capacity(site) - ram_a = self.get_ram_available(site) - ram_c = self.get_ram_capacity(site) - disk_a = self.get_disk_available(site) - disk_c = self.get_disk_capacity(site) - nic_basic_a = self.get_component_available(site, "SharedNIC-ConnectX-6") - nic_basic_c = self.get_component_capacity(site, "SharedNIC-ConnectX-6") - nic_cx6_a = self.get_component_available(site, "SmartNIC-ConnectX-6") - nic_cx6_c = self.get_component_capacity(site, "SmartNIC-ConnectX-6") - nic_cx5_a = self.get_component_available(site, "SmartNIC-ConnectX-5") - nic_cx5_c = self.get_component_capacity(site, "SmartNIC-ConnectX-5") - nvme_a = self.get_component_available(site, "NVME-P4510") - nvme_c = self.get_component_capacity(site, "NVME-P4510") - tesla_t4_a = self.get_component_available(site, "GPU-Tesla T4") - tesla_t4_c = self.get_component_capacity(site, "GPU-Tesla T4") - rtx6000_a = self.get_component_available(site, "GPU-RTX6000") - rtx6000_c = self.get_component_capacity(site, "GPU-RTX6000") - a30_a = self.get_component_available(site, "GPU-A30") - a30_c = self.get_component_capacity(site, "GPU-A30") - a40_a = self.get_component_available(site, "GPU-A40") - a40_c = self.get_component_capacity(site, "GPU-A40") - u280_a = self.get_component_available(site, "FPGA-Xilinx-U280") - u280_c = self.get_component_capacity(site, "FPGA-Xilinx-U280") - ptp = self.get_ptp_capable(site) - - d = { - "name": site.name if isinstance(site, node.Node) else site.get_name(), - "state": self.get_state(site), - "address": self.get_location_postal(site), - "location": self.get_location_lat_long(site) if latlon else "", - "ptp_capable": ptp, - "hosts": self.get_host_capacity(site), - "cpus": self.get_cpu_capacity(site), - "cores_available": core_a, - "cores_capacity": core_c, - "cores_allocated": core_c - core_a, - "ram_available": ram_a, - "ram_capacity": ram_c, - "ram_allocated": ram_c - ram_a, - "disk_available": disk_a, - "disk_capacity": disk_c, - "disk_allocated": disk_c - disk_a, - "nic_basic_available": nic_basic_a, - "nic_basic_capacity": nic_basic_c, - "nic_basic_allocated": nic_basic_c - nic_basic_a, - "nic_connectx_6_available": nic_cx6_a, - "nic_connectx_6_capacity": nic_cx6_c, - "nic_connectx_6_allocated": nic_cx6_c - nic_cx6_a, - "nic_connectx_5_available": nic_cx5_a, - "nic_connectx_5_capacity": nic_cx5_c, - "nic_connectx_5_allocated": nic_cx5_c - nic_cx5_a, - "nvme_available": nvme_a, - "nvme_capacity": nvme_c, - "nvme_allocated": nvme_c - nvme_a, - "tesla_t4_available": tesla_t4_a, - "tesla_t4_capacity": tesla_t4_c, - "tesla_t4_allocated": tesla_t4_c - tesla_t4_a, - "rtx6000_available": rtx6000_a, - "rtx6000_capacity": rtx6000_c, - "rtx6000_allocated": rtx6000_c - rtx6000_a, - "a30_available": a30_a, - "a30_capacity": a30_c, - "a30_allocated": a30_c - a30_a, - "a40_available": a40_a, - "a40_capacity": a40_c, - "a40_allocated": a40_c - a40_a, - "fpga_u280_available": u280_a, - "fpga_u280_capacity": u280_c, - "fpga_u280_allocated": u280_c - u280_a, - } - if not latlon: - d.pop("location") - return d + :param site: Name of the site or site object. + :type site: str or node.Node - def site_to_dictXXX(self, site): - site_name = site.name - return { - "name": {"pretty_name": "Name", "value": site.name}, - "address": { - "pretty_name": "Address", - "value": self.get_location_postal(site_name), - }, - "location": { - "pretty_name": "Location", - "value": self.get_location_lat_long(site_name), - }, - "ptp": { - "pretty_name": "PTP Capable", - "value": self.get_ptp_capable(), - }, - "hosts": { - "pretty_name": "Hosts", - "value": self.get_host_capacity(site_name), - }, - "cpus": {"pretty_name": "CPUs", "value": self.get_cpu_capacity(site_name)}, - "cores_available": { - "pretty_name": "Cores Available", - "value": self.get_core_available(site_name), - }, - "cores_capacity": { - "pretty_name": "Cores Capacity", - "value": self.get_core_capacity(site_name), - }, - "cores_allocated": { - "pretty_name": "Cores Allocated", - "value": self.get_core_capacity(site_name) - - self.get_core_available(site_name), - }, - "ram_available": { - "pretty_name": "RAM Available", - "value": self.get_ram_available(site_name), - }, - "ram_capacity": { - "pretty_name": "RAM Capacity", - "value": self.get_ram_capacity(site_name), - }, - "ram_allocated": { - "pretty_name": "RAM Allocated", - "value": self.get_ram_capacity(site_name) - - self.get_ram_available(site_name), - }, - "disk_available": { - "pretty_name": "Disk Available", - "value": self.get_disk_available(site_name), - }, - "disk_capacity": { - "pretty_name": "Disk Capacity", - "value": self.get_disk_capacity(site_name), - }, - "disk_allocated": { - "pretty_name": "Disk Allocated", - "value": self.get_disk_capacity(site_name) - - self.get_disk_available(site_name), - }, - "nic_basic_available": { - "pretty_name": "Basic NIC Available", - "value": self.get_component_available( - site_name, "SharedNIC-ConnectX-6" - ), - }, - "nic_basic_capacity": { - "pretty_name": "Basic NIC Capacity", - "value": self.get_component_capacity(site_name, "SharedNIC-ConnectX-6"), - }, - "nic_basic_allocated": { - "pretty_name": "Basic NIC Allocated", - "value": self.get_component_capacity(site_name, "SharedNIC-ConnectX-6") - - self.get_component_available(site_name, "SharedNIC-ConnectX-6"), - }, - "nic_connectx_6_available": { - "pretty_name": "ConnectX-6 Available", - "value": self.get_component_available(site_name, "SmartNIC-ConnectX-6"), - }, - "nic_connectx_6_capacity": { - "pretty_name": "ConnectX-6 Capacity", - "value": self.get_component_capacity(site_name, "SmartNIC-ConnectX-6"), - }, - "nic_connectx_6_allocated": { - "pretty_name": "ConnectX-6 Allocated", - "value": self.get_component_capacity(site_name, "SmartNIC-ConnectX-6") - - self.get_component_available(site_name, "SmartNIC-ConnectX-6"), - }, - "nic_connectx_5_available": { - "pretty_name": "ConnectX-5 Available", - "value": self.get_component_available(site_name, "SmartNIC-ConnectX-5"), - }, - "nic_connectx_5_capacity": { - "pretty_name": "ConnectX-5 Capacity", - "value": self.get_component_capacity(site_name, "SmartNIC-ConnectX-5"), - }, - "nic_connectx_5_allocated": { - "pretty_name": "ConnectX-5 Allocated", - "value": self.get_component_capacity(site_name, "SmartNIC-ConnectX-5") - - self.get_component_available(site_name, "SmartNIC-ConnectX-5"), - }, - "nvme_available": { - "pretty_name": "NVMe Available", - "value": self.get_component_available(site_name, "NVME-P4510"), - }, - "nvme_capacity": { - "pretty_name": "NVMe Capacity", - "value": self.get_component_capacity(site_name, "NVME-P4510"), - }, - "nvme_allocated": { - "pretty_name": "NVMe Allocated", - "value": self.get_component_capacity(site_name, "NVME-P4510") - - self.get_component_available(site_name, "NVME-P4510"), - }, - "tesla_t4_available": { - "pretty_name": "Tesla T4 Available", - "value": self.get_component_available(site_name, "GPU-Tesla T4"), - }, - "tesla_t4_capacity": { - "pretty_name": "Tesla T4 Capacity", - "value": self.get_component_capacity(site_name, "GPU-Tesla T4"), - }, - "tesla_t4_allocated": { - "pretty_name": "Tesla T4 Allocated", - "value": self.get_component_capacity(site_name, "GPU-Tesla T4") - - self.get_component_available(site_name, "GPU-Tesla T4"), - }, - "rtx6000_available": { - "pretty_name": "RTX6000 Available", - "value": self.get_component_available(site_name, "GPU-RTX6000"), - }, - "rtx6000_capacity": { - "pretty_name": "RTX6000 Capacity", - "value": self.get_component_capacity(site_name, "GPU-RTX6000"), - }, - "rtx6000_allocated": { - "pretty_name": "RTX6000 Allocated", - "value": self.get_component_capacity(site_name, "GPU-RTX6000") - - self.get_component_available(site_name, "GPU-RTX6000"), - }, - "a30_available": { - "pretty_name": "A30 Available", - "value": self.get_component_available(site_name, "GPU-A30"), - }, - "a30_capacity": { - "pretty_name": "A30 Capacity", - "value": self.get_component_capacity(site_name, "GPU-A30"), - }, - "a30_allocated": { - "pretty_name": "A30 Allocated", - "value": self.get_component_capacity(site_name, "GPU-A30") - - self.get_component_available(site_name, "GPU-A30"), - }, - "a40_available": { - "pretty_name": "A40 Available", - "value": self.get_component_available(site_name, "GPU-A40"), - }, - "a40_capacity": { - "pretty_name": "A40 Capacity", - "value": self.get_component_capacity(site_name, "GPU-A40"), - }, - "a40_allocated": { - "pretty_name": "A40 Allocated", - "value": self.get_component_capacity(site_name, "GPU-A40") - - self.get_component_available(site_name, "GPU-A40"), - }, - "fpga_u280_available": { - "pretty_name": "FPGA U280 Available", - "value": self.get_component_available(site_name, "FPGA-Xilinx-U280"), - }, - "fpga_u280_capacity": { - "pretty_name": "FPGA U280 Capacity", - "value": self.get_component_capacity(site_name, "FPGA-Xilinx-U280"), - }, - "fpga_u280_allocated": { - "pretty_name": "FPGA U280 Allocated", - "value": self.get_component_capacity(site_name, "FPGA-Xilinx-U280") - - self.get_component_available(site_name, "FPGA-Xilinx-U280"), - }, - } + :param latlon: Flag indicating whether to convert address to latitude and longitude. + :type latlon: bool + + :return: Dictionary representation of the site information. + :rtype: dict + """ + if isinstance(site, str): + site = self.get_site(site_name=site) + elif isinstance(site, node.Node): + site = Site(site=site, fablib_manager=self.fablib_manager) + return site.to_dict() def list_sites( self, @@ -960,12 +698,38 @@ def list_sites( pretty_names=True, latlon=True, ): + """ + List information about sites. + + :param output: Output type for listing information. + :type output: Any, optional + + :param fields: List of fields to include in the output. + :type fields: Optional[List[str]], optional + + :param quiet: Flag indicating whether to display output quietly. + :type quiet: bool, optional + + :param filter_function: Function to filter the output. + :type filter_function: Optional[Callable[[Dict], bool]], optional + + :param pretty_names: Flag indicating whether to use pretty names for fields. + :type pretty_names: bool, optional + + :param latlon: Flag indicating whether to convert address to latitude and longitude. + :type latlon: bool, optional + + :return: Table listing information about sites. + :rtype: str + """ table = [] - for site_name, site in self.topology.sites.items(): - table.append(self.site_to_dict(site.get_sliver(), latlon=latlon)) + for site_name, site in self.sites.items(): + site_dict = site.to_dict() + if site_dict.get("hosts"): + table.append(site_dict) if pretty_names: - pretty_names_dict = self.site_pretty_names + pretty_names_dict = ResourceConstants.pretty_names else: pretty_names_dict = {} @@ -979,6 +743,89 @@ def list_sites( pretty_names_dict=pretty_names_dict, ) + def list_hosts( + self, + output=None, + fields=None, + quiet=False, + filter_function=None, + pretty_names=True, + ): + """ + List information about hosts. + + :param output: Output type for listing information. + :type output: Any, optional + + :param fields: List of fields to include in the output. + :type fields: Optional[List[str]], optional + + :param quiet: Flag indicating whether to display output quietly. + :type quiet: bool, optional + + :param filter_function: Function to filter the output. + :type filter_function: Optional[Callable[[Dict], bool]], optional + + :param pretty_names: Flag indicating whether to use pretty names for fields. + :type pretty_names: bool, optional + + :return: Table listing information about hosts. + :rtype: str + """ + table = [] + for site_name, site in self.sites.items(): + for host_name, host in site.get_hosts().items(): + host_dict = host.to_dict() + table.append(host_dict) + + if pretty_names: + pretty_names_dict = ResourceConstants.pretty_names + else: + pretty_names_dict = {} + + return self.get_fablib_manager().list_table( + table, + fields=fields, + title="Hosts", + output=output, + quiet=quiet, + filter_function=filter_function, + pretty_names_dict=pretty_names_dict, + ) + + def validate_requested_ero_path(self, source: str, end: str, hops: List[str]): + """ + Validate a requested network path between two sites for layer 2 network connection + :param source: Source site + :type source: str + :param end: Target site + :type end: str + :param hops: requested hops + :type hops: List[str + + :raises Exception in case of error or if requested path does not exist or is invalid + """ + hop_sites_node_ids = [] + for hop in hops: + ns = self.get_topology().network_services.get(f"{hop.upper()}_ns") + if not ns: + raise Exception(f"Hop: {hop} is not found in the available sites!") + hop_sites_node_ids.append(ns.node_id) + + source_site = self.__get_topology_site(site_name=source) + end_site = self.__get_topology_site(site_name=end) + + if not source_site or not end_site: + raise Exception(f"Source {source} or End: {end} is not found!") + + path = self.get_fim().graph_model.get_nodes_on_path_with_hops( + node_a=source_site.node_id, node_z=end_site.node_id, hops=hop_sites_node_ids + ) + if not path or not len(path): + raise Exception( + f"Requested path via {hops} between {source} and {end} is invalid!" + ) + class Links(Resources): link_pretty_names = { @@ -1087,6 +934,7 @@ class FacilityPorts(Resources): "site_name": "Site", "node_id": "Interface Name", "vlan_range": "VLAN Range", + "allocated_vlan_range": "Allocated VLAN Range", "local_name": "Local Name", "device_name": "Device Name", "region": "Region", @@ -1142,6 +990,7 @@ def __str__(self) -> str: "site_name", "node_id", "vlan_range", + "allocated_vlan_range", "local_name", "device_name", "region", @@ -1157,11 +1006,15 @@ def fp_to_dict(self, iface: interface.Interface, name: str, site: str) -> dict: :return: collection of link properties :rtype: dict """ + label_allocations = iface.get_property("label_allocations") return { "name": name, "site_name": site, "node_id": iface.node_id, "vlan_range": iface.labels.vlan_range if iface.labels else "N/A", + "allocated_vlan_range": ( + label_allocations.vlan if label_allocations else "N/A" + ), "local_name": ( iface.labels.local_name if iface.labels and iface.labels.local_name @@ -1188,7 +1041,22 @@ def list_facility_ports( """ Print a table of link resources in pretty format. - :return: formatted table of resources + :param output: Output type for listing information. + :type output: Any, optional + + :param fields: List of fields to include in the output. + :type fields: Optional[List[str]], optional + + :param quiet: Flag indicating whether to display output quietly. + :type quiet: bool, optional + + :param filter_function: Function to filter the output. + :type filter_function: Optional[Callable[[Dict], bool]], optional + + :param pretty_names: Flag indicating whether to use pretty names for fields. + :type pretty_names: bool, optional + + :return: Formatted table of resources. :rtype: object """ table = [] diff --git a/fabrictestbed_extensions/fablib/site.py b/fabrictestbed_extensions/fablib/site.py new file mode 100644 index 00000000..b3943581 --- /dev/null +++ b/fabrictestbed_extensions/fablib/site.py @@ -0,0 +1,1218 @@ +#!/usr/bin/env python3 +# MIT License +# +# Copyright (c) 2020 FABRIC Testbed +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Author: Komal Thareja(kthare10@renci.org) + +from __future__ import annotations + +import json +import logging +import traceback +from typing import Dict, List, Tuple + +from fabrictestbed.slice_editor import Capacities +from fim.user import Component, node +from fim.user.composite_node import CompositeNode +from fim.view_only_dict import ViewOnlyDict + +from fabrictestbed_extensions.fablib.constants import Constants + + +class ResourceConstants: + attribute_name_mappings = { + Constants.CORES.lower(): { + Constants.NON_PRETTY_NAME: Constants.CORES.lower(), + Constants.PRETTY_NAME: Constants.CORES, + Constants.HEADER_NAME: Constants.CORES, + }, + Constants.RAM.lower(): { + Constants.NON_PRETTY_NAME: Constants.RAM.lower(), + Constants.PRETTY_NAME: Constants.RAM, + Constants.HEADER_NAME: f"{Constants.RAM} ({Capacities.UNITS[Constants.RAM.lower()]})", + }, + Constants.DISK: { + Constants.NON_PRETTY_NAME: Constants.DISK.lower(), + Constants.PRETTY_NAME: Constants.DISK, + Constants.HEADER_NAME: f"{Constants.DISK} ({Capacities.UNITS[Constants.DISK.lower()]})", + }, + Constants.NIC_SHARED_CONNECTX_6: { + Constants.NON_PRETTY_NAME: "nic_basic", + Constants.PRETTY_NAME: "Basic NIC", + Constants.HEADER_NAME: "Basic (100 Gbps NIC)", + }, + Constants.P4_SWITCH: { + Constants.NON_PRETTY_NAME: Constants.P4_SWITCH.lower(), + Constants.PRETTY_NAME: Constants.P4_SWITCH, + Constants.HEADER_NAME: Constants.P4_SWITCH, + }, + Constants.SMART_NIC_CONNECTX_6: { + Constants.NON_PRETTY_NAME: "nic_connectx_6", + Constants.PRETTY_NAME: "ConnectX-6", + Constants.HEADER_NAME: "ConnectX-6 (100 Gbps x2 NIC)", + }, + Constants.SMART_NIC_CONNECTX_5: { + Constants.NON_PRETTY_NAME: "nic_connectx_5", + Constants.PRETTY_NAME: "ConnectX-5", + Constants.HEADER_NAME: "ConnectX-5 (25 Gbps x2 NIC)", + }, + Constants.NVME_P4510: { + Constants.NON_PRETTY_NAME: "nvme", + Constants.PRETTY_NAME: "NVMe", + Constants.HEADER_NAME: "P4510 (NVMe 1TB)", + }, + Constants.GPU_TESLA_T4: { + Constants.NON_PRETTY_NAME: "tesla_t4", + Constants.PRETTY_NAME: "Tesla T4", + Constants.HEADER_NAME: "Tesla T4 (GPU)", + }, + Constants.GPU_RTX6000: { + Constants.NON_PRETTY_NAME: "rtx6000", + Constants.PRETTY_NAME: "RTX6000", + Constants.HEADER_NAME: "RTX6000 (GPU)", + }, + Constants.GPU_A30: { + Constants.NON_PRETTY_NAME: "a30", + Constants.PRETTY_NAME: "A30", + Constants.HEADER_NAME: "A30 (GPU)", + }, + Constants.GPU_A40: { + Constants.NON_PRETTY_NAME: "a40", + Constants.PRETTY_NAME: "A40", + Constants.HEADER_NAME: "A40 (GPU)", + }, + Constants.FPGA_XILINX_U280: { + Constants.NON_PRETTY_NAME: "fpga_u280", + Constants.PRETTY_NAME: "U280", + Constants.HEADER_NAME: "FPGA-Xilinx-U280", + }, + } + pretty_names = { + "name": "Name", + "state": "State", + "address": "Address", + "location": "Location", + "ptp_capable": "PTP Capable", + Constants.HOSTS.lower(): Constants.HOSTS, + Constants.CPUS.lower(): Constants.CPUS, + } + for attribute, names in attribute_name_mappings.items(): + pretty_name = names.get(Constants.PRETTY_NAME) + non_pretty_name = names.get(Constants.NON_PRETTY_NAME) + if pretty_name not in pretty_names: + pretty_names[f"{non_pretty_name}_{Constants.AVAILABLE.lower()}"] = ( + f"{pretty_name} {Constants.AVAILABLE}" + ) + pretty_names[f"{non_pretty_name}_{Constants.ALLOCATED.lower()}"] = ( + f"{pretty_name} {Constants.ALLOCATED}" + ) + pretty_names[f"{non_pretty_name}_{Constants.CAPACITY.lower()}"] = ( + f"{pretty_name} {Constants.CAPACITY}" + ) + + +class Switch: + def __init__(self, switch: node.Node, fablib_manager): + """ + Initialize a Switch object. + + :param switch: The node representing the switch. + :type switch: node.Node + + :param fablib_manager: The manager for the Fabric library. + :type fablib_manager: Any + + """ + self.switch = switch + self.fablib_manager = fablib_manager + + def get_capacity(self) -> int: + """ + Get the capacity of the switch. + + :return: The capacity of the switch. + :rtype: int + """ + try: + return self.switch.capacities.unit + except Exception: + return 0 + + def get_allocated(self) -> int: + """ + Get the allocated capacity of the switch. + + :return: The allocated capacity of the switch. + :rtype: int + """ + try: + return self.switch.capacity_allocations.unit + except Exception: + return 0 + + def get_available(self) -> int: + """ + Get the available capacity of the switch. + + :return: The available capacity of the switch. + :rtype: int + """ + return self.get_capacity() - self.get_allocated() + + def get_fim(self) -> node.Node: + """ + Get the FIM object of the Switch. + + :return: The FIM of the Switch. + :rtype: node.Node + """ + return self.switch + + +class Host: + def __init__(self, host: node.Node, state: str, ptp: bool, fablib_manager): + """ + Initialize a Host object. + + :param host: The node representing the host. + :type host: node.Node + + :param state: The state of the host. + :type state: str + + :param ptp: Boolean indicating if the host is PTP capable. + :type ptp: bool + + :param fablib_manager: The manager for the Fabric library. + :type fablib_manager: Any + + :return: None + """ + self.host = host + self.state = state + self.ptp = ptp + self.fablib_manager = fablib_manager + self.host_info = {} + self.__load() + + def get_fablib_manager(self): + """ + Get the Fabric library manager associated with the host. + + :return: The Fabric library manager. + :rtype: Any + """ + return self.fablib_manager + + def __str__(self): + """ + Convert the Host object to a string representation in JSON format. + + :return: JSON string representation of the Host object. + :rtype: str + """ + return self.to_json() + + def to_dict(self) -> dict: + """ + Convert the Host object to a dictionary. + + :return: Dictionary representation of the Host object. + :rtype: dict + """ + d = { + "name": self.get_name(), + "state": self.get_state(), + "address": self.get_location_postal(), + "location": self.get_location_lat_long(), + "ptp_capable": self.get_ptp_capable(), + } + + for attribute, names in ResourceConstants.attribute_name_mappings.items(): + if attribute in Constants.P4_SWITCH: + continue + capacity = self.host_info.get(attribute.lower(), {}).get( + Constants.CAPACITY.lower(), 0 + ) + allocated = self.host_info.get(attribute.lower(), {}).get( + Constants.ALLOCATED.lower(), 0 + ) + available = capacity - allocated + d[ + f"{names.get(Constants.NON_PRETTY_NAME)}_{Constants.AVAILABLE.lower()}" + ] = available + d[ + f"{names.get(Constants.NON_PRETTY_NAME)}_{Constants.CAPACITY.lower()}" + ] = capacity + d[ + f"{names.get(Constants.NON_PRETTY_NAME)}_{Constants.ALLOCATED.lower()}" + ] = allocated + + return d + + def to_json(self) -> str: + """ + Convert the Host object to a JSON string. + + :return: JSON string representation of the Host object. + :rtype: str + """ + return json.dumps(self.to_dict(), indent=4) + + def get_state(self) -> str: + """ + Get the state of the host. + + :return: The state of the host. + :rtype: str + """ + if not self.state: + return "" + return self.state + + def get_fim(self) -> node.Node: + """ + Get the FIM object of the host. + + :return: The FIM of the host. + :rtype: node.Node + """ + return self.host + + def __load(self): + """ + Load information about the host. + + :return: None + """ + try: + self.host_info[Constants.CORES.lower()] = { + Constants.CAPACITY.lower(): self.get_core_capacity(), + Constants.ALLOCATED.lower(): self.get_core_allocated(), + } + self.host_info[Constants.RAM.lower()] = { + Constants.CAPACITY.lower(): self.get_ram_capacity(), + Constants.ALLOCATED.lower(): self.get_ram_allocated(), + } + self.host_info[Constants.DISK.lower()] = { + Constants.CAPACITY.lower(): self.get_disk_capacity(), + Constants.ALLOCATED.lower(): self.get_disk_allocated(), + } + + if self.host.components: + for component_model_name, c in self.host.components.items(): + comp_cap = self.host_info.setdefault( + component_model_name.lower(), {} + ) + comp_cap.setdefault(Constants.CAPACITY.lower(), 0) + comp_cap.setdefault(Constants.ALLOCATED.lower(), 0) + comp_cap[Constants.CAPACITY.lower()] += c.capacities.unit + if c.capacity_allocations: + comp_cap[ + Constants.ALLOCATED.lower() + ] += c.capacity_allocations.unit + except Exception as e: + # logging.error(f"Failed to get {component_model_name} capacity {site}: {e}") + pass + + def get_components(self) -> ViewOnlyDict: + """ + Get the components associated with the host. + + :return: Dictionary-like view of the components associated with the host. + :rtype: ViewOnlyDict + """ + try: + return self.host.components + except Exception as e: + pass + + def get_component(self, comp_model_type: str) -> Component: + """ + Get a specific component associated with the host. + + :param comp_model_type: The type of component to retrieve. + :type comp_model_type: str + + :return: The specified component. + :rtype: Component + """ + try: + return self.host.components.get(comp_model_type) + except Exception as e: + pass + + def show( + self, + output: str = None, + fields: list[str] = None, + quiet: bool = False, + pretty_names=True, + ) -> str: + """ + Creates a tabulated string of all the available resources at a specific host. + + Intended for printing available resources at a host. + + :param output: Output type + :type output: str + :param fields: List of fields to include + :type fields: List + :param quiet: flag indicating verbose or quiet display + :type quiet: bool + :param pretty_names: flag indicating if pretty names for the fields to be used or not + :type pretty_names: bool + + :return: Tabulated string of available resources + :rtype: String + """ + + data = self.to_dict() + + if pretty_names: + pretty_names_dict = ResourceConstants.pretty_names + else: + pretty_names_dict = {} + + host_table = self.get_fablib_manager().show_table( + data, + fields=fields, + title="Host", + output=output, + quiet=quiet, + pretty_names_dict=pretty_names_dict, + ) + + return host_table + + def get_location_postal(self) -> str: + """ + Gets the location of a site by postal address + + :param site: site name or site object + :type site: String or Node or NodeSliver + :return: postal address of the site + :rtype: String + """ + try: + return self.host.location.postal + except Exception as e: + # logging.debug(f"Failed to get postal address for {site}") + return "" + + def get_location_lat_long(self) -> Tuple[float, float]: + """ + Gets gets location of a site in latitude and longitude + + :return: latitude and longitude of the site + :rtype: Tuple(float,float) + """ + try: + return self.host.location.to_latlon() + except Exception as e: + # logging.debug(f"Failed to get latitude and longitude for {site}") + return 0, 0 + + def get_ptp_capable(self) -> bool: + """ + Gets the PTP flag of the site - if it has a native PTP capability + :param site: site name or object + :type site: String or Node or NodeSliver + :return: boolean flag + :rtype: bool + """ + try: + return self.ptp + except Exception as e: + # logging.debug(f"Failed to get PTP status for {site}") + return False + + def get_name(self): + """ + Gets the host name + + :return: str + """ + try: + return self.host.name + except Exception as e: + # logging.debug(f"Failed to get name for {host}") + return "" + + def get_core_capacity(self) -> int: + """ + Gets the total number of cores at the site + + :return: core count + :rtype: int + """ + try: + return self.host.capacities.core + except Exception as e: + # logging.debug(f"Failed to get core capacity {site}") + return 0 + + def get_core_allocated(self) -> int: + """ + Gets the number of currently allocated cores at the site + + :return: core count + :rtype: int + """ + try: + return self.host.capacity_allocations.core + except Exception as e: + # logging.debug(f"Failed to get cores allocated {site}") + return 0 + + def get_core_available(self) -> int: + """ + Gets the number of currently available cores at the site + :return: core count + :rtype: int + """ + try: + return self.get_core_capacity() - self.get_core_allocated() + except Exception as e: + # logging.debug(f"Failed to get cores available {site}") + return self.get_core_capacity() + + def get_ram_capacity(self) -> int: + """ + Gets the total amount of memory at the site in GB + + :return: ram in GB + :rtype: int + """ + try: + return self.host.capacities.ram + except Exception as e: + # logging.debug(f"Failed to get ram capacity {site}") + return 0 + + def get_ram_allocated(self) -> int: + """ + Gets the amount of memory currently allocated the site in GB + + :param site: site name or object + :type site: String or Node or NodeSliver + :return: ram in GB + :rtype: int + """ + try: + return self.host.capacity_allocations.ram + except Exception as e: + # logging.debug(f"Failed to get ram allocated {site}") + return 0 + + def get_ram_available(self) -> int: + """ + Gets the amount of memory currently available the site in GB + + :param site: site name or object + :type site: String or Node or NodeSliver + :return: ram in GB + :rtype: int + """ + try: + return self.get_ram_capacity() - self.get_ram_allocated() + except Exception as e: + # logging.debug(f"Failed to get ram available {site_name}") + return self.get_ram_capacity() + + def get_disk_capacity(self) -> int: + """ + Gets the total amount of disk available the site in GB + + :return: disk in GB + :rtype: int + """ + try: + return self.host.capacities.disk + except Exception as e: + # logging.debug(f"Failed to get disk capacity {site}") + return 0 + + def get_disk_allocated(self) -> int: + """ + Gets the amount of disk allocated the site in GB + + :return: disk in GB + :rtype: int + """ + try: + return self.host.capacity_allocations.disk + except Exception as e: + # logging.debug(f"Failed to get disk allocated {site}") + return 0 + + def get_disk_available(self) -> int: + """ + Gets the amount of disk available the site in GB + + :param site: site name or object + :type site: String or Node or NodeSliver + :return: disk in GB + :rtype: int + """ + try: + return self.get_disk_capacity() - self.get_disk_allocated() + except Exception as e: + # logging.debug(f"Failed to get disk available {site_name}") + return self.get_disk_capacity() + + def get_component_capacity( + self, + component_model_name: str, + ) -> int: + """ + Gets the total site capacity of a component by model name. + + :param component_model_name: component model name + :type component_model_name: String + :return: total component capacity + :rtype: int + """ + component_capacity = 0 + try: + if component_model_name in self.host.components: + component_capacity += self.host.components[ + component_model_name + ].capacities.unit + return component_capacity + except Exception as e: + # logging.error(f"Failed to get {component_model_name} capacity {site}: {e}") + return component_capacity + + def get_component_allocated( + self, + component_model_name: str, + ) -> int: + """ + Gets gets number of currently allocated components on a the site + by the component by model name. + + :param component_model_name: component model name + :type component_model_name: String + :return: currently allocated component of this model + :rtype: int + """ + component_allocated = 0 + try: + if ( + component_model_name in self.host.components + and self.host.components[component_model_name].capacity_allocations + ): + component_allocated += self.host.components[ + component_model_name + ].capacity_allocations.unit + return component_allocated + except Exception as e: + # logging.error(f"Failed to get {component_model_name} allocated {site}: {e}") + return component_allocated + + def get_component_available( + self, + component_model_name: str, + ) -> int: + """ + Gets gets number of currently available components on the site + by the component by model name. + + :param component_model_name: component model name + :type component_model_name: String + :return: currently available component of this model + :rtype: int + """ + try: + return self.get_component_capacity( + component_model_name + ) - self.get_component_allocated(component_model_name) + except Exception as e: + # logging.debug(f"Failed to get {component_model_name} available {site}") + return self.get_component_capacity(component_model_name) + + +class Site: + def __init__(self, site: CompositeNode, fablib_manager): + """ + Initialize a Site object. + + :param site: The node representing the site. + :type site: node.Node + + :param fablib_manager: The manager for the Fabric library. + :type fablib_manager: Any + + :return: None + """ + super().__init__() + self.site = site + self.fablib_manager = fablib_manager + self.hosts = {} + self.switches = {} + self.site_info = {} + self.__load() + + def get_hosts(self) -> Dict[str, Host]: + """ + Get the hosts associated with the site. + + :return: Dictionary of hosts associated with the site. + :rtype: Dict[str, Host] + """ + return self.hosts + + def __load(self): + """ + Load information about the site. + + :return: None + """ + self.__load_hosts() + self.__load_site_info() + + def __load_hosts(self): + """ + Load Hosts and Switches for a site. + + :return: None + """ + try: + from fim.user import NodeType + + for c_name, child in self.site.children.items(): + if child.type == NodeType.Server: + self.hosts[child.name] = Host( + host=child, + state=self.get_state(child.name), + ptp=self.get_ptp_capable(), + fablib_manager=self.fablib_manager, + ) + elif child.type == NodeType.Switch: + self.switches[child.name] = Switch( + switch=child, fablib_manager=self.get_fablib_manager() + ) + except Exception as e: + logging.error(f"Error occurred - {e}") + logging.error(traceback.format_exc()) + + def to_json(self) -> str: + """ + Convert the Site object to a JSON string. + + :return: JSON string representation of the Site object. + :rtype: str + """ + return json.dumps(self.to_dict(), indent=4) + + def get_fablib_manager(self): + """ + Get the Fabric library manager associated with the site. + + :return: The Fabric library manager. + :rtype: Any + """ + return self.fablib_manager + + def to_row(self) -> Tuple[list, list]: + """ + Convert the Site object to a row for tabular display. + + :return: Tuple containing headers and row for tabular display. + :rtype: Tuple[list, list] + """ + headers = [ + "Name", + "PTP Capable", + Constants.CPUS, + ] + row = [ + self.get_name(), + self.get_ptp_capable(), + self.get_cpu_capacity(), + ] + + for attribute, names in ResourceConstants.attribute_name_mappings.items(): + allocated = self.site_info.get(attribute, {}).get( + Constants.ALLOCATED.lower(), 0 + ) + capacity = self.site_info.get(attribute, {}).get( + Constants.CAPACITY.lower(), 0 + ) + available = capacity - allocated + row.append(f"{available}/{capacity}") + headers.append(names.get(Constants.HEADER_NAME)) + return headers, row + + def get_host(self, name: str) -> Host: + """ + Get a specific host associated with the site. + + :param name: The name of the host to retrieve. + :type name: str + + :return: The specified host. + :rtype: Host + """ + return self.hosts.get(name) + + def __str__(self) -> str: + """ + Convert the Site object to a string representation in JSON format. + + :return: JSON string representation of the Site object. + :rtype: str + """ + return self.to_json() + + def get_name(self) -> str: + """ + Gets the site name + + :return: str(MaintenanceState) + """ + try: + return self.site.name + except Exception as e: + # logging.debug(f"Failed to get name for {site}") + return "" + + def get_state(self, host: str = None): + """ + Gets the maintenance state of the node + + :return: str(MaintenanceState) + """ + try: + if not host: + return str(self.site.maintenance_info.get(self.site.name).state) + else: + if self.site.maintenance_info.get(host): + return str(self.site.maintenance_info.get(host).state) + else: + return "Active" + except Exception as e: + # logging.debug(f"Failed to get maintenance state for {site}") + return "" + + def get_location_postal(self) -> str: + """ + Gets the location of a site by postal address + + :param site: site name or site object + :type site: String or Node or NodeSliver + :return: postal address of the site + :rtype: String + """ + try: + return self.site.location.postal + except Exception as e: + # logging.debug(f"Failed to get postal address for {site}") + return "" + + def get_location_lat_long(self) -> Tuple[float, float]: + """ + Gets gets location of a site in latitude and longitude + + :return: latitude and longitude of the site + :rtype: Tuple(float,float) + """ + try: + return self.site.location.to_latlon() + except Exception as e: + # logging.debug(f"Failed to get latitude and longitude for {site}") + return 0, 0 + + def get_ptp_capable(self) -> bool: + """ + Gets the PTP flag of the site - if it has a native PTP capability + :param site: site name or object + :type site: String or Node or NodeSliver + :return: boolean flag + :rtype: bool + """ + try: + return self.site.flags.ptp + except Exception as e: + # logging.debug(f"Failed to get PTP status for {site}") + return False + + def get_host_capacity(self) -> int: + """ + Gets the number of hosts at the site + + :param site: site name or site object + :type site: String or Node or NodeSliver + :return: host count + :rtype: int + """ + try: + return self.site.capacities.unit + except Exception as e: + # logging.debug(f"Failed to get host count {site}") + return 0 + + def get_cpu_capacity(self) -> int: + """ + Gets the total number of cpus at the site + + :param site: site name or site object + :type site: String or node.Node or NodeSliver + :return: cpu count + :rtype: int + """ + try: + return self.site.capacities.cpu + except Exception as e: + # logging.debug(f"Failed to get cpu capacity {site}") + return 0 + + def to_dict(self) -> dict: + """ + Convert site information into a dictionary + """ + d = { + "name": self.get_name(), + "state": self.get_state(), + "address": self.get_location_postal(), + "location": self.get_location_lat_long(), + "ptp_capable": self.get_ptp_capable(), + "hosts": self.get_host_capacity(), + "cpus": self.get_cpu_capacity(), + } + + for attribute, names in ResourceConstants.attribute_name_mappings.items(): + capacity = self.site_info.get(attribute.lower(), {}).get( + Constants.CAPACITY.lower(), 0 + ) + allocated = self.site_info.get(attribute.lower(), {}).get( + Constants.ALLOCATED.lower(), 0 + ) + available = capacity - allocated + d[ + f"{names.get(Constants.NON_PRETTY_NAME)}_{Constants.AVAILABLE.lower()}" + ] = available + d[ + f"{names.get(Constants.NON_PRETTY_NAME)}_{Constants.CAPACITY.lower()}" + ] = capacity + d[ + f"{names.get(Constants.NON_PRETTY_NAME)}_{Constants.ALLOCATED.lower()}" + ] = allocated + + return d + + def __load_site_info(self): + """ + Load the total site capacity of all components for a site + """ + try: + self.site_info[Constants.CORES.lower()] = { + Constants.CAPACITY.lower(): self.get_core_capacity(), + Constants.ALLOCATED.lower(): self.get_core_allocated(), + } + self.site_info[Constants.RAM.lower()] = { + Constants.CAPACITY.lower(): self.get_ram_capacity(), + Constants.ALLOCATED.lower(): self.get_ram_allocated(), + } + self.site_info[Constants.DISK.lower()] = { + Constants.CAPACITY.lower(): self.get_disk_capacity(), + Constants.ALLOCATED.lower(): self.get_disk_allocated(), + } + + for h in self.hosts.values(): + if h.get_components(): + for component_model_name, c in h.get_components().items(): + comp_cap = self.site_info.setdefault( + component_model_name.lower(), {} + ) + comp_cap.setdefault(Constants.CAPACITY.lower(), 0) + comp_cap.setdefault(Constants.ALLOCATED.lower(), 0) + comp_cap[Constants.CAPACITY.lower()] += c.capacities.unit + if c.capacity_allocations: + comp_cap[ + Constants.ALLOCATED.lower() + ] += c.capacity_allocations.unit + + p4_mappings = ResourceConstants.attribute_name_mappings.get( + Constants.P4_SWITCH + ) + for s in self.switches.values(): + self.site_info[p4_mappings.get(Constants.NON_PRETTY_NAME)] = { + Constants.CAPACITY.lower(): s.get_capacity(), + Constants.ALLOCATED.lower(): s.get_allocated(), + } + + except Exception as e: + # logging.error(f"Failed to get {component_model_name} capacity {site}: {e}") + pass + + def show( + self, + output: str = None, + fields: list[str] = None, + quiet: bool = False, + pretty_names=True, + ) -> str: + """ + Creates a tabulated string of all the available resources at a specific site. + + Intended for printing available resources at a site. + + :param output: Output type + :type output: str + :param fields: List of fields to include + :type fields: List + :param quiet: flag indicating verbose or quiet display + :type quiet: bool + :param pretty_names: flag indicating if pretty names for the fields to be used or not + :type pretty_names: bool + + :return: Tabulated string of available resources + :rtype: String + """ + + data = self.to_dict() + + if pretty_names: + pretty_names_dict = ResourceConstants.pretty_names + else: + pretty_names_dict = {} + + site_table = self.get_fablib_manager().show_table( + data, + fields=fields, + title="Site", + output=output, + quiet=quiet, + pretty_names_dict=pretty_names_dict, + ) + + return site_table + + def get_component_capacity( + self, + component_model_name: str, + ) -> int: + """ + Gets the total site capacity of a component by model name. + + :param component_model_name: component model name + :type component_model_name: String + :return: total component capacity + :rtype: int + """ + component_capacity = 0 + try: + for h in self.hosts.values(): + component_capacity += h.get_component_capacity( + component_model_name=component_model_name + ) + return component_capacity + except Exception as e: + # logging.error(f"Failed to get {component_model_name} capacity {site}: {e}") + return component_capacity + + def get_component_allocated( + self, + component_model_name: str, + ) -> int: + """ + Gets gets number of currently allocated components on a the site + by the component by model name. + + :param component_model_name: component model name + :type component_model_name: String + :return: currently allocated component of this model + :rtype: int + """ + component_allocated = 0 + try: + for h in self.hosts.values(): + component_allocated += h.get_component_allocated( + component_model_name=component_model_name + ) + return component_allocated + except Exception as e: + # logging.error(f"Failed to get {component_model_name} allocated {site}: {e}") + return component_allocated + + def get_component_available( + self, + component_model_name: str, + ) -> int: + """ + Gets gets number of currently available components on the site + by the component by model name. + + :param component_model_name: component model name + :type component_model_name: String + :return: currently available component of this model + :rtype: int + """ + try: + return self.get_component_capacity( + component_model_name + ) - self.get_component_allocated(component_model_name) + except Exception as e: + # logging.debug(f"Failed to get {component_model_name} available {site}") + return self.get_component_capacity(component_model_name) + + def get_fim(self) -> node.Node: + """ + Get the FIM object of the site. + + :return: The FIM of the site. + :rtype: node.Node + """ + return self.site + + def get_core_capacity(self) -> int: + """ + Gets the total number of cores at the site + + :return: core count + :rtype: int + """ + try: + return self.site.capacities.core + except Exception as e: + # logging.debug(f"Failed to get core capacity {site}") + return 0 + + def get_core_allocated(self) -> int: + """ + Gets the number of currently allocated cores at the site + + :return: core count + :rtype: int + """ + try: + return self.site.capacity_allocations.core + except Exception as e: + # logging.debug(f"Failed to get cores allocated {site}") + return 0 + + def get_core_available(self) -> int: + """ + Gets the number of currently available cores at the site + :return: core count + :rtype: int + """ + try: + return self.get_core_capacity() - self.get_core_allocated() + except Exception as e: + # logging.debug(f"Failed to get cores available {site}") + return self.get_core_capacity() + + def get_ram_capacity(self) -> int: + """ + Gets the total amount of memory at the site in GB + + :return: ram in GB + :rtype: int + """ + try: + return self.site.capacities.ram + except Exception as e: + # logging.debug(f"Failed to get ram capacity {site}") + return 0 + + def get_ram_allocated(self) -> int: + """ + Gets the amount of memory currently allocated the site in GB + + :param site: site name or object + :type site: String or Node or NodeSliver + :return: ram in GB + :rtype: int + """ + try: + return self.site.capacity_allocations.ram + except Exception as e: + # logging.debug(f"Failed to get ram allocated {site}") + return 0 + + def get_ram_available(self) -> int: + """ + Gets the amount of memory currently available the site in GB + + :param site: site name or object + :type site: String or Node or NodeSliver + :return: ram in GB + :rtype: int + """ + try: + return self.get_ram_capacity() - self.get_ram_allocated() + except Exception as e: + # logging.debug(f"Failed to get ram available {site_name}") + return self.get_ram_capacity() + + def get_disk_capacity(self) -> int: + """ + Gets the total amount of disk available the site in GB + + :return: disk in GB + :rtype: int + """ + try: + return self.site.capacities.disk + except Exception as e: + # logging.debug(f"Failed to get disk capacity {site}") + return 0 + + def get_disk_allocated(self) -> int: + """ + Gets the amount of disk allocated the site in GB + + :return: disk in GB + :rtype: int + """ + try: + return self.site.capacity_allocations.disk + except Exception as e: + # logging.debug(f"Failed to get disk allocated {site}") + return 0 + + def get_disk_available(self) -> int: + """ + Gets the amount of disk available the site in GB + + :param site: site name or object + :type site: String or Node or NodeSliver + :return: disk in GB + :rtype: int + """ + try: + return self.get_disk_capacity() - self.get_disk_allocated() + except Exception as e: + # logging.debug(f"Failed to get disk available {site_name}") + return self.get_disk_capacity() + + def get_host_names(self) -> List[str]: + """ + Gets a list of all currently available hosts + + :return: list of host names + :rtype: List[String] + """ + return list(self.hosts.keys()) diff --git a/fabrictestbed_extensions/fablib/slice.py b/fabrictestbed_extensions/fablib/slice.py index 907bb049..dd94b37e 100644 --- a/fabrictestbed_extensions/fablib/slice.py +++ b/fabrictestbed_extensions/fablib/slice.py @@ -56,14 +56,16 @@ import time from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Tuple import pandas as pd +from fim.user import Capacities, Labels, NodeType from fss_utils.sshkey import FABRICSSHKey from IPython.core.display_functions import display from fabrictestbed_extensions.fablib.constants import Constants from fabrictestbed_extensions.fablib.facility_port import FacilityPort +from fabrictestbed_extensions.fablib.switch import Switch if TYPE_CHECKING: from fabric_cf.orchestrator.swagger_client import ( @@ -204,10 +206,14 @@ def state_color(val): color = f"{Constants.SUCCESS_LIGHT_COLOR}" elif val == "ModifyOK": color = f"{Constants.IN_PROGRESS_LIGHT_COLOR}" + elif val == "AllocatedOK": + color = f"{Constants.IN_PROGRESS_LIGHT_COLOR}" elif val == "StableError": color = f"{Constants.ERROR_LIGHT_COLOR}" elif val == "ModifyError": color = f"{Constants.ERROR_LIGHT_COLOR}" + elif val == "AllocatedError": + color = f"{Constants.ERROR_LIGHT_COLOR}" elif val == "Configuring": color = f"{Constants.IN_PROGRESS_LIGHT_COLOR}" elif val == "Modifying": @@ -760,11 +766,49 @@ def get_slice_private_key(self) -> str: return self.fablib_manager.get_default_slice_private_key() def is_dead_or_closing(self): + """ + Tests is the slice is Dead or Closing state. + + :return: True if slice is Dead or Closing state, False otherwise + :rtype: Bool + """ if self.get_state() in ["Closing", "Dead"]: return True else: return False + def is_advanced_allocation(self) -> bool: + """ + Checks if slice is requested in future + + :return: True if slice is Allocated and starts in future, False otherwise + :rtype: Bool + """ + now = datetime.now(timezone.utc) + lease_start = ( + datetime.strptime(self.get_lease_start(), Constants.LEASE_TIME_FORMAT) + if self.get_lease_start() + else None + ) + if lease_start and lease_start > now and self.is_allocated(): + return True + return False + + def is_allocated(self) -> bool: + """ + Tests is the slice is in Allocated State. + + :return: True if slice is Allocated, False otherwise + :rtype: Bool + """ + if ( + self.get_state() in ["AllocatedOK", "AllocatedError"] + and self.get_lease_start() + ): + return True + else: + return False + def isStable(self) -> bool: """ Tests is the slice is stable. Stable means all requests for @@ -794,18 +838,13 @@ def get_state(self) -> str: :rtype: str """ - if self.sm_slice == None: - state = None - else: - try: - state = self.sm_slice.state - except Exception as e: - logging.warning( - f"Exception in get_state from non-None sm_slice. Returning None state: {e}" - ) - state = None - - return state + try: + if self.sm_slice is not None: + return self.sm_slice.state + except Exception as e: + logging.warning( + f"Exception in get_state from non-None sm_slice. Returning None state: {e}" + ) def get_name(self) -> str: """ @@ -833,18 +872,13 @@ def get_lease_end(self) -> str: :rtype: String """ - if self.sm_slice is None: - lease_end_time = None - else: - try: - lease_end_time = self.sm_slice.lease_end_time - except Exception as e: - logging.warning( - f"Exception in get_lease_end from non-None sm_slice. Returning None state: {e}" - ) - lease_end_time = None - - return lease_end_time + try: + if self.sm_slice is not None: + return self.sm_slice.lease_end_time + except Exception as e: + logging.warning( + f"Exception in get_lease_end from non-None sm_slice. Returning None state: {e}" + ) def get_lease_start(self) -> str: """ @@ -853,19 +887,13 @@ def get_lease_start(self) -> str: :return: timestamp when lease starts :rtype: String """ - - if self.sm_slice is None: - lease_start_time = None - else: - try: - lease_start_time = self.sm_slice.lease_start_time - except Exception as e: - logging.warning( - f"Exception in get_lease_start from non-None sm_slice. Returning None state: {e}" - ) - lease_start_time = None - - return lease_start_time + try: + if self.sm_slice is not None: + return self.sm_slice.lease_start_time + except Exception as e: + logging.warning( + f"Exception in get_lease_end from non-None sm_slice. Returning None state: {e}" + ) def get_project_id(self) -> str: """ @@ -881,6 +909,7 @@ def add_port_mirror_service( name: str, mirror_interface_name: str, receive_interface: Interface or None = None, + mirror_interface_vlan: str = None, mirror_direction: str = "both", ) -> NetworkService: """ @@ -893,6 +922,7 @@ def add_port_mirror_service( :param name: Name of the service :param mirror_interface_name: Name of the interface on the dataplane switch to mirror + :param mirror_interface_vlan: Vlan of the interface :param receive_interface: Interface in the topology belonging to a SmartNIC component :param mirror_direction: String 'rx', 'tx' or 'both' @@ -904,6 +934,7 @@ def add_port_mirror_service( slice=self, name=name, mirror_interface_name=mirror_interface_name, + mirror_interface_vlan=mirror_interface_vlan, receive_interface=receive_interface, mirror_direction=mirror_direction, ) @@ -990,6 +1021,7 @@ def add_l3network( interfaces: List[Interface] = [], type: str = "IPv4", user_data: dict = {}, + technology: str = None, ) -> NetworkService: """ Adds a new L3 network service to this slice. @@ -1039,6 +1071,8 @@ def add_l3network( :param user_data :type user_data: dict + :param technology: Specify the technology used should be set to AL2S when using for AL2S peering; otherwise None + :type technology: str :return: a new L3 network service :rtype: NetworkService @@ -1052,11 +1086,19 @@ def add_l3network( interfaces=interfaces, type=type, user_data=user_data, + technology=technology, ) def add_facility_port( - self, name: str = None, site: str = None, vlan: Union[str, list] = None - ) -> NetworkService: + self, + name: str = None, + site: str = None, + vlan: Union[str, list] = None, + labels: Labels = None, + peer_labels: Labels = None, + bandwidth: int = 10, + mtu: int = None, + ) -> FacilityPort: """ Adds a new L2 facility port to this slice @@ -1066,11 +1108,26 @@ def add_facility_port( :type site: String :param vlan: vlan :type vlan: String + :param labels: labels for the facility port such as VLAN, ip sub net + :type: labels: Labels + :param peer_labels: peer labels for the facility port such as VLAN, ip sub net, bgp key - used for AL2S Peering + :type: peer_labels: Labels + :param bandwidth: bandwidth + :type: bandwidth: int + :param mtu: MTU size + :type: mtu: int :return: a new L2 facility port :rtype: NetworkService """ return FacilityPort.new_facility_port( - slice=self, name=name, site=site, vlan=vlan + slice=self, + name=name, + site=site, + vlan=vlan, + labels=labels, + peer_labels=peer_labels, + bandwidth=bandwidth, + mtu=mtu, ) def add_node( @@ -1085,6 +1142,8 @@ def add_node( host: str = None, user_data: dict = {}, avoid: List[str] = [], + validate: bool = False, + raise_exception: bool = False, ) -> Node: """ Creates a new node on this fablib slice. @@ -1128,10 +1187,23 @@ def add_node( random site. :type avoid: List[String] + :param validate: Validate node can be allocated w.r.t available resources + :type validate: bool + + :param raise_exception: Raise exception in case of Failure + :type raise_exception: bool + :return: a new node :rtype: Node """ - node = Node.new_node(slice=self, name=name, site=site, avoid=avoid) + node = Node.new_node( + slice=self, + name=name, + site=site, + avoid=avoid, + validate=validate, + raise_exception=raise_exception, + ) node.init_fablib_data() @@ -1154,6 +1226,83 @@ def add_node( self.nodes = None self.interfaces = None + if validate: + status, error = self.get_fablib_manager().validate_node(node=node) + if not status: + node.delete() + node = None + logging.warning(error) + if raise_exception: + raise ValueError(error) + return node + + def add_switch( + self, + name: str, + site: str = None, + user_data: dict = None, + avoid: List[str] = None, + validate: bool = False, + raise_exception: bool = False, + ) -> Switch: + """ + Creates a new switch on this fablib slice. + + :param name: Name of the new switch + :type name: String + + :param site: (Optional) Name of the site to deploy the node + on. Default to a random site. + :type site: String + + :param user_data + :type user_data: dict + + :param avoid: (Optional) A list of sites to avoid is allowing + random site. + :type avoid: List[String] + + :param validate: Validate node can be allocated w.r.t available resources + :type validate: bool + + :param raise_exception: Raise exception in case of Failure + :type raise_exception: bool + + :return: a new node + :rtype: Node + """ + if not user_data: + user_data = {} + if not avoid: + avoid = [] + + node = Switch.new_switch( + slice=self, + name=name, + site=site, + avoid=avoid, + validate=validate, + raise_exception=raise_exception, + ) + + node.init_fablib_data() + + user_data_working = node.get_user_data() + for k, v in user_data.items(): + user_data_working[k] = v + node.set_user_data(user_data_working) + + self.nodes = None + self.interfaces = None + + if validate: + status, error = self.get_fablib_manager().validate_node(node=node) + if not status: + node.delete() + node = None + logging.warning(error) + if raise_exception: + raise ValueError(error) return node def get_object_by_reservation( @@ -1350,7 +1499,6 @@ def get_l3network(self, name: str = None) -> Union[NetworkService or None]: return NetworkService.get_l3network_service(self, name) except Exception as e: logging.info(e, exc_info=True) - return None def get_l2networks(self) -> List[NetworkService]: """ @@ -1379,7 +1527,6 @@ def get_l2network(self, name: str = None) -> NetworkService or None: return NetworkService.get_l2network_service(self, name) except Exception as e: logging.info(e, exc_info=True) - return None def get_network_services(self) -> List[NetworkService]: """ @@ -1435,7 +1582,6 @@ def get_network(self, name: str = None) -> NetworkService or None: return NetworkService.get_network_service(self, name) except Exception as e: logging.info(e, exc_info=True) - return None def delete(self): """ @@ -1443,6 +1589,9 @@ def delete(self): :raises Exception: if deleting the slice fails """ + if not self.sm_slice: + self.topology = None + return return_status, result = self.fablib_manager.get_slice_manager().delete( slice_object=self.sm_slice ) @@ -1472,19 +1621,11 @@ def renew(self, end_date: str = None, days: int = None): if end_date is None and days is None: raise Exception("Either end_date or days must be specified!") - if days is not None: - end_date = (datetime.now(timezone.utc) + timedelta(days=days)).strftime( - "%Y-%m-%d %H:%M:%S %z" - ) - - return_status, result = self.fablib_manager.get_slice_manager().renew( - slice_object=self.sm_slice, new_lease_end_time=end_date - ) + if end_date is not None: + end = datetime.strptime(end_date, "%Y-%m-%d %H:%M:%S %z") + days = (end - datetime.now(timezone.utc)).days - if return_status != Status.OK: - raise Exception( - "Failed to renew slice: {}, {}".format(return_status, result) - ) + self.submit(lease_in_days=days) def build_error_exception_string(self) -> str: """ @@ -1677,7 +1818,7 @@ def post_boot_config(self): Only use this method after a non-blocking submit call and only call it once. """ - if self.is_dead_or_closing(): + if self.is_dead_or_closing() or self.is_allocated(): print( f"FAILURE: Slice is in {self.get_state()} state; cannot do post boot config" ) @@ -1733,8 +1874,8 @@ def post_boot_config(self): ) # ({time.time() - start:.0f} sec)") for thread in concurrent.futures.as_completed(threads.keys()): + node = threads[thread] try: - node = threads[thread] result = thread.result() # print(result) print( @@ -1789,7 +1930,7 @@ def isReady(self, update=False): if ( node.get_reservation_state() == "Active" - and node.get_management_ip() == None + and node.get_management_ip() is None ): logging.warning( f"slice not ready: node {node.get_name()} management ip: {node.get_management_ip()}" @@ -1804,7 +1945,7 @@ def isReady(self, update=False): in [ipaddress.IPv4Network, ipaddress.IPv6Network] or not type(net.get_gateway()) in [ipaddress.IPv4Address, ipaddress.IPv6Address] - or net.get_available_ips() == None + or net.get_available_ips() is None ): logging.warning( f"slice not ready: net {net.get_name()}, subnet: {net.get_subnet()}, available_ips: {net.get_available_ips()}" @@ -1859,6 +2000,7 @@ def wait_jupyter(self, timeout: int = 1800, interval: int = 30, verbose=False): time.sleep(interval) stable = False + allocated = False self.update_slice() self.update_slivers() @@ -1871,6 +2013,14 @@ def wait_jupyter(self, timeout: int = 1800, interval: int = 30, verbose=False): hasNetworks = False if self.isReady(): break + elif self.is_advanced_allocation(): + allocated = True + self.update() + if len(self.get_interfaces()) > 0: + hasNetworks = True + else: + hasNetworks = False + break else: if verbose: self.update() @@ -1909,7 +2059,6 @@ def wait_jupyter(self, timeout: int = 1800, interval: int = 30, verbose=False): display(node_table) if hasNetworks and network_table: display(network_table) - else: if slice_show_table: display(slice_show_table) @@ -1942,11 +2091,14 @@ def wait_jupyter(self, timeout: int = 1800, interval: int = 30, verbose=False): if hasNetworks and network_table: display(network_table) - print(f"\nTime to stable {time.time() - start:.0f} seconds") + print(f"\nTime to {self.get_state()} {time.time() - start:.0f} seconds") - print("Running post_boot_config ... ") - self.post_boot_config() - print(f"Time to post boot config {time.time() - start:.0f} seconds") + if stable: + print("Running post_boot_config ... ") + self.post_boot_config() + print(f"Time to post boot config {time.time() - start:.0f} seconds") + elif allocated: + print("Future allocation - skipping post_boot_config ... ") # Last update to get final data for display # no longer needed because post_boot_config does this @@ -1983,7 +2135,9 @@ def submit( post_boot_config: bool = True, wait_ssh: bool = True, extra_ssh_keys: List[str] = None, + lease_start_time: datetime = None, lease_in_days: int = None, + validate: bool = False, ) -> str: """ Submits a slice request to FABRIC. @@ -1996,32 +2150,77 @@ def submit( :param wait: indicator for whether to wait for the slice's resources to be active + :type wait: bool + :param wait_timeout: how many seconds to wait on the slice resources + :type wait_timeout: int + :param wait_interval: how often to check on the slice resources + :type wait_interval: int + :param progress: indicator for whether to show progress while waiting + :type progress: bool + :param wait_jupyter: Special wait for jupyter notebooks. + :type wait_jupyter: str + :param post_boot_config: + :type post_boot_config: bool + :param wait_ssh: + :type wait_ssh: bool + :param extra_ssh_keys: Optional list of additional SSH public keys to be installed in the slivers of this slice + :type extra_ssh_keys: List[str] + + :param lease_start_time: Optional lease start in UTC time format: %Y-%m-%d %H:%M:%S %z + :type lease_start_time: datetime + :param lease_in_days: Optional lease duration in days, by default the slice is active for 24 hours i.e 1 day, only used for create. + :type lease_in_days: int + + :param validate: Validate node can be allocated w.r.t available resources + :type validate: bool + :return: slice_id """ if not wait: progress = False + if validate: + self.validate() + # Generate Slice Graph slice_graph = self.get_fim_topology().serialize() + lease_start_time_str = None + if lease_start_time: + lease_start_time_str = lease_start_time.strftime("%Y-%m-%d %H:%M:%S %z") + + lease_end_time = None + if lease_in_days: + start_time = ( + lease_start_time if lease_end_time else datetime.now(timezone.utc) + ) + lease_end_time = (start_time + timedelta(days=lease_in_days)).strftime( + "%Y-%m-%d %H:%M:%S %z" + ) + # Request slice from Orchestrator if self._is_modify(): - ( - return_status, - slice_reservations, - ) = self.fablib_manager.get_slice_manager().modify( - slice_id=self.slice_id, slice_graph=slice_graph - ) + if lease_in_days: + return_status, result = self.fablib_manager.get_slice_manager().renew( + slice_object=self.sm_slice, new_lease_end_time=lease_end_time + ) + else: + ( + return_status, + slice_reservations, + ) = self.fablib_manager.get_slice_manager().modify( + slice_id=self.slice_id, slice_graph=slice_graph + ) else: # retrieve and validate SSH keys ssh_keys = list() @@ -2041,12 +2240,6 @@ def submit( # this will throw an informative exception FABRICSSHKey.get_key_length(ssh_key) - lease_end_time = None - if lease_in_days: - lease_end_time = ( - datetime.now(timezone.utc) + timedelta(days=lease_in_days) - ).strftime("%Y-%m-%d %H:%M:%S %z") - ( return_status, slice_reservations, @@ -2055,6 +2248,7 @@ def submit( slice_graph=slice_graph, ssh_key=ssh_keys, lease_end_time=lease_end_time, + lease_start_time=lease_start_time_str, ) if return_status == Status.OK: logging.info( @@ -2096,10 +2290,14 @@ def submit( timeout=wait_timeout, interval=wait_interval, progress=progress ) + advance_allocation = self.is_advanced_allocation() if progress: - print("Running post boot config ... ", end="") + if advance_allocation: + print("Future allocation - skipping post_boot_config ... ") + else: + print("Running post boot config ... ", end="") - if post_boot_config: + if advance_allocation and post_boot_config: self.post_boot_config() else: self.update() @@ -2612,3 +2810,31 @@ def _is_modify(self) -> bool: return False else: return True + + def validate(self, raise_exception: bool = True) -> Tuple[bool, Dict[str, str]]: + """ + Validate the slice w.r.t available resources before submission + + :param raise_exception: raise exception if validation fails + :type raise_exception: bool + + :return: Tuple indicating status for validation and dictionary of the errors corresponding to + each requested node + :rtype: Tuple[bool, Dict[str, str]] + """ + allocated = {} + errors = {} + nodes_to_remove = [] + for n in self.get_nodes(): + status, error = self.get_fablib_manager().validate_node( + node=n, allocated=allocated + ) + if not status: + nodes_to_remove.append(n) + errors[n.get_name()] = error + logging.warning(f"{n.get_name()} - {error}") + for n in nodes_to_remove: + n.delete() + if raise_exception and len(errors): + raise Exception(f"Slice validation failed - {errors}!") + return len(errors) == 0, errors diff --git a/fabrictestbed_extensions/fablib/switch.py b/fabrictestbed_extensions/fablib/switch.py new file mode 100644 index 00000000..22f7494b --- /dev/null +++ b/fabrictestbed_extensions/fablib/switch.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +# MIT License +# +# Copyright (c) 2020 FABRIC Testbed +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Author: Komal Thareja (kthare10@renci.org) +from __future__ import annotations + +import json +import logging +from typing import TYPE_CHECKING, List + +import jinja2 +from IPython.core.display_functions import display +from tabulate import tabulate + +from fabrictestbed_extensions.fablib.interface import Interface +from fabrictestbed_extensions.fablib.node import Node + +if TYPE_CHECKING: + from fabrictestbed_extensions.fablib.slice import Slice + +from fabrictestbed.slice_editor import Capacities +from fabrictestbed.slice_editor import Node as FimNode + + +class Switch(Node): + def __init__( + self, + slice: Slice, + node: FimNode, + validate: bool = False, + raise_exception: bool = False, + ): + """ + Node constructor, usually invoked by ``Slice.add_node()``. + + :param slice: the fablib slice to have this node on + :type slice: Slice + + :param node: the FIM node that this Node represents + :type node: Node + + :param validate: Validate node can be allocated w.r.t available resources + :type validate: bool + + :param raise_exception: Raise exception in case validation failes + :type raise_exception: bool + + """ + super(Switch, self).__init__( + slice=slice, node=node, validate=validate, raise_exception=raise_exception + ) + self.username = "rare" + + def __str__(self): + """ + Creates a tabulated string describing the properties of the + node. + + Intended for printing node information. + + :return: Tabulated string of node information + :rtype: String + """ + table = [ + ["ID", self.get_reservation_id()], + ["Name", self.get_name()], + ["Site", self.get_site()], + ["Management IP", self.get_management_ip()], + ["Reservation State", self.get_reservation_state()], + ["Error Message", self.get_error_message()], + ["SSH Command", self.get_ssh_command()], + ] + + return tabulate(table) # , headers=["Property", "Value"]) + + @staticmethod + def new_switch( + slice: Slice = None, + name: str = None, + site: str = None, + avoid: List[str] = None, + validate: bool = False, + raise_exception: bool = False, + ) -> Switch: + """ + Not intended for API call. See: Slice.add_node() + + Creates a new FABRIC node and returns a fablib node with the + new node. + + :param slice: the fablib slice to build the new node on + :type slice: Slice + + :param name: the name of the new node + :type name: str + + :param site: the name of the site to build the node on + :type site: str + + :param avoid: a list of node names to avoid + :type avoid: List[str] + + :param validate: Validate node can be allocated w.r.t available resources + :type validate: bool + + :param raise_exception: Raise exception in case of failure + :type raise_exception: bool + + :return: a new fablib node + :rtype: Node + """ + if not avoid: + avoid = [] + + if site is None: + [site] = slice.get_fablib_manager().get_random_sites( + avoid=avoid, + ) + + logging.info(f"Adding node: {name}, slice: {slice.get_name()}, site: {site}") + node = Switch( + slice, + slice.topology.add_switch(name=name, site=site), + validate=validate, + raise_exception=raise_exception, + ) + node.__set_capacities(unit=1) + + node.init_fablib_data() + + return node + + def toJson(self): + """ + Returns the node attributes as a JSON string + + :return: slice attributes as JSON string + :rtype: str + """ + return json.dumps(self.toDict(), indent=4) + + @staticmethod + def get_pretty_name_dict(): + return { + "id": "ID", + "name": "Name", + "site": "Site", + "username": "Username", + "management_ip": "Management IP", + "state": "State", + "error": "Error", + "ssh_command": "SSH Command", + "public_ssh_key_file": "Public SSH Key File", + "private_ssh_key_file": "Private SSH Key File", + } + + def toDict(self, skip: list = None): + """ + Returns the node attributes as a dictionary + + :return: slice attributes as dictionary + :rtype: dict + """ + if not skip: + skip = [] + + rtn_dict = {} + + if "id" not in skip: + rtn_dict["id"] = str(self.get_reservation_id()) + if "name" not in skip: + rtn_dict["name"] = str(self.get_name()) + if "site" not in skip: + rtn_dict["site"] = str(self.get_site()) + if "management_ip" not in skip: + rtn_dict["management_ip"] = ( + str(self.get_management_ip()).strip() + if str(self.get_reservation_state()) == "Active" + and self.get_management_ip() + else "" + ) # str(self.get_management_ip()) + if "state" not in skip: + rtn_dict["state"] = str(self.get_reservation_state()) + if "error" not in skip: + rtn_dict["error"] = str(self.get_error_message()) + if "ssh_command" not in skip: + if str(self.get_reservation_state()) == "Active": + rtn_dict["ssh_command"] = str(self.get_ssh_command()) + else: + rtn_dict["ssh_command"] = "" + if "public_ssh_key_file" not in skip: + rtn_dict["public_ssh_key_file"] = str(self.get_public_key_file()) + if "private_ssh_key_file" not in skip: + rtn_dict["private_ssh_key_file"] = str(self.get_private_key_file()) + + return rtn_dict + + def generate_template_context(self): + context = self.toDict(skip=["ssh_command"]) + context["components"] = [] + return context + + def get_template_context(self, skip: List[str] = None): + if not skip: + skip = ["ssh_command"] + + return self.get_slice().get_template_context(self, skip=skip) + + def render_template(self, input_string, skip: List[str] = None): + if not skip: + skip = ["ssh_command"] + + environment = jinja2.Environment() + # environment.json_encoder = json.JSONEncoder(ensure_ascii=False) + template = environment.from_string(input_string) + output_string = template.render(self.get_template_context(skip=skip)) + + return output_string + + def show( + self, fields=None, output=None, quiet=False, colors=False, pretty_names=True + ): + """ + Show a table containing the current node attributes. + + There are several output options: ``"text"``, ``"pandas"``, + and ``"json"`` that determine the format of the output that is + returned and (optionally) displayed/printed. + + :param output: output format. Options are: + + - ``"text"``: string formatted with tabular + + - ``"pandas"``: pandas dataframe + + - ``"json"``: string in json format + + :type output: str + + :param fields: List of fields to show. JSON output will + include all available fields. + :type fields: List[str] + + :param quiet: True to specify printing/display + :type quiet: bool + + :param colors: True to specify state colors for pandas output + :type colors: bool + + :return: table in format specified by output parameter + :rtype: Object + + Here's an example of ``fields``:: + + fields=['Name','State'] + """ + + data = self.toDict() + + def state_color(val): + if val == "Active": + color = f"{self.get_fablib_manager().SUCCESS_LIGHT_COLOR}" + elif val == "Configuring": + color = f"{self.get_fablib_manager().IN_PROGRESS_LIGHT_COLOR}" + elif val == "Closed": + color = f"{self.get_fablib_manager().ERROR_LIGHT_COLOR}" + else: + color = "" + return "background-color: %s" % color + + if pretty_names: + pretty_names_dict = self.get_pretty_name_dict() + else: + pretty_names_dict = {} + + if colors and self.get_fablib_manager().is_jupyter_notebook(): + table = self.get_fablib_manager().show_table( + data, + fields=fields, + title="Switch", + output="pandas", + quiet=True, + pretty_names_dict=pretty_names_dict, + ) + table.applymap(state_color) + + if not quiet: + display(table) + else: + table = self.get_fablib_manager().show_table( + data, + fields=fields, + title="Switch", + output=output, + quiet=quiet, + pretty_names_dict=pretty_names_dict, + ) + + return table + + def get_fim(self) -> FimNode: + """ + Not recommended for most users. + + Gets the node's FABRIC Information Model (fim) object. This method + is used to access data at a lower level than FABlib. + + :return: the FABRIC model node + :rtype: FIMNode + """ + return self.fim_node + + def __set_capacities(self, unit: int = 1): + """ + Sets the capacities of the FABRIC node. + """ + cap = Capacities(unit=unit) + self.get_fim().set_properties(capacities=cap) + + def delete(self): + """ + Remove the switch from the slice. All components and interfaces associated with + the Node are removed from the Slice. + """ + self.get_slice().get_fim_topology().remove_switch(name=self.get_name()) + + def get_interfaces(self) -> List[Interface] or None: + """ + Gets a list of the interfaces associated with the FABRIC node. + + :return: a list of interfaces on the node + :rtype: List[Interface] + """ + interfaces = [] + for name, ifs in self.get_fim().interfaces.items(): + interfaces.append(Interface(node=self, fim_interface=ifs, model="NIC_P4")) + + return interfaces diff --git a/pyproject.toml b/pyproject.toml index e56736f9..8cb855a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi" [project] name = "fabrictestbed-extensions" -version = "1.6.5" +version = "1.7.0" description = "FABRIC Python Client Library and CLI Extensions" authors = [ { name = "Paul Ruth", email = "pruth@renci.org" }, @@ -20,7 +20,7 @@ dependencies = [ "ipyleaflet", "ipycytoscape", "tabulate", - "fabrictestbed==1.6.9", + "fabrictestbed==1.7.1", "paramiko", "jinja2>=3.0.0", "pandas",