From cf73238cf479dac49fb21ce3bdcff667c87d5125 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Mon, 11 Mar 2024 12:19:46 -0700 Subject: [PATCH 01/75] cloudlab quick fix when a user modifies config --- fabfed/provider/cloudlab/cloudlab_provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fabfed/provider/cloudlab/cloudlab_provider.py b/fabfed/provider/cloudlab/cloudlab_provider.py index a392f268..db24222a 100644 --- a/fabfed/provider/cloudlab/cloudlab_provider.py +++ b/fabfed/provider/cloudlab/cloudlab_provider.py @@ -127,11 +127,11 @@ def do_add_resource(self, *, resource: dict): rtype = resource.get(Constants.RES_TYPE) label = resource.get(Constants.LABEL) - if rtype == Constants.RES_TYPE_NETWORK: + if rtype == Constants.RES_TYPE_NETWORK and \ + self.retrieve_attribute_from_saved_state(resource, self.resource_name(resource), attribute='interface'): net_name = self.resource_name(resource) interface = self.retrieve_attribute_from_saved_state(resource, net_name, attribute='interface') - assert interface, "expecting a vlan interface in saved state for {label}" vlan = interface['vlan'] if isinstance(vlan, str): From 8804144a819c1367bfd10390fb8fddd0aa8e0ab7 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Mon, 11 Mar 2024 15:17:44 -0700 Subject: [PATCH 02/75] now fabric nodes can depend on the network instead of the reverse --- fabfed/provider/api/dependency_util.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fabfed/provider/api/dependency_util.py b/fabfed/provider/api/dependency_util.py index 1906279a..62fe5cac 100644 --- a/fabfed/provider/api/dependency_util.py +++ b/fabfed/provider/api/dependency_util.py @@ -12,6 +12,9 @@ def has_resolved_internal_dependencies(*, resource, attribute): def get_values_for_dependency(*, resource, attribute): + assert has_resolved_external_dependencies(resource=resource, attribute=attribute) \ + or has_resolved_internal_dependencies(resource=resource, attribute=attribute) + values = [] for temp in resource.get(attribute): @@ -20,6 +23,10 @@ def get_values_for_dependency(*, resource, attribute): return values +def get_single_value_for_dependency_if_any(*, resource, attribute): + values = get_values_for_dependency(resource=resource, attribute=attribute) + + return values[0] def get_single_value_for_dependency(*, resource, attribute): values = get_values_for_dependency(resource=resource, attribute=attribute) From 5dde6a5c69b9fc0a141318fdd3d30c49b3ca58fc Mon Sep 17 00:00:00 2001 From: abessiari Date: Mon, 11 Mar 2024 18:21:47 -0700 Subject: [PATCH 03/75] Tested fablib 1.4 and allowed for fabric nodes to point network. (#120) * testing fablib 1.5 * Update requirements.txt to 1.6.5 * Update requirements.txt * Update config.fab * Update config.fab --- cicd/test_configs/fabric_l2_vpn/config.fab | 7 +-- fabfed/provider/fabric/fabric_slice.py | 54 ++++++++++++++++------ requirements.txt | 2 +- 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/cicd/test_configs/fabric_l2_vpn/config.fab b/cicd/test_configs/fabric_l2_vpn/config.fab index cb8d29fd..9d941ffd 100644 --- a/cicd/test_configs/fabric_l2_vpn/config.fab +++ b/cicd/test_configs/fabric_l2_vpn/config.fab @@ -24,14 +24,9 @@ resource: image: default_rocky_8 nic_model: NIC_Basic flavor: {'cores': 2, 'ram': 8, 'disk': 10} - - ubuntu_node: - provider: '{{ fabric.fabric_provider }}' - site: '{{ var.fabric_site }}' - count: 1 - image: default_ubuntu_22 + network: '{{ network.fabric_network }}' - network: - fabric_network: provider: '{{ fabric.fabric_provider }}' layer3: "{{ layer3.my_layer }}" - interface: [ '{{ node.rocky_node }}', '{{ node.ubuntu_node }}' ] count: 1 diff --git a/fabfed/provider/fabric/fabric_slice.py b/fabfed/provider/fabric/fabric_slice.py index 1a5ce160..aae1c73e 100644 --- a/fabfed/provider/fabric/fabric_slice.py +++ b/fabfed/provider/fabric/fabric_slice.py @@ -109,32 +109,60 @@ def _add_network(self, resource: dict): def _add_node(self, resource: dict): node_count = resource[Constants.RES_COUNT] label = resource[Constants.LABEL] - states = resource[Constants.SAVED_STATES] - state_map = {} - - for state in states: - state_map[state.attributes['name']] = state.attributes for i in range(node_count): name = self.provider.resource_name(resource, i) if name in self.existing_nodes: - delegate = self.slice_object.get_node(name) + from fabrictestbed_extensions.fablib.node import Node as NodeDelegate + + delegate: NodeDelegate = self.slice_object.get_node(name) assert delegate is not None, "expected to find node {name} in slice {self.name}" - node = FabricNode(label=label, delegate=delegate, network_label="") + network_label = self.provider.retrieve_attribute_from_saved_state(resource, name, 'network_label') + dataplane_ipv4 = self.provider.retrieve_attribute_from_saved_state(resource, name, 'dataplane_ipv4') + node = FabricNode(label=label, delegate=delegate, network_label=network_label) - if name in state_map: - from ipaddress import IPv4Address + if not network_label: + if util.has_resolved_internal_dependencies(resource=resource, attribute='network'): + network = util.get_single_value_for_dependency(resource=resource, attribute='network') + + if network: + # TODO: Need to talk to fablib team and make sure an existing node's interface + # TODO: can be added to a network after it has been created. - attributes = state_map[name] - assert 'dataplane_ipv4' in attributes + from fabrictestbed_extensions.fablib.network_service import NetworkService - if attributes['dataplane_ipv4']: - node.set_used_dataplane_ipv4(IPv4Address(attributes['dataplane_ipv4'])) + itf = node.delegate.add_component(model="NIC_Basic", + name=FABRIC_STITCH_NET_IFACE_NAME).get_interfaces()[0] + self.logger.info(f"Added interface {itf.get_name()} to node {node.get_name()}") + delegate: NetworkService = network.delegate + self.logger.info(f"adding interface {itf.get_name()} to {delegate.get_name()}") + delegate.add_interface(itf) + node.set_network_label(network.label) + + if dataplane_ipv4: + from ipaddress import IPv4Address + + node.set_used_dataplane_ipv4(IPv4Address(dataplane_ipv4)) else: node_builder = NodeBuilder(label, self.slice_object, name, resource) node = node_builder.build() self.slice_modified = True + network: Union[Network, None] = None + + if util.has_resolved_internal_dependencies(resource=resource, attribute='network'): + network = util.get_single_value_for_dependency(resource=resource, attribute='network') + + if network: + from fabrictestbed_extensions.fablib.network_service import NetworkService + + itf = node.delegate.add_component(model="NIC_Basic", + name=FABRIC_STITCH_NET_IFACE_NAME).get_interfaces()[0] + self.logger.info(f"Added interface {itf.get_name()} to node {node.get_name()}") + delegate: NetworkService = network.delegate + self.logger.info(f"adding interface {itf.get_name()} to {delegate.get_name()}") + delegate.add_interface(itf) + node.set_network_label(network.label) self.nodes.append(node) diff --git a/requirements.txt b/requirements.txt index 37a0abc7..e471600e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -fabrictestbed-extensions==1.6.3 +fabrictestbed-extensions==1.6.4 python-chi sense-o-api==1.26 ansible==7.1.0 From 101f1434331c35724e077272faa65a07b3449bf0 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Mon, 11 Mar 2024 22:48:34 -0700 Subject: [PATCH 04/75] fxing corner case for destroy --- fabfed/controller/controller.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/fabfed/controller/controller.py b/fabfed/controller/controller.py index 5f0e6e99..b1ce5925 100644 --- a/fabfed/controller/controller.py +++ b/fabfed/controller/controller.py @@ -392,20 +392,6 @@ def destroy(self, *, provider_states: List[ProviderState]): skip_resources.update([external_state.label for external_state in external_states]) continue - fabric_work_around = False - # TODO: THIS FABRIC SPECIFIC AS WE DON"T SUPPORT SLICE MODIFY API JUST YET - for remaining_resource in remaining_resources: - if provider_label == remaining_resource.provider.label \ - and "@fabric" in remaining_resource.provider.label: - fabric_work_around = True - break - - if fabric_work_around: - self.logger.warning(f"Skipping deleting fabric resource: {resource} with {provider_label}") - remaining_resources.append(resource) - skip_resources.update([external_state.label for external_state in external_states]) - continue - try: provider.delete_resource(resource=resource.attributes) except Exception as e: @@ -431,10 +417,11 @@ def destroy(self, *, provider_states: List[ProviderState]): provider_state.failed = provider.failed for remaining_resource in [r for r in remaining_resources if r.provider.label == provider_state.label]: - resource_states = resource_state_map[remaining_resource.label] - provider_state.add_all(resource_states) + if remaining_resource.label in resource_state_map: + resource_states = resource_state_map[remaining_resource.label] + provider_state.add_all(resource_states) - if provider_state.states(): + if provider_state.states() or provider_state.failed: provider_states.append(provider_state) if exceptions: From 405a79eecf2b393447f2c36cfaa59afa89910d96 Mon Sep 17 00:00:00 2001 From: abessiari Date: Mon, 11 Mar 2024 23:03:38 -0700 Subject: [PATCH 05/75] Update README.md --- examples/stitch/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/stitch/README.md b/examples/stitch/README.md index 95d680a9..ff83ca92 100644 --- a/examples/stitch/README.md +++ b/examples/stitch/README.md @@ -21,11 +21,17 @@ fabfed workflow --help ## validate config fabfed workflow --session -validate +## shows the stitch info ... +fabfed workflow --session -init [-json] + +## prints a summary of what resources will be added or deleted +fabfed workflow --session -plan [-json] + ## provision resources fabfed workflow --session -apply ## display state. -fabfed workflow --session -show +fabfed workflow --session -show [-json] ## display sessions fabfed sessions -show From 77eb9296c7732bfc4f1cdbfb162805e07b712c25 Mon Sep 17 00:00:00 2001 From: abessiari Date: Mon, 11 Mar 2024 23:04:26 -0700 Subject: [PATCH 06/75] Update README.md --- examples/stitch/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/stitch/README.md b/examples/stitch/README.md index ff83ca92..76fbd192 100644 --- a/examples/stitch/README.md +++ b/examples/stitch/README.md @@ -36,6 +36,6 @@ fabfed workflow --session -show [-json] ## display sessions fabfed sessions -show -## destroy Resources. +## destroy resources and the session if sucessful fabfed workflow --session -destroy ``` From 155700b376ab2580a44a35fd45c2f8207e76fd8e Mon Sep 17 00:00:00 2001 From: abessiari Date: Mon, 11 Mar 2024 23:18:21 -0700 Subject: [PATCH 07/75] Update workflow_design.md --- docs/workflow_design.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/workflow_design.md b/docs/workflow_design.md index 3569e3cb..52c7dbc6 100644 --- a/docs/workflow_design.md +++ b/docs/workflow_design.md @@ -102,6 +102,9 @@ chi: key_pair: user: password: + project_id: + tacc: + uc: slice-private-key-location: slice-public-key-location: ``` @@ -241,11 +244,9 @@ resource: # Class ``` # Dependencies -A resource can refer to other resources that it depends on. Line 9 in the example below, states that the fabric_network depends -on the fabric_node. This is an `internal dependency` as both resources are handled by the same provider. The fabfed -controller detects this internal dependency and processes the fabric_node before the fabric_network. -Also note that the `interface` attribute implies that the fabric_network is interested in the `interfaces` -of the fabric_network. +A resource can refer to other resources that it depends on. Line 14 in the example below, states that the fabric_nonde depends +on the fabric_network. This is an `internal dependency` as both resources are handled by the same provider. The fabfed +controller detects this internal dependency and processes the fabric_network before the fabric_network. ``` 1. resource: @@ -256,11 +257,12 @@ of the fabric_network. 6. stitch_with: '{{ network.fabric_network }}' # External Dependency. Normally this would mean fabric_network would be processed first. 7. - fabric_network: 8. provider: '{{ fabric.fabric_provider }}' -9. interface: '{{ node.fabric_node }}' # Internal dependency -10. layer3: "{{ layer3.my_layer }}" +9. layer3: "{{ layer3.my_layer }}" +10. 11 - fabric_node: 12 provider: '{{ fabric.fabric_provider }}' 13 site: '{{ var.fabric_site }}' +14 network: '{{ network.fabric_network }}' # Internal dependency 14 image: default_rocky_8 15 count: 1 ``` From ed95aad2a445d875687e57a557fad5ef7df44af3 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Tue, 12 Mar 2024 01:13:14 -0700 Subject: [PATCH 08/75] fix for removing extra networks --- fabfed/provider/fabric/fabric_slice.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fabfed/provider/fabric/fabric_slice.py b/fabfed/provider/fabric/fabric_slice.py index aae1c73e..0fd43daa 100644 --- a/fabfed/provider/fabric/fabric_slice.py +++ b/fabfed/provider/fabric/fabric_slice.py @@ -35,7 +35,15 @@ def init(self, destroy_phase): if not destroy_phase: self.slice_created = self.slice_object.get_state() == "StableOK" self.existing_nodes = [node.get_name() for node in self.slice_object.get_nodes()] - self.existing_networks = [net.get_name() for net in self.slice_object.get_networks()] + self.existing_networks = [] + + for net in self.slice_object.get_networks(): + net_name = net.get_name() + + if "_aux" in net_name or FABRIC_IPV4_NET_NAME in net_name or FABRIC_IPV6_NET_NAME in net_name: + continue + + self.existing_networks.append(net_name) @property def name(self) -> str: From bcb2a4e5956411d075730ceb4bbe7189ac50bbe1 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Tue, 12 Mar 2024 19:23:01 -0700 Subject: [PATCH 09/75] patch for id token. waiting for ssh per komal --- examples/fabric/config.fab | 28 +--------- examples/stitch/nodes.fab | 2 +- fabfed/provider/fabric/fabric_constants.py | 4 +- fabfed/provider/fabric/fabric_node.py | 4 ++ fabfed/provider/fabric/fabric_provider.py | 11 +++- fabfed/provider/fabric/fabric_slice.py | 54 +++++++++++-------- fabfed/provider/fabric/fabric_slice_helper.py | 36 +++++++++++++ 7 files changed, 85 insertions(+), 54 deletions(-) diff --git a/examples/fabric/config.fab b/examples/fabric/config.fab index 2cdab1a6..590fe3e3 100644 --- a/examples/fabric/config.fab +++ b/examples/fabric/config.fab @@ -1,37 +1,11 @@ -variable: - - fabric_site: - default: WASH - provider: - fabric: - fabric_provider: credential_file: ~/.fabfed/fabfed_credentials.yml profile: fabric -config: - - layer3: - - my_layer: - subnet: 192.168.1.0/24 - gateway: 192.168.1.1 - ip_start: 192.168.1.2 - ip_end: 192.168.1.254 - resource: - node: - rocky_node: provider: '{{ fabric.fabric_provider }}' - site: '{{ var.fabric_site }}' + site: WASH count: 1 - image: default_rocky_8 - nic_model: NIC_Basic - flavor: {'cores': 2, 'ram': 8, 'disk': 10} - - ubuntu_node: - provider: '{{ fabric.fabric_provider }}' - site: '{{ var.fabric_site }}' - count: 1 - image: default_ubuntu_22 - - network: - - fabric_network: - provider: '{{ fabric.fabric_provider }}' - layer3: "{{ layer3.my_layer }}" - interface: [ '{{ node.rocky_node }}', '{{ node.ubuntu_node }}' ] - count: 1 diff --git a/examples/stitch/nodes.fab b/examples/stitch/nodes.fab index 7346c1cd..ca567183 100644 --- a/examples/stitch/nodes.fab +++ b/examples/stitch/nodes.fab @@ -4,7 +4,7 @@ resource: provider: '{{ fabric.fabric_provider }}' count: 1 image: default_rocky_8 - nic_model: NIC_ConnectX_5 + nic_model: NIC_Basic - chi_node: provider: '{{ chi.chi_provider }}' diff --git a/fabfed/provider/fabric/fabric_constants.py b/fabfed/provider/fabric/fabric_constants.py index 56147ae6..8ac31e23 100644 --- a/fabfed/provider/fabric/fabric_constants.py +++ b/fabfed/provider/fabric/fabric_constants.py @@ -40,6 +40,8 @@ # FABRIC public ipv6 subnet FABRIC_PUBLIC_IPV6_SUBNET = IPv6Network('2602:FCFB::0/40') -INCLUDE_FABNETS = False # TODO This should be customizable +INCLUDE_FABNETS = False # TODO This should be customizable FABRIC_SLEEP_AFTER_SUBMIT_OK = 120 # In seconds + +PATCH_FOR_TOKENS = True diff --git a/fabfed/provider/fabric/fabric_node.py b/fabfed/provider/fabric/fabric_node.py index 43e2cba5..9e0a0215 100644 --- a/fabfed/provider/fabric/fabric_node.py +++ b/fabfed/provider/fabric/fabric_node.py @@ -176,8 +176,12 @@ def __init__(self, label, slice_object: Slice, name: str, resource: dict): if INCLUDE_FABNETS: net_iface_v4 = self.node.add_component(model='NIC_Basic', name=FABRIC_IPV4_NET_IFACE_NAME).get_interfaces()[0] + # TODO: KOMAL: + net_iface_v4.set_mode("auto") net_iface_v6 = self.node.add_component(model='NIC_Basic', name=FABRIC_IPV6_NET_IFACE_NAME).get_interfaces()[0] + # TODO: KOMAL + net_iface_v6.set_mode("auto") slice_object.add_l3network(name=f"{name}-{FABRIC_IPV4_NET_NAME}", interfaces=[net_iface_v4], type='IPv4') slice_object.add_l3network(name=f"{name}-{FABRIC_IPV6_NET_NAME}", interfaces=[net_iface_v6], type='IPv6') diff --git a/fabfed/provider/fabric/fabric_provider.py b/fabfed/provider/fabric/fabric_provider.py index 95a13244..8f76660f 100644 --- a/fabfed/provider/fabric/fabric_provider.py +++ b/fabfed/provider/fabric/fabric_provider.py @@ -79,10 +79,14 @@ def setup_environment(self): os.environ['FABRIC_TOKEN_LOCATION'] = token_location + from . import fabric_slice_helper + + fabric_slice_helper.patch_for_token() + + def _init_slice(self, destroy_phase=False): if not self.slice_init: self.logger.info(f"Initializing slice {self.name}") - self.slice_init = True import time from fabfed.util.utils import get_log_level, get_log_location @@ -115,9 +119,12 @@ def _init_slice(self, destroy_phase=False): if attempt == self.retry - 1: raise e - self.logger.info(f"Initializing slice {self.name}. Going to sleep. Will retry ...") + self.logger.info(f"Initializing slice {self.name}. Going to sleep. Will retry ...{e}") time.sleep(2) + self.logger.info(f"Initialized slice {self.name}") + self.slice_init = True + def supports_modify(self): return True diff --git a/fabfed/provider/fabric/fabric_slice.py b/fabfed/provider/fabric/fabric_slice.py index 0fd43daa..bf3e7bd8 100644 --- a/fabfed/provider/fabric/fabric_slice.py +++ b/fabfed/provider/fabric/fabric_slice.py @@ -89,10 +89,7 @@ def _add_network(self, resource: dict): node.set_network_label(label) self.provider.networks.append(net) - - if self.resource_listener: - self.resource_listener.on_added(source=self, provider=self.provider, resource=net) - + self.resource_listener.on_added(source=self, provider=self.provider, resource=net) return network_builder = NetworkBuilder(label, self.provider, self.slice_object, net_name, resource) @@ -110,9 +107,7 @@ def _add_network(self, resource: dict): net = network_builder.build() self.provider.networks.append(net) self.slice_modified = self.slice_created - - if self.resource_listener: - self.resource_listener.on_added(source=self, provider=self.provider, resource=net) + self.resource_listener.on_added(source=self, provider=self.provider, resource=net) def _add_node(self, resource: dict): node_count = resource[Constants.RES_COUNT] @@ -173,9 +168,7 @@ def _add_node(self, resource: dict): node.set_network_label(network.label) self.nodes.append(node) - - if self.resource_listener: - self.resource_listener.on_added(source=self, provider=self.provider, resource=node) + self.resource_listener.on_added(source=self, provider=self.provider, resource=node) def add_resource(self, *, resource: dict): rtype = resource.get(Constants.RES_TYPE) @@ -209,11 +202,15 @@ def _submit_and_wait(self) -> str or None: self.logger.warning(f"Exception occurred while waiting state={state}:{e}") raise e - # try: - # self.slice_object.update() - # self.slice_object.post_boot_config() - # except Exception as e: - # self.logger.warning(f"Exception occurred while update/post_boot_config: {e}") + try: + self.slice_object.wait_ssh() + except Exception as e: + self.logger.warning(f"Exception occurred while waiting on ssh: {e}") + + try: + self.slice_object.post_boot_config() + except Exception as e: + self.logger.warning(f"Exception occurred while update/post_boot_config: {e}") self.logger.info(f"Slice provisioning successful {self.slice_object.get_state()}") @@ -298,6 +295,8 @@ def _do_handle_node_networking(self): self._ensure_management_ips() + # TODO KOMAL + ''' for node in self.nodes: for attempt in range(self.retry): try: @@ -320,6 +319,7 @@ def _do_handle_node_networking(self): self.slice_object = fablib.get_slice(name=self.provider.name) for node in self.nodes: fabric_slice_helper.setup_fabric_networks(self.slice_object, node, node.v4net_name, node.v6net_name) + ''' for network in self.networks: from ipaddress import IPv4Network @@ -446,7 +446,7 @@ def create_resource(self, *, resource: dict): else: self.logger.debug(f"already provisioned. {self.name}: state={state}") - if not self.notified_create and self.resource_listener: + if not self.notified_create: self._handle_node_networking() for node in self.nodes: @@ -462,19 +462,27 @@ def create_resource(self, *, resource: dict): self.slice_created = True self.slice_modified = False self.existing_nodes = [n.name for n in self.nodes] - self.existing_networks = [n.get_name() for n in self.networks] + # self.existing_networks = [n.get_name() for n in self.networks] + self.existing_networks = [] + + for net in self.slice_object.get_networks(): + net_name = net.get_name() + + if "_aux" in net_name or FABRIC_IPV4_NET_NAME in net_name or FABRIC_IPV6_NET_NAME in net_name: + continue + + self.existing_networks.append(net_name) if self.nodes: self._handle_node_networking() - if self.resource_listener: - for node in self.nodes: - self.resource_listener.on_created(source=self, provider=self.provider, resource=node) + for node in self.nodes: + self.resource_listener.on_created(source=self, provider=self.provider, resource=node) - for net in self.networks: - self.resource_listener.on_created(source=self, provider=self.provider, resource=net) + for net in self.networks: + self.resource_listener.on_created(source=self, provider=self.provider, resource=net) - self.notified_create = True + self.notified_create = True def delete_resource(self, *, resource: dict): label = resource.get(Constants.LABEL) diff --git a/fabfed/provider/fabric/fabric_slice_helper.py b/fabfed/provider/fabric/fabric_slice_helper.py index 97c1707c..819be3d5 100644 --- a/fabfed/provider/fabric/fabric_slice_helper.py +++ b/fabfed/provider/fabric/fabric_slice_helper.py @@ -159,3 +159,39 @@ def init_slice(name: str, destroy_phase): assert slice_object.get_state() == "StableOK" return slice_object + + +def patch_for_token(): + if not PATCH_FOR_TOKENS: + return + + def load_tokens(self, refresh: bool = True): + """ + Load Fabric Tokens from the tokens.json if it exists + Otherwise, this is the first attempt, create the tokens and save them + @note this function is invoked when reloading the tokens to ensure tokens + from the token file are read instead of the local variables + """ + import os + import json + + # Load the tokens from the JSON + if os.path.exists(self.token_location): + with open(self.token_location, 'r') as stream: + self.tokens = json.loads(stream.read()) + refresh_token = self.get_refresh_token() + else: + # First time login, use environment variable to load the tokens + from fabrictestbed.util.constants import Constants + + refresh_token = os.environ.get(Constants.CILOGON_REFRESH_TOKEN) + if refresh_token is None: + return + # Renew the tokens to ensure any project_id changes are taken into account + if refresh and self.auto_refresh: + self.refresh_tokens(refresh_token=refresh_token) + + from fabrictestbed.slice_manager.slice_manager import SliceManager + + mngr = SliceManager + mngr._SliceManager__load_tokens = load_tokens From 2fa2579a851598e8520325d9b5135e3b7ae6510d Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Tue, 12 Mar 2024 19:54:12 -0700 Subject: [PATCH 10/75] now cicd will fail as we can support long term token --- cicd/run-fabfed.sh | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/cicd/run-fabfed.sh b/cicd/run-fabfed.sh index 90ab0ecc..bb53eee4 100755 --- a/cicd/run-fabfed.sh +++ b/cicd/run-fabfed.sh @@ -22,12 +22,44 @@ then options="-v $3" fi +echo "***************** APPLYING ****************" echo fabfed workflow -c $conf_dir $options -s $session -apply fabfed workflow -c $conf_dir $options -s $session -apply -fabfed workflow -c $conf_dir $options -s $session -show +ret1=$? + +echo "***************** APPLY SUMMARY ****************" +echo fabfed workflow -c $conf_dir $options -s $session -show -summary +summary=`fabfed workflow -c $conf_dir $options -s $session -show -summary` +fabfed workflow -c $conf_dir $options -s $session -show -summary fabfed sessions -show + echo "***************** DESTOYING ****************" echo fabfed workflow -c $conf_dir $options -s $session -destroy fabfed workflow -c $conf_dir $options -s $session -destroy +ret2=$? + +echo "***************** DESTROY SUMMARY ****************" +fabfed workflow -c $conf_dir $options -s $session -show -summary fabfed sessions -show -exit $? + +echo "***************** APPLY RESULTS ****************" +echo "APPLY RESULTS:" +echo $summary + +if [[ ! $ret1 -eq 0 ]]; then + echo "Apply failed ....." +fi + +if [[ ! $ret2 -eq 0 ]]; then + echo "Destroy failed ....." +fi + +if [[ ! $ret1 -eq 0 ]]; then + exit 1 +fi + +if [[ ! $ret2 -eq 0 ]]; then + exit 2 +fi + +exit 0 From 95f57f9e7dbe5b9839f5ad4ed3ec89980569c282 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Tue, 12 Mar 2024 20:14:33 -0700 Subject: [PATCH 11/75] bridge cicd --- cicd/test_configs/fabric/config.fab | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/cicd/test_configs/fabric/config.fab b/cicd/test_configs/fabric/config.fab index 0ef6556c..36fff00f 100644 --- a/cicd/test_configs/fabric/config.fab +++ b/cicd/test_configs/fabric/config.fab @@ -1,6 +1,8 @@ variable: - fabric_site: default: WASH + - other_fabric_site: + default: SALT provider: - fabric: @@ -18,21 +20,14 @@ config: resource: - node: - rocky_node: - provider: '{{ fabric.fabric_provider }}' + provider: '{{ fabric.fabric_provider }}' site: '{{ var.fabric_site }}' - count: 1 - image: default_rocky_8 - nic_model: NIC_Basic - flavor: {'cores': 2, 'ram': 8, 'disk': 10} - - ubuntu_node: - provider: '{{ fabric.fabric_provider }}' - site: '{{ var.fabric_site }}' - count: 1 - # image: default_ubuntu_22 - image: default_rocky_8 + - other_node: + provider: '{{ fabric.fabric_provider }}' + site: '{{ var.other_fabric_site }}' - network: - fabric_network: provider: '{{ fabric.fabric_provider }}' layer3: "{{ layer3.my_layer }}" - interface: [ '{{ node.rocky_node }}', '{{ node.ubuntu_node }}' ] + interface: [ '{{ node.rocky_node }}', '{{ node.other_node }}' ] count: 1 From b57abc75f8a3446d080510cdcb73095c1537d6c3 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Wed, 13 Mar 2024 00:47:19 -0700 Subject: [PATCH 12/75] refactoring high level fabfed api --- fabfed/controller/controller.py | 13 ++++++++----- fabfed/util/config.py | 24 +++++++++++++----------- fabfed/util/parser.py | 7 +++---- tests/test_workflow.py | 6 +++--- tools/fabfed.py | 20 +++++++++----------- 5 files changed, 36 insertions(+), 34 deletions(-) diff --git a/fabfed/controller/controller.py b/fabfed/controller/controller.py index b1ce5925..d7ca650b 100644 --- a/fabfed/controller/controller.py +++ b/fabfed/controller/controller.py @@ -10,14 +10,17 @@ from ..util.constants import Constants from ..util.config_models import ResourceConfig from ..util.stats import ProviderStats, Duration, Stages +from fabfed.util.utils import get_logger class Controller: - def __init__(self, *, config: WorkflowConfig, logger: logging.Logger, + def __init__(self, *, config: WorkflowConfig, logger: Union[logging.Logger, None] = None, policy: Union[Dict[str, ProviderPolicy], None] = None, use_local_policy=True): - self.config = config - self.logger = logger + import copy + + self.config = copy.deepcopy(config) + self.logger = logger or get_logger() self.provider_factory: Union[ProviderFactory, None] = None self.resources: List[ResourceConfig] = [] self.policy = policy @@ -27,7 +30,7 @@ def __init__(self, *, config: WorkflowConfig, logger: logging.Logger, def init(self, *, session: str, provider_factory: ProviderFactory, provider_states: List[ProviderState]): init_provider_map: Dict[str, bool] = dict() - for provider_config in self.config.get_provider_config(): + for provider_config in self.config.get_provider_configs(): init_provider_map[provider_config.label] = False for resource in self.config.get_resource_configs(): @@ -38,7 +41,7 @@ def init(self, *, session: str, provider_factory: ProviderFactory, provider_stat init_provider_map[provider_state.label] = init_provider_map[provider_state.label] \ or len(provider_state.states()) > 0 - for provider_config in self.config.get_provider_config(): + for provider_config in self.config.get_provider_configs(): if not init_provider_map[provider_config.label]: self.logger.warning(f"Skipping initialization of {provider_config.label}: no resources") continue diff --git a/fabfed/util/config.py b/fabfed/util/config.py index 80db4aa0..b76f9cdd 100644 --- a/fabfed/util/config.py +++ b/fabfed/util/config.py @@ -1,19 +1,21 @@ from .parser import Parser from .config_models import ResourceConfig, ProviderConfig -from typing import List +from typing import List, Union, Dict class WorkflowConfig: - def __init__(self, *, dir_path=None, content=None, var_dict=None): - import time + def __init__(self, *, provider_configs: List[ProviderConfig], resource_configs: List[ResourceConfig]): + self.provider_configs = provider_configs + self.resource_configs = resource_configs - start = time.time() - self.providers, self.resources = Parser.parse(dir_path=dir_path, content=content, var_dict=var_dict) - end = time.time() - self.parse_and_validate_config_duration = end - start - - def get_provider_config(self) -> List[ProviderConfig]: - return self.providers + def get_provider_configs(self) -> List[ProviderConfig]: + return self.provider_configs def get_resource_configs(self) -> List[ResourceConfig]: - return self.resources + return self.resource_configs + + @staticmethod + def parse(*, dir_path: Union[str, None] = None, content: Union[str, None] = None, + var_dict: Union[Dict, None] = None): + provider_configs, resource_configs = Parser.parse(dir_path=dir_path, content=content, var_dict=var_dict) + return WorkflowConfig(provider_configs=provider_configs, resource_configs=resource_configs) diff --git a/fabfed/util/parser.py b/fabfed/util/parser.py index 62f74298..cec01b13 100644 --- a/fabfed/util/parser.py +++ b/fabfed/util/parser.py @@ -1,5 +1,5 @@ from types import SimpleNamespace -from typing import List, Tuple +from typing import List, Tuple, Union from .config_models import * from .constants import Constants @@ -220,9 +220,8 @@ def parse_resource_base_configs(ns_list: List[SimpleNamespace]) -> List[BaseConf return resource_base_configs @staticmethod - def parse(*, dir_path=None, - content=None, - var_dict=None) -> Tuple[List[ProviderConfig], List[ResourceConfig]]: + def parse(*, dir_path: Union[str, None] = None, content: Union[str, None] = None, + var_dict: Union[Dict, None] = None) -> Tuple[List[ProviderConfig], List[ResourceConfig]]: from .utils import load_as_ns_from_yaml diff --git a/tests/test_workflow.py b/tests/test_workflow.py index bdd68e34..f9cc1db5 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -22,7 +22,7 @@ def get_stats(*, states: List[ProviderState]): def run_plan_workflow(*, session, config_str): - config = WorkflowConfig(content=config_str) + config = WorkflowConfig.parse(content=config_str) # logger = utils.init_logger() logger = logging.getLogger(__name__) @@ -35,7 +35,7 @@ def run_plan_workflow(*, session, config_str): def run_apply_workflow(*, session, config_str) -> List[ProviderState]: - config = WorkflowConfig(content=config_str) + config = WorkflowConfig.parse(content=config_str) # logger = utils.init_logger() logger = logging.getLogger(__name__) @@ -56,7 +56,7 @@ def run_apply_workflow(*, session, config_str) -> List[ProviderState]: def run_destroy_workflow(*, session, config_str) -> List[ProviderState]: - config = WorkflowConfig(content=config_str) + config = WorkflowConfig.parse(content=config_str) logger = logging.getLogger(__name__) controller = Controller(config=config, logger=logger) states = sutil.load_states(session) diff --git a/tools/fabfed.py b/tools/fabfed.py index a876fb6f..acf42ce5 100644 --- a/tools/fabfed.py +++ b/tools/fabfed.py @@ -33,7 +33,7 @@ def manage_workflow(args): if args.validate: try: - WorkflowConfig(dir_path=config_dir, var_dict=var_dict) + WorkflowConfig.parse(dir_path=config_dir, var_dict=var_dict) logger.info("config looks ok") except Exception as e: logger.error(f"Validation failed .... {type(e)} {e}") @@ -45,12 +45,12 @@ def manage_workflow(args): import time start = time.time() - config = WorkflowConfig(dir_path=config_dir, var_dict=var_dict) + config = WorkflowConfig.parse(dir_path=config_dir, var_dict=var_dict) + parse_and_validate_config_duration = time.time() - start controller_duration_start = time.time() try: controller = Controller(config=config, - logger=logger, policy=policy, use_local_policy=not args.use_remote_policy) except Exception as e: @@ -121,7 +121,7 @@ def manage_workflow(args): comment="time spent in controller. It does not include time spent in providers") providers_duration = Duration(duration=providers_duration, comment="time spent in all providers") - validate_config_duration = Duration(duration=config.parse_and_validate_config_duration, + validate_config_duration = Duration(duration=parse_and_validate_config_duration, comment="time spent in parsing and validating config") fabfed_stats = FabfedStats(action="apply", @@ -137,9 +137,8 @@ def manage_workflow(args): return if args.init: - config = WorkflowConfig(dir_path=config_dir, var_dict=var_dict) + config = WorkflowConfig.parse(dir_path=config_dir, var_dict=var_dict) controller = Controller(config=config, - logger=logger, policy=policy, use_local_policy=not args.use_remote_policy) states = sutil.load_states(args.session) @@ -148,9 +147,8 @@ def manage_workflow(args): return if args.plan: - config = WorkflowConfig(dir_path=config_dir, var_dict=var_dict) + config = WorkflowConfig.parse(dir_path=config_dir, var_dict=var_dict) controller = Controller(config=config, - logger=logger, policy=policy, use_local_policy=not args.use_remote_policy) states = sutil.load_states(args.session) @@ -186,12 +184,12 @@ def manage_workflow(args): import time start = time.time() - config = WorkflowConfig(dir_path=config_dir, var_dict=var_dict) + config = WorkflowConfig.parse(dir_path=config_dir, var_dict=var_dict) + parse_and_validate_config_duration = time.time() - start controller_duration_start = time.time() try: controller = Controller(config=config, - logger=logger, policy=policy, use_local_policy=not args.use_remote_policy) controller.init(session=args.session, provider_factory=default_provider_factory, provider_states=states) @@ -228,7 +226,7 @@ def manage_workflow(args): controller_duration = Duration(duration=controller_duration, comment="time spent in controller. It does not include time spent in providers") providers_duration = Duration(duration=providers_duration, comment="time spent in all providers") - validate_config_duration = Duration(duration=config.parse_and_validate_config_duration, + validate_config_duration = Duration(duration=parse_and_validate_config_duration, comment="time spent in parsing and validating config") provider_stats = controller.get_stats() From fa2077dda36fafe2b286fcc30980f0e65066d749 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Wed, 13 Mar 2024 09:44:35 -0700 Subject: [PATCH 13/75] function name changed --- fabfed/policy/policy_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabfed/policy/policy_helper.py b/fabfed/policy/policy_helper.py index 8bc987a7..f12688ea 100644 --- a/fabfed/policy/policy_helper.py +++ b/fabfed/policy/policy_helper.py @@ -501,7 +501,7 @@ def handle_stitch_info(config, policy, resources): from fabfed.util.resource_dependency_helper import ResourceDependencyEvaluator, order_resources - dependency_evaluator = ResourceDependencyEvaluator(resources, config.get_provider_config()) + dependency_evaluator = ResourceDependencyEvaluator(resources, config.get_provider_configs()) dependency_map = dependency_evaluator.evaluate() resources = order_resources(dependency_map) From ec025fed189dcac3a6f5d5a8a61094966440e430 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Wed, 13 Mar 2024 09:47:40 -0700 Subject: [PATCH 14/75] using device name --- examples/demos/sense-aws/config.fab | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/demos/sense-aws/config.fab b/examples/demos/sense-aws/config.fab index 6e98d712..d9104943 100644 --- a/examples/demos/sense-aws/config.fab +++ b/examples/demos/sense-aws/config.fab @@ -29,7 +29,7 @@ config: cloud_facility: local_device: local_port: - cloud_region: + cloud_region: "us-east-1" # FOR SENSE and FABRIC local_asn: "55038" # customer_asn (hard coded for now) @@ -58,7 +58,8 @@ resource: count: 1 stitch_with: '{{ network.fabric_network }}' stitch_option: - port_name: agg3.ashb + device_name: agg3.ashb + - fabric_network: provider: '{{ fabric.fabric_provider }}' layer3: "{{ layer3.fab_layer }}" From e8daa3ed277b5e7ac2f454cc7af5eb1a6fca68ba Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Thu, 7 Mar 2024 10:24:48 -0600 Subject: [PATCH 15/75] The initial structure of Jupyter notebook scripts. --- notebooks/examples/README.md | 0 notebooks/examples/native-aws/README.md | 0 notebooks/examples/native-aws/config.fab | 60 ++++++ notebooks/examples/native-gcp/README.md | 0 notebooks/examples/native-gcp/config.fab | 56 ++++++ .../fabfed_credentials_template.yml | 62 ++++++ notebooks/native-aws.ipynb | 183 ++++++++++++++++++ notebooks/native-gcp.ipynb | 175 +++++++++++++++++ 8 files changed, 536 insertions(+) create mode 100644 notebooks/examples/README.md create mode 100644 notebooks/examples/native-aws/README.md create mode 100644 notebooks/examples/native-aws/config.fab create mode 100644 notebooks/examples/native-gcp/README.md create mode 100644 notebooks/examples/native-gcp/config.fab create mode 100644 notebooks/fabfed_config/fabfed_credentials_template.yml create mode 100644 notebooks/native-aws.ipynb create mode 100644 notebooks/native-gcp.ipynb diff --git a/notebooks/examples/README.md b/notebooks/examples/README.md new file mode 100644 index 00000000..e69de29b diff --git a/notebooks/examples/native-aws/README.md b/notebooks/examples/native-aws/README.md new file mode 100644 index 00000000..e69de29b diff --git a/notebooks/examples/native-aws/config.fab b/notebooks/examples/native-aws/config.fab new file mode 100644 index 00000000..c732c2d4 --- /dev/null +++ b/notebooks/examples/native-aws/config.fab @@ -0,0 +1,60 @@ +provider: + - aws: + - aws_provider: + - credential_file: ~/.fabfed/fabfed_credentials.yml + profile: aws + - fabric: + - fabric_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: fabric + +config: + - layer3: + - fab_layer: + subnet: 192.168.10.0/24 + gateway: 192.168.10.1 + ip_start: 192.168.10.2 + ip_end: 192.168.10.254 + - aws_layer: + subnet: 10.0.1.0/24 + - peering: + - my_peering: + # FOR FABRIC + cloud_account: "296256999979" + + # FOR AWS and FABRIC + cloud_region: "us-east-1" + + cloud_vpc: "vpc-0936b973cf039f794" + remote_asn: 64512 # amazon_asn + local_asn: 55038 # customer_asn + + # FOR FABRIC + local_address: "192.168.1.1/30" # customer_ip + remote_address: "192.168.1.2/30" # amazon_ip + +resource: + - network: + - aws_net: + provider: '{{ aws.aws_provider }}' + name: aws-net + layer3: "{{ layer3.aws_layer }}" + peering: "{{ peering.my_peering }}" + stitch_with: '{{ network.fabric_network }}' + stitch_option: + device_name: agg3.ashb.net.internet2.edu + count: 1 + + - fabric_network: + provider: '{{ fabric.fabric_provider }}' + layer3: "{{ layer3.fab_layer }}" + peering: "{{ peering.my_peering }}" + interface: '{{ node.fabric_node }}' + count: 1 + - node: + - fabric_node: + provider: '{{ fabric.fabric_provider }}' + site: MAX # Use RENC when deploying to fabric beta environment + image: default_rocky_8 + count: 1 + nic_model: NIC_Basic diff --git a/notebooks/examples/native-gcp/README.md b/notebooks/examples/native-gcp/README.md new file mode 100644 index 00000000..e69de29b diff --git a/notebooks/examples/native-gcp/config.fab b/notebooks/examples/native-gcp/config.fab new file mode 100644 index 00000000..25aa1647 --- /dev/null +++ b/notebooks/examples/native-gcp/config.fab @@ -0,0 +1,56 @@ +provider: + - gcp: + - gcp_provider: + - credential_file: ~/.fabfed/fabfed_credentials.yml + profile: gcp + - fabric: + - fabric_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: fabric + +config: + - layer3: + - fab_layer: + subnet: 192.168.10.0/24 + gateway: 192.168.10.1 + ip_start: 192.168.10.2 + ip_end: 192.168.10.254 + - gcp_layer: + subnet: 10.100.0.0/24 # subnet.cidr and vpc.cidr + - peering: + - my_peering: + # FOR GCP + cloud_region: "us-east4" + cloud_vpc: "vpc-69acc1d9-8c24-47cd-90b8-33be57167dbf" + + # uncomment and override cloud_vlan with a value between 3 and 4095. The default is 4. + # cloud_vlan: + + # FOR GCP AND FABRIC. + remote_asn: 16550 # google_asn + + # FOR FABRIC + local_address: "192.168.1.1/30" # customer_ip + remote_address: "192.168.1.2/30" # google_ip +resource: + - network: + - gcp_net: + provider: '{{ gcp.gcp_provider }}' + name: gcp-net + layer3: "{{ layer3.gcp_layer }}" + peering: "{{ peering.my_peering }}" + stitch_with: '{{ network.fabric_network }}' + count: 1 + - fabric_network: + provider: '{{ fabric.fabric_provider }}' + layer3: "{{ layer3.fab_layer }}" + peering: "{{ peering.my_peering }}" + interface: '{{ node.fabric_node }}' + count: 1 + - node: + - fabric_node: + provider: '{{ fabric.fabric_provider }}' + site: MAX + image: default_rocky_8 + count: 1 + nic_model: NIC_Basic diff --git a/notebooks/fabfed_config/fabfed_credentials_template.yml b/notebooks/fabfed_config/fabfed_credentials_template.yml new file mode 100644 index 00000000..3550eb64 --- /dev/null +++ b/notebooks/fabfed_config/fabfed_credentials_template.yml @@ -0,0 +1,62 @@ +# Fabric +# https://portal.fabric-testbed.net +# Create and download FABRIC at https://portal.fabric-testbed.net/ via Experiments -> Manage Tokens +# Create a Bastion Key and User FABRIC at https://portal.fabric-testbed.net/ via Experiments -> Manage SSH Keys +fabric: + bastion-user-name: + token-location: + bastion-key-location: + project_id: + slice-private-key-location: + slice-public-key-location: + +# Chameleon +# https://www.chameleoncloud.org/ +# Create a CLI Password as described here: https://chameleoncloud.readthedocs.io/en/latest/technical/cli.html +# Project Name can be found https://www.chameleoncloud.org/ +chi: + project_name: + key_pair: + user: + password: + slice-private-key-location: + slice-public-key-location: + +# https://sense-o.es.net:8443/StackV-web/portal/ +sense: + auth_endpoint: https://sense-o.es.net:8543/auth/realms/StackV/protocol/openid-connect/token + api_endpoint: https://sense-o-dev.es.net:8443/StackV-web/restapi + client_id: StackV + username: + password: + secret: + verify: False + slice-private-key-location: # THIS IS NEEDED IF USING JANUS SERVICE + + +janus: + url: + username: + password: + token: + +# Cloudlab +# https://cloudlab.us +# Create an account and join project fabfed +# Must download certificate and reference it below +cloudlab: + project: + certificate: + user: + slice-private-key-location: + +# Amazon AWS +aws: + access_key: + secret_key: + +# Google Cloud +# Create a service account and downdload it to the path below +gcp: + service_key_path: + project: diff --git a/notebooks/native-aws.ipynb b/notebooks/native-aws.ipynb new file mode 100644 index 00000000..854a7ea3 --- /dev/null +++ b/notebooks/native-aws.ipynb @@ -0,0 +1,183 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", + "metadata": {}, + "source": [ + "# Native AWS + Fabric\n", + "Stitching native AWS and Fabric resources." + ] + }, + { + "cell_type": "markdown", + "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", + "metadata": {}, + "source": [ + "## Assign session name and .fab path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8efd310-300a-4f56-bb26-5431bdec3969", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "session_name = \"native-aws-lzhang9\"\n", + "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", + "fab_file_path = \"./examples/native-aws\"\n", + "\n", + "print(session_name)\n", + "print(fab_file_path)" + ] + }, + { + "cell_type": "markdown", + "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", + "metadata": {}, + "source": [ + "## Validate .fab files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72148135-c66f-41ed-9a36-b037c7fbb966", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -validate" + ] + }, + { + "cell_type": "markdown", + "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", + "metadata": {}, + "source": [ + "## Add resources to providers and validate resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0aa67fe6-34b4-4a20-b9b3-a5fb6921efd2", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -plan" + ] + }, + { + "cell_type": "markdown", + "id": "c29bf401-66d9-4b8b-9c68-fd2ebbd85ee2", + "metadata": {}, + "source": [ + "## View stitching information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d3324c9-004e-4c29-b768-08b2de294ed3", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" + ] + }, + { + "cell_type": "markdown", + "id": "d19fef3d-2d37-4ffc-8f8f-07446bc5a484", + "metadata": {}, + "source": [ + "## Create resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5e16ead-338b-4e3c-9e38-61203a2489f4", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -apply" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4f5add4c-2e74-4934-9086-50cc9aab69a9", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", + "metadata": {}, + "source": [ + "## View the state of the workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -show" + ] + }, + { + "cell_type": "markdown", + "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", + "metadata": {}, + "source": [ + "## Delete all resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57305ac9-fa7e-4578-a200-ecdcb74a2a4c", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -destroy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee0f1096-c026-46ca-b39b-4c8a59e3d5f8", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/native-gcp.ipynb b/notebooks/native-gcp.ipynb new file mode 100644 index 00000000..c1540cff --- /dev/null +++ b/notebooks/native-gcp.ipynb @@ -0,0 +1,175 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", + "metadata": {}, + "source": [ + "# Native GCP + Fabric\n", + "Stitching native GCP and Fabric resources." + ] + }, + { + "cell_type": "markdown", + "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", + "metadata": {}, + "source": [ + "## Assign session name and .fab path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8efd310-300a-4f56-bb26-5431bdec3969", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "session_name = \"native-gcp-lzhang9\"\n", + "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", + "fab_file_path = \"./examples/native-gcp\"\n", + "\n", + "print(session_name)\n", + "print(fab_file_path)" + ] + }, + { + "cell_type": "markdown", + "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", + "metadata": {}, + "source": [ + "## Validate .fab files " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72148135-c66f-41ed-9a36-b037c7fbb966", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -validate" + ] + }, + { + "cell_type": "markdown", + "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", + "metadata": {}, + "source": [ + "## Add resources to providers and validate resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0aa67fe6-34b4-4a20-b9b3-a5fb6921efd2", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -plan" + ] + }, + { + "cell_type": "markdown", + "id": "c29bf401-66d9-4b8b-9c68-fd2ebbd85ee2", + "metadata": {}, + "source": [ + "## View stitching information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d3324c9-004e-4c29-b768-08b2de294ed3", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" + ] + }, + { + "cell_type": "markdown", + "id": "d19fef3d-2d37-4ffc-8f8f-07446bc5a484", + "metadata": {}, + "source": [ + "## Create resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5e16ead-338b-4e3c-9e38-61203a2489f4", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -apply" + ] + }, + { + "cell_type": "markdown", + "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", + "metadata": {}, + "source": [ + "## View the state of the workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -show" + ] + }, + { + "cell_type": "markdown", + "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", + "metadata": {}, + "source": [ + "## Delete all resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57305ac9-fa7e-4578-a200-ecdcb74a2a4c", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -destroy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee0f1096-c026-46ca-b39b-4c8a59e3d5f8", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From d7b41f7f37d4c217361d7a171491fceeacb74497 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Thu, 7 Mar 2024 17:54:27 -0600 Subject: [PATCH 16/75] Update aws config.fab --- notebooks/examples/native-aws/config.fab | 25 +++++++++++------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/notebooks/examples/native-aws/config.fab b/notebooks/examples/native-aws/config.fab index c732c2d4..7ec7245d 100644 --- a/notebooks/examples/native-aws/config.fab +++ b/notebooks/examples/native-aws/config.fab @@ -1,13 +1,15 @@ +variable: + - vpc: + default: vpc-0936b973cf039f794 provider: - aws: - aws_provider: - - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: aws + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: aws - fabric: - fabric_provider: credential_file: ~/.fabfed/fabfed_credentials.yml profile: fabric - config: - layer3: - fab_layer: @@ -20,19 +22,15 @@ config: - peering: - my_peering: # FOR FABRIC - cloud_account: "296256999979" - + cloud_account: "296256999979" # FOR AWS and FABRIC cloud_region: "us-east-1" - - cloud_vpc: "vpc-0936b973cf039f794" + cloud_vpc: '{{ var.vpc }}' remote_asn: 64512 # amazon_asn - local_asn: 55038 # customer_asn - + local_asn: 55038 # customer_asn # FOR FABRIC local_address: "192.168.1.1/30" # customer_ip remote_address: "192.168.1.2/30" # amazon_ip - resource: - network: - aws_net: @@ -42,15 +40,14 @@ resource: peering: "{{ peering.my_peering }}" stitch_with: '{{ network.fabric_network }}' stitch_option: - device_name: agg3.ashb.net.internet2.edu - count: 1 - + device_name: agg3.ashb + count: 1 - fabric_network: provider: '{{ fabric.fabric_provider }}' layer3: "{{ layer3.fab_layer }}" peering: "{{ peering.my_peering }}" interface: '{{ node.fabric_node }}' - count: 1 + count: 1 - node: - fabric_node: provider: '{{ fabric.fabric_provider }}' From 84738c492464d9fbcd2923bbe72f58f9bc8e81ba Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 8 Mar 2024 19:22:21 -0600 Subject: [PATCH 17/75] Updated native gcp config.fab --- notebooks/examples/native-gcp/config.fab | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/notebooks/examples/native-gcp/config.fab b/notebooks/examples/native-gcp/config.fab index 25aa1647..03c1192d 100644 --- a/notebooks/examples/native-gcp/config.fab +++ b/notebooks/examples/native-gcp/config.fab @@ -1,13 +1,15 @@ +variable: + - vpc: + default: vpc-69acc1d9-8c24-47cd-90b8-33be57167dbf provider: - gcp: - gcp_provider: - - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: gcp + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: gcp - fabric: - fabric_provider: credential_file: ~/.fabfed/fabfed_credentials.yml profile: fabric - config: - layer3: - fab_layer: @@ -21,14 +23,12 @@ config: - my_peering: # FOR GCP cloud_region: "us-east4" - cloud_vpc: "vpc-69acc1d9-8c24-47cd-90b8-33be57167dbf" - + cloud_vpc: '{{ var.vpc }}' # uncomment and override cloud_vlan with a value between 3 and 4095. The default is 4. # cloud_vlan: # FOR GCP AND FABRIC. remote_asn: 16550 # google_asn - # FOR FABRIC local_address: "192.168.1.1/30" # customer_ip remote_address: "192.168.1.2/30" # google_ip @@ -52,5 +52,5 @@ resource: provider: '{{ fabric.fabric_provider }}' site: MAX image: default_rocky_8 - count: 1 + count: 1 nic_model: NIC_Basic From 38e9fcd65aa213f396ee1009913397a30bfffa04 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 8 Mar 2024 19:22:29 -0600 Subject: [PATCH 18/75] Update native-gcp.ipynb --- notebooks/native-gcp.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/native-gcp.ipynb b/notebooks/native-gcp.ipynb index c1540cff..e13fc5ed 100644 --- a/notebooks/native-gcp.ipynb +++ b/notebooks/native-gcp.ipynb @@ -27,7 +27,7 @@ "import os\n", "\n", "session_name = \"native-gcp-lzhang9\"\n", - "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", + "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/gcp\"\n", "fab_file_path = \"./examples/native-gcp\"\n", "\n", "print(session_name)\n", From d3b0e7ab75dd2110b3623cdeb3021b215930abac Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 8 Mar 2024 20:15:45 -0600 Subject: [PATCH 19/75] Update native-aws.ipynb --- notebooks/native-aws.ipynb | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/notebooks/native-aws.ipynb b/notebooks/native-aws.ipynb index 854a7ea3..54e01085 100644 --- a/notebooks/native-aws.ipynb +++ b/notebooks/native-aws.ipynb @@ -100,20 +100,14 @@ "cell_type": "code", "execution_count": null, "id": "e5e16ead-338b-4e3c-9e38-61203a2489f4", - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "!fabfed workflow -s $session_name -c $fab_file_path -apply" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "4f5add4c-2e74-4934-9086-50cc9aab69a9", - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "markdown", "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", @@ -151,12 +145,15 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "id": "ee0f1096-c026-46ca-b39b-4c8a59e3d5f8", + "cell_type": "markdown", + "id": "138944ac-3892-41e2-a9cc-2c59383b5d63", "metadata": {}, - "outputs": [], - "source": [] + "source": [ + "## Note: Amazon Instance Info\n", + "- EC2/Instances/i-078a7be6f4a3df5d6\n", + "- Private IPv4 10.0.1.106\n", + "- Public IPv4 54.242.87.34" + ] } ], "metadata": { From 78f6ad5e362d3645776350daef0a84a4a51dde1e Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 8 Mar 2024 21:22:24 -0600 Subject: [PATCH 20/75] Create sense-gcp config.fab --- notebooks/examples/sense-gcp/config.fab | 78 +++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 notebooks/examples/sense-gcp/config.fab diff --git a/notebooks/examples/sense-gcp/config.fab b/notebooks/examples/sense-gcp/config.fab new file mode 100644 index 00000000..6ae27417 --- /dev/null +++ b/notebooks/examples/sense-gcp/config.fab @@ -0,0 +1,78 @@ +provider: + - sense: + - sense_provider: + - credential_file: ~/.fabfed/fabfed_credentials.yml + profile: sense + - fabric: + - fabric_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: fabric + - janus: + - janus_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: janus +config: + - layer3: + - fab_layer: + subnet: 192.168.10.0/24 # subnet: "10.0.0.0/24" # for sense subnet.cidr + gateway: 192.168.10.1 + ip_start: 192.168.10.2 # optional: auto generate if subnet is given + ip_end: 192.168.10.254 + - sense_layer: + subnet: 10.200.1.0/24 # subnet.cidr and vpc.cidr + - peering: + - my_peering: + # FOR FABRIC + # cloud_account: "296256999979" # pairing key + # cloud_facility: + # local_device: "agg3.ashb.net.internet2.edu" + # local_port: "Bundle-Ether5" + # cloud_region: "us-east4" + # cloud_vlan: + + # FOR SENSE and FABRIC + local_asn: "55038" # customer_asn (hard coded for now) + local_address: "192.168.1.1/30" # customer_ip (ignore) + remote_asn: "16550" # google_asn (default) + remote_address: "192.168.1.2/30" # google_ip (ignore) +resource: + - node: + - sense_node: + provider: '{{ sense.sense_provider }}' + name: gcp-node + network: "{{ network.sense_net }}" + service: + count: 1 + - fabric_node: + provider: '{{ fabric.fabric_provider }}' + site: MAX + image: default_rocky_8 + count: 1 + nic_model: NIC_Basic + - network: + - sense_net: + provider: '{{ sense.sense_provider }}' + name: gcp-net + layer3: "{{ layer3.sense_layer }}" + peering: "{{ peering.my_peering }}" + count: 1 + profile: FABRIC-GCP-INTERCON + stitch_with: '{{ network.fabric_network }}' + profile: FABRIC-GCP-INTERCON + stitch_option: + group_name: GCP + + - fabric_network: + provider: '{{ fabric.fabric_provider }}' + layer3: "{{ layer3.fab_layer }}" + peering: "{{ peering.my_peering }}" + interface: '{{ node.fabric_node }}' + count: 1 + - service: + - dtn_service: + provider: '{{ janus.janus_provider }}' + node: [ '{{ node.sense_node }}', '{{ node.fabric_node }}' ] + controller: '{{ node.fabric_node }}' + image: dtnaas/tools + profile: fabfed + count: 0 # SET THIS TO 1 if you want use janus service From 4074ce586518b764077399551352ef8d74db8e62 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 8 Mar 2024 21:28:37 -0600 Subject: [PATCH 21/75] Create sense-gcp.ipynb --- notebooks/sense-gcp.ipynb | 178 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 notebooks/sense-gcp.ipynb diff --git a/notebooks/sense-gcp.ipynb b/notebooks/sense-gcp.ipynb new file mode 100644 index 00000000..beabdcb8 --- /dev/null +++ b/notebooks/sense-gcp.ipynb @@ -0,0 +1,178 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", + "metadata": {}, + "source": [ + "# SENSE GCP + Fabric\n", + "Stitching SENSE GCP and Fabric resources." + ] + }, + { + "cell_type": "markdown", + "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", + "metadata": {}, + "source": [ + "## Assign session name and .fab path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8efd310-300a-4f56-bb26-5431bdec3969", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "session_name = \"sense-gcp-lzhang9\"\n", + "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/gcp\"\n", + "fab_file_path = \"./examples/sense-gcp\"\n", + "\n", + "print(session_name)\n", + "print(fab_file_path)" + ] + }, + { + "cell_type": "markdown", + "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", + "metadata": {}, + "source": [ + "## Validate .fab files " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72148135-c66f-41ed-9a36-b037c7fbb966", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -validate" + ] + }, + { + "cell_type": "markdown", + "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", + "metadata": {}, + "source": [ + "## Add resources to providers and validate resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0aa67fe6-34b4-4a20-b9b3-a5fb6921efd2", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -plan" + ] + }, + { + "cell_type": "markdown", + "id": "c29bf401-66d9-4b8b-9c68-fd2ebbd85ee2", + "metadata": {}, + "source": [ + "## View stitching information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d3324c9-004e-4c29-b768-08b2de294ed3", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" + ] + }, + { + "cell_type": "markdown", + "id": "d19fef3d-2d37-4ffc-8f8f-07446bc5a484", + "metadata": {}, + "source": [ + "## Create resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5e16ead-338b-4e3c-9e38-61203a2489f4", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -apply" + ] + }, + { + "cell_type": "markdown", + "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", + "metadata": {}, + "source": [ + "## View the state of the workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -show" + ] + }, + { + "cell_type": "markdown", + "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", + "metadata": {}, + "source": [ + "## Delete all resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57305ac9-fa7e-4578-a200-ecdcb74a2a4c", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -destroy" + ] + }, + { + "cell_type": "markdown", + "id": "67883063-3a57-4cdd-b14e-af35ea912f74", + "metadata": {}, + "source": [ + "## Note: VM Instance Info\n", + "- EC2/Instances/i-078a7be6f4a3df5d6\n", + "- Private IPv4 10.200.1.2\n", + "- Public IPv4 34.86.164.172" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 7641b2a25dc61929d66f4564a8a0ccdf3c7c17e2 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Sat, 9 Mar 2024 20:36:25 -0600 Subject: [PATCH 22/75] Create sense-aws.ipynb --- notebooks/sense-aws.ipynb | 180 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 notebooks/sense-aws.ipynb diff --git a/notebooks/sense-aws.ipynb b/notebooks/sense-aws.ipynb new file mode 100644 index 00000000..5a532c81 --- /dev/null +++ b/notebooks/sense-aws.ipynb @@ -0,0 +1,180 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", + "metadata": {}, + "source": [ + "# SENSE AWS + Fabric\n", + "Stitching SENSE AWS and Fabric resources." + ] + }, + { + "cell_type": "markdown", + "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", + "metadata": {}, + "source": [ + "## Assign session name and .fab path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8efd310-300a-4f56-bb26-5431bdec3969", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "session_name = \"sense-aws-lzhang9\"\n", + "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", + "fab_file_path = \"./examples/sense-aws\"\n", + "\n", + "print(session_name)\n", + "print(fab_file_path)" + ] + }, + { + "cell_type": "markdown", + "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", + "metadata": {}, + "source": [ + "## Validate .fab files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72148135-c66f-41ed-9a36-b037c7fbb966", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -validate" + ] + }, + { + "cell_type": "markdown", + "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", + "metadata": {}, + "source": [ + "## Add resources to providers and validate resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0aa67fe6-34b4-4a20-b9b3-a5fb6921efd2", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -plan" + ] + }, + { + "cell_type": "markdown", + "id": "c29bf401-66d9-4b8b-9c68-fd2ebbd85ee2", + "metadata": {}, + "source": [ + "## View stitching information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d3324c9-004e-4c29-b768-08b2de294ed3", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" + ] + }, + { + "cell_type": "markdown", + "id": "d19fef3d-2d37-4ffc-8f8f-07446bc5a484", + "metadata": {}, + "source": [ + "## Create resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5e16ead-338b-4e3c-9e38-61203a2489f4", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -apply" + ] + }, + { + "cell_type": "markdown", + "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", + "metadata": {}, + "source": [ + "## View the state of the workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -show" + ] + }, + { + "cell_type": "markdown", + "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", + "metadata": {}, + "source": [ + "## Delete all resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57305ac9-fa7e-4578-a200-ecdcb74a2a4c", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed workflow -s $session_name -c $fab_file_path -destroy" + ] + }, + { + "cell_type": "markdown", + "id": "138944ac-3892-41e2-a9cc-2c59383b5d63", + "metadata": {}, + "source": [ + "## Note: Amazon Instance Info\n", + "- EC2/Instances/i-078a7be6f4a3df5d6\n", + "- Private IPv4 10.0.1.106\n", + "- Public IPv4 54.242.87.34" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 8c4ef17ff4d07395e3fb8c9b3d31c88e4e33becb Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Sat, 9 Mar 2024 20:39:14 -0600 Subject: [PATCH 23/75] Create config.fab --- notebooks/examples/sense-aws/config.fab | 63 +++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 notebooks/examples/sense-aws/config.fab diff --git a/notebooks/examples/sense-aws/config.fab b/notebooks/examples/sense-aws/config.fab new file mode 100644 index 00000000..a6333942 --- /dev/null +++ b/notebooks/examples/sense-aws/config.fab @@ -0,0 +1,63 @@ +provider: + - sense: + - sense_provider: + - credential_file: ~/.fabfed/fabfed_credentials.yml + profile: sense + - fabric: + - fabric_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: fabric +config: + - layer3: + - fab_layer: + subnet: 192.168.10.0/24 # subnet: "10.0.0.0/24" # for sense subnet.cidr + gateway: 192.168.10.1 + ip_start: 192.168.10.2 # optional: auto generate if subnet is given + ip_end: 192.168.10.254 + - sense_layer: + subnet: 10.200.1.0/24 # subnet.cidr and vpc.cidr + - peering: + - my_peering: + # FOR FABRIC + cloud_account: "296256999979" + cloud_facility: + local_device: + local_port: + cloud_region: "us-east-1" + + # FOR SENSE and FABRIC + local_asn: "55038" # customer_asn (hard coded for now) + local_address: "192.168.1.1/30" # customer_ip + remote_asn: "64512" # amazon_asn + remote_address: "192.168.1.2/30" # amazon_ip +resource: + - node: + - sense_node: + provider: '{{ sense.sense_provider }}' + name: aws-node + network: "{{ network.sense_net }}" + service: + count: 1 + - fabric_node: + provider: '{{ fabric.fabric_provider }}' + site: MAX # Use RENC when deploying to fabric beta environment + image: default_rocky_8 + count: 1 + nic_model: NIC_Basic + - network: + - sense_net: + provider: '{{ sense.sense_provider }}' + name: aws-net + layer3: "{{ layer3.sense_layer }}" + peering: "{{ peering.my_peering }}" + count: 1 + stitch_with: '{{ network.fabric_network }}' + stitch_option: + device_name: agg3.ashb + + - fabric_network: + provider: '{{ fabric.fabric_provider }}' + layer3: "{{ layer3.fab_layer }}" + peering: "{{ peering.my_peering }}" + interface: '{{ node.fabric_node }}' + count: 1 From 32f788266181dd87926d0f82cd16c28aa839be8d Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Sat, 9 Mar 2024 20:39:28 -0600 Subject: [PATCH 24/75] Create fablib-tools.ipynb --- notebooks/fablib-tools.ipynb | 308 +++++++++++++++++++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 notebooks/fablib-tools.ipynb diff --git a/notebooks/fablib-tools.ipynb b/notebooks/fablib-tools.ipynb new file mode 100644 index 00000000..f5199337 --- /dev/null +++ b/notebooks/fablib-tools.ipynb @@ -0,0 +1,308 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "57c387fd-49d1-4d47-ab64-910d94ec0244", + "metadata": {}, + "source": [ + "# FABLIB Tools\n" + ] + }, + { + "cell_type": "markdown", + "id": "1c84b882-9d63-4c45-a8ae-5cb22c1191a2", + "metadata": {}, + "source": [ + "## Import the FABlib Library\n", + "Chose one of two working enviroments to import FABlib.\n", + "- fabric-testbed\n", + "- local host" + ] + }, + { + "cell_type": "markdown", + "id": "04c03230-61f4-48e5-a5bc-031f494e5c2a", + "metadata": {}, + "source": [ + "### Environment: Fabric-Testbed" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25e4df8b-4eed-4fdf-b6ac-2a801945b251", + "metadata": {}, + "outputs": [], + "source": [ + "from fabrictestbed_extensions.fablib.fablib import FablibManager as fablib_manager\n", + "import json\n", + "\n", + "try:\n", + " fablib = fablib_manager()\n", + " \n", + " fablib.show_config();\n", + "except Exception as e:\n", + " print(f\"Exception: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "1df07de8-ac57-41da-966f-8865d4cb42e7", + "metadata": {}, + "source": [ + "### Environment: Local Host" + ] + }, + { + "cell_type": "markdown", + "id": "07c7fbc9-730c-4cb9-a1b4-03afb1ad5fa8", + "metadata": {}, + "source": [ + "Get Fabric configuration from FABFED credential file\n", + "- Set the path to fabfed credential file\n", + "- Read the fabric configuration from fabfed crential file\n", + "- Set the path to ssh_config the same as that of the bastion_ssh_key\n", + "- Generate fabric_rc\n", + "- Import Fablib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3dde8e65-5de4-4d47-a9cf-08c09d9f9394", + "metadata": {}, + "outputs": [], + "source": [ + "# Path to your YAML file\n", + "credential_file_path = 'fabfed_config/fabfed_credentials.yml'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "73906428-445a-47d9-850c-859eac86e8d1", + "metadata": {}, + "outputs": [], + "source": [ + "import yaml\n", + "\n", + "# Read the YAML file\n", + "with open(credential_file_path, 'r') as file:\n", + " try:\n", + " yaml_data = yaml.safe_load(file)\n", + " except yaml.YAMLError as exc:\n", + " print(exc)\n", + "\n", + "fabric_config = yaml_data['fabric']\n", + "\n", + "def print_dict(dictionary, indent=0):\n", + " for key, value in dictionary.items():\n", + " if isinstance(value, dict):\n", + " print(' ' * indent + str(key) + ':')\n", + " print_dict(value, indent + 1)\n", + " else:\n", + " print(' ' * indent + str(key) + ': ' + str(value))\n", + " \n", + "print_dict(fabric_config)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a87ec9b-5c69-4b61-b7e6-c6d78355fbfe", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "home_directory = os.path.expanduser(\"~\")\n", + "print(\"Home directory:\", home_directory)\n", + "\n", + "bastion_ssh_config_file = os.path.dirname(fabric_config['bastion-key-location']) + \"/ssh-config\"\n", + "print(f\"ssh_config path: {bastion_ssh_config_file}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60790e7d-245a-416a-bcee-0ed3d5c49306", + "metadata": {}, + "outputs": [], + "source": [ + "# Data to write to the file\n", + "data = f\"\"\"\n", + "export FABRIC_CREDMGR_HOST=cm.fabric-testbed.net\n", + "export FABRIC_ORCHESTRATOR_HOST=orchestrator.fabric-testbed.net\n", + "export FABRIC_PROJECT_ID={fabric_config['project_id']}\n", + "export FABRIC_TOKEN_LOCATION={fabric_config['token-location']}\n", + "\n", + "export FABRIC_BASTION_HOST=bastion.fabric-testbed.net\n", + "export FABRIC_BASTION_USERNAME={fabric_config['bastion-user-name']}\n", + "\n", + "export FABRIC_BASTION_KEY_LOCATION={fabric_config['bastion-key-location']}\n", + "#export FABRIC_BASTION_KEY_PASSPHRASE=\n", + "\n", + "export FABRIC_BASTION_SSH_CONFIG_FILE={bastion_ssh_config_file}\n", + "\n", + "export FABRIC_SLICE_PRIVATE_KEY_FILE={fabric_config['slice-private-key-location']}\n", + "export FABRIC_SLICE_PUBLIC_KEY_FILE={fabric_config['slice-public-key-location']} \n", + "#export FABRIC_SLICE_PRIVATE_KEY_PASSPHRASE=\n", + "\n", + "export FABRIC_LOG_FILE={home_directory}/fablib.log\n", + "export FABRIC_LOG_LEVEL=INFO \n", + "\n", + "export FABRIC_AVOID=''\n", + "\n", + "export FABRIC_SSH_COMMAND_LINE=\"ssh -i {{ _self_.private_ssh_key_file }} -F {bastion_ssh_config_file} {{ _self_.username }}@{{ _self_.management_ip }}\"\n", + "\"\"\"\n", + "\n", + "# Specify the file path\n", + "fabric_rc_file = \"./fabric_config/fabric_rc\"\n", + "\n", + "# Delete if it exists\n", + "if os.path.exists(fabric_rc_file):\n", + " # Delete the file\n", + " os.remove(fabric_rc_file)\n", + " \n", + "# Open the file in write mode\n", + "with open(fabric_rc_file, \"w\") as file:\n", + " # Write the data to the file\n", + " file.write(data)\n", + "\n", + "print(\"Data has been written to\", fabric_rc_file)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2714cb84-3915-4272-9e43-f8eac4d06994", + "metadata": {}, + "outputs": [], + "source": [ + "from fabrictestbed_extensions.fablib.fablib import FablibManager as fablib_manager\n", + "import json\n", + "\n", + "try:\n", + " fablib = fablib_manager(fabric_rc=fabric_rc_file)\n", + " \n", + " fablib.show_config();\n", + "except Exception as e:\n", + " print(f\"Exception: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "5895ddb9-1a78-4efc-a700-5cdd0a191ca6", + "metadata": {}, + "source": [ + "## Show Slice Information" + ] + }, + { + "cell_type": "markdown", + "id": "cf501be0-3613-4217-81a7-e4b6011c9222", + "metadata": {}, + "source": [ + "### List all slices" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0efa1db9-151f-42b8-be43-411405e667c1", + "metadata": {}, + "outputs": [], + "source": [ + "fablib.list_slices()" + ] + }, + { + "cell_type": "markdown", + "id": "fbf13418-e0e7-414e-9aea-56b81bc7f567", + "metadata": {}, + "source": [ + "### Show specific slice" + ] + }, + { + "cell_type": "markdown", + "id": "d6a6e4a0-f10f-4723-9330-34f1e6724b43", + "metadata": {}, + "source": [ + "Observe the specific slice's attributes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd1dfea3-55cf-4ef6-92ef-c115b8846bed", + "metadata": {}, + "outputs": [], + "source": [ + "slice_name = \"native-aws-lzhang9-1\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5465749-e292-4de1-a73e-6377adb9e9d9", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " slice = fablib.get_slice(name=slice_name)\n", + " slice.show()\n", + " slice.list_nodes()\n", + " slice.list_networks()\n", + " slice.list_interfaces()\n", + "except Exception as e:\n", + " print(f\"Exception: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "60eee875-942d-4653-b80d-4de6e65e19d3", + "metadata": {}, + "source": [ + "## Delete the Slice\n", + "\n", + "Please delete your slice when you are done with your experiment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cf9de91d-7045-4b07-ba7d-1c7eae1c0f17", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " slice = fablib.get_slice(name=slice_name)\n", + " slice.delete()\n", + "except Exception as e:\n", + " print(f\"Exception: {e}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From debfa91e02af3dda6a81f0a97d42dd729e743637 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Mon, 11 Mar 2024 00:07:23 -0500 Subject: [PATCH 25/75] Change directory name from notebooks to jupyter --- notebooks/examples/README.md | 0 notebooks/examples/native-aws/README.md | 0 notebooks/examples/native-aws/config.fab | 57 ---- notebooks/examples/native-gcp/README.md | 0 notebooks/examples/native-gcp/config.fab | 56 ---- notebooks/examples/sense-aws/config.fab | 63 ---- notebooks/examples/sense-gcp/config.fab | 78 ----- .../fabfed_credentials_template.yml | 62 ---- notebooks/fablib-tools.ipynb | 308 ------------------ notebooks/native-aws.ipynb | 180 ---------- notebooks/native-gcp.ipynb | 175 ---------- notebooks/sense-aws.ipynb | 180 ---------- notebooks/sense-gcp.ipynb | 178 ---------- 13 files changed, 1337 deletions(-) delete mode 100644 notebooks/examples/README.md delete mode 100644 notebooks/examples/native-aws/README.md delete mode 100644 notebooks/examples/native-aws/config.fab delete mode 100644 notebooks/examples/native-gcp/README.md delete mode 100644 notebooks/examples/native-gcp/config.fab delete mode 100644 notebooks/examples/sense-aws/config.fab delete mode 100644 notebooks/examples/sense-gcp/config.fab delete mode 100644 notebooks/fabfed_config/fabfed_credentials_template.yml delete mode 100644 notebooks/fablib-tools.ipynb delete mode 100644 notebooks/native-aws.ipynb delete mode 100644 notebooks/native-gcp.ipynb delete mode 100644 notebooks/sense-aws.ipynb delete mode 100644 notebooks/sense-gcp.ipynb diff --git a/notebooks/examples/README.md b/notebooks/examples/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/notebooks/examples/native-aws/README.md b/notebooks/examples/native-aws/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/notebooks/examples/native-aws/config.fab b/notebooks/examples/native-aws/config.fab deleted file mode 100644 index 7ec7245d..00000000 --- a/notebooks/examples/native-aws/config.fab +++ /dev/null @@ -1,57 +0,0 @@ -variable: - - vpc: - default: vpc-0936b973cf039f794 -provider: - - aws: - - aws_provider: - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: aws - - fabric: - - fabric_provider: - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: fabric -config: - - layer3: - - fab_layer: - subnet: 192.168.10.0/24 - gateway: 192.168.10.1 - ip_start: 192.168.10.2 - ip_end: 192.168.10.254 - - aws_layer: - subnet: 10.0.1.0/24 - - peering: - - my_peering: - # FOR FABRIC - cloud_account: "296256999979" - # FOR AWS and FABRIC - cloud_region: "us-east-1" - cloud_vpc: '{{ var.vpc }}' - remote_asn: 64512 # amazon_asn - local_asn: 55038 # customer_asn - # FOR FABRIC - local_address: "192.168.1.1/30" # customer_ip - remote_address: "192.168.1.2/30" # amazon_ip -resource: - - network: - - aws_net: - provider: '{{ aws.aws_provider }}' - name: aws-net - layer3: "{{ layer3.aws_layer }}" - peering: "{{ peering.my_peering }}" - stitch_with: '{{ network.fabric_network }}' - stitch_option: - device_name: agg3.ashb - count: 1 - - fabric_network: - provider: '{{ fabric.fabric_provider }}' - layer3: "{{ layer3.fab_layer }}" - peering: "{{ peering.my_peering }}" - interface: '{{ node.fabric_node }}' - count: 1 - - node: - - fabric_node: - provider: '{{ fabric.fabric_provider }}' - site: MAX # Use RENC when deploying to fabric beta environment - image: default_rocky_8 - count: 1 - nic_model: NIC_Basic diff --git a/notebooks/examples/native-gcp/README.md b/notebooks/examples/native-gcp/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/notebooks/examples/native-gcp/config.fab b/notebooks/examples/native-gcp/config.fab deleted file mode 100644 index 03c1192d..00000000 --- a/notebooks/examples/native-gcp/config.fab +++ /dev/null @@ -1,56 +0,0 @@ -variable: - - vpc: - default: vpc-69acc1d9-8c24-47cd-90b8-33be57167dbf -provider: - - gcp: - - gcp_provider: - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: gcp - - fabric: - - fabric_provider: - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: fabric -config: - - layer3: - - fab_layer: - subnet: 192.168.10.0/24 - gateway: 192.168.10.1 - ip_start: 192.168.10.2 - ip_end: 192.168.10.254 - - gcp_layer: - subnet: 10.100.0.0/24 # subnet.cidr and vpc.cidr - - peering: - - my_peering: - # FOR GCP - cloud_region: "us-east4" - cloud_vpc: '{{ var.vpc }}' - # uncomment and override cloud_vlan with a value between 3 and 4095. The default is 4. - # cloud_vlan: - - # FOR GCP AND FABRIC. - remote_asn: 16550 # google_asn - # FOR FABRIC - local_address: "192.168.1.1/30" # customer_ip - remote_address: "192.168.1.2/30" # google_ip -resource: - - network: - - gcp_net: - provider: '{{ gcp.gcp_provider }}' - name: gcp-net - layer3: "{{ layer3.gcp_layer }}" - peering: "{{ peering.my_peering }}" - stitch_with: '{{ network.fabric_network }}' - count: 1 - - fabric_network: - provider: '{{ fabric.fabric_provider }}' - layer3: "{{ layer3.fab_layer }}" - peering: "{{ peering.my_peering }}" - interface: '{{ node.fabric_node }}' - count: 1 - - node: - - fabric_node: - provider: '{{ fabric.fabric_provider }}' - site: MAX - image: default_rocky_8 - count: 1 - nic_model: NIC_Basic diff --git a/notebooks/examples/sense-aws/config.fab b/notebooks/examples/sense-aws/config.fab deleted file mode 100644 index a6333942..00000000 --- a/notebooks/examples/sense-aws/config.fab +++ /dev/null @@ -1,63 +0,0 @@ -provider: - - sense: - - sense_provider: - - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: sense - - fabric: - - fabric_provider: - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: fabric -config: - - layer3: - - fab_layer: - subnet: 192.168.10.0/24 # subnet: "10.0.0.0/24" # for sense subnet.cidr - gateway: 192.168.10.1 - ip_start: 192.168.10.2 # optional: auto generate if subnet is given - ip_end: 192.168.10.254 - - sense_layer: - subnet: 10.200.1.0/24 # subnet.cidr and vpc.cidr - - peering: - - my_peering: - # FOR FABRIC - cloud_account: "296256999979" - cloud_facility: - local_device: - local_port: - cloud_region: "us-east-1" - - # FOR SENSE and FABRIC - local_asn: "55038" # customer_asn (hard coded for now) - local_address: "192.168.1.1/30" # customer_ip - remote_asn: "64512" # amazon_asn - remote_address: "192.168.1.2/30" # amazon_ip -resource: - - node: - - sense_node: - provider: '{{ sense.sense_provider }}' - name: aws-node - network: "{{ network.sense_net }}" - service: - count: 1 - - fabric_node: - provider: '{{ fabric.fabric_provider }}' - site: MAX # Use RENC when deploying to fabric beta environment - image: default_rocky_8 - count: 1 - nic_model: NIC_Basic - - network: - - sense_net: - provider: '{{ sense.sense_provider }}' - name: aws-net - layer3: "{{ layer3.sense_layer }}" - peering: "{{ peering.my_peering }}" - count: 1 - stitch_with: '{{ network.fabric_network }}' - stitch_option: - device_name: agg3.ashb - - - fabric_network: - provider: '{{ fabric.fabric_provider }}' - layer3: "{{ layer3.fab_layer }}" - peering: "{{ peering.my_peering }}" - interface: '{{ node.fabric_node }}' - count: 1 diff --git a/notebooks/examples/sense-gcp/config.fab b/notebooks/examples/sense-gcp/config.fab deleted file mode 100644 index 6ae27417..00000000 --- a/notebooks/examples/sense-gcp/config.fab +++ /dev/null @@ -1,78 +0,0 @@ -provider: - - sense: - - sense_provider: - - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: sense - - fabric: - - fabric_provider: - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: fabric - - janus: - - janus_provider: - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: janus -config: - - layer3: - - fab_layer: - subnet: 192.168.10.0/24 # subnet: "10.0.0.0/24" # for sense subnet.cidr - gateway: 192.168.10.1 - ip_start: 192.168.10.2 # optional: auto generate if subnet is given - ip_end: 192.168.10.254 - - sense_layer: - subnet: 10.200.1.0/24 # subnet.cidr and vpc.cidr - - peering: - - my_peering: - # FOR FABRIC - # cloud_account: "296256999979" # pairing key - # cloud_facility: - # local_device: "agg3.ashb.net.internet2.edu" - # local_port: "Bundle-Ether5" - # cloud_region: "us-east4" - # cloud_vlan: - - # FOR SENSE and FABRIC - local_asn: "55038" # customer_asn (hard coded for now) - local_address: "192.168.1.1/30" # customer_ip (ignore) - remote_asn: "16550" # google_asn (default) - remote_address: "192.168.1.2/30" # google_ip (ignore) -resource: - - node: - - sense_node: - provider: '{{ sense.sense_provider }}' - name: gcp-node - network: "{{ network.sense_net }}" - service: - count: 1 - - fabric_node: - provider: '{{ fabric.fabric_provider }}' - site: MAX - image: default_rocky_8 - count: 1 - nic_model: NIC_Basic - - network: - - sense_net: - provider: '{{ sense.sense_provider }}' - name: gcp-net - layer3: "{{ layer3.sense_layer }}" - peering: "{{ peering.my_peering }}" - count: 1 - profile: FABRIC-GCP-INTERCON - stitch_with: '{{ network.fabric_network }}' - profile: FABRIC-GCP-INTERCON - stitch_option: - group_name: GCP - - - fabric_network: - provider: '{{ fabric.fabric_provider }}' - layer3: "{{ layer3.fab_layer }}" - peering: "{{ peering.my_peering }}" - interface: '{{ node.fabric_node }}' - count: 1 - - service: - - dtn_service: - provider: '{{ janus.janus_provider }}' - node: [ '{{ node.sense_node }}', '{{ node.fabric_node }}' ] - controller: '{{ node.fabric_node }}' - image: dtnaas/tools - profile: fabfed - count: 0 # SET THIS TO 1 if you want use janus service diff --git a/notebooks/fabfed_config/fabfed_credentials_template.yml b/notebooks/fabfed_config/fabfed_credentials_template.yml deleted file mode 100644 index 3550eb64..00000000 --- a/notebooks/fabfed_config/fabfed_credentials_template.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Fabric -# https://portal.fabric-testbed.net -# Create and download FABRIC at https://portal.fabric-testbed.net/ via Experiments -> Manage Tokens -# Create a Bastion Key and User FABRIC at https://portal.fabric-testbed.net/ via Experiments -> Manage SSH Keys -fabric: - bastion-user-name: - token-location: - bastion-key-location: - project_id: - slice-private-key-location: - slice-public-key-location: - -# Chameleon -# https://www.chameleoncloud.org/ -# Create a CLI Password as described here: https://chameleoncloud.readthedocs.io/en/latest/technical/cli.html -# Project Name can be found https://www.chameleoncloud.org/ -chi: - project_name: - key_pair: - user: - password: - slice-private-key-location: - slice-public-key-location: - -# https://sense-o.es.net:8443/StackV-web/portal/ -sense: - auth_endpoint: https://sense-o.es.net:8543/auth/realms/StackV/protocol/openid-connect/token - api_endpoint: https://sense-o-dev.es.net:8443/StackV-web/restapi - client_id: StackV - username: - password: - secret: - verify: False - slice-private-key-location: # THIS IS NEEDED IF USING JANUS SERVICE - - -janus: - url: - username: - password: - token: - -# Cloudlab -# https://cloudlab.us -# Create an account and join project fabfed -# Must download certificate and reference it below -cloudlab: - project: - certificate: - user: - slice-private-key-location: - -# Amazon AWS -aws: - access_key: - secret_key: - -# Google Cloud -# Create a service account and downdload it to the path below -gcp: - service_key_path: - project: diff --git a/notebooks/fablib-tools.ipynb b/notebooks/fablib-tools.ipynb deleted file mode 100644 index f5199337..00000000 --- a/notebooks/fablib-tools.ipynb +++ /dev/null @@ -1,308 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "57c387fd-49d1-4d47-ab64-910d94ec0244", - "metadata": {}, - "source": [ - "# FABLIB Tools\n" - ] - }, - { - "cell_type": "markdown", - "id": "1c84b882-9d63-4c45-a8ae-5cb22c1191a2", - "metadata": {}, - "source": [ - "## Import the FABlib Library\n", - "Chose one of two working enviroments to import FABlib.\n", - "- fabric-testbed\n", - "- local host" - ] - }, - { - "cell_type": "markdown", - "id": "04c03230-61f4-48e5-a5bc-031f494e5c2a", - "metadata": {}, - "source": [ - "### Environment: Fabric-Testbed" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "25e4df8b-4eed-4fdf-b6ac-2a801945b251", - "metadata": {}, - "outputs": [], - "source": [ - "from fabrictestbed_extensions.fablib.fablib import FablibManager as fablib_manager\n", - "import json\n", - "\n", - "try:\n", - " fablib = fablib_manager()\n", - " \n", - " fablib.show_config();\n", - "except Exception as e:\n", - " print(f\"Exception: {e}\")" - ] - }, - { - "cell_type": "markdown", - "id": "1df07de8-ac57-41da-966f-8865d4cb42e7", - "metadata": {}, - "source": [ - "### Environment: Local Host" - ] - }, - { - "cell_type": "markdown", - "id": "07c7fbc9-730c-4cb9-a1b4-03afb1ad5fa8", - "metadata": {}, - "source": [ - "Get Fabric configuration from FABFED credential file\n", - "- Set the path to fabfed credential file\n", - "- Read the fabric configuration from fabfed crential file\n", - "- Set the path to ssh_config the same as that of the bastion_ssh_key\n", - "- Generate fabric_rc\n", - "- Import Fablib" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3dde8e65-5de4-4d47-a9cf-08c09d9f9394", - "metadata": {}, - "outputs": [], - "source": [ - "# Path to your YAML file\n", - "credential_file_path = 'fabfed_config/fabfed_credentials.yml'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "73906428-445a-47d9-850c-859eac86e8d1", - "metadata": {}, - "outputs": [], - "source": [ - "import yaml\n", - "\n", - "# Read the YAML file\n", - "with open(credential_file_path, 'r') as file:\n", - " try:\n", - " yaml_data = yaml.safe_load(file)\n", - " except yaml.YAMLError as exc:\n", - " print(exc)\n", - "\n", - "fabric_config = yaml_data['fabric']\n", - "\n", - "def print_dict(dictionary, indent=0):\n", - " for key, value in dictionary.items():\n", - " if isinstance(value, dict):\n", - " print(' ' * indent + str(key) + ':')\n", - " print_dict(value, indent + 1)\n", - " else:\n", - " print(' ' * indent + str(key) + ': ' + str(value))\n", - " \n", - "print_dict(fabric_config)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5a87ec9b-5c69-4b61-b7e6-c6d78355fbfe", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "home_directory = os.path.expanduser(\"~\")\n", - "print(\"Home directory:\", home_directory)\n", - "\n", - "bastion_ssh_config_file = os.path.dirname(fabric_config['bastion-key-location']) + \"/ssh-config\"\n", - "print(f\"ssh_config path: {bastion_ssh_config_file}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "60790e7d-245a-416a-bcee-0ed3d5c49306", - "metadata": {}, - "outputs": [], - "source": [ - "# Data to write to the file\n", - "data = f\"\"\"\n", - "export FABRIC_CREDMGR_HOST=cm.fabric-testbed.net\n", - "export FABRIC_ORCHESTRATOR_HOST=orchestrator.fabric-testbed.net\n", - "export FABRIC_PROJECT_ID={fabric_config['project_id']}\n", - "export FABRIC_TOKEN_LOCATION={fabric_config['token-location']}\n", - "\n", - "export FABRIC_BASTION_HOST=bastion.fabric-testbed.net\n", - "export FABRIC_BASTION_USERNAME={fabric_config['bastion-user-name']}\n", - "\n", - "export FABRIC_BASTION_KEY_LOCATION={fabric_config['bastion-key-location']}\n", - "#export FABRIC_BASTION_KEY_PASSPHRASE=\n", - "\n", - "export FABRIC_BASTION_SSH_CONFIG_FILE={bastion_ssh_config_file}\n", - "\n", - "export FABRIC_SLICE_PRIVATE_KEY_FILE={fabric_config['slice-private-key-location']}\n", - "export FABRIC_SLICE_PUBLIC_KEY_FILE={fabric_config['slice-public-key-location']} \n", - "#export FABRIC_SLICE_PRIVATE_KEY_PASSPHRASE=\n", - "\n", - "export FABRIC_LOG_FILE={home_directory}/fablib.log\n", - "export FABRIC_LOG_LEVEL=INFO \n", - "\n", - "export FABRIC_AVOID=''\n", - "\n", - "export FABRIC_SSH_COMMAND_LINE=\"ssh -i {{ _self_.private_ssh_key_file }} -F {bastion_ssh_config_file} {{ _self_.username }}@{{ _self_.management_ip }}\"\n", - "\"\"\"\n", - "\n", - "# Specify the file path\n", - "fabric_rc_file = \"./fabric_config/fabric_rc\"\n", - "\n", - "# Delete if it exists\n", - "if os.path.exists(fabric_rc_file):\n", - " # Delete the file\n", - " os.remove(fabric_rc_file)\n", - " \n", - "# Open the file in write mode\n", - "with open(fabric_rc_file, \"w\") as file:\n", - " # Write the data to the file\n", - " file.write(data)\n", - "\n", - "print(\"Data has been written to\", fabric_rc_file)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2714cb84-3915-4272-9e43-f8eac4d06994", - "metadata": {}, - "outputs": [], - "source": [ - "from fabrictestbed_extensions.fablib.fablib import FablibManager as fablib_manager\n", - "import json\n", - "\n", - "try:\n", - " fablib = fablib_manager(fabric_rc=fabric_rc_file)\n", - " \n", - " fablib.show_config();\n", - "except Exception as e:\n", - " print(f\"Exception: {e}\")" - ] - }, - { - "cell_type": "markdown", - "id": "5895ddb9-1a78-4efc-a700-5cdd0a191ca6", - "metadata": {}, - "source": [ - "## Show Slice Information" - ] - }, - { - "cell_type": "markdown", - "id": "cf501be0-3613-4217-81a7-e4b6011c9222", - "metadata": {}, - "source": [ - "### List all slices" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0efa1db9-151f-42b8-be43-411405e667c1", - "metadata": {}, - "outputs": [], - "source": [ - "fablib.list_slices()" - ] - }, - { - "cell_type": "markdown", - "id": "fbf13418-e0e7-414e-9aea-56b81bc7f567", - "metadata": {}, - "source": [ - "### Show specific slice" - ] - }, - { - "cell_type": "markdown", - "id": "d6a6e4a0-f10f-4723-9330-34f1e6724b43", - "metadata": {}, - "source": [ - "Observe the specific slice's attributes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cd1dfea3-55cf-4ef6-92ef-c115b8846bed", - "metadata": {}, - "outputs": [], - "source": [ - "slice_name = \"native-aws-lzhang9-1\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a5465749-e292-4de1-a73e-6377adb9e9d9", - "metadata": {}, - "outputs": [], - "source": [ - "try:\n", - " slice = fablib.get_slice(name=slice_name)\n", - " slice.show()\n", - " slice.list_nodes()\n", - " slice.list_networks()\n", - " slice.list_interfaces()\n", - "except Exception as e:\n", - " print(f\"Exception: {e}\")" - ] - }, - { - "cell_type": "markdown", - "id": "60eee875-942d-4653-b80d-4de6e65e19d3", - "metadata": {}, - "source": [ - "## Delete the Slice\n", - "\n", - "Please delete your slice when you are done with your experiment." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cf9de91d-7045-4b07-ba7d-1c7eae1c0f17", - "metadata": {}, - "outputs": [], - "source": [ - "try:\n", - " slice = fablib.get_slice(name=slice_name)\n", - " slice.delete()\n", - "except Exception as e:\n", - " print(f\"Exception: {e}\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.18" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/native-aws.ipynb b/notebooks/native-aws.ipynb deleted file mode 100644 index 54e01085..00000000 --- a/notebooks/native-aws.ipynb +++ /dev/null @@ -1,180 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", - "metadata": {}, - "source": [ - "# Native AWS + Fabric\n", - "Stitching native AWS and Fabric resources." - ] - }, - { - "cell_type": "markdown", - "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", - "metadata": {}, - "source": [ - "## Assign session name and .fab path" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d8efd310-300a-4f56-bb26-5431bdec3969", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "session_name = \"native-aws-lzhang9\"\n", - "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", - "fab_file_path = \"./examples/native-aws\"\n", - "\n", - "print(session_name)\n", - "print(fab_file_path)" - ] - }, - { - "cell_type": "markdown", - "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", - "metadata": {}, - "source": [ - "## Validate .fab files" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72148135-c66f-41ed-9a36-b037c7fbb966", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -validate" - ] - }, - { - "cell_type": "markdown", - "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", - "metadata": {}, - "source": [ - "## Add resources to providers and validate resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0aa67fe6-34b4-4a20-b9b3-a5fb6921efd2", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -plan" - ] - }, - { - "cell_type": "markdown", - "id": "c29bf401-66d9-4b8b-9c68-fd2ebbd85ee2", - "metadata": {}, - "source": [ - "## View stitching information" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9d3324c9-004e-4c29-b768-08b2de294ed3", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" - ] - }, - { - "cell_type": "markdown", - "id": "d19fef3d-2d37-4ffc-8f8f-07446bc5a484", - "metadata": {}, - "source": [ - "## Create resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e5e16ead-338b-4e3c-9e38-61203a2489f4", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -apply" - ] - }, - { - "cell_type": "markdown", - "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", - "metadata": {}, - "source": [ - "## View the state of the workflow" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -show" - ] - }, - { - "cell_type": "markdown", - "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", - "metadata": {}, - "source": [ - "## Delete all resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57305ac9-fa7e-4578-a200-ecdcb74a2a4c", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -destroy" - ] - }, - { - "cell_type": "markdown", - "id": "138944ac-3892-41e2-a9cc-2c59383b5d63", - "metadata": {}, - "source": [ - "## Note: Amazon Instance Info\n", - "- EC2/Instances/i-078a7be6f4a3df5d6\n", - "- Private IPv4 10.0.1.106\n", - "- Public IPv4 54.242.87.34" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.18" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/native-gcp.ipynb b/notebooks/native-gcp.ipynb deleted file mode 100644 index e13fc5ed..00000000 --- a/notebooks/native-gcp.ipynb +++ /dev/null @@ -1,175 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", - "metadata": {}, - "source": [ - "# Native GCP + Fabric\n", - "Stitching native GCP and Fabric resources." - ] - }, - { - "cell_type": "markdown", - "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", - "metadata": {}, - "source": [ - "## Assign session name and .fab path" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d8efd310-300a-4f56-bb26-5431bdec3969", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "session_name = \"native-gcp-lzhang9\"\n", - "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/gcp\"\n", - "fab_file_path = \"./examples/native-gcp\"\n", - "\n", - "print(session_name)\n", - "print(fab_file_path)" - ] - }, - { - "cell_type": "markdown", - "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", - "metadata": {}, - "source": [ - "## Validate .fab files " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72148135-c66f-41ed-9a36-b037c7fbb966", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -validate" - ] - }, - { - "cell_type": "markdown", - "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", - "metadata": {}, - "source": [ - "## Add resources to providers and validate resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0aa67fe6-34b4-4a20-b9b3-a5fb6921efd2", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -plan" - ] - }, - { - "cell_type": "markdown", - "id": "c29bf401-66d9-4b8b-9c68-fd2ebbd85ee2", - "metadata": {}, - "source": [ - "## View stitching information" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9d3324c9-004e-4c29-b768-08b2de294ed3", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" - ] - }, - { - "cell_type": "markdown", - "id": "d19fef3d-2d37-4ffc-8f8f-07446bc5a484", - "metadata": {}, - "source": [ - "## Create resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e5e16ead-338b-4e3c-9e38-61203a2489f4", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -apply" - ] - }, - { - "cell_type": "markdown", - "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", - "metadata": {}, - "source": [ - "## View the state of the workflow" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -show" - ] - }, - { - "cell_type": "markdown", - "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", - "metadata": {}, - "source": [ - "## Delete all resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57305ac9-fa7e-4578-a200-ecdcb74a2a4c", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -destroy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ee0f1096-c026-46ca-b39b-4c8a59e3d5f8", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.18" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/sense-aws.ipynb b/notebooks/sense-aws.ipynb deleted file mode 100644 index 5a532c81..00000000 --- a/notebooks/sense-aws.ipynb +++ /dev/null @@ -1,180 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", - "metadata": {}, - "source": [ - "# SENSE AWS + Fabric\n", - "Stitching SENSE AWS and Fabric resources." - ] - }, - { - "cell_type": "markdown", - "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", - "metadata": {}, - "source": [ - "## Assign session name and .fab path" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d8efd310-300a-4f56-bb26-5431bdec3969", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "session_name = \"sense-aws-lzhang9\"\n", - "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", - "fab_file_path = \"./examples/sense-aws\"\n", - "\n", - "print(session_name)\n", - "print(fab_file_path)" - ] - }, - { - "cell_type": "markdown", - "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", - "metadata": {}, - "source": [ - "## Validate .fab files" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72148135-c66f-41ed-9a36-b037c7fbb966", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -validate" - ] - }, - { - "cell_type": "markdown", - "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", - "metadata": {}, - "source": [ - "## Add resources to providers and validate resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0aa67fe6-34b4-4a20-b9b3-a5fb6921efd2", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -plan" - ] - }, - { - "cell_type": "markdown", - "id": "c29bf401-66d9-4b8b-9c68-fd2ebbd85ee2", - "metadata": {}, - "source": [ - "## View stitching information" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9d3324c9-004e-4c29-b768-08b2de294ed3", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" - ] - }, - { - "cell_type": "markdown", - "id": "d19fef3d-2d37-4ffc-8f8f-07446bc5a484", - "metadata": {}, - "source": [ - "## Create resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e5e16ead-338b-4e3c-9e38-61203a2489f4", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -apply" - ] - }, - { - "cell_type": "markdown", - "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", - "metadata": {}, - "source": [ - "## View the state of the workflow" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -show" - ] - }, - { - "cell_type": "markdown", - "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", - "metadata": {}, - "source": [ - "## Delete all resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57305ac9-fa7e-4578-a200-ecdcb74a2a4c", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -destroy" - ] - }, - { - "cell_type": "markdown", - "id": "138944ac-3892-41e2-a9cc-2c59383b5d63", - "metadata": {}, - "source": [ - "## Note: Amazon Instance Info\n", - "- EC2/Instances/i-078a7be6f4a3df5d6\n", - "- Private IPv4 10.0.1.106\n", - "- Public IPv4 54.242.87.34" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.18" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/sense-gcp.ipynb b/notebooks/sense-gcp.ipynb deleted file mode 100644 index beabdcb8..00000000 --- a/notebooks/sense-gcp.ipynb +++ /dev/null @@ -1,178 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", - "metadata": {}, - "source": [ - "# SENSE GCP + Fabric\n", - "Stitching SENSE GCP and Fabric resources." - ] - }, - { - "cell_type": "markdown", - "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", - "metadata": {}, - "source": [ - "## Assign session name and .fab path" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d8efd310-300a-4f56-bb26-5431bdec3969", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "session_name = \"sense-gcp-lzhang9\"\n", - "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/gcp\"\n", - "fab_file_path = \"./examples/sense-gcp\"\n", - "\n", - "print(session_name)\n", - "print(fab_file_path)" - ] - }, - { - "cell_type": "markdown", - "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", - "metadata": {}, - "source": [ - "## Validate .fab files " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72148135-c66f-41ed-9a36-b037c7fbb966", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -validate" - ] - }, - { - "cell_type": "markdown", - "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", - "metadata": {}, - "source": [ - "## Add resources to providers and validate resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0aa67fe6-34b4-4a20-b9b3-a5fb6921efd2", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -plan" - ] - }, - { - "cell_type": "markdown", - "id": "c29bf401-66d9-4b8b-9c68-fd2ebbd85ee2", - "metadata": {}, - "source": [ - "## View stitching information" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9d3324c9-004e-4c29-b768-08b2de294ed3", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" - ] - }, - { - "cell_type": "markdown", - "id": "d19fef3d-2d37-4ffc-8f8f-07446bc5a484", - "metadata": {}, - "source": [ - "## Create resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e5e16ead-338b-4e3c-9e38-61203a2489f4", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -apply" - ] - }, - { - "cell_type": "markdown", - "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", - "metadata": {}, - "source": [ - "## View the state of the workflow" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -show" - ] - }, - { - "cell_type": "markdown", - "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", - "metadata": {}, - "source": [ - "## Delete all resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57305ac9-fa7e-4578-a200-ecdcb74a2a4c", - "metadata": {}, - "outputs": [], - "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -destroy" - ] - }, - { - "cell_type": "markdown", - "id": "67883063-3a57-4cdd-b14e-af35ea912f74", - "metadata": {}, - "source": [ - "## Note: VM Instance Info\n", - "- EC2/Instances/i-078a7be6f4a3df5d6\n", - "- Private IPv4 10.200.1.2\n", - "- Public IPv4 34.86.164.172" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.18" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From 02744573e4eff987dd21b7a0c9b6aa4217ce7346 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Tue, 12 Mar 2024 08:52:49 -0500 Subject: [PATCH 26/75] Update .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 8c5620ea..761138ad 100644 --- a/.gitignore +++ b/.gitignore @@ -153,3 +153,6 @@ venv # emacs auto-save *.*~ + +# jupyter artifacts +*/.ipynb_checkpoints From cbb050883379071d1373937acdc13f2c929c30b1 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Tue, 12 Mar 2024 23:05:36 -0500 Subject: [PATCH 27/75] Changed the network prefix To have more hosts in the same subnet. --- jupyter/examples/native-aws/config.fab | 4 ++-- jupyter/examples/native-gcp/config.fab | 4 ++-- jupyter/examples/sense-aws/config.fab | 4 ++-- jupyter/examples/sense-gcp/config.fab | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/jupyter/examples/native-aws/config.fab b/jupyter/examples/native-aws/config.fab index 7ec7245d..a2b29b01 100644 --- a/jupyter/examples/native-aws/config.fab +++ b/jupyter/examples/native-aws/config.fab @@ -29,8 +29,8 @@ config: remote_asn: 64512 # amazon_asn local_asn: 55038 # customer_asn # FOR FABRIC - local_address: "192.168.1.1/30" # customer_ip - remote_address: "192.168.1.2/30" # amazon_ip + local_address: "192.168.1.1/16" # customer_ip + remote_address: "192.168.1.2/16" # amazon_ip resource: - network: - aws_net: diff --git a/jupyter/examples/native-gcp/config.fab b/jupyter/examples/native-gcp/config.fab index 03c1192d..b4dcb010 100644 --- a/jupyter/examples/native-gcp/config.fab +++ b/jupyter/examples/native-gcp/config.fab @@ -30,8 +30,8 @@ config: # FOR GCP AND FABRIC. remote_asn: 16550 # google_asn # FOR FABRIC - local_address: "192.168.1.1/30" # customer_ip - remote_address: "192.168.1.2/30" # google_ip + local_address: "192.168.1.1/16" # customer_ip + remote_address: "192.168.1.2/16" # google_ip resource: - network: - gcp_net: diff --git a/jupyter/examples/sense-aws/config.fab b/jupyter/examples/sense-aws/config.fab index a6333942..dbd6ef69 100644 --- a/jupyter/examples/sense-aws/config.fab +++ b/jupyter/examples/sense-aws/config.fab @@ -27,9 +27,9 @@ config: # FOR SENSE and FABRIC local_asn: "55038" # customer_asn (hard coded for now) - local_address: "192.168.1.1/30" # customer_ip + local_address: "192.168.1.1/16" # customer_ip remote_asn: "64512" # amazon_asn - remote_address: "192.168.1.2/30" # amazon_ip + remote_address: "192.168.1.2/16" # amazon_ip resource: - node: - sense_node: diff --git a/jupyter/examples/sense-gcp/config.fab b/jupyter/examples/sense-gcp/config.fab index 6ae27417..dcfe7c96 100644 --- a/jupyter/examples/sense-gcp/config.fab +++ b/jupyter/examples/sense-gcp/config.fab @@ -32,9 +32,9 @@ config: # FOR SENSE and FABRIC local_asn: "55038" # customer_asn (hard coded for now) - local_address: "192.168.1.1/30" # customer_ip (ignore) + local_address: "192.168.1.1/16" # customer_ip (ignore) remote_asn: "16550" # google_asn (default) - remote_address: "192.168.1.2/30" # google_ip (ignore) + remote_address: "192.168.1.2/16" # google_ip (ignore) resource: - node: - sense_node: From 06213e3240248da168903d93594066dae66f758a Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Tue, 12 Mar 2024 23:06:08 -0500 Subject: [PATCH 28/75] Update config.fab --- jupyter/examples/chameleon/star/config.fab | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter/examples/chameleon/star/config.fab b/jupyter/examples/chameleon/star/config.fab index fbcdcfa2..283c9d4f 100644 --- a/jupyter/examples/chameleon/star/config.fab +++ b/jupyter/examples/chameleon/star/config.fab @@ -52,4 +52,4 @@ resource: controller: '{{ node.fabric_node }}' image: dtnaas/tools profile: fabfed - count: 1 + count: 0 From d0e457dd955622c7991963618ff1d6c1531f785a Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Wed, 13 Mar 2024 10:21:02 -0500 Subject: [PATCH 29/75] Add varfile.yml files For native-gcp and native-aws to read VPCs from varfile. --- jupyter/examples/native-aws/varfile.yml | 3 +++ jupyter/examples/native-gcp/varfile.yml | 3 +++ jupyter/native-aws.ipynb | 2 +- jupyter/native-gcp.ipynb | 2 +- jupyter/welcom.ipynb | 9 ++++++--- 5 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 jupyter/examples/native-aws/varfile.yml create mode 100644 jupyter/examples/native-gcp/varfile.yml diff --git a/jupyter/examples/native-aws/varfile.yml b/jupyter/examples/native-aws/varfile.yml new file mode 100644 index 00000000..a4045c56 --- /dev/null +++ b/jupyter/examples/native-aws/varfile.yml @@ -0,0 +1,3 @@ +vpc: vpc-0936b973cf039f794 +#vpc: vpc-034da8fe2c9380fec +#vpc: vpc-05bf1ae332ca8b212 \ No newline at end of file diff --git a/jupyter/examples/native-gcp/varfile.yml b/jupyter/examples/native-gcp/varfile.yml new file mode 100644 index 00000000..8f43e285 --- /dev/null +++ b/jupyter/examples/native-gcp/varfile.yml @@ -0,0 +1,3 @@ +vpc: vpc-69acc1d9-8c24-47cd-90b8-33be57167dbf +#vpc: vpc-00d79c7f-debd-472e-8053-bb646c216183 +#vpc: vpc-33f21cdf-60af-4a96-bc48-3b0fb33dba5c \ No newline at end of file diff --git a/jupyter/native-aws.ipynb b/jupyter/native-aws.ipynb index 54e01085..f4ab3c1f 100644 --- a/jupyter/native-aws.ipynb +++ b/jupyter/native-aws.ipynb @@ -105,7 +105,7 @@ }, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -apply" + "!fabfed workflow -s $session_name -c $fab_file_path -v $fab_file_path/varfile.yml -apply" ] }, { diff --git a/jupyter/native-gcp.ipynb b/jupyter/native-gcp.ipynb index fefdecee..f6cbf543 100644 --- a/jupyter/native-gcp.ipynb +++ b/jupyter/native-gcp.ipynb @@ -103,7 +103,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -apply" + "!fabfed workflow -s $session_name -c $fab_file_path -v $fab_file_path/varfile.yml -apply" ] }, { diff --git a/jupyter/welcom.ipynb b/jupyter/welcom.ipynb index 79d8c005..ae9e2381 100644 --- a/jupyter/welcom.ipynb +++ b/jupyter/welcom.ipynb @@ -93,15 +93,18 @@ "id": "8127923d-a615-4472-981a-fc20d01aca35", "metadata": {}, "source": [ - "## Examples of Stitching Workflows\n", - "Examples of stitching different provider resources:\n", + "## Examples of FabFed Workflows\n", + "Examples of workflows using FabFed CLI:\n", "\n", "- [Native aws](./native-aws.ipynb): Stiching native AWS and Fabric\n", "- [Native gcp](./native-gcp.ipynb): Stiching native GCP and Fabric\n", "- [SENSE aws](./sense-aws.ipynb): Stiching SENSE AWS and Fabric\n", "- [SENSE gcp](./sense-gcp.ipynb): Stiching SENSE GCP and Fabric\n", "- [Cloudlab](./cloudlab.ipynb): Stiching Cloudlab and Fabric\n", - "- [Chameleon](./chameleon.ipynb): Stiching Chameleon and Fabric" + "- [Chameleon](./chameleon.ipynb): Stiching Chameleon and Fabric\n", + "\n", + "Examples of workflows using FabFed library:\n", + "- [Native gcp](./native-gcp.ipynb): Stiching native GCP and Fabric" ] }, { From f392ed39f0098b2f6f290e0af3bb7460f183de8b Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Wed, 13 Mar 2024 12:14:48 -0500 Subject: [PATCH 30/75] Add notebooks for workflows using fabfed library --- jupyter/chameleon-lib.ipynb | 288 +++++++++++++++++++++++++++++++++ jupyter/cloudlab-lib.ipynb | 288 +++++++++++++++++++++++++++++++++ jupyter/native-aws-lib.ipynb | 300 +++++++++++++++++++++++++++++++++++ jupyter/native-gcp-lib.ipynb | 300 +++++++++++++++++++++++++++++++++++ jupyter/sense-aws-lib.ipynb | 300 +++++++++++++++++++++++++++++++++++ jupyter/sense-gcp-lib.ipynb | 300 +++++++++++++++++++++++++++++++++++ jupyter/welcom.ipynb | 23 ++- 7 files changed, 1795 insertions(+), 4 deletions(-) create mode 100644 jupyter/chameleon-lib.ipynb create mode 100644 jupyter/cloudlab-lib.ipynb create mode 100644 jupyter/native-aws-lib.ipynb create mode 100644 jupyter/native-gcp-lib.ipynb create mode 100644 jupyter/sense-aws-lib.ipynb create mode 100644 jupyter/sense-gcp-lib.ipynb diff --git a/jupyter/chameleon-lib.ipynb b/jupyter/chameleon-lib.ipynb new file mode 100644 index 00000000..ec96314e --- /dev/null +++ b/jupyter/chameleon-lib.ipynb @@ -0,0 +1,288 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", + "metadata": {}, + "source": [ + "# Chameleon + Fabric\n", + "Stitching Chameleon and Fabric resources using FabFed library." + ] + }, + { + "cell_type": "markdown", + "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", + "metadata": {}, + "source": [ + "## Assign session name and .fab path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8efd310-300a-4f56-bb26-5431bdec3969", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "session_name = \"chameleon-lzhang9-lib\"\n", + "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", + "fab_file_path = \"./examples/chameleon/star\"\n", + "\n", + "print(session_name)\n", + "print(fab_file_path)" + ] + }, + { + "cell_type": "markdown", + "id": "a6e6acf4-b348-4e08-9808-896f794bb01d", + "metadata": {}, + "source": [ + "## Import FabFeb library" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79a3a114-3c67-4a79-a6fb-8227b64dc429", + "metadata": {}, + "outputs": [], + "source": [ + "from fabfed.util.config import WorkflowConfig\n", + "from fabfed.util import state as sutil\n", + "from fabfed.controller.controller import Controller\n", + "from fabfed.controller.provider_factory import default_provider_factory" + ] + }, + { + "cell_type": "markdown", + "id": "73a535d1-4f1a-4606-b4a4-128c675255db", + "metadata": {}, + "source": [ + "## Read .fab files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce52aa52-99af-4391-bb42-3ab5a9554b9a", + "metadata": {}, + "outputs": [], + "source": [ + "def read_file_to_str(file_path):\n", + " try:\n", + " with open(file_path, 'r') as file:\n", + " file_content = file.read()\n", + " return file_content\n", + " except FileNotFoundError:\n", + " print(f\"File '{file_path}' not found.\")\n", + " return None\n", + "\n", + "# Read config.fab to config_str:\n", + "config_str = read_file_to_str(f\"{fab_file_path}/config.fab\")\n", + "if config_str:\n", + " print(\"File content:\")\n", + " print(config_str)\n", + "\n", + "print(config_str)" + ] + }, + { + "cell_type": "markdown", + "id": "f6106b4e-076f-4349-b215-c6db497c97a9", + "metadata": {}, + "source": [ + "## Create a workflow config object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba75ff1a-7442-4e05-a4ca-b1128397983e", + "metadata": {}, + "outputs": [], + "source": [ + "config = WorkflowConfig(content=config_str)" + ] + }, + { + "cell_type": "markdown", + "id": "c87f8329-054e-45f3-9d69-3c4acc25252f", + "metadata": {}, + "source": [ + "## Create a controller" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d161b4f-4b6b-4993-80d3-26f59034c2cb", + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "controller = Controller(config=config, logger=logger)" + ] + }, + { + "cell_type": "markdown", + "id": "4e1aa491-7ddc-491b-82f6-ec9d41b08c3a", + "metadata": {}, + "source": [ + "## Init stitching information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7da5efbb-0c83-4842-b533-c81340028013", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session)\n", + "\n", + "controller.init(session=session_name, provider_factory=default_provider_factory, provider_states=states)" + ] + }, + { + "cell_type": "markdown", + "id": "465ea762-1e62-4532-a0e3-aa5bef0e23c4", + "metadata": {}, + "source": [ + "## Add resources to providers and validate resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8ae4d51-a812-4d80-997c-11c7ef5548a2", + "metadata": {}, + "outputs": [], + "source": [ + "controller.plan(provider_states=[])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a1a864d-bdd9-4422-84a6-cade3a7c485c", + "metadata": {}, + "outputs": [], + "source": [ + "controller.add(provider_states=[])" + ] + }, + { + "cell_type": "markdown", + "id": "78419199-2802-481e-b0c8-08b362f658e1", + "metadata": {}, + "source": [ + "## Create resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4db644c5-b7d5-4092-8329-cca89a13aef6", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " controller.apply(provider_states=[])\n", + "except ControllerException as e:\n", + " assert isinstance(e.exceptions[0], DummyFailCreateException)" + ] + }, + { + "cell_type": "markdown", + "id": "beee11c5-5adc-4756-8133-a8eda0253969", + "metadata": {}, + "source": [ + "## Save the workflow state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ca3360b-c033-4af8-9f1d-0acf12b1c0d0", + "metadata": {}, + "outputs": [], + "source": [ + "states = controller.get_states()\n", + "sutil.save_states(states, session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", + "metadata": {}, + "source": [ + "## View the state of the workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", + "metadata": {}, + "source": [ + "## Delete all resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc2c921a-4f27-470a-ac8b-a7c957559b2d", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session_name)\n", + "\n", + "controller.destroy(provider_states=states)\n", + "\n", + "sutil.save_states(states, session_name)\n", + "if not states:\n", + " sutil.destroy_session(session_name)\n", + "\n", + "print(states)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/jupyter/cloudlab-lib.ipynb b/jupyter/cloudlab-lib.ipynb new file mode 100644 index 00000000..19dcec71 --- /dev/null +++ b/jupyter/cloudlab-lib.ipynb @@ -0,0 +1,288 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", + "metadata": {}, + "source": [ + "# Cloudlab + Fabric\n", + "Stitching Cloudlab and Fabric resources using FabFed library." + ] + }, + { + "cell_type": "markdown", + "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", + "metadata": {}, + "source": [ + "## Assign session name and .fab path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8efd310-300a-4f56-bb26-5431bdec3969", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "session_name = \"cloudlab-lzhang9-lib\"\n", + "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/cloudlab\"\n", + "fab_file_path = \"./examples/cloudlab\"\n", + "\n", + "print(session_name)\n", + "print(fab_file_path)" + ] + }, + { + "cell_type": "markdown", + "id": "a6e6acf4-b348-4e08-9808-896f794bb01d", + "metadata": {}, + "source": [ + "## Import FabFeb library" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79a3a114-3c67-4a79-a6fb-8227b64dc429", + "metadata": {}, + "outputs": [], + "source": [ + "from fabfed.util.config import WorkflowConfig\n", + "from fabfed.util import state as sutil\n", + "from fabfed.controller.controller import Controller\n", + "from fabfed.controller.provider_factory import default_provider_factory" + ] + }, + { + "cell_type": "markdown", + "id": "73a535d1-4f1a-4606-b4a4-128c675255db", + "metadata": {}, + "source": [ + "## Read .fab files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce52aa52-99af-4391-bb42-3ab5a9554b9a", + "metadata": {}, + "outputs": [], + "source": [ + "def read_file_to_str(file_path):\n", + " try:\n", + " with open(file_path, 'r') as file:\n", + " file_content = file.read()\n", + " return file_content\n", + " except FileNotFoundError:\n", + " print(f\"File '{file_path}' not found.\")\n", + " return None\n", + "\n", + "# Read config.fab to config_str:\n", + "config_str = read_file_to_str(f\"{fab_file_path}/config.fab\")\n", + "if config_str:\n", + " print(\"File content:\")\n", + " print(config_str)\n", + "\n", + "print(config_str)" + ] + }, + { + "cell_type": "markdown", + "id": "f6106b4e-076f-4349-b215-c6db497c97a9", + "metadata": {}, + "source": [ + "## Create a workflow config object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba75ff1a-7442-4e05-a4ca-b1128397983e", + "metadata": {}, + "outputs": [], + "source": [ + "config = WorkflowConfig(content=config_str)" + ] + }, + { + "cell_type": "markdown", + "id": "c87f8329-054e-45f3-9d69-3c4acc25252f", + "metadata": {}, + "source": [ + "## Create a controller" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d161b4f-4b6b-4993-80d3-26f59034c2cb", + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "controller = Controller(config=config, logger=logger)" + ] + }, + { + "cell_type": "markdown", + "id": "4e1aa491-7ddc-491b-82f6-ec9d41b08c3a", + "metadata": {}, + "source": [ + "## Init stitching information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7da5efbb-0c83-4842-b533-c81340028013", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session)\n", + "\n", + "controller.init(session=session_name, provider_factory=default_provider_factory, provider_states=states)" + ] + }, + { + "cell_type": "markdown", + "id": "465ea762-1e62-4532-a0e3-aa5bef0e23c4", + "metadata": {}, + "source": [ + "## Add resources to providers and validate resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8ae4d51-a812-4d80-997c-11c7ef5548a2", + "metadata": {}, + "outputs": [], + "source": [ + "controller.plan(provider_states=[])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a1a864d-bdd9-4422-84a6-cade3a7c485c", + "metadata": {}, + "outputs": [], + "source": [ + "controller.add(provider_states=[])" + ] + }, + { + "cell_type": "markdown", + "id": "78419199-2802-481e-b0c8-08b362f658e1", + "metadata": {}, + "source": [ + "## Create resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4db644c5-b7d5-4092-8329-cca89a13aef6", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " controller.apply(provider_states=[])\n", + "except ControllerException as e:\n", + " assert isinstance(e.exceptions[0], DummyFailCreateException)" + ] + }, + { + "cell_type": "markdown", + "id": "beee11c5-5adc-4756-8133-a8eda0253969", + "metadata": {}, + "source": [ + "## Save the workflow state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ca3360b-c033-4af8-9f1d-0acf12b1c0d0", + "metadata": {}, + "outputs": [], + "source": [ + "states = controller.get_states()\n", + "sutil.save_states(states, session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", + "metadata": {}, + "source": [ + "## View the state of the workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", + "metadata": {}, + "source": [ + "## Delete all resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc2c921a-4f27-470a-ac8b-a7c957559b2d", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session_name)\n", + "\n", + "controller.destroy(provider_states=states)\n", + "\n", + "sutil.save_states(states, session_name)\n", + "if not states:\n", + " sutil.destroy_session(session_name)\n", + "\n", + "print(states)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/jupyter/native-aws-lib.ipynb b/jupyter/native-aws-lib.ipynb new file mode 100644 index 00000000..184041d5 --- /dev/null +++ b/jupyter/native-aws-lib.ipynb @@ -0,0 +1,300 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", + "metadata": {}, + "source": [ + "# Native AWS + Fabric\n", + "Stitching Native AWS and Fabric resources using FabFed library." + ] + }, + { + "cell_type": "markdown", + "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", + "metadata": {}, + "source": [ + "## Assign session name and .fab path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8efd310-300a-4f56-bb26-5431bdec3969", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "session_name = \"native-aws-lzhang9-lib\"\n", + "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", + "fab_file_path = \"./examples/native-aws\"\n", + "\n", + "print(session_name)\n", + "print(fab_file_path)" + ] + }, + { + "cell_type": "markdown", + "id": "a6e6acf4-b348-4e08-9808-896f794bb01d", + "metadata": {}, + "source": [ + "## Import FabFeb library" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79a3a114-3c67-4a79-a6fb-8227b64dc429", + "metadata": {}, + "outputs": [], + "source": [ + "from fabfed.util.config import WorkflowConfig\n", + "from fabfed.util import state as sutil\n", + "from fabfed.controller.controller import Controller\n", + "from fabfed.controller.provider_factory import default_provider_factory" + ] + }, + { + "cell_type": "markdown", + "id": "73a535d1-4f1a-4606-b4a4-128c675255db", + "metadata": {}, + "source": [ + "## Read .fab files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce52aa52-99af-4391-bb42-3ab5a9554b9a", + "metadata": {}, + "outputs": [], + "source": [ + "def read_file_to_str(file_path):\n", + " try:\n", + " with open(file_path, 'r') as file:\n", + " file_content = file.read()\n", + " return file_content\n", + " except FileNotFoundError:\n", + " print(f\"File '{file_path}' not found.\")\n", + " return None\n", + "\n", + "# Read config.fab to config_str:\n", + "config_str = read_file_to_str(f\"{fab_file_path}/config.fab\")\n", + "if config_str:\n", + " print(\"File content:\")\n", + " print(config_str)\n", + "\n", + "print(config_str)" + ] + }, + { + "cell_type": "markdown", + "id": "f6106b4e-076f-4349-b215-c6db497c97a9", + "metadata": {}, + "source": [ + "## Create a workflow config object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba75ff1a-7442-4e05-a4ca-b1128397983e", + "metadata": {}, + "outputs": [], + "source": [ + "config = WorkflowConfig(content=config_str)" + ] + }, + { + "cell_type": "markdown", + "id": "c87f8329-054e-45f3-9d69-3c4acc25252f", + "metadata": {}, + "source": [ + "## Create a controller" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d161b4f-4b6b-4993-80d3-26f59034c2cb", + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "controller = Controller(config=config, logger=logger)" + ] + }, + { + "cell_type": "markdown", + "id": "4e1aa491-7ddc-491b-82f6-ec9d41b08c3a", + "metadata": {}, + "source": [ + "## Init stitching information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7da5efbb-0c83-4842-b533-c81340028013", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session)\n", + "\n", + "controller.init(session=session_name, provider_factory=default_provider_factory, provider_states=states)" + ] + }, + { + "cell_type": "markdown", + "id": "465ea762-1e62-4532-a0e3-aa5bef0e23c4", + "metadata": {}, + "source": [ + "## Add resources to providers and validate resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8ae4d51-a812-4d80-997c-11c7ef5548a2", + "metadata": {}, + "outputs": [], + "source": [ + "controller.plan(provider_states=[])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a1a864d-bdd9-4422-84a6-cade3a7c485c", + "metadata": {}, + "outputs": [], + "source": [ + "controller.add(provider_states=[])" + ] + }, + { + "cell_type": "markdown", + "id": "78419199-2802-481e-b0c8-08b362f658e1", + "metadata": {}, + "source": [ + "## Create resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4db644c5-b7d5-4092-8329-cca89a13aef6", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " controller.apply(provider_states=[])\n", + "except ControllerException as e:\n", + " assert isinstance(e.exceptions[0], DummyFailCreateException)" + ] + }, + { + "cell_type": "markdown", + "id": "beee11c5-5adc-4756-8133-a8eda0253969", + "metadata": {}, + "source": [ + "## Save the workflow state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ca3360b-c033-4af8-9f1d-0acf12b1c0d0", + "metadata": {}, + "outputs": [], + "source": [ + "states = controller.get_states()\n", + "sutil.save_states(states, session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", + "metadata": {}, + "source": [ + "## View the state of the workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", + "metadata": {}, + "source": [ + "## Delete all resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc2c921a-4f27-470a-ac8b-a7c957559b2d", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session_name)\n", + "\n", + "controller.destroy(provider_states=states)\n", + "\n", + "sutil.save_states(states, session_name)\n", + "if not states:\n", + " sutil.destroy_session(session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "366bbf7f-4d16-4aa4-8845-18edb4c389a6", + "metadata": {}, + "source": [ + "## Note: Amazon Instance Info\n", + "\n", + "- EC2/Instances/i-078a7be6f4a3df5d6\n", + "- Private IPv4 10.0.1.106\n", + "- Public IPv4 54.242.87.34" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/jupyter/native-gcp-lib.ipynb b/jupyter/native-gcp-lib.ipynb new file mode 100644 index 00000000..291d643f --- /dev/null +++ b/jupyter/native-gcp-lib.ipynb @@ -0,0 +1,300 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", + "metadata": {}, + "source": [ + "# Native GCP + Fabric\n", + "Stitching native GCP and Fabric resources using FabFed library." + ] + }, + { + "cell_type": "markdown", + "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", + "metadata": {}, + "source": [ + "## Assign session name and .fab path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8efd310-300a-4f56-bb26-5431bdec3969", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "session_name = \"native-gcp-lzhang9-lib\"\n", + "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/gcp\"\n", + "fab_file_path = \"./examples/native-gcp\"\n", + "\n", + "print(session_name)\n", + "print(fab_file_path)" + ] + }, + { + "cell_type": "markdown", + "id": "a6e6acf4-b348-4e08-9808-896f794bb01d", + "metadata": {}, + "source": [ + "## Import FabFeb library" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79a3a114-3c67-4a79-a6fb-8227b64dc429", + "metadata": {}, + "outputs": [], + "source": [ + "from fabfed.util.config import WorkflowConfig\n", + "from fabfed.util import state as sutil\n", + "from fabfed.controller.controller import Controller\n", + "from fabfed.controller.provider_factory import default_provider_factory" + ] + }, + { + "cell_type": "markdown", + "id": "73a535d1-4f1a-4606-b4a4-128c675255db", + "metadata": {}, + "source": [ + "## Read .fab files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce52aa52-99af-4391-bb42-3ab5a9554b9a", + "metadata": {}, + "outputs": [], + "source": [ + "def read_file_to_str(file_path):\n", + " try:\n", + " with open(file_path, 'r') as file:\n", + " file_content = file.read()\n", + " return file_content\n", + " except FileNotFoundError:\n", + " print(f\"File '{file_path}' not found.\")\n", + " return None\n", + "\n", + "# Read config.fab to config_str:\n", + "config_str = read_file_to_str(f\"{fab_file_path}/config.fab\")\n", + "if config_str:\n", + " print(\"File content:\")\n", + " print(config_str)\n", + "\n", + "print(config_str)" + ] + }, + { + "cell_type": "markdown", + "id": "f6106b4e-076f-4349-b215-c6db497c97a9", + "metadata": {}, + "source": [ + "## Create a workflow config object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba75ff1a-7442-4e05-a4ca-b1128397983e", + "metadata": {}, + "outputs": [], + "source": [ + "config = WorkflowConfig(content=config_str)" + ] + }, + { + "cell_type": "markdown", + "id": "c87f8329-054e-45f3-9d69-3c4acc25252f", + "metadata": {}, + "source": [ + "## Create a controller" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d161b4f-4b6b-4993-80d3-26f59034c2cb", + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "controller = Controller(config=config, logger=logger)" + ] + }, + { + "cell_type": "markdown", + "id": "4e1aa491-7ddc-491b-82f6-ec9d41b08c3a", + "metadata": {}, + "source": [ + "## Init stitching information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7da5efbb-0c83-4842-b533-c81340028013", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session)\n", + "\n", + "controller.init(session=session_name, provider_factory=default_provider_factory, provider_states=states)" + ] + }, + { + "cell_type": "markdown", + "id": "465ea762-1e62-4532-a0e3-aa5bef0e23c4", + "metadata": {}, + "source": [ + "## Add resources to providers and validate resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8ae4d51-a812-4d80-997c-11c7ef5548a2", + "metadata": {}, + "outputs": [], + "source": [ + "controller.plan(provider_states=[])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a1a864d-bdd9-4422-84a6-cade3a7c485c", + "metadata": {}, + "outputs": [], + "source": [ + "controller.add(provider_states=[])" + ] + }, + { + "cell_type": "markdown", + "id": "78419199-2802-481e-b0c8-08b362f658e1", + "metadata": {}, + "source": [ + "## Create resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4db644c5-b7d5-4092-8329-cca89a13aef6", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " controller.apply(provider_states=[])\n", + "except ControllerException as e:\n", + " assert isinstance(e.exceptions[0], DummyFailCreateException)" + ] + }, + { + "cell_type": "markdown", + "id": "beee11c5-5adc-4756-8133-a8eda0253969", + "metadata": {}, + "source": [ + "## Save the workflow state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ca3360b-c033-4af8-9f1d-0acf12b1c0d0", + "metadata": {}, + "outputs": [], + "source": [ + "states = controller.get_states()\n", + "sutil.save_states(states, session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", + "metadata": {}, + "source": [ + "## View the state of the workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", + "metadata": {}, + "source": [ + "## Delete all resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc2c921a-4f27-470a-ac8b-a7c957559b2d", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session_name)\n", + "\n", + "controller.destroy(provider_states=states)\n", + "\n", + "sutil.save_states(states, session_name)\n", + "if not states:\n", + " sutil.destroy_session(session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "366bbf7f-4d16-4aa4-8845-18edb4c389a6", + "metadata": {}, + "source": [ + "## Note: GCP Instance Info\n", + "- vm-69acc1d9-8c24-47cd-90b8-33be57167dbf---vm-1\n", + "- Private IPv4 10.100.0.2\n", + "- Public IPv4 34.150.230.154\n", + "- ssh -i ~/.fabfed/kp-sense-private kp-sense@34.150.230.154" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/jupyter/sense-aws-lib.ipynb b/jupyter/sense-aws-lib.ipynb new file mode 100644 index 00000000..769f5a58 --- /dev/null +++ b/jupyter/sense-aws-lib.ipynb @@ -0,0 +1,300 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", + "metadata": {}, + "source": [ + "# SENSE AWS + Fabric\n", + "Stitching SENSE AWS and Fabric resources using FabFed library." + ] + }, + { + "cell_type": "markdown", + "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", + "metadata": {}, + "source": [ + "## Assign session name and .fab path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8efd310-300a-4f56-bb26-5431bdec3969", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "session_name = \"sense-aws-lzhang9-lib\"\n", + "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", + "fab_file_path = \"./examples/sense-aws\"\n", + "\n", + "print(session_name)\n", + "print(fab_file_path)" + ] + }, + { + "cell_type": "markdown", + "id": "a6e6acf4-b348-4e08-9808-896f794bb01d", + "metadata": {}, + "source": [ + "## Import FabFeb library" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79a3a114-3c67-4a79-a6fb-8227b64dc429", + "metadata": {}, + "outputs": [], + "source": [ + "from fabfed.util.config import WorkflowConfig\n", + "from fabfed.util import state as sutil\n", + "from fabfed.controller.controller import Controller\n", + "from fabfed.controller.provider_factory import default_provider_factory" + ] + }, + { + "cell_type": "markdown", + "id": "73a535d1-4f1a-4606-b4a4-128c675255db", + "metadata": {}, + "source": [ + "## Read .fab files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce52aa52-99af-4391-bb42-3ab5a9554b9a", + "metadata": {}, + "outputs": [], + "source": [ + "def read_file_to_str(file_path):\n", + " try:\n", + " with open(file_path, 'r') as file:\n", + " file_content = file.read()\n", + " return file_content\n", + " except FileNotFoundError:\n", + " print(f\"File '{file_path}' not found.\")\n", + " return None\n", + "\n", + "# Read config.fab to config_str:\n", + "config_str = read_file_to_str(f\"{fab_file_path}/config.fab\")\n", + "if config_str:\n", + " print(\"File content:\")\n", + " print(config_str)\n", + "\n", + "print(config_str)" + ] + }, + { + "cell_type": "markdown", + "id": "f6106b4e-076f-4349-b215-c6db497c97a9", + "metadata": {}, + "source": [ + "## Create a workflow config object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba75ff1a-7442-4e05-a4ca-b1128397983e", + "metadata": {}, + "outputs": [], + "source": [ + "config = WorkflowConfig(content=config_str)" + ] + }, + { + "cell_type": "markdown", + "id": "c87f8329-054e-45f3-9d69-3c4acc25252f", + "metadata": {}, + "source": [ + "## Create a controller" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d161b4f-4b6b-4993-80d3-26f59034c2cb", + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "controller = Controller(config=config, logger=logger)" + ] + }, + { + "cell_type": "markdown", + "id": "4e1aa491-7ddc-491b-82f6-ec9d41b08c3a", + "metadata": {}, + "source": [ + "## Init stitching information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7da5efbb-0c83-4842-b533-c81340028013", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session)\n", + "\n", + "controller.init(session=session_name, provider_factory=default_provider_factory, provider_states=states)" + ] + }, + { + "cell_type": "markdown", + "id": "465ea762-1e62-4532-a0e3-aa5bef0e23c4", + "metadata": {}, + "source": [ + "## Add resources to providers and validate resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8ae4d51-a812-4d80-997c-11c7ef5548a2", + "metadata": {}, + "outputs": [], + "source": [ + "controller.plan(provider_states=[])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a1a864d-bdd9-4422-84a6-cade3a7c485c", + "metadata": {}, + "outputs": [], + "source": [ + "controller.add(provider_states=[])" + ] + }, + { + "cell_type": "markdown", + "id": "78419199-2802-481e-b0c8-08b362f658e1", + "metadata": {}, + "source": [ + "## Create resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4db644c5-b7d5-4092-8329-cca89a13aef6", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " controller.apply(provider_states=[])\n", + "except ControllerException as e:\n", + " assert isinstance(e.exceptions[0], DummyFailCreateException)" + ] + }, + { + "cell_type": "markdown", + "id": "beee11c5-5adc-4756-8133-a8eda0253969", + "metadata": {}, + "source": [ + "## Save the workflow state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ca3360b-c033-4af8-9f1d-0acf12b1c0d0", + "metadata": {}, + "outputs": [], + "source": [ + "states = controller.get_states()\n", + "sutil.save_states(states, session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", + "metadata": {}, + "source": [ + "## View the state of the workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", + "metadata": {}, + "source": [ + "## Delete all resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc2c921a-4f27-470a-ac8b-a7c957559b2d", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session_name)\n", + "\n", + "controller.destroy(provider_states=states)\n", + "\n", + "sutil.save_states(states, session_name)\n", + "if not states:\n", + " sutil.destroy_session(session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "366bbf7f-4d16-4aa4-8845-18edb4c389a6", + "metadata": {}, + "source": [ + "## Note: Amazon Instance Info\n", + "\n", + "- EC2/Instances/i-078a7be6f4a3df5d6\n", + "- Private IPv4 10.0.1.106\n", + "- Public IPv4 54.242.87.34" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/jupyter/sense-gcp-lib.ipynb b/jupyter/sense-gcp-lib.ipynb new file mode 100644 index 00000000..99e088f9 --- /dev/null +++ b/jupyter/sense-gcp-lib.ipynb @@ -0,0 +1,300 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4ed9aa05-01b2-4a98-9992-acd8f38615e2", + "metadata": {}, + "source": [ + "# SENSE GCP + Fabric\n", + "Stitching SENSE GCP and Fabric resources using FabFed library." + ] + }, + { + "cell_type": "markdown", + "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", + "metadata": {}, + "source": [ + "## Assign session name and .fab path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8efd310-300a-4f56-bb26-5431bdec3969", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "session_name = \"sense-gcp-lzhang9-lib\"\n", + "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/sense-gcp\"\n", + "fab_file_path = \"./examples/sense-gcp\"\n", + "\n", + "print(session_name)\n", + "print(fab_file_path)" + ] + }, + { + "cell_type": "markdown", + "id": "a6e6acf4-b348-4e08-9808-896f794bb01d", + "metadata": {}, + "source": [ + "## Import FabFeb library" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79a3a114-3c67-4a79-a6fb-8227b64dc429", + "metadata": {}, + "outputs": [], + "source": [ + "from fabfed.util.config import WorkflowConfig\n", + "from fabfed.util import state as sutil\n", + "from fabfed.controller.controller import Controller\n", + "from fabfed.controller.provider_factory import default_provider_factory" + ] + }, + { + "cell_type": "markdown", + "id": "73a535d1-4f1a-4606-b4a4-128c675255db", + "metadata": {}, + "source": [ + "## Read .fab files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce52aa52-99af-4391-bb42-3ab5a9554b9a", + "metadata": {}, + "outputs": [], + "source": [ + "def read_file_to_str(file_path):\n", + " try:\n", + " with open(file_path, 'r') as file:\n", + " file_content = file.read()\n", + " return file_content\n", + " except FileNotFoundError:\n", + " print(f\"File '{file_path}' not found.\")\n", + " return None\n", + "\n", + "# Read config.fab to config_str:\n", + "config_str = read_file_to_str(f\"{fab_file_path}/config.fab\")\n", + "if config_str:\n", + " print(\"File content:\")\n", + " print(config_str)\n", + "\n", + "print(config_str)" + ] + }, + { + "cell_type": "markdown", + "id": "f6106b4e-076f-4349-b215-c6db497c97a9", + "metadata": {}, + "source": [ + "## Create a workflow config object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba75ff1a-7442-4e05-a4ca-b1128397983e", + "metadata": {}, + "outputs": [], + "source": [ + "config = WorkflowConfig(content=config_str)" + ] + }, + { + "cell_type": "markdown", + "id": "c87f8329-054e-45f3-9d69-3c4acc25252f", + "metadata": {}, + "source": [ + "## Create a controller" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d161b4f-4b6b-4993-80d3-26f59034c2cb", + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "controller = Controller(config=config, logger=logger)" + ] + }, + { + "cell_type": "markdown", + "id": "4e1aa491-7ddc-491b-82f6-ec9d41b08c3a", + "metadata": {}, + "source": [ + "## Init stitching information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7da5efbb-0c83-4842-b533-c81340028013", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session)\n", + "\n", + "controller.init(session=session_name, provider_factory=default_provider_factory, provider_states=states)" + ] + }, + { + "cell_type": "markdown", + "id": "465ea762-1e62-4532-a0e3-aa5bef0e23c4", + "metadata": {}, + "source": [ + "## Add resources to providers and validate resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8ae4d51-a812-4d80-997c-11c7ef5548a2", + "metadata": {}, + "outputs": [], + "source": [ + "controller.plan(provider_states=[])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a1a864d-bdd9-4422-84a6-cade3a7c485c", + "metadata": {}, + "outputs": [], + "source": [ + "controller.add(provider_states=[])" + ] + }, + { + "cell_type": "markdown", + "id": "78419199-2802-481e-b0c8-08b362f658e1", + "metadata": {}, + "source": [ + "## Create resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4db644c5-b7d5-4092-8329-cca89a13aef6", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " controller.apply(provider_states=[])\n", + "except ControllerException as e:\n", + " assert isinstance(e.exceptions[0], DummyFailCreateException)" + ] + }, + { + "cell_type": "markdown", + "id": "beee11c5-5adc-4756-8133-a8eda0253969", + "metadata": {}, + "source": [ + "## Save the workflow state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ca3360b-c033-4af8-9f1d-0acf12b1c0d0", + "metadata": {}, + "outputs": [], + "source": [ + "states = controller.get_states()\n", + "sutil.save_states(states, session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", + "metadata": {}, + "source": [ + "## View the state of the workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", + "metadata": {}, + "source": [ + "## Delete all resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc2c921a-4f27-470a-ac8b-a7c957559b2d", + "metadata": {}, + "outputs": [], + "source": [ + "states = sutil.load_states(session_name)\n", + "\n", + "controller.destroy(provider_states=states)\n", + "\n", + "sutil.save_states(states, session_name)\n", + "if not states:\n", + " sutil.destroy_session(session_name)\n", + "\n", + "print(states)" + ] + }, + { + "cell_type": "markdown", + "id": "366bbf7f-4d16-4aa4-8845-18edb4c389a6", + "metadata": {}, + "source": [ + "## Note: GCP Instance Info\n", + "- vm-69acc1d9-8c24-47cd-90b8-33be57167dbf---vm-1\n", + "- Private IPv4 10.100.0.2\n", + "- Public IPv4 34.150.230.154\n", + "- ssh -i ~/.fabfed/kp-sense-private kp-sense@34.150.230.154" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/jupyter/welcom.ipynb b/jupyter/welcom.ipynb index ae9e2381..a4f41e58 100644 --- a/jupyter/welcom.ipynb +++ b/jupyter/welcom.ipynb @@ -94,7 +94,7 @@ "metadata": {}, "source": [ "## Examples of FabFed Workflows\n", - "Examples of workflows using FabFed CLI:\n", + "### Using FabFed CLI:\n", "\n", "- [Native aws](./native-aws.ipynb): Stiching native AWS and Fabric\n", "- [Native gcp](./native-gcp.ipynb): Stiching native GCP and Fabric\n", @@ -103,8 +103,13 @@ "- [Cloudlab](./cloudlab.ipynb): Stiching Cloudlab and Fabric\n", "- [Chameleon](./chameleon.ipynb): Stiching Chameleon and Fabric\n", "\n", - "Examples of workflows using FabFed library:\n", - "- [Native gcp](./native-gcp.ipynb): Stiching native GCP and Fabric" + "### Using FabFed library:\n", + "- [Native aws](./native-aws-lib.ipynb): Stiching native AWS and Fabric\n", + "- [Native gcp](./native-gcp-lib.ipynb): Stiching native GCP and Fabric\n", + "- [SENSE aws](./sense-aws-lib.ipynb): Stiching SENSE AWS and Fabric\n", + "- [SENSE gcp](./sense-gcp-lib.ipynb): Stiching SENSE GCP and Fabric\n", + "- [Cloudlab](./cloudlab-lib.ipynb): Stiching Cloudlab and Fabric\n", + "- [Chameleon](./chameleon-lib.ipynb): Stiching Chameleon and Fabric" ] }, { @@ -114,8 +119,18 @@ "source": [ "## Tools\n", "Collection of utility tools for monitoring resources\n", - "- [fablib](./fablib-tools.ipynb): Utilities of fablib to manage Fabric slices." + "- [fablib](./fablib-tools.ipynb): Utilities of fablib to manage Fabric slices.\n", + "- [GCP](./fablib-tools.ipynb): Utilities of GCP to manage Google Cloud resources. (upcoming)\n", + "- [AWS](./fablib-tools.ipynb): Utilities of GCP to manage AWS Cloud resources. (upcoming)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3f660dbb-6c2d-4ec7-bc79-e4c19f8e3bc5", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From 355efaf28ff3f9779dca3f742a0fc8d27be90155 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Wed, 13 Mar 2024 14:23:13 -0500 Subject: [PATCH 31/75] Update cloudlab.ipynb --- jupyter/cloudlab.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter/cloudlab.ipynb b/jupyter/cloudlab.ipynb index 01ae9bd6..582bfd91 100644 --- a/jupyter/cloudlab.ipynb +++ b/jupyter/cloudlab.ipynb @@ -27,7 +27,7 @@ "import os\n", "\n", "session_name = \"cloudlab-lzhang9\"\n", - "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", + "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/cloudlab\"\n", "fab_file_path = \"./examples/cloudlab\"\n", "\n", "print(session_name)\n", From 549d54c59ed8758dac384ec82f023b53a8f5ec31 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Wed, 13 Mar 2024 13:33:39 -0700 Subject: [PATCH 32/75] added stitch info option to the command line for a workflow --- README.md | 2 ++ fabfed/util/state.py | 14 ++++++++++++ fabfed/util/utils.py | 1 + tools/fabfed.py | 54 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+) diff --git a/README.md b/README.md index 991d2774..f8e50cd6 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ fabfed workflow --config-dir some_dir [--var-file some_var_file.yml] --session s fabfed workflow --config-dir some_dir [--var-file some_var_file.yml] --session some_session -init [-summary] [-json] +fabfed workflow --config-dir some_dir [--var-file some_var_file.yml] --session some_session -stitch-info [-summary] [-json] + fabfed workflow --config-dir some_dir [--var-file some_var_file.yml] --session some_session -plan [-summary] [-json] fabfed workflow --config-dir some_dir [--var-file some_var_file.yml] --session some_session -apply diff --git a/fabfed/util/state.py b/fabfed/util/state.py index 2928f9f5..eb5456f9 100644 --- a/fabfed/util/state.py +++ b/fabfed/util/state.py @@ -191,6 +191,20 @@ def dump_resources(*, resources, to_json: bool, summary: bool = False): sys.stdout.write(yaml.dump(resources, Dumper=get_dumper(), default_flow_style=False, sort_keys=False)) +def dump_objects(objects, to_json: bool): + import sys + + if to_json: + import json + + sys.stdout.write(json.dumps(objects, cls=SetEncoder, indent=3)) + else: + import yaml + from fabfed.model.state import get_dumper + + sys.stdout.write(yaml.dump(objects, Dumper=get_dumper(), default_flow_style=False, sort_keys=False)) + + def dump_states(states, to_json: bool, summary: bool = False): import sys from fabfed.model.state import get_dumper diff --git a/fabfed/util/utils.py b/fabfed/util/utils.py index d5310557..e9cb203d 100644 --- a/fabfed/util/utils.py +++ b/fabfed/util/utils.py @@ -43,6 +43,7 @@ def build_parser(*, manage_workflow, manage_sessions, display_stitch_info): help='assembles and validates all .fab files in the config directory') workflow_parser.add_argument('-apply', action='store_true', default=False, help='create resources') workflow_parser.add_argument('-init', action='store_true', default=False, help='display resource ordering') + workflow_parser.add_argument('-stitch-info', action='store_true', default=False, help='display network stitch-info') workflow_parser.add_argument('-plan', action='store_true', default=False, help='shows plan') workflow_parser.add_argument('-use-remote-policy', action='store_true', default=False, help='use remote policy') workflow_parser.add_argument('-show', action='store_true', default=False, help='display resource.') diff --git a/tools/fabfed.py b/tools/fabfed.py index acf42ce5..7cdf3ac8 100644 --- a/tools/fabfed.py +++ b/tools/fabfed.py @@ -146,6 +146,60 @@ def manage_workflow(args): sutil.dump_resources(resources=controller.resources, to_json=args.json, summary=args.summary) return + if args.stitch_info: + config = WorkflowConfig.parse(dir_path=config_dir, var_dict=var_dict) + controller = Controller(config=config, + policy=policy, + use_local_policy=not args.use_remote_policy) + states = sutil.load_states(args.session) + controller.init(session=args.session, provider_factory=default_provider_factory, provider_states=states) + resources = controller.resources + + from collections import namedtuple + + if not args.summary: + stitch_info_details = [] + + StitchInfoDetails = namedtuple("StitchInfoDetails", "label provider_label stitch_info") + + for network in filter(lambda n: n.is_network, resources): + details = StitchInfoDetails(label=network.label, + provider_label=network.provider.label, + stitch_info=network.attributes.get(Constants.RES_STITCH_INFO)) + stitch_info_details.append(details) + + stitch_info_details = dict(StitchNetworkDetails = stitch_info_details ) + sutil.dump_objects(objects=stitch_info_details, to_json=args.json) + + NetworkInfo = namedtuple("NetworkInfo", "label provider_label") + StitchInfoSummary = namedtuple("StitchInfoSummary", "network_infos stitch_info") + + stitch_info_summaries = [] + stitch_info_map = {} + stitch_info_network_info_map = {} + + + for network in filter(lambda n: n.is_network and n.attributes.get(Constants.RES_STITCH_INFO), resources): + stitch_info = network.attributes.get(Constants.RES_STITCH_INFO) + + if stitch_info: + network_info = NetworkInfo(label=network.label, provider_label=network.provider.label) + stitch_port_name = stitch_info.stitch_port['name'] + stitch_info_map[stitch_port_name] = stitch_info + + if stitch_port_name not in stitch_info_network_info_map: + stitch_info_network_info_map[stitch_port_name] = [] + + stitch_info_network_info_map[stitch_port_name].append(network_info) + + for k, v in stitch_info_network_info_map.items(): + stitch_info_summary = StitchInfoSummary(network_infos=v, stitch_info=stitch_info_map[k]) + stitch_info_summaries.append(stitch_info_summary) + + stitch_info_summaries = dict(StitchInfoSummary = stitch_info_summaries) + sutil.dump_objects(objects=stitch_info_summaries, to_json=args.json) + return + if args.plan: config = WorkflowConfig.parse(dir_path=config_dir, var_dict=var_dict) controller = Controller(config=config, From 537ac4254e1b9fd165ca1dc342fbcb3dce4783b6 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Wed, 13 Mar 2024 14:35:57 -0700 Subject: [PATCH 33/75] better plan outoput adding sense aws to cicd fix for sense aws when deleting and fabfed exits with code 1 now --- .github/workflows/ubuntu_22_and_test.yml | 28 ++++++++- README.md | 4 +- cicd/docker/Dockerfile.jupyter | 17 ++++++ cicd/fabfed_credentials.yml.cicd | 10 ++++ cicd/run-fabfed.sh | 10 +++- cicd/test_configs/fabric_sense_aws/config.fab | 58 +++++++++++++++++++ examples/demos/sense-aws/config.fab | 1 - fabfed/controller/controller.py | 9 +-- fabfed/provider/sense/sense_constants.py | 9 +++ fabfed/provider/sense/sense_provider.py | 27 +++++---- fabfed/util/state.py | 21 +++---- tools/fabfed.py | 19 +++--- 12 files changed, 168 insertions(+), 45 deletions(-) create mode 100644 cicd/docker/Dockerfile.jupyter create mode 100644 cicd/test_configs/fabric_sense_aws/config.fab diff --git a/.github/workflows/ubuntu_22_and_test.yml b/.github/workflows/ubuntu_22_and_test.yml index 389f9463..af064e48 100644 --- a/.github/workflows/ubuntu_22_and_test.yml +++ b/.github/workflows/ubuntu_22_and_test.yml @@ -18,6 +18,12 @@ env: FABRIC_PROJECT: ${{ secrets.FABRIC_PROJECT }} FABRIC_USER: ${{ secrets.FABRIC_USER }} + SENSE_USER: ${{ secrets.SENSE_USER }} + SENSE_PASSWORD: ${{ secrets.SENSE_PASSWORD }} + SENSE_SECRET: ${{ secrets.SENSE_SECRET }} + SENSE_SLIVER_KEY: ${{ secrets.SENSE_SLIVER_KEY }} + RUN_FABRIC_AWS_SENSE: ${{ vars.RUN_FABRIC_AWS_SENSE }} + jobs: UbuntuTest: runs-on: ubuntu-latest @@ -44,6 +50,16 @@ jobs: pip install --no-cache-dir --ignore-requires-python -r requirements.txt python3 -m pip install --no-cache-dir . + - name: Run Unit Tests + run: | + echo "*******************" + echo ${{ env.RUN_FABRIC_AWS_SENSE }} + echo "*******************" + echo ${{ env.RUN_FABRIC_AWS_SENSE == 'false' }} + echo "*******************" + echo ${{ env.RUN_FABRIC_AWS_SENSE == false }} + echo "*******************" + - name: Run Unit Tests run: | pytest tests @@ -64,13 +80,21 @@ jobs: echo ${{ env.FABRIC_BASTION_KEY }} | base64 --decode > ${{ github.workspace }}/creds/bastion echo ${{ env.FABRIC_SLIVER_KEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver echo ${{ env.FABRIC_SLIVER_PUBKEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver.pub + echo ${{ env.SENSE_SLIVER_KEY }} | base64 --decode > ${{ github.workspace }}/creds/sense - name: Test Fabric FacilityPort + if: ${{ env.RUN_FABRIC_AWS_SENSE == 'false' || env.RUN_FABRIC_AWS_SENSE == false }} run: | - session=fabric-facility-port + session=cicd-fabric-facility-port echo "vlan: 3102" > $session-varfile.yml ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml - name: Test L2VPN with nodes. + if: ${{ env.RUN_FABRIC_AWS_SENSE == 'false' || env.RUN_FABRIC_AWS_SENSE == false }} + run: | + ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_l2_vpn cicd-fabric-l2-vpn + + - name: Test SENSE/AWS. + if: ${{ env.RUN_FABRIC_AWS_SENSE == 'true' || env.RUN_FABRIC_AWS_SENSE == true }} run: | - ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_l2_vpn fabric-l2-vpn + ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_sense_aws cicd-sense-aws diff --git a/README.md b/README.md index f8e50cd6..2066ed49 100644 --- a/README.md +++ b/README.md @@ -64,10 +64,8 @@ the --config-dir. If this option is not present, the current directory i # Example to view stitch policy from cloudlab to fabric fabfed stitch-policy -providers "fabric,cloudlab" +# Validation fabfed workflow --config-dir some_dir [--var-file some_var_file.yml] --session some_session -validate - -fabfed workflow --config-dir some_dir [--var-file some_var_file.yml] --session some_session -init [-summary] [-json] - fabfed workflow --config-dir some_dir [--var-file some_var_file.yml] --session some_session -stitch-info [-summary] [-json] fabfed workflow --config-dir some_dir [--var-file some_var_file.yml] --session some_session -plan [-summary] [-json] diff --git a/cicd/docker/Dockerfile.jupyter b/cicd/docker/Dockerfile.jupyter new file mode 100644 index 00000000..652f8236 --- /dev/null +++ b/cicd/docker/Dockerfile.jupyter @@ -0,0 +1,17 @@ +ARG base_image=ubuntu:24.04 + +FROM ${base_image} + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && apt-get -u -y install lsb-release python3 binutils python3-venv python3-pip wget git bash-completion vim yq jq \ + && apt-get -qq purge && apt-get -qq clean && rm -rf /var/lib/apt/lists/* + +RUN pip install --break-system-packages jupyterlab +COPY /requirements.txt /tmp/ +RUN pip install --no-cache-dir --break-system-packages --ignore-requires-python -r /tmp/requirements.txt + +COPY ./setup.py ./tools ./README.md ./MANIFEST.in /requirements.txt / +COPY ./fabfed /fabfed +COPY ./tools /tools +RUN python3 -m pip install --break-system-packages --no-cache-dir . diff --git a/cicd/fabfed_credentials.yml.cicd b/cicd/fabfed_credentials.yml.cicd index 49f38261..9768b129 100644 --- a/cicd/fabfed_credentials.yml.cicd +++ b/cicd/fabfed_credentials.yml.cicd @@ -13,3 +13,13 @@ aws: gcp: service_key_path: GCP_SERVICE_KEY_PATH project: GCP_PROJECT + +sense: + auth_endpoint: https://sense-o.es.net:8543/auth/realms/StackV/protocol/openid-connect/token + api_endpoint: https://sense-o-dev.es.net:8443/StackV-web/restapi + client_id: StackV + username: SENSE_USER + password: SENSE_PASSWORD + secret: SENSE_SECRET + verify: False + slice-private-key-location: creds/sense diff --git a/cicd/run-fabfed.sh b/cicd/run-fabfed.sh index bb53eee4..2517e510 100755 --- a/cicd/run-fabfed.sh +++ b/cicd/run-fabfed.sh @@ -17,6 +17,12 @@ cp $script_dir/fabfed_credentials.yml.cicd ~/.fabfed/ sed -i "s/FABRIC_PROJECT/$FABRIC_PROJECT/" ~/.fabfed/fabfed_credentials.yml.cicd sed -i "s/FABRIC_USER/$FABRIC_USER/" ~/.fabfed/fabfed_credentials.yml.cicd +# SENSE +sed -i "s/SENSE_USER/$SENSE_USER/" ~/.fabfed/fabfed_credentials.yml.cicd +sed -i "s/SENSE_PASSWORD/$SENSE_PASSWORD/" ~/.fabfed/fabfed_credentials.yml.cicd +sed -i "s/SENSE_SECRET/$SENSE_SECRET/" ~/.fabfed/fabfed_credentials.yml.cicd + + if [ -n "$var_file" ] then options="-v $3" @@ -29,7 +35,7 @@ ret1=$? echo "***************** APPLY SUMMARY ****************" echo fabfed workflow -c $conf_dir $options -s $session -show -summary -summary=`fabfed workflow -c $conf_dir $options -s $session -show -summary` +fabfed workflow -c $conf_dir $options -s $session -show -summary > $session-apply-state.yaml fabfed workflow -c $conf_dir $options -s $session -show -summary fabfed sessions -show @@ -44,7 +50,7 @@ fabfed sessions -show echo "***************** APPLY RESULTS ****************" echo "APPLY RESULTS:" -echo $summary +cat $session-apply-state.yaml if [[ ! $ret1 -eq 0 ]]; then echo "Apply failed ....." diff --git a/cicd/test_configs/fabric_sense_aws/config.fab b/cicd/test_configs/fabric_sense_aws/config.fab new file mode 100644 index 00000000..fff54f92 --- /dev/null +++ b/cicd/test_configs/fabric_sense_aws/config.fab @@ -0,0 +1,58 @@ +provider: + - sense: + - sense_provider: + - credential_file: ~/.fabfed/fabfed_credentials.yml.cicd + profile: sense + - fabric: + - fabric_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml.cicd + profile: fabric + +config: + - layer3: + - fab_layer: + subnet: 192.168.10.0/24 # subnet: "10.0.0.0/24" # for sense subnet.cidr + gateway: 192.168.10.1 + ip_start: 192.168.10.2 # optional: auto generate if subnet is given + ip_end: 192.168.10.254 + - sense_layer: + subnet: 10.200.1.0/24 # subnet.cidr and vpc.cidr + - peering: + - my_peering: + # FOR FABRIC + cloud_account: "296256999979" + cloud_facility: + local_device: + local_port: + cloud_region: "us-east-1" + + # FOR SENSE and FABRIC + local_asn: "55038" # customer_asn (hard coded for now) + local_address: "192.168.1.1/30" # customer_ip + remote_asn: "64512" # amazon_asn + remote_address: "192.168.1.2/30" # amazon_ip +resource: + - node: + - sense_node: + provider: '{{ sense.sense_provider }}' + network: "{{ network.sense_net }}" + service: + - fabric_node: + provider: '{{ fabric.fabric_provider }}' + site: MAX # Use RENC when deploying to fabric beta environment + image: default_rocky_8 + nic_model: NIC_Basic + - network: + - sense_net: + provider: '{{ sense.sense_provider }}' + layer3: "{{ layer3.sense_layer }}" + peering: "{{ peering.my_peering }}" + stitch_with: '{{ network.fabric_network }}' + stitch_option: + device_name: agg3.ashb + + - fabric_network: + provider: '{{ fabric.fabric_provider }}' + layer3: "{{ layer3.fab_layer }}" + peering: "{{ peering.my_peering }}" + interface: '{{ node.fabric_node }}' diff --git a/examples/demos/sense-aws/config.fab b/examples/demos/sense-aws/config.fab index d9104943..05074eb3 100644 --- a/examples/demos/sense-aws/config.fab +++ b/examples/demos/sense-aws/config.fab @@ -52,7 +52,6 @@ resource: - network: - sense_net: provider: '{{ sense.sense_provider }}' - name: aws-net layer3: "{{ layer3.sense_layer }}" peering: "{{ peering.my_peering }}" count: 1 diff --git a/fabfed/controller/controller.py b/fabfed/controller/controller.py index d7ca650b..7cf37eb6 100644 --- a/fabfed/controller/controller.py +++ b/fabfed/controller/controller.py @@ -145,15 +145,10 @@ def init(self, *, session: str, provider_factory: ProviderFactory, provider_stat if Constants.RES_LAYER3 in other.attributes: network.attributes[Constants.RES_PEER_LAYER3].append(other.attributes[Constants.RES_LAYER3]) - for network in [net for net in networks if net.attributes.get(Constants.NETWORK_STITCH_WITH)]: - temp_label = network.attributes[Constants.LABEL] - stitch_with = network.attributes[Constants.NETWORK_STITCH_WITH] + for network in [net for net in networks if net.attributes.get(Constants.RES_STITCH_INFO)]: stitch_info = network.attributes.get(Constants.RES_STITCH_INFO) - assert stitch_info is not None, f"network {temp_label} does not have {stitch_info}" - self.logger.info(f"{network}:stitch_with={stitch_with}: stitch_info={stitch_info}") - - self.resources = [r for r in self.resources if r.attributes[Constants.RES_COUNT] > 0] + self.logger.info(f"{network}: stitch_info={stitch_info}") def plan(self, provider_states: List[ProviderState]): resources = self.resources diff --git a/fabfed/provider/sense/sense_constants.py b/fabfed/provider/sense/sense_constants.py index 06b68115..9e465771 100644 --- a/fabfed/provider/sense/sense_constants.py +++ b/fabfed/provider/sense/sense_constants.py @@ -1,7 +1,16 @@ from fabfed.util.constants import Constants import enum +AUTH_ENDPOINT = "auth_endpoint" +API_ENDPOINT = "api_endpoint" +CLIENT_ID = "client_id" +USERNAME = "username" +PASSWORD = "password" +SECRET = "secret" SENSE_SLICE_PRIVATE_KEY_LOCATION = "slice-private-key-location" + +SENSE_CONF_ATTRS = [AUTH_ENDPOINT, API_ENDPOINT, CLIENT_ID, USERNAME, PASSWORD, SECRET, SENSE_SLICE_PRIVATE_KEY_LOCATION] + SENSE_PROFILE_UID = "service_profile_uuid" SENSE_ALIAS = "alias" SENSE_EDIT = "options" diff --git a/fabfed/provider/sense/sense_provider.py b/fabfed/provider/sense/sense_provider.py index f7406ede..51028e75 100644 --- a/fabfed/provider/sense/sense_provider.py +++ b/fabfed/provider/sense/sense_provider.py @@ -1,9 +1,9 @@ from fabfed.policy.policy_helper import get_stitch_port_for_provider -from fabfed.exceptions import ResourceTypeNotSupported +from fabfed.exceptions import ResourceTypeNotSupported, ProviderException from fabfed.provider.api.provider import Provider -from fabfed.util.constants import Constants from fabfed.util.utils import get_logger from .sense_exceptions import SenseException +from .sense_constants import * logger = get_logger() @@ -16,20 +16,20 @@ def __init__(self, *, type, label, name, config: dict): self._handled_modify = False def setup_environment(self): - from fabfed.util import utils + for attr in SENSE_CONF_ATTRS: + if self.config.get(attr.lower()) is None and self.config.get(attr.upper()) is None: + raise ProviderException(f"{self.name}: Expecting a value for {attr}") - credential_file = self.config.get(Constants.CREDENTIAL_FILE) - profile = self.config.get(Constants.PROFILE) - config = utils.load_yaml_from_file(credential_file) + pkey = self.config[SENSE_SLICE_PRIVATE_KEY_LOCATION] - if profile not in config: - from fabfed.exceptions import ProviderException + from fabfed.util.utils import can_read, is_private_key, absolute_path - raise ProviderException( - f"credential file {credential_file} does not have a section for keyword {profile}" - ) + pkey = absolute_path(pkey) - self.config = config[profile] + if not can_read(pkey) or not is_private_key(pkey): + raise ProviderException(f"{self.name}: unable to read/parse ssh key in {pkey}") + + self.config[SENSE_SLICE_PRIVATE_KEY_LOCATION] = pkey @property def private_key_file_location(self): @@ -62,9 +62,11 @@ def _handle_peering_config(self, resource): def _init_client(self): if not self.initialized: + self.logger.info(f"{self.name}: Initializing sense client") from .sense_client import init_client init_client(self.config) + self.logger.info(f"{self.name}: Initialized sense client") self.initialized = True def do_add_resource(self, *, resource: dict): @@ -190,6 +192,7 @@ def do_create_resource(self, *, resource: dict): self.logger.debug(f"Created network: {vars(net)}") def do_delete_resource(self, *, resource: dict): + self._init_client() rtype = resource.get(Constants.RES_TYPE) assert rtype in self.supported_resources label = resource.get(Constants.LABEL) diff --git a/fabfed/util/state.py b/fabfed/util/state.py index eb5456f9..5b64ae69 100644 --- a/fabfed/util/state.py +++ b/fabfed/util/state.py @@ -19,6 +19,7 @@ def dump_plan(*, resources, to_json: bool, summary: bool = False): import sys plan = {} + ResourceSummary = namedtuple("ResourceSummary", "label attributes") if not summary: summaries = [] @@ -55,21 +56,21 @@ def dump_plan(*, resources, to_json: bool, summary: bool = False): summaries.append(ResourceDetails(label=label, attributes=resource_dict)) plan['resource_details'] = summaries - ResourceSummary = namedtuple("ResourceSummary", "label attributes") - summaries = [] + summaries = [] - for resource in resources: - resource_dict = {} + for resource in resources: + resource_dict = {} - label = resource.attributes[Constants.LABEL] + label = resource.attributes[Constants.LABEL] - import copy + import copy - details = copy.deepcopy(resource.attributes[Constants.RES_CREATION_DETAILS]) - resource_dict[Constants.RES_CREATION_DETAILS] = details - summaries.append(ResourceSummary(label=label, attributes=resource_dict)) + details = copy.deepcopy(resource.attributes[Constants.RES_CREATION_DETAILS]) + resource_dict[Constants.RES_CREATION_DETAILS] = details + summaries.append(ResourceSummary(label=label, attributes=resource_dict)) + + plan['resource_creation_details'] = summaries - plan['resource_summaries'] = summaries summaries = [] to_be_created = 0 to_be_deleted = 0 diff --git a/tools/fabfed.py b/tools/fabfed.py index 7cdf3ac8..234fc729 100644 --- a/tools/fabfed.py +++ b/tools/fabfed.py @@ -94,6 +94,9 @@ def manage_workflow(args): except ControllerException as ce: logger.error(f"Exceptions while creating resources ... {ce}") workflow_failed = True + except Exception as e: + logger.error(f"Unknown error while creating resources ... {e}") + workflow_failed = True controller_duration = time.time() - controller_duration_start providers_duration = 0 @@ -105,9 +108,7 @@ def manage_workflow(args): states = controller.get_states() nodes, networks, services, pending, failed = utils.get_counters(states=states) - - if pending or failed: - workflow_failed = True + workflow_failed = workflow_failed or pending or failed if Constants.RECONCILE_STATES: states = sutil.reconcile_states(states, args.session) @@ -134,7 +135,7 @@ def manage_workflow(args): logger.info(f"STATS:duration_in_seconds={workflow_duration}") logger.info(f"nodes={nodes}, networks={networks}, services={services}, pending={pending}, failed={failed}") sutil.save_stats(dict(comment="all durations are in seconds", stats=fabfed_stats), args.session) - return + sys.exit(1 if workflow_failed else 0) if args.init: config = WorkflowConfig.parse(dir_path=config_dir, var_dict=var_dict) @@ -168,7 +169,7 @@ def manage_workflow(args): stitch_info=network.attributes.get(Constants.RES_STITCH_INFO)) stitch_info_details.append(details) - stitch_info_details = dict(StitchNetworkDetails = stitch_info_details ) + stitch_info_details = dict(StitchNetworkDetails=stitch_info_details) sutil.dump_objects(objects=stitch_info_details, to_json=args.json) NetworkInfo = namedtuple("NetworkInfo", "label provider_label") @@ -178,7 +179,6 @@ def manage_workflow(args): stitch_info_map = {} stitch_info_network_info_map = {} - for network in filter(lambda n: n.is_network and n.attributes.get(Constants.RES_STITCH_INFO), resources): stitch_info = network.attributes.get(Constants.RES_STITCH_INFO) @@ -196,7 +196,7 @@ def manage_workflow(args): stitch_info_summary = StitchInfoSummary(network_infos=v, stitch_info=stitch_info_map[k]) stitch_info_summaries.append(stitch_info_summary) - stitch_info_summaries = dict(StitchInfoSummary = stitch_info_summaries) + stitch_info_summaries = dict(StitchInfoSummary=stitch_info_summaries) sutil.dump_objects(objects=stitch_info_summaries, to_json=args.json) return @@ -251,10 +251,13 @@ def manage_workflow(args): logger.error(f"Exceptions while initializing controller .... {e}") sys.exit(1) + destroy_failed = False + try: controller.destroy(provider_states=states) except ControllerException as e: logger.error(f"Exceptions while deleting resources ...{e}") + destroy_failed = True except KeyboardInterrupt as kie: logger.error(f"Keyboard Interrupt while deleting resources ... {kie}") sys.exit(1) @@ -293,7 +296,7 @@ def manage_workflow(args): provider_stats=provider_stats) logger.info(f"STATS:duration_in_seconds={workflow_duration}") sutil.save_stats(dict(comment="all durations are in seconds", stats=fabfed_stats), args.session) - return + sys.exit(1 if destroy_failed else 0) def manage_sessions(args): From f0795411673472f76a1acb2151249dc1e1315b90 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Thu, 14 Mar 2024 12:16:31 -0700 Subject: [PATCH 34/75] adding fabfed_manager --- fabfed/fabfed_manager.py | 202 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 fabfed/fabfed_manager.py diff --git a/fabfed/fabfed_manager.py b/fabfed/fabfed_manager.py new file mode 100644 index 00000000..35b2b6fc --- /dev/null +++ b/fabfed/fabfed_manager.py @@ -0,0 +1,202 @@ +from fabfed.util import utils +from fabfed.util.config import WorkflowConfig +from typing import Union, Dict, List, Any +from fabfed.controller.controller import Controller +from fabfed.util import state as sutil +from fabfed.model.state import ProviderState +from fabfed.exceptions import ControllerException +from fabfed.util.constants import Constants + +logger = utils.init_logger() + +''' +from fabfed.fabfed_manager import FabfedManager + +config_dir = "examples/fabric" +session="some_session" +fabfed_manager = FabfedManager(config_dir=config_dir) +fabfed_manager.validate() +fabfed_manager.stitch_info(session=session) +to_be_created_count, to_be_deleted_count = fabfed_manager.plan(session=session) +status_code = fabfed_manager.apply(session=session) +fabfed_manager.show(session=session) +fabfed_manager.show_sessions() +status_code = fabfed_manager.destroy(session=session) +fabfed_manager.show(session=session) +fabfed_manager.show_sessions() + +''' + + +class FabfedManager: + def __init__(self, *, config_dir: str, var_dict: Union[Dict[str, str], None] = None): + self.config_dir = config_dir + self.var_dict = var_dict or dict() + self.controller: Union[Controller, None] = None + self.provider_states: List[ProviderState] = list() + self.sessions: List[Any] = list() + + def _load_sessions(self): + import os + from pathlib import Path + + base_dir = os.path.join(str(Path.home()), '.fabfed', 'sessions') + os.makedirs(base_dir, exist_ok=True) + sessions = os.listdir(base_dir) + self.sessions = [dict(session=s, config_dir=sutil.load_meta_data(s, 'config_dir')) for s in sessions] + + def _delete_session_if_empty(self, *, session): + self.provider_states = sutil.load_states(session) + + if not self.provider_states: + sutil.destroy_session(session) + + self._load_sessions() + + def _init_controller(self, *, session: str): + self.provider_states = sutil.load_states(session) + config = WorkflowConfig.parse(dir_path=self.config_dir, var_dict=self.var_dict) + controller: Controller = Controller(config=config) + + from fabfed.controller.provider_factory import default_provider_factory + controller.init(session=session, + provider_factory=default_provider_factory, + provider_states=self.provider_states) + self.controller = controller + + def validate(self): + config = WorkflowConfig.parse(dir_path=self.config_dir, var_dict=self.var_dict) + return config + + def plan(self, *, session: str, to_json: bool = False, summary: bool = True): + self._init_controller(session=session) + self.controller.plan(provider_states=self.provider_states) + resources = self.controller.resources + cr, dl = sutil.dump_plan(resources=resources, to_json=to_json, summary=summary) + + logger.warning(f"Applying this plan would create {cr} resource(s) and destroy {dl} resource(s)") + self._delete_session_if_empty(session=session) + return cr, dl + + def apply(self, *, session: str): + self._init_controller(session=session) + self.controller.plan(provider_states=self.provider_states) + self.controller.add(provider_states=self.provider_states) + workflow_failed = False + + sutil.save_meta_data(dict(config_dir=self.config_dir), session) + + try: + self.controller.apply(provider_states=self.provider_states) + except KeyboardInterrupt as kie: + logger.error(f"Keyboard Interrupt while creating resources ... {kie}") + workflow_failed = True + except ControllerException as ce: + logger.error(f"Exceptions while creating resources ... {ce}") + workflow_failed = True + except Exception as e: + logger.error(f"Unknown error while creating resources ... {e}") + workflow_failed = True + + self.provider_states = self.controller.get_states() + nodes, networks, services, pending, failed = utils.get_counters(states=self.provider_states) + workflow_failed = workflow_failed or pending or failed + self.provider_states = sutil.reconcile_states(self.provider_states, session) + sutil.save_states(self.provider_states, session) + logger.info(f"nodes={nodes}, networks={networks}, services={services}, pending={pending}, failed={failed}") + return 1 if workflow_failed else 0 + + def show(self, *, session: str, to_json: bool = False, summary: bool = True): + self._load_sessions() + session_names = [session_meta['session'] for session_meta in self.sessions] + self.provider_states = sutil.load_states(session) if session in session_names else [] + sutil.dump_states(self.provider_states, to_json, summary) + self._delete_session_if_empty(session=session) + + def stitch_info(self, session: str, to_json: bool = False, summary: bool = True): + self._init_controller(session=session) + + resources = self.controller.resources + + from collections import namedtuple + + if not summary: + stitch_info_details = [] + + StitchInfoDetails = namedtuple("StitchInfoDetails", "label provider_label stitch_info") + + for network in filter(lambda n: n.is_network, resources): + details = StitchInfoDetails(label=network.label, + provider_label=network.provider.label, + stitch_info=network.attributes.get(Constants.RES_STITCH_INFO)) + stitch_info_details.append(details) + + stitch_info_details = dict(StitchNetworkDetails=stitch_info_details) + sutil.dump_objects(objects=stitch_info_details, to_json=to_json) + + NetworkInfo = namedtuple("NetworkInfo", "label provider_label") + StitchInfoSummary = namedtuple("StitchInfoSummary", "network_infos stitch_info") + + stitch_info_summaries = [] + stitch_info_map = {} + stitch_info_network_info_map = {} + + for network in filter(lambda n: n.is_network and n.attributes.get(Constants.RES_STITCH_INFO), resources): + stitch_info = network.attributes.get(Constants.RES_STITCH_INFO) + + if stitch_info: + network_info = NetworkInfo(label=network.label, provider_label=network.provider.label) + stitch_port_name = stitch_info.stitch_port['name'] + stitch_info_map[stitch_port_name] = stitch_info + + if stitch_port_name not in stitch_info_network_info_map: + stitch_info_network_info_map[stitch_port_name] = [] + + stitch_info_network_info_map[stitch_port_name].append(network_info) + + for k, v in stitch_info_network_info_map.items(): + stitch_info_summary = StitchInfoSummary(network_infos=v, stitch_info=stitch_info_map[k]) + stitch_info_summaries.append(stitch_info_summary) + + stitch_info_summaries = dict(StitchInfoSummary=stitch_info_summaries) + sutil.dump_objects(objects=stitch_info_summaries, to_json=to_json) + + def destroy(self, *, session: str): + self._load_sessions() + session_names = [session_meta['session'] for session_meta in self.sessions] + + if session not in session_names: + return + + self.provider_states = sutil.load_states(session) + + if not self.provider_states: + sutil.destroy_session(session) + self.provider_states = [] + self._load_sessions() + return + + self._init_controller(session=session) + + destroy_failed = False + + try: + self.controller.destroy(provider_states=self.provider_states) + except ControllerException as e: + logger.error(f"Exceptions while deleting resources ...{e}") + destroy_failed = True + except KeyboardInterrupt as kie: + logger.error(f"Keyboard Interrupt while deleting resources ... {kie}") + return 1 + + if not self.provider_states: + logger.info(f"Destroying session {session} ...") + sutil.destroy_session(session) + self.controller = None + else: + sutil.save_states(self.provider_states, session) + + return 1 if destroy_failed else 0 + + def show_sessions(self, *, to_json: bool = False): + self.sessions = utils.dump_sessions(to_json) From 165e0250dece001515256635915ad1105ba729ce Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Thu, 14 Mar 2024 14:07:25 -0700 Subject: [PATCH 35/75] aws_fabric_native to cicd --- .github/workflows/ubuntu_22_and_test.yml | 9 ++- .../test_configs/fabric_native_aws/config.fab | 60 +++++++++++++++++++ cicd/test_configs/fabric_sense_aws/config.fab | 1 - fabfed/controller/controller.py | 3 + fabfed/policy/policy_helper.py | 6 ++ 5 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 cicd/test_configs/fabric_native_aws/config.fab diff --git a/.github/workflows/ubuntu_22_and_test.yml b/.github/workflows/ubuntu_22_and_test.yml index af064e48..7dc22632 100644 --- a/.github/workflows/ubuntu_22_and_test.yml +++ b/.github/workflows/ubuntu_22_and_test.yml @@ -50,7 +50,7 @@ jobs: pip install --no-cache-dir --ignore-requires-python -r requirements.txt python3 -m pip install --no-cache-dir . - - name: Run Unit Tests + - name: Print Env Vars run: | echo "*******************" echo ${{ env.RUN_FABRIC_AWS_SENSE }} @@ -94,6 +94,13 @@ jobs: run: | ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_l2_vpn cicd-fabric-l2-vpn + - name: Test Fabric FacilityPort + if: ${{ env.RUN_FABRIC_AWS_SENSE == 'false' || env.RUN_FABRIC_AWS_SENSE == false }} + run: | + session=aes-aws-native + echo "vlan: 4" > $session-varfile.yml + ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_native_aws $session $session-varfile.yml + - name: Test SENSE/AWS. if: ${{ env.RUN_FABRIC_AWS_SENSE == 'true' || env.RUN_FABRIC_AWS_SENSE == true }} run: | diff --git a/cicd/test_configs/fabric_native_aws/config.fab b/cicd/test_configs/fabric_native_aws/config.fab new file mode 100644 index 00000000..5ce6861e --- /dev/null +++ b/cicd/test_configs/fabric_native_aws/config.fab @@ -0,0 +1,60 @@ +variable: + - vlan: + default: 4 + +provider: + - fabric: + - fabric_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml.cicd + profile: fabric + +config: + - layer3: + - fab_layer: + subnet: 192.168.10.0/24 + gateway: 192.168.10.1 + ip_start: 192.168.10.2 + ip_end: 192.168.10.254 + - aws_layer: + subnet: 10.0.1.0/24 + - peering: + - my_peering: + # FOR FABRIC + cloud_account: "296256999979" + cloud_facility: + local_device: + local_port: + cloud_region: "us-east-1" + cloud_vlan: '{{ var.vlan }}' + + # FOR SENSE and FABRIC + local_asn: "55038" + local_address: "192.168.1.1/30" + remote_asn: "64512" + remote_address: "192.168.1.2/30" +resource: + - node: + - fabric_node: + provider: '{{ fabric.fabric_provider }}' + site: MAX + + - network: + - fabric_network: + provider: '{{ fabric.fabric_provider }}' + layer3: "{{ layer3.fab_layer }}" + peer_layer3: [ "{{ layer3.aws_layer }}" ] + peering: "{{ peering.my_peering }}" + interface: '{{ node.fabric_node }}' + stitch_info: + stitch_port: + name: AWS_PORTS + profile: FABRIC-AWS-DX-VGW + provider: aws + peer: + profile: Cloud-Facility-AWS + provider: fabric + device_name: agg3.ashb + local_name: TenGigE0/0/0/11/3 + site: AWS + producer: fabric + consumer: aws diff --git a/cicd/test_configs/fabric_sense_aws/config.fab b/cicd/test_configs/fabric_sense_aws/config.fab index fff54f92..3aeb9a1e 100644 --- a/cicd/test_configs/fabric_sense_aws/config.fab +++ b/cicd/test_configs/fabric_sense_aws/config.fab @@ -36,7 +36,6 @@ resource: - sense_node: provider: '{{ sense.sense_provider }}' network: "{{ network.sense_net }}" - service: - fabric_node: provider: '{{ fabric.fabric_provider }}' site: MAX # Use RENC when deploying to fabric beta environment diff --git a/fabfed/controller/controller.py b/fabfed/controller/controller.py index 7cf37eb6..89b67751 100644 --- a/fabfed/controller/controller.py +++ b/fabfed/controller/controller.py @@ -130,6 +130,9 @@ def init(self, *, session: str, provider_factory: ProviderFactory, provider_stat peering_to_network_mapping = {} for network in networks: + if Constants.RES_PEER_LAYER3 in network.attributes: # This is for testing + continue + network.attributes[Constants.RES_PEER_LAYER3] = [] peering = network.attributes.get(Constants.RES_PEERING) diff --git a/fabfed/policy/policy_helper.py b/fabfed/policy/policy_helper.py index f12688ea..e93e7ede 100644 --- a/fabfed/policy/policy_helper.py +++ b/fabfed/policy/policy_helper.py @@ -564,6 +564,12 @@ def get_stitch_port_for_provider(*, resource: dict, provider: str): if not stitch_info: return None + if isinstance(stitch_info, dict): # This is for testing purposes. + producer = stitch_info['producer'] + consumer = stitch_info['consumer'] + stitch_info = StitchInfo(stitch_port=stitch_info['stitch_port'], producer=producer, consumer=consumer) + resource[Constants.RES_STITCH_INFO] = stitch_info + stitch_ports = [stitch_info.stitch_port] if PEER in stitch_info.stitch_port: From 6fb675747a7afde437fbfc10af507cbb6a54bc98 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Thu, 14 Mar 2024 18:02:13 -0700 Subject: [PATCH 36/75] adding cloudlab to cicd --- .github/workflows/ubuntu_22_and_test.yml | 9 ++++++ cicd/fabfed_credentials.yml.cicd | 6 ++++ cicd/run-fabfed.sh | 2 ++ cicd/test_configs/cloudlab/config.fab | 38 ++++++++++++++++++++++++ 4 files changed, 55 insertions(+) create mode 100644 cicd/test_configs/cloudlab/config.fab diff --git a/.github/workflows/ubuntu_22_and_test.yml b/.github/workflows/ubuntu_22_and_test.yml index 7dc22632..87e1d28e 100644 --- a/.github/workflows/ubuntu_22_and_test.yml +++ b/.github/workflows/ubuntu_22_and_test.yml @@ -81,6 +81,8 @@ jobs: echo ${{ env.FABRIC_SLIVER_KEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver echo ${{ env.FABRIC_SLIVER_PUBKEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver.pub echo ${{ env.SENSE_SLIVER_KEY }} | base64 --decode > ${{ github.workspace }}/creds/sense + echo ${{ env.CLAB_SLIVER_KEY }} | base64 --decode > ${{ github.workspace }}/creds/cloudlab + echo ${{ env.CLAB_PEM_FILE }} | base64 --decode > ${{ github.workspace }}/creds/cloudlab.pem - name: Test Fabric FacilityPort if: ${{ env.RUN_FABRIC_AWS_SENSE == 'false' || env.RUN_FABRIC_AWS_SENSE == false }} @@ -101,6 +103,13 @@ jobs: echo "vlan: 4" > $session-varfile.yml ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_native_aws $session $session-varfile.yml + - name: Test Cloudlab + if: ${{ env.RUN_FABRIC_AWS_SENSE == 'false' || env.RUN_FABRIC_AWS_SENSE == false }} + run: | + session=cicd-clab + echo "site: CLEM" > $session-varfile.yml + ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/cloudlab $session $session-varfile.yml + - name: Test SENSE/AWS. if: ${{ env.RUN_FABRIC_AWS_SENSE == 'true' || env.RUN_FABRIC_AWS_SENSE == true }} run: | diff --git a/cicd/fabfed_credentials.yml.cicd b/cicd/fabfed_credentials.yml.cicd index 9768b129..f29f7eef 100644 --- a/cicd/fabfed_credentials.yml.cicd +++ b/cicd/fabfed_credentials.yml.cicd @@ -14,6 +14,12 @@ gcp: service_key_path: GCP_SERVICE_KEY_PATH project: GCP_PROJECT +cloudlab: + project: fabfed + certificate: creds/cloudlab.pem + user: CLAB_USER + slice-private-key-location: creds/cloudlab + sense: auth_endpoint: https://sense-o.es.net:8543/auth/realms/StackV/protocol/openid-connect/token api_endpoint: https://sense-o-dev.es.net:8443/StackV-web/restapi diff --git a/cicd/run-fabfed.sh b/cicd/run-fabfed.sh index 2517e510..985f75e1 100755 --- a/cicd/run-fabfed.sh +++ b/cicd/run-fabfed.sh @@ -22,6 +22,8 @@ sed -i "s/SENSE_USER/$SENSE_USER/" ~/.fabfed/fabfed_credentials.yml.cicd sed -i "s/SENSE_PASSWORD/$SENSE_PASSWORD/" ~/.fabfed/fabfed_credentials.yml.cicd sed -i "s/SENSE_SECRET/$SENSE_SECRET/" ~/.fabfed/fabfed_credentials.yml.cicd +# CLAB +sed -i "s/CLAB_USER/$CLAB_USER/" ~/.fabfed/fabfed_credentials.yml.cicd if [ -n "$var_file" ] then diff --git a/cicd/test_configs/cloudlab/config.fab b/cicd/test_configs/cloudlab/config.fab new file mode 100644 index 00000000..e548003d --- /dev/null +++ b/cicd/test_configs/cloudlab/config.fab @@ -0,0 +1,38 @@ +variable: + - site: + default: STAR + +provider: + - cloudlab: + - cloudlab_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml.cicd + profile: cloudlab + - fabric: + - fabric_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml.cicd + profile: fabric +config: + - layer3: + - my_layer: + subnet: 192.168.1.0/24 + gateway: 192.168.1.1 + ip_start: 192.168.1.2 + ip_end: 192.168.1.254 +resource: + - network: + - cnet: + provider: '{{cloudlab.cloudlab_provider }}' + layer3: "{{ layer3.my_layer }}" + - fabric_network: + provider: '{{ fabric.fabric_provider }}' + layer3: "{{ layer3.my_layer }}" + # interface: '{{ node.fabric_node }}' + stitch_with: '{{ network.cnet }}' + - node: + - fabric_node: + provider: '{{ fabric.fabric_provider }}' + site: '{{ var.site }}' + network: "{{ network.fabric_network }}" + - cnode: + provider: '{{ cloudlab.cloudlab_provider }}' + network: "{{ network.cnet }}" From c809d7aae158f3879e11150586e98e5a44db03f4 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Thu, 14 Mar 2024 18:19:09 -0700 Subject: [PATCH 37/75] cicd for cloudlab and minor fix that triggers delete an immature + apply --- .github/workflows/ubuntu_22_and_test.yml | 3 ++- fabfed/provider/cloudlab/cloudlab_provider.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_22_and_test.yml b/.github/workflows/ubuntu_22_and_test.yml index 87e1d28e..e52cfbed 100644 --- a/.github/workflows/ubuntu_22_and_test.yml +++ b/.github/workflows/ubuntu_22_and_test.yml @@ -23,6 +23,7 @@ env: SENSE_SECRET: ${{ secrets.SENSE_SECRET }} SENSE_SLIVER_KEY: ${{ secrets.SENSE_SLIVER_KEY }} RUN_FABRIC_AWS_SENSE: ${{ vars.RUN_FABRIC_AWS_SENSE }} + RUN_CLAB: ${{ vars.RUN_CLAB }} jobs: UbuntuTest: @@ -104,7 +105,7 @@ jobs: ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_native_aws $session $session-varfile.yml - name: Test Cloudlab - if: ${{ env.RUN_FABRIC_AWS_SENSE == 'false' || env.RUN_FABRIC_AWS_SENSE == false }} + if: ${{ env.RUN_CLAB == 'false' }} run: | session=cicd-clab echo "site: CLEM" > $session-varfile.yml diff --git a/fabfed/provider/cloudlab/cloudlab_provider.py b/fabfed/provider/cloudlab/cloudlab_provider.py index db24222a..c4b0af6c 100644 --- a/fabfed/provider/cloudlab/cloudlab_provider.py +++ b/fabfed/provider/cloudlab/cloudlab_provider.py @@ -155,7 +155,7 @@ def do_add_resource(self, *, resource: dict): for idx in range(0, node_count): node_name = self.resource_name(resource, idx) - node = CloudlabNode(label=label, name=f'{node_name}-{idx}', provider=self, network=net) + node = CloudlabNode(label=label, name=f'{node_name}', provider=self, network=net) self._nodes.append(node) self.resource_listener.on_added(source=self, provider=self, resource=node) return From 66cb278579f836462767fd8c85f6bf72f9784f82 Mon Sep 17 00:00:00 2001 From: abessiari Date: Thu, 14 Mar 2024 18:42:25 -0700 Subject: [PATCH 38/75] Update ubuntu_22_and_test.yml --- .github/workflows/ubuntu_22_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_22_and_test.yml b/.github/workflows/ubuntu_22_and_test.yml index e52cfbed..9b66284a 100644 --- a/.github/workflows/ubuntu_22_and_test.yml +++ b/.github/workflows/ubuntu_22_and_test.yml @@ -105,7 +105,7 @@ jobs: ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_native_aws $session $session-varfile.yml - name: Test Cloudlab - if: ${{ env.RUN_CLAB == 'false' }} + if: ${{ env.RUN_CLAB == 'true' }} run: | session=cicd-clab echo "site: CLEM" > $session-varfile.yml From 6de060fb081271c25565987e9c2c4a485406b2bd Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Thu, 14 Mar 2024 18:51:24 -0700 Subject: [PATCH 39/75] renaming a test in cide for aws native --- .github/workflows/ubuntu_22_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_22_and_test.yml b/.github/workflows/ubuntu_22_and_test.yml index 9b66284a..dee806a4 100644 --- a/.github/workflows/ubuntu_22_and_test.yml +++ b/.github/workflows/ubuntu_22_and_test.yml @@ -97,7 +97,7 @@ jobs: run: | ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_l2_vpn cicd-fabric-l2-vpn - - name: Test Fabric FacilityPort + - name: Test Fabric AWS (Fabric Only) if: ${{ env.RUN_FABRIC_AWS_SENSE == 'false' || env.RUN_FABRIC_AWS_SENSE == false }} run: | session=aes-aws-native From 5a3c08889ac20bb318748c6e5479ce85a740d342 Mon Sep 17 00:00:00 2001 From: abessiari Date: Thu, 14 Mar 2024 19:16:53 -0700 Subject: [PATCH 40/75] Update ubuntu_22_and_test.yml --- .github/workflows/ubuntu_22_and_test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_22_and_test.yml b/.github/workflows/ubuntu_22_and_test.yml index dee806a4..8bcb741d 100644 --- a/.github/workflows/ubuntu_22_and_test.yml +++ b/.github/workflows/ubuntu_22_and_test.yml @@ -10,7 +10,6 @@ on: workflow_dispatch: env: - fabfed_image: fabfed:cicd FABRIC_TOKEN: ${{ secrets.FABRIC_TOKEN }} FABRIC_BASTION_KEY: ${{ secrets.FABRIC_BASTION_KEY }} FABRIC_SLIVER_KEY: ${{ secrets.FABRIC_SLIVER_KEY }} @@ -22,6 +21,9 @@ env: SENSE_PASSWORD: ${{ secrets.SENSE_PASSWORD }} SENSE_SECRET: ${{ secrets.SENSE_SECRET }} SENSE_SLIVER_KEY: ${{ secrets.SENSE_SLIVER_KEY }} + + CLAB_USER: ${{ secrets.CLAB_USER }} + RUN_FABRIC_AWS_SENSE: ${{ vars.RUN_FABRIC_AWS_SENSE }} RUN_CLAB: ${{ vars.RUN_CLAB }} From 27f9192127faa8ad1f795e993a4f93caa245cf71 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Thu, 14 Mar 2024 19:43:37 -0700 Subject: [PATCH 41/75] added var site to cloudlab config --- jupyter/examples/cloudlab/config.fab | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jupyter/examples/cloudlab/config.fab b/jupyter/examples/cloudlab/config.fab index b7d835fa..e5787d08 100644 --- a/jupyter/examples/cloudlab/config.fab +++ b/jupyter/examples/cloudlab/config.fab @@ -1,3 +1,7 @@ +variable: + - site: + default: UTAH # CLEM + provider: - cloudlab: - cloudlab_provider: @@ -32,7 +36,7 @@ resource: - node: - fabric_node: provider: '{{ fabric.fabric_provider }}' - site: UTAH # CLEM # UTAH + site: '{{ var.site }}' image: default_rocky_8 nic_model: NIC_Basic count: 1 From bfb7fb836fc5588d2b5d58078a4dee9eec3210e4 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Thu, 14 Mar 2024 19:56:57 -0700 Subject: [PATCH 42/75] cicd ssh keys for clouldlab --- .github/workflows/ubuntu_22_and_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu_22_and_test.yml b/.github/workflows/ubuntu_22_and_test.yml index 8bcb741d..7aeb9a3b 100644 --- a/.github/workflows/ubuntu_22_and_test.yml +++ b/.github/workflows/ubuntu_22_and_test.yml @@ -23,6 +23,8 @@ env: SENSE_SLIVER_KEY: ${{ secrets.SENSE_SLIVER_KEY }} CLAB_USER: ${{ secrets.CLAB_USER }} + CLAB_SLIVER_KEY: ${{ secrets.CLAB_SLIVER_KEY }} + CLAB_PEM_FILE: ${{ secrets.CLAB_PEM_FILE }} RUN_FABRIC_AWS_SENSE: ${{ vars.RUN_FABRIC_AWS_SENSE }} RUN_CLAB: ${{ vars.RUN_CLAB }} From c7a98e83c848fef362664ab16548cf66a406d725 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Thu, 14 Mar 2024 20:05:31 -0700 Subject: [PATCH 43/75] add project ids to chameleon credentials --- config/fabfed_credentials_template.yml | 3 +++ jupyter/fabfed_config/fabfed_credentials_template.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/config/fabfed_credentials_template.yml b/config/fabfed_credentials_template.yml index 3550eb64..c1d70981 100644 --- a/config/fabfed_credentials_template.yml +++ b/config/fabfed_credentials_template.yml @@ -21,6 +21,9 @@ chi: password: slice-private-key-location: slice-public-key-location: + project_id: + tacc: + uc: # https://sense-o.es.net:8443/StackV-web/portal/ sense: diff --git a/jupyter/fabfed_config/fabfed_credentials_template.yml b/jupyter/fabfed_config/fabfed_credentials_template.yml index 3550eb64..c1d70981 100644 --- a/jupyter/fabfed_config/fabfed_credentials_template.yml +++ b/jupyter/fabfed_config/fabfed_credentials_template.yml @@ -21,6 +21,9 @@ chi: password: slice-private-key-location: slice-public-key-location: + project_id: + tacc: + uc: # https://sense-o.es.net:8443/StackV-web/portal/ sense: From ab5a7ff4205db166a5f8b2525a3dfaea544ebca6 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Thu, 14 Mar 2024 20:15:57 -0700 Subject: [PATCH 44/75] installing cloudlab --- .github/workflows/ubuntu_22_and_test.yml | 28 ++++++++++-------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ubuntu_22_and_test.yml b/.github/workflows/ubuntu_22_and_test.yml index 7aeb9a3b..6ed57745 100644 --- a/.github/workflows/ubuntu_22_and_test.yml +++ b/.github/workflows/ubuntu_22_and_test.yml @@ -54,16 +54,7 @@ jobs: pip list -v | grep neo4j pip install --no-cache-dir --ignore-requires-python -r requirements.txt python3 -m pip install --no-cache-dir . - - - name: Print Env Vars - run: | - echo "*******************" - echo ${{ env.RUN_FABRIC_AWS_SENSE }} - echo "*******************" - echo ${{ env.RUN_FABRIC_AWS_SENSE == 'false' }} - echo "*******************" - echo ${{ env.RUN_FABRIC_AWS_SENSE == false }} - echo "*******************" + pip install --no-cache-dir git+https://gitlab.flux.utah.edu/stoller/portal-tools.git - name: Run Unit Tests run: | @@ -72,6 +63,9 @@ jobs: - name: Test Stitch Policy run: | fabfed stitch-policy -providers "chi,fabric" + fabfed stitch-policy -providers "cloudlab,fabric" + fabfed stitch-policy -providers "aws,fabric" + fabfed stitch-policy -providers "gcp,fabric" - name: Show Sessions run: | @@ -89,6 +83,13 @@ jobs: echo ${{ env.CLAB_SLIVER_KEY }} | base64 --decode > ${{ github.workspace }}/creds/cloudlab echo ${{ env.CLAB_PEM_FILE }} | base64 --decode > ${{ github.workspace }}/creds/cloudlab.pem + - name: Test Cloudlab + if: ${{ env.RUN_CLAB == 'true' }} + run: | + session=cicd-clab + echo "site: CLEM" > $session-varfile.yml + ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/cloudlab $session $session-varfile.yml + - name: Test Fabric FacilityPort if: ${{ env.RUN_FABRIC_AWS_SENSE == 'false' || env.RUN_FABRIC_AWS_SENSE == false }} run: | @@ -108,13 +109,6 @@ jobs: echo "vlan: 4" > $session-varfile.yml ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_native_aws $session $session-varfile.yml - - name: Test Cloudlab - if: ${{ env.RUN_CLAB == 'true' }} - run: | - session=cicd-clab - echo "site: CLEM" > $session-varfile.yml - ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/cloudlab $session $session-varfile.yml - - name: Test SENSE/AWS. if: ${{ env.RUN_FABRIC_AWS_SENSE == 'true' || env.RUN_FABRIC_AWS_SENSE == true }} run: | From 49a14fdc59f915ceffa4f2bb581064e044aa570e Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Thu, 14 Mar 2024 22:20:07 -0500 Subject: [PATCH 45/75] Update cloudlab-lib.ipynb --- jupyter/cloudlab-lib.ipynb | 181 +++++++++++++------------------------ 1 file changed, 65 insertions(+), 116 deletions(-) diff --git a/jupyter/cloudlab-lib.ipynb b/jupyter/cloudlab-lib.ipynb index 19dcec71..15ef3bd3 100644 --- a/jupyter/cloudlab-lib.ipynb +++ b/jupyter/cloudlab-lib.ipynb @@ -11,120 +11,92 @@ }, { "cell_type": "markdown", - "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", + "id": "d644302e-f1a2-452c-a252-a30e6a579d05", "metadata": {}, "source": [ - "## Assign session name and .fab path" + "## Install Dependencies" ] }, { "cell_type": "code", "execution_count": null, - "id": "d8efd310-300a-4f56-bb26-5431bdec3969", + "id": "44e7c5f5-74e5-4ad8-ac54-eafe859e36a6", "metadata": {}, "outputs": [], "source": [ - "import os\n", - "\n", - "session_name = \"cloudlab-lzhang9-lib\"\n", - "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/cloudlab\"\n", - "fab_file_path = \"./examples/cloudlab\"\n", - "\n", - "print(session_name)\n", - "print(fab_file_path)" + "!pip install git+https://gitlab.flux.utah.edu/stoller/portal-tools.git" ] }, { "cell_type": "markdown", - "id": "a6e6acf4-b348-4e08-9808-896f794bb01d", - "metadata": {}, - "source": [ - "## Import FabFeb library" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "79a3a114-3c67-4a79-a6fb-8227b64dc429", - "metadata": {}, - "outputs": [], - "source": [ - "from fabfed.util.config import WorkflowConfig\n", - "from fabfed.util import state as sutil\n", - "from fabfed.controller.controller import Controller\n", - "from fabfed.controller.provider_factory import default_provider_factory" - ] - }, - { - "cell_type": "markdown", - "id": "73a535d1-4f1a-4606-b4a4-128c675255db", + "id": "cfe860e3-340c-4d54-8ce7-c883e00486a1", "metadata": {}, "source": [ - "## Read .fab files" + "## Assign session name and .fab path" ] }, { "cell_type": "code", "execution_count": null, - "id": "ce52aa52-99af-4391-bb42-3ab5a9554b9a", + "id": "d8efd310-300a-4f56-bb26-5431bdec3969", "metadata": {}, "outputs": [], "source": [ - "def read_file_to_str(file_path):\n", - " try:\n", - " with open(file_path, 'r') as file:\n", - " file_content = file.read()\n", - " return file_content\n", - " except FileNotFoundError:\n", - " print(f\"File '{file_path}' not found.\")\n", - " return None\n", + "import os\n", "\n", - "# Read config.fab to config_str:\n", - "config_str = read_file_to_str(f\"{fab_file_path}/config.fab\")\n", - "if config_str:\n", - " print(\"File content:\")\n", - " print(config_str)\n", + "session_name = \"cloudlab-lzhang9-lib\"\n", + "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/cloudlab\"\n", + "fab_file_path = \"./examples/cloudlab\"\n", "\n", - "print(config_str)" + "print(session_name)\n", + "print(fab_file_path)" ] }, { "cell_type": "markdown", - "id": "f6106b4e-076f-4349-b215-c6db497c97a9", + "id": "a6e6acf4-b348-4e08-9808-896f794bb01d", "metadata": {}, "source": [ - "## Create a workflow config object" + "## Import FabFeb library" ] }, { "cell_type": "code", "execution_count": null, - "id": "ba75ff1a-7442-4e05-a4ca-b1128397983e", + "id": "83e8b131-3f19-474f-a73b-33788a6ac2a2", "metadata": {}, "outputs": [], "source": [ - "config = WorkflowConfig(content=config_str)" + "try:\n", + " from fabfed.fabfed_manager import FabfedManager\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", + " \n", + "print(\"Complete!\")" ] }, { "cell_type": "markdown", - "id": "c87f8329-054e-45f3-9d69-3c4acc25252f", + "id": "71f1c05a-cdda-4a6a-9d8b-0479fcc25b98", "metadata": {}, "source": [ - "## Create a controller" + "## Validate .fab files" ] }, { "cell_type": "code", "execution_count": null, - "id": "5d161b4f-4b6b-4993-80d3-26f59034c2cb", + "id": "3e608892-5673-4bf7-99df-981bfee99ba6", "metadata": {}, "outputs": [], "source": [ - "import logging\n", - "logger = logging.getLogger(__name__)\n", - "\n", - "controller = Controller(config=config, logger=logger)" + "try:\n", + " fabfed_manager = FabfedManager(config_dir=fab_file_path)\n", + " fabfed_manager.validate()\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", + " \n", + "print(\"Complete!\")" ] }, { @@ -132,47 +104,47 @@ "id": "4e1aa491-7ddc-491b-82f6-ec9d41b08c3a", "metadata": {}, "source": [ - "## Init stitching information" + "## Stitching information" ] }, { "cell_type": "code", "execution_count": null, - "id": "7da5efbb-0c83-4842-b533-c81340028013", + "id": "2b4e6db0-d789-4c7d-bf30-556b9f2b8e24", "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session)\n", + "try:\n", + " fabfed_manager.stitch_info(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", "\n", - "controller.init(session=session_name, provider_factory=default_provider_factory, provider_states=states)" + "print(\"Complete!\")" ] }, { "cell_type": "markdown", - "id": "465ea762-1e62-4532-a0e3-aa5bef0e23c4", - "metadata": {}, - "source": [ - "## Add resources to providers and validate resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f8ae4d51-a812-4d80-997c-11c7ef5548a2", + "id": "061cfe97-f32f-4b24-9a80-85593ca87d4c", "metadata": {}, - "outputs": [], "source": [ - "controller.plan(provider_states=[])" + "## List resources for creation and deletion" ] }, { "cell_type": "code", "execution_count": null, - "id": "7a1a864d-bdd9-4422-84a6-cade3a7c485c", + "id": "34bc3815-3246-42b7-b661-10e4695ac928", "metadata": {}, "outputs": [], "source": [ - "controller.add(provider_states=[])" + "try:\n", + " to_be_created_count, to_be_deleted_count = fabfed_manager.plan(session=session_name)\n", + "except Exception as e:\n", + " print(\"Error: {e}\")\n", + "\n", + "print(\"\\n=== Total ===\")\n", + "print(f\"to_be_created: {to_be_created_count}\")\n", + "print(f\"to_be_deleted: {to_be_deleted_count}\")" ] }, { @@ -186,55 +158,35 @@ { "cell_type": "code", "execution_count": null, - "id": "4db644c5-b7d5-4092-8329-cca89a13aef6", + "id": "30acfbf9-a3bc-4d54-9c52-a408be59d9c1", "metadata": {}, "outputs": [], "source": [ "try:\n", - " controller.apply(provider_states=[])\n", - "except ControllerException as e:\n", - " assert isinstance(e.exceptions[0], DummyFailCreateException)" - ] - }, - { - "cell_type": "markdown", - "id": "beee11c5-5adc-4756-8133-a8eda0253969", - "metadata": {}, - "source": [ - "## Save the workflow state" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9ca3360b-c033-4af8-9f1d-0acf12b1c0d0", - "metadata": {}, - "outputs": [], - "source": [ - "states = controller.get_states()\n", - "sutil.save_states(states, session_name)\n", + " status_code = fabfed_manager.apply(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", "\n", - "print(states)" + "print(\"Complete!\")" ] }, { "cell_type": "markdown", - "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", + "id": "1ddb719e-caf7-4973-b7c8-f42417aa4202", "metadata": {}, "source": [ - "## View the state of the workflow" + "## Show sessions" ] }, { "cell_type": "code", "execution_count": null, - "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", + "id": "8142c79c-3d88-4fe9-b5ec-c9d296607007", "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session_name)\n", - "\n", - "print(states)" + "fabfed_manager.show(session=session_name)\n", + "fabfed_manager.show_sessions()" ] }, { @@ -252,15 +204,12 @@ "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session_name)\n", - "\n", - "controller.destroy(provider_states=states)\n", - "\n", - "sutil.save_states(states, session_name)\n", - "if not states:\n", - " sutil.destroy_session(session_name)\n", + "try:\n", + " fabfed_manager.destroy(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", "\n", - "print(states)" + "print(\"Complete!\")" ] } ], From 4ebb249acba63fb3f70659d9f5b3537b6b02d5e3 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Thu, 14 Mar 2024 22:05:39 -0700 Subject: [PATCH 46/75] adding gcp to cicd --- .github/workflows/ubuntu_22_and_test_gcp.yml | 71 ++++++++++++++++++++ cicd/fabfed_credentials.yml.cicd | 4 +- cicd/test_configs/gcp/config.fab | 58 ++++++++++++++++ 3 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ubuntu_22_and_test_gcp.yml create mode 100644 cicd/test_configs/gcp/config.fab diff --git a/.github/workflows/ubuntu_22_and_test_gcp.yml b/.github/workflows/ubuntu_22_and_test_gcp.yml new file mode 100644 index 00000000..5c894d21 --- /dev/null +++ b/.github/workflows/ubuntu_22_and_test_gcp.yml @@ -0,0 +1,71 @@ +name: GCP-Ubuntu22.04 Test + +on: + push: + branches: + - knit8 + workflow_dispatch: + +env: + FABRIC_TOKEN: ${{ secrets.FABRIC_TOKEN }} + FABRIC_BASTION_KEY: ${{ secrets.FABRIC_BASTION_KEY }} + FABRIC_SLIVER_KEY: ${{ secrets.FABRIC_SLIVER_KEY }} + FABRIC_SLIVER_PUBKEY: ${{ secrets.FABRIC_SLIVER_PUBKEY }} + FABRIC_PROJECT: ${{ secrets.FABRIC_PROJECT }} + FABRIC_USER: ${{ secrets.FABRIC_USER }} + + RUN_GCP: ${{ vars.RUN_GCP }} + +jobs: + GcpOnUbuntuTest: + runs-on: ubuntu-latest + steps: + + - name: Checkout + uses: actions/checkout@v4 + + - name: Print Versions + run: | + python3 --version + pip --version + pwd + whoami + echo ${{ github.workspace }} + echo $HOME + + - name: Install Fabfed And Requirements + run: | + pip install --upgrade pip + pip install setuptools --upgrade + pip install --no-cache-dir --ignore-requires-python neo4j==5.18.0 + pip list -v | grep neo4j + pip install --no-cache-dir --ignore-requires-python -r requirements.txt + python3 -m pip install --no-cache-dir . + + - name: Run Unit Tests + run: | + pytest tests + + - name: Test Stitch Policy + run: | + fabfed stitch-policy -providers "gcp,fabric" + + - name: Show Sessions + run: | + fabfed sessions -show + fabfed sessions -show -json + + - name: Save Credentials + run: | + mkdir -p ${{ github.workspace }}/creds + echo ${{ env.FABRIC_TOKEN }} | base64 --decode > ${{ github.workspace }}/creds/token.json + echo ${{ env.FABRIC_BASTION_KEY }} | base64 --decode > ${{ github.workspace }}/creds/bastion + echo ${{ env.FABRIC_SLIVER_KEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver + echo ${{ env.FABRIC_SLIVER_PUBKEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver.pub + echo ${{ env.GCP_SERVICE_KEY_PATH }} | base64 --decode > ${{ github.workspace }}/creds/gcp.json + + - name: Test GCP + if: ${{ env.RUN_GCP == 'true' }} + run: | + session=cicd-gcp + ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/gcp $session diff --git a/cicd/fabfed_credentials.yml.cicd b/cicd/fabfed_credentials.yml.cicd index f29f7eef..8f03370b 100644 --- a/cicd/fabfed_credentials.yml.cicd +++ b/cicd/fabfed_credentials.yml.cicd @@ -11,8 +11,8 @@ aws: secret_key: AWS_SECRET_KEY gcp: - service_key_path: GCP_SERVICE_KEY_PATH - project: GCP_PROJECT + service_key_path: creds/gcp.json + project: fabfed cloudlab: project: fabfed diff --git a/cicd/test_configs/gcp/config.fab b/cicd/test_configs/gcp/config.fab new file mode 100644 index 00000000..083f7893 --- /dev/null +++ b/cicd/test_configs/gcp/config.fab @@ -0,0 +1,58 @@ +variable: + - vpc: + default: vpc-69acc1d9-8c24-47cd-90b8-33be57167dbf + - vlan: + default: 5 # a value between 3 and 4095. The default is 4. + +provider: + - gcp: + - gcp_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml.cicd + profile: gcp + - fabric: + - fabric_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml.cicd + profile: fabric +config: + - layer3: + - fab_layer: + subnet: 192.168.10.0/24 + gateway: 192.168.10.1 + ip_start: 192.168.10.2 + ip_end: 192.168.10.254 + - gcp_layer: + subnet: 10.100.0.0/24 # subnet.cidr and vpc.cidr + - peering: + - my_peering: + # FOR GCP + cloud_region: "us-east4" + cloud_vpc: '{{ var.vpc }}' + cloud_vlan: '{{ var.vlan }}' + + # FOR GCP AND FABRIC. + remote_asn: 16550 # google_asn + # FOR FABRIC + local_address: "192.168.1.1/16" # customer_ip + remote_address: "192.168.1.2/16" # google_ip +resource: + - network: + - gcp_net: + provider: '{{ gcp.gcp_provider }}' + name: gcp-net + layer3: "{{ layer3.gcp_layer }}" + peering: "{{ peering.my_peering }}" + stitch_with: '{{ network.fabric_network }}' + count: 1 + - fabric_network: + provider: '{{ fabric.fabric_provider }}' + layer3: "{{ layer3.fab_layer }}" + peering: "{{ peering.my_peering }}" + interface: '{{ node.fabric_node }}' + count: 1 + - node: + - fabric_node: + provider: '{{ fabric.fabric_provider }}' + site: MAX + image: default_rocky_8 + count: 1 + nic_model: NIC_Basic From 8822dc27404a100fce9d1a06f9c08885314cadb4 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Thu, 14 Mar 2024 22:30:25 -0700 Subject: [PATCH 47/75] added config validation for gcp provider --- fabfed/provider/gcp/gcp_provider.py | 39 ++++++++++++++++++----------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/fabfed/provider/gcp/gcp_provider.py b/fabfed/provider/gcp/gcp_provider.py index 52aca932..36545309 100644 --- a/fabfed/provider/gcp/gcp_provider.py +++ b/fabfed/provider/gcp/gcp_provider.py @@ -1,4 +1,4 @@ -from fabfed.exceptions import ResourceTypeNotSupported +from fabfed.exceptions import ResourceTypeNotSupported, ProviderException from fabfed.provider.api.provider import Provider from fabfed.util.constants import Constants from fabfed.util.utils import get_logger @@ -21,22 +21,12 @@ def service_key_path(self): return self.config.get(gcp_constants.SERVICE_KEY_PATH) def setup_environment(self): - from fabfed.util import utils - - credential_file = self.config.get(Constants.CREDENTIAL_FILE) - profile = self.config.get(Constants.PROFILE) - config = utils.load_yaml_from_file(credential_file) - - if profile not in config: - from fabfed.exceptions import ProviderException - - raise ProviderException( - f"credential file {credential_file} does not have a section for keyword {profile}" - ) - - self.config = config[profile] normalized_config = {} + for attr in [gcp_constants.PROJECT, gcp_constants.SERVICE_KEY_PATH]: + if self.config.get(attr) is None: + raise ProviderException(f"{self.name}: Expecting a value for {attr}") + for k, v in self.config.items(): normalized_config[k.upper()] = v @@ -44,6 +34,25 @@ def setup_environment(self): assert self.config.get(gcp_constants.SERVICE_KEY_PATH) assert self.config.get(gcp_constants.PROJECT) + skey = self.config[gcp_constants.SERVICE_KEY_PATH] + + from fabfed.util.utils import can_read, absolute_path + + skey = absolute_path(skey) + + if not can_read(skey): + raise ProviderException(f"{self.name}: unable to read service key in {skey}") + + try: + with open(skey, 'r') as fp: + import json + + json.load(fp) + except Exception as e: + raise ProviderException(f"{self.name}:Unable to parse {skey} to json:{e}") + + self.config[gcp_constants.SERVICE_KEY_PATH] = skey + def do_add_resource(self, *, resource: dict): label = resource.get(Constants.LABEL) rtype = resource.get(Constants.RES_TYPE) From 0734ecf64eb6fea2dda65a41ed0d8cd74d1a9171 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Thu, 14 Mar 2024 22:35:06 -0700 Subject: [PATCH 48/75] added config validation for gcp provider --- fabfed/provider/gcp/gcp_provider.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fabfed/provider/gcp/gcp_provider.py b/fabfed/provider/gcp/gcp_provider.py index 36545309..d6a94074 100644 --- a/fabfed/provider/gcp/gcp_provider.py +++ b/fabfed/provider/gcp/gcp_provider.py @@ -23,10 +23,6 @@ def service_key_path(self): def setup_environment(self): normalized_config = {} - for attr in [gcp_constants.PROJECT, gcp_constants.SERVICE_KEY_PATH]: - if self.config.get(attr) is None: - raise ProviderException(f"{self.name}: Expecting a value for {attr}") - for k, v in self.config.items(): normalized_config[k.upper()] = v @@ -34,6 +30,10 @@ def setup_environment(self): assert self.config.get(gcp_constants.SERVICE_KEY_PATH) assert self.config.get(gcp_constants.PROJECT) + for attr in [gcp_constants.PROJECT, gcp_constants.SERVICE_KEY_PATH]: + if self.config.get(attr) is None: + raise ProviderException(f"{self.name}: Expecting a value for {attr}") + skey = self.config[gcp_constants.SERVICE_KEY_PATH] from fabfed.util.utils import can_read, absolute_path From 35177e81bb69b5c0262d9e35d0c5bf853d3a6e03 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Thu, 14 Mar 2024 22:41:09 -0700 Subject: [PATCH 49/75] added config validation for gcp provider --- .github/workflows/ubuntu_22_and_test_gcp.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu_22_and_test_gcp.yml b/.github/workflows/ubuntu_22_and_test_gcp.yml index 5c894d21..a341f7dc 100644 --- a/.github/workflows/ubuntu_22_and_test_gcp.yml +++ b/.github/workflows/ubuntu_22_and_test_gcp.yml @@ -14,6 +14,8 @@ env: FABRIC_PROJECT: ${{ secrets.FABRIC_PROJECT }} FABRIC_USER: ${{ secrets.FABRIC_USER }} + GCP_SERVICE_KEY_PATH: ${{ secrets.GCP_SERVICE_KEY_PATH }} + RUN_GCP: ${{ vars.RUN_GCP }} jobs: From 9a21bda23269ced8eee75d18497d34c79fa8be8e Mon Sep 17 00:00:00 2001 From: abessiari Date: Thu, 14 Mar 2024 23:34:01 -0700 Subject: [PATCH 50/75] Update ubuntu_22_and_test_gcp.yml --- .github/workflows/ubuntu_22_and_test_gcp.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ubuntu_22_and_test_gcp.yml b/.github/workflows/ubuntu_22_and_test_gcp.yml index a341f7dc..5435bbf6 100644 --- a/.github/workflows/ubuntu_22_and_test_gcp.yml +++ b/.github/workflows/ubuntu_22_and_test_gcp.yml @@ -1,9 +1,6 @@ name: GCP-Ubuntu22.04 Test on: - push: - branches: - - knit8 workflow_dispatch: env: From 5019244eaa8cbb8cc33f5ad8bb595a9af6d268da Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 15 Mar 2024 09:59:49 -0500 Subject: [PATCH 51/75] Update cloudlab-lib.ipynb --- jupyter/cloudlab-lib.ipynb | 57 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/jupyter/cloudlab-lib.ipynb b/jupyter/cloudlab-lib.ipynb index 15ef3bd3..20065d7d 100644 --- a/jupyter/cloudlab-lib.ipynb +++ b/jupyter/cloudlab-lib.ipynb @@ -175,7 +175,7 @@ "id": "1ddb719e-caf7-4973-b7c8-f42417aa4202", "metadata": {}, "source": [ - "## Show sessions" + "## Show session state" ] }, { @@ -185,7 +185,24 @@ "metadata": {}, "outputs": [], "source": [ - "fabfed_manager.show(session=session_name)\n", + "fabfed_manager.show(session=session_name)" + ] + }, + { + "cell_type": "markdown", + "id": "d6bd757c-ec87-409c-b7d2-58e20375d015", + "metadata": {}, + "source": [ + "## Show all sessions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3991e1b-c2fa-4b1f-8e9e-d66d5713230b", + "metadata": {}, + "outputs": [], + "source": [ "fabfed_manager.show_sessions()" ] }, @@ -211,6 +228,42 @@ "\n", "print(\"Complete!\")" ] + }, + { + "cell_type": "markdown", + "id": "c66b1992-88e7-44dc-af64-ba82c1a975da", + "metadata": {}, + "source": [ + "## Show sesstion state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cbd1b833-3ad0-423f-94f9-b6592ba26c20", + "metadata": {}, + "outputs": [], + "source": [ + "fabfed_manager.show(session=session_name)" + ] + }, + { + "cell_type": "markdown", + "id": "b2eeae12-ad35-4b18-b868-9975cd6dccae", + "metadata": {}, + "source": [ + "## Show all sessions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c85f08e9-57ff-4528-9127-aa5cb15d238a", + "metadata": {}, + "outputs": [], + "source": [ + "fabfed_manager.show_sessions()" + ] } ], "metadata": { From 12bf80aa6b5643dd0de7f2f4cb513e428f6ea328 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Fri, 15 Mar 2024 10:02:15 -0700 Subject: [PATCH 52/75] added show_available_stitch_ports to manager --- fabfed/fabfed_manager.py | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/fabfed/fabfed_manager.py b/fabfed/fabfed_manager.py index 35b2b6fc..55e502ef 100644 --- a/fabfed/fabfed_manager.py +++ b/fabfed/fabfed_manager.py @@ -16,6 +16,7 @@ session="some_session" fabfed_manager = FabfedManager(config_dir=config_dir) fabfed_manager.validate() +fabfed_manager.show_available_stitch_ports(from_provider='chi', to_provider="fabric") fabfed_manager.stitch_info(session=session) to_be_created_count, to_be_deleted_count = fabfed_manager.plan(session=session) status_code = fabfed_manager.apply(session=session) @@ -200,3 +201,44 @@ def destroy(self, *, session: str): def show_sessions(self, *, to_json: bool = False): self.sessions = utils.dump_sessions(to_json) + + def show_available_stitch_ports(self, *, from_provider, to_provider): + from fabfed.policy.policy_helper import load_policy + + policy = load_policy() + + from fabfed.policy.policy_helper import find_stitch_port_for_providers, peer_stitch_ports + + providers = [from_provider, to_provider] + stitch_infos = find_stitch_port_for_providers(policy, providers) + stitch_infos = peer_stitch_ports(stitch_infos) + attrs = ["preference", "member-of", 'name'] + names = [] + + for stitch_info in stitch_infos: + names.append(stitch_info.stitch_port.get("name")) + + for attr in attrs: + stitch_info.stitch_port.pop(attr, None) + + peer = stitch_info.stitch_port.get('peer', {}) + + for attr in attrs: + peer.pop(attr, None) + + import yaml + from fabfed.util.constants import Constants + + stitch_port_configs = [] + + for i, stitch_info in enumerate(stitch_infos): + producer = stitch_info.producer + consumer = stitch_info.consumer + stitch_port = stitch_info.stitch_port + c = {"config": [{Constants.NETWORK_STITCH_CONFIG: [{f"si_from_{names[i]}": {"producer": producer, + "consumer": consumer, + "stitch_port": stitch_port}}]}]} + stitch_port_configs.append(c) + + rep = yaml.dump(stitch_port_configs, default_flow_style=False, sort_keys=False) + print(rep) From e1cb55e6610ee95bfddd00da5443843f6e681011 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Fri, 15 Mar 2024 15:21:31 -0700 Subject: [PATCH 53/75] tested tacc. improved output for chi node --- fabfed/provider/chi/chi_network.py | 1 + fabfed/provider/fabric/fabric_slice.py | 2 ++ tools/fabfed.py | 16 ++++++++++++---- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/fabfed/provider/chi/chi_network.py b/fabfed/provider/chi/chi_network.py index 40fcbf81..67db8e82 100644 --- a/fabfed/provider/chi/chi_network.py +++ b/fabfed/provider/chi/chi_network.py @@ -22,6 +22,7 @@ class ChiNetwork(Network): def __init__(self, *, label, name: str, site: str, project_name: str, layer3: Config, stitch_provider: str): super().__init__(label=label, name=name, site=site) self.project_name = project_name + self.layer3 = layer3 self.subnet = layer3.attributes.get(Constants.RES_SUBNET) self.ip_start = layer3.attributes.get(Constants.RES_LAYER3_DHCP_START) self.ip_end = layer3.attributes.get(Constants.RES_LAYER3_DHCP_END) diff --git a/fabfed/provider/fabric/fabric_slice.py b/fabfed/provider/fabric/fabric_slice.py index bf3e7bd8..70466eb5 100644 --- a/fabfed/provider/fabric/fabric_slice.py +++ b/fabfed/provider/fabric/fabric_slice.py @@ -475,6 +475,8 @@ def create_resource(self, *, resource: dict): if self.nodes: self._handle_node_networking() + else: + self._reload_networks() for node in self.nodes: self.resource_listener.on_created(source=self, provider=self.provider, resource=node) diff --git a/tools/fabfed.py b/tools/fabfed.py index 234fc729..b915a739 100644 --- a/tools/fabfed.py +++ b/tools/fabfed.py @@ -10,6 +10,13 @@ from fabfed.util.constants import Constants +def delete_session_if_empty(*, session): + provider_states = sutil.load_states(session) + + if not provider_states: + sutil.destroy_session(session) + + def manage_workflow(args): logger = utils.init_logger() config_dir = utils.absolute_path(args.config_dir) @@ -21,10 +28,6 @@ def manage_workflow(args): logger.warning(f"ATTN: The CORRECT config dir for session {args.session} is {config_dir_from_meta}!!!!!!!") sys.exit(1) - # we skip -show and -destroy - if args.apply or args.init or args.plan or args.validate: - sutil.save_meta_data(dict(config_dir=config_dir), args.session) - var_dict = utils.load_vars(args.var_file) if args.var_file else {} from fabfed.policy.policy_helper import load_policy @@ -41,6 +44,7 @@ def manage_workflow(args): sys.exit(1) if args.apply: + sutil.save_meta_data(dict(config_dir=config_dir), args.session) sutil.delete_stats(args.session) import time @@ -145,6 +149,7 @@ def manage_workflow(args): states = sutil.load_states(args.session) controller.init(session=args.session, provider_factory=default_provider_factory, provider_states=states) sutil.dump_resources(resources=controller.resources, to_json=args.json, summary=args.summary) + delete_session_if_empty(session=args.session) return if args.stitch_info: @@ -198,6 +203,7 @@ def manage_workflow(args): stitch_info_summaries = dict(StitchInfoSummary=stitch_info_summaries) sutil.dump_objects(objects=stitch_info_summaries, to_json=args.json) + delete_session_if_empty(session=args.session) return if args.plan: @@ -211,6 +217,7 @@ def manage_workflow(args): cr, dl = sutil.dump_plan(resources=controller.resources, to_json=args.json, summary=args.summary) logger.warning(f"Applying this plan would create {cr} resource(s) and destroy {dl} resource(s)") + delete_session_if_empty(session=args.session) return if args.show: @@ -221,6 +228,7 @@ def manage_workflow(args): if args.stats: stats = sutil.load_stats(args.session) sutil.dump_stats(stats, args.json) + delete_session_if_empty(session=args.session) return if args.destroy: From c34b8e82b3cb20425896f745528c99b2cf057217 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Fri, 15 Mar 2024 10:02:15 -0700 Subject: [PATCH 54/75] added show_available_stitch_ports to manager tested tacc. improved output for chi node one file for start and tacc + node count can be set zero example showing policy override better logging for sense provider example for supplying policy --- .../demos/chameleon/{star => }/config.fab | 20 +++--- examples/demos/chameleon/tacc/config.fab | 54 ---------------- examples/demos/cloudlab/mass/config.fab | 13 ++-- examples/supply_stitch_policy/config.fab | 63 +++++++++++++++++++ fabfed/fabfed_manager.py | 42 +++++++++++++ fabfed/provider/chi/chi_network.py | 1 + fabfed/provider/fabric/fabric_slice.py | 2 + fabfed/provider/sense/sense_provider.py | 3 +- tools/fabfed.py | 16 +++-- 9 files changed, 141 insertions(+), 73 deletions(-) rename examples/demos/chameleon/{star => }/config.fab (84%) delete mode 100644 examples/demos/chameleon/tacc/config.fab create mode 100644 examples/supply_stitch_policy/config.fab diff --git a/examples/demos/chameleon/star/config.fab b/examples/demos/chameleon/config.fab similarity index 84% rename from examples/demos/chameleon/star/config.fab rename to examples/demos/chameleon/config.fab index fbcdcfa2..edaa5409 100644 --- a/examples/demos/chameleon/star/config.fab +++ b/examples/demos/chameleon/config.fab @@ -1,3 +1,9 @@ +variable: + - site: + default: STAR # TACC + - node_count: + default: 1 + provider: - fabric: - fabric_provider: @@ -25,25 +31,25 @@ resource: provider: '{{ chi.chi_provider }}' name: stitch_net layer3: "{{ layer3.my_layer }}" - count: 1 - fabric_network: provider: '{{ fabric.fabric_provider }}' - interface: '{{ node.fabric_node }}' layer3: "{{ layer3.my_layer }}" - count: 1 stitch_with: '{{ network.chi_network }}' + count: 1 stitch_option: - site: STAR + site: '{{ var.site }}' - node: - fabric_node: provider: '{{ fabric.fabric_provider }}' - count: 1 + count: '{{ var.node_count }}' image: default_rocky_8 + network: '{{ network.fabric_network }}' + - chi_node: provider: '{{ chi.chi_provider }}' image: CC-Ubuntu20.04 network: '{{ network.chi_network }}' - count: 1 + count: '{{ var.node_count }}' flavor: m1.medium - service: - dtn_service: @@ -52,4 +58,4 @@ resource: controller: '{{ node.fabric_node }}' image: dtnaas/tools profile: fabfed - count: 1 + count: 0 diff --git a/examples/demos/chameleon/tacc/config.fab b/examples/demos/chameleon/tacc/config.fab deleted file mode 100644 index da9de9fb..00000000 --- a/examples/demos/chameleon/tacc/config.fab +++ /dev/null @@ -1,54 +0,0 @@ -provider: - - fabric: - - fabric_provider: - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: fabric - - chi: - - chi_provider: - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: chi - - janus: - - janus_provider: - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: janus - -config: - - layer3: - - my_layer: - subnet: 192.168.100.0/24 - gateway: 192.168.100.1 - ip_start: 192.168.100.100 - ip_end: 192.168.100.250 -resource: - - network: - - chi_network: - provider: '{{ chi.chi_provider }}' - name: stitch_net - layer3: "{{ layer3.my_layer }}" - - fabric_network: - provider: '{{ fabric.fabric_provider }}' - interface: '{{ node.fabric_node }}' - layer3: "{{ layer3.my_layer }}" - stitch_with: '{{ network.chi_network }}' - count: 1 - stitch_option: - site: TACC - - node: - - fabric_node: - provider: '{{ fabric.fabric_provider }}' - count: 1 - image: default_rocky_8 - - chi_node: - provider: '{{ chi.chi_provider }}' - image: CC-Ubuntu20.04 - network: '{{ network.chi_network }}' - count: 1 - flavor: m1.medium - - service: - - dtn_service: - provider: '{{ janus.janus_provider }}' - node: [ '{{ node.chi_node }}', '{{ node.fabric_node }}' ] - controller: '{{ node.fabric_node }}' - image: dtnaas/tools - profile: fabfed - count: 1 diff --git a/examples/demos/cloudlab/mass/config.fab b/examples/demos/cloudlab/mass/config.fab index 2a9807e2..730d97b2 100644 --- a/examples/demos/cloudlab/mass/config.fab +++ b/examples/demos/cloudlab/mass/config.fab @@ -1,4 +1,6 @@ variable: + - node_count: + default: 1 - vlan: default: 3110 @@ -18,6 +20,7 @@ config: producer: cloudlab consumer: fabric stitch_port: + name: clab-mass profile: OCT-MGHPCC provider: fabric device_name: OCT-MGHPCC @@ -45,20 +48,16 @@ resource: - cloudlab_node: provider: '{{ cloudlab.cloudlab_provider }}' network: "{{ network.cnet }}" - count: 1 + count: '{{ var.node_count }}' - node: - fabric_node: provider: '{{ fabric.fabric_provider }}' - image: default_rocky_8 - nic_model: NIC_ConnectX_5 - count: 1 + network: "{{ network.fabric_network }}" + count: '{{ var.node_count }}' - network: - fabric_network: provider: '{{ fabric.fabric_provider }}' layer3: "{{ layer3.my_layer }}" - interface: '{{ node.fabric_node }}' stitch_with: '{{ network.cnet }}' stitch_option: - device_name: CloudLab-Clemson policy: '{{ policy.my_stitch_policy }}' - count: 1 diff --git a/examples/supply_stitch_policy/config.fab b/examples/supply_stitch_policy/config.fab new file mode 100644 index 00000000..730d97b2 --- /dev/null +++ b/examples/supply_stitch_policy/config.fab @@ -0,0 +1,63 @@ +variable: + - node_count: + default: 1 + - vlan: + default: 3110 + +provider: + - cloudlab: + - cloudlab_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: cloudlab + - fabric: + - fabric_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: fabric + +config: + - policy: + - my_stitch_policy: + producer: cloudlab + consumer: fabric + stitch_port: + name: clab-mass + profile: OCT-MGHPCC + provider: fabric + device_name: OCT-MGHPCC + site: MASS + peer: + profile: fabfed-stitch-v2 + provider: cloudlab + option: + cluster: urn:publicid:IDN+cloudlab.umass.edu+authority+cm + interface: + - vlan: '{{ var.vlan }}' + + - layer3: + - my_layer: + subnet: 192.168.1.0/24 + gateway: 192.168.1.1 + ip_start: 192.168.1.2 + ip_end: 192.168.1.254 +resource: + - network: + - cnet: + provider: '{{cloudlab.cloudlab_provider }}' + layer3: "{{ layer3.my_layer }}" + - node: + - cloudlab_node: + provider: '{{ cloudlab.cloudlab_provider }}' + network: "{{ network.cnet }}" + count: '{{ var.node_count }}' + - node: + - fabric_node: + provider: '{{ fabric.fabric_provider }}' + network: "{{ network.fabric_network }}" + count: '{{ var.node_count }}' + - network: + - fabric_network: + provider: '{{ fabric.fabric_provider }}' + layer3: "{{ layer3.my_layer }}" + stitch_with: '{{ network.cnet }}' + stitch_option: + policy: '{{ policy.my_stitch_policy }}' diff --git a/fabfed/fabfed_manager.py b/fabfed/fabfed_manager.py index 35b2b6fc..55e502ef 100644 --- a/fabfed/fabfed_manager.py +++ b/fabfed/fabfed_manager.py @@ -16,6 +16,7 @@ session="some_session" fabfed_manager = FabfedManager(config_dir=config_dir) fabfed_manager.validate() +fabfed_manager.show_available_stitch_ports(from_provider='chi', to_provider="fabric") fabfed_manager.stitch_info(session=session) to_be_created_count, to_be_deleted_count = fabfed_manager.plan(session=session) status_code = fabfed_manager.apply(session=session) @@ -200,3 +201,44 @@ def destroy(self, *, session: str): def show_sessions(self, *, to_json: bool = False): self.sessions = utils.dump_sessions(to_json) + + def show_available_stitch_ports(self, *, from_provider, to_provider): + from fabfed.policy.policy_helper import load_policy + + policy = load_policy() + + from fabfed.policy.policy_helper import find_stitch_port_for_providers, peer_stitch_ports + + providers = [from_provider, to_provider] + stitch_infos = find_stitch_port_for_providers(policy, providers) + stitch_infos = peer_stitch_ports(stitch_infos) + attrs = ["preference", "member-of", 'name'] + names = [] + + for stitch_info in stitch_infos: + names.append(stitch_info.stitch_port.get("name")) + + for attr in attrs: + stitch_info.stitch_port.pop(attr, None) + + peer = stitch_info.stitch_port.get('peer', {}) + + for attr in attrs: + peer.pop(attr, None) + + import yaml + from fabfed.util.constants import Constants + + stitch_port_configs = [] + + for i, stitch_info in enumerate(stitch_infos): + producer = stitch_info.producer + consumer = stitch_info.consumer + stitch_port = stitch_info.stitch_port + c = {"config": [{Constants.NETWORK_STITCH_CONFIG: [{f"si_from_{names[i]}": {"producer": producer, + "consumer": consumer, + "stitch_port": stitch_port}}]}]} + stitch_port_configs.append(c) + + rep = yaml.dump(stitch_port_configs, default_flow_style=False, sort_keys=False) + print(rep) diff --git a/fabfed/provider/chi/chi_network.py b/fabfed/provider/chi/chi_network.py index 40fcbf81..67db8e82 100644 --- a/fabfed/provider/chi/chi_network.py +++ b/fabfed/provider/chi/chi_network.py @@ -22,6 +22,7 @@ class ChiNetwork(Network): def __init__(self, *, label, name: str, site: str, project_name: str, layer3: Config, stitch_provider: str): super().__init__(label=label, name=name, site=site) self.project_name = project_name + self.layer3 = layer3 self.subnet = layer3.attributes.get(Constants.RES_SUBNET) self.ip_start = layer3.attributes.get(Constants.RES_LAYER3_DHCP_START) self.ip_end = layer3.attributes.get(Constants.RES_LAYER3_DHCP_END) diff --git a/fabfed/provider/fabric/fabric_slice.py b/fabfed/provider/fabric/fabric_slice.py index bf3e7bd8..70466eb5 100644 --- a/fabfed/provider/fabric/fabric_slice.py +++ b/fabfed/provider/fabric/fabric_slice.py @@ -475,6 +475,8 @@ def create_resource(self, *, resource: dict): if self.nodes: self._handle_node_networking() + else: + self._reload_networks() for node in self.nodes: self.resource_listener.on_created(source=self, provider=self.provider, resource=node) diff --git a/fabfed/provider/sense/sense_provider.py b/fabfed/provider/sense/sense_provider.py index 51028e75..61213b45 100644 --- a/fabfed/provider/sense/sense_provider.py +++ b/fabfed/provider/sense/sense_provider.py @@ -149,12 +149,13 @@ def do_create_resource(self, *, resource: dict): rtype = resource.get(Constants.RES_TYPE) assert rtype in self.supported_resources label = resource.get(Constants.LABEL) + self.logger.info(f"{self.name}: Creating resource {label} ....") if not self._handled_modify and self.modified: assert rtype == Constants.RES_TYPE_NETWORK, "sense expects network to be created first" self._handled_modify = True - self.logger.info(f"Deleting sense resources ....") + self.logger.warning(f"{self.name}:Modified state: Deleting sense resources ...") from .sense_network import SenseNetwork diff --git a/tools/fabfed.py b/tools/fabfed.py index 234fc729..b915a739 100644 --- a/tools/fabfed.py +++ b/tools/fabfed.py @@ -10,6 +10,13 @@ from fabfed.util.constants import Constants +def delete_session_if_empty(*, session): + provider_states = sutil.load_states(session) + + if not provider_states: + sutil.destroy_session(session) + + def manage_workflow(args): logger = utils.init_logger() config_dir = utils.absolute_path(args.config_dir) @@ -21,10 +28,6 @@ def manage_workflow(args): logger.warning(f"ATTN: The CORRECT config dir for session {args.session} is {config_dir_from_meta}!!!!!!!") sys.exit(1) - # we skip -show and -destroy - if args.apply or args.init or args.plan or args.validate: - sutil.save_meta_data(dict(config_dir=config_dir), args.session) - var_dict = utils.load_vars(args.var_file) if args.var_file else {} from fabfed.policy.policy_helper import load_policy @@ -41,6 +44,7 @@ def manage_workflow(args): sys.exit(1) if args.apply: + sutil.save_meta_data(dict(config_dir=config_dir), args.session) sutil.delete_stats(args.session) import time @@ -145,6 +149,7 @@ def manage_workflow(args): states = sutil.load_states(args.session) controller.init(session=args.session, provider_factory=default_provider_factory, provider_states=states) sutil.dump_resources(resources=controller.resources, to_json=args.json, summary=args.summary) + delete_session_if_empty(session=args.session) return if args.stitch_info: @@ -198,6 +203,7 @@ def manage_workflow(args): stitch_info_summaries = dict(StitchInfoSummary=stitch_info_summaries) sutil.dump_objects(objects=stitch_info_summaries, to_json=args.json) + delete_session_if_empty(session=args.session) return if args.plan: @@ -211,6 +217,7 @@ def manage_workflow(args): cr, dl = sutil.dump_plan(resources=controller.resources, to_json=args.json, summary=args.summary) logger.warning(f"Applying this plan would create {cr} resource(s) and destroy {dl} resource(s)") + delete_session_if_empty(session=args.session) return if args.show: @@ -221,6 +228,7 @@ def manage_workflow(args): if args.stats: stats = sutil.load_stats(args.session) sutil.dump_stats(stats, args.json) + delete_session_if_empty(session=args.session) return if args.destroy: From 10c8531b0e2acf4c8e454ea621f16b38bf0e86a8 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Fri, 15 Mar 2024 17:22:06 -0700 Subject: [PATCH 55/75] updating design doc --- docs/workflow_design.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/workflow_design.md b/docs/workflow_design.md index 52c7dbc6..fae8a513 100644 --- a/docs/workflow_design.md +++ b/docs/workflow_design.md @@ -37,6 +37,7 @@ The `provider` class supports the following types. The `config` class support the following types: - [ ] layer3 - [ ] peering +- [ ] policy The `resource` class support three types: - [ ] node @@ -324,9 +325,9 @@ Also note the `stitch_option` right below the `stitch_with`. If `stitch_option` ``` fabfed stitch-policy -providers "fabric,aws" ``` - To view the stitch port that would be selected use the -init option before you -apply. + To view the stitch port that would be selected use the -stitch-info option before you -apply. ``` -fabfed workflow -s my-aws-test -init -summary +fabfed workflow -s my-aws-test -stitch-info -summary ``` ``` - network: @@ -343,12 +344,11 @@ fabfed workflow -s my-aws-test -init -summary provider: '{{ fabric.fabric_provider }}' layer3: "{{ layer3.fab_layer }}" peering: "{{ peering.my_peering }}" - interface: '{{ node.fabric_node }}' count: 1 ``` # Overriding Stitching Policy - Fabfed uses system defined stitching. Here we discuss the several ways one can override the stich coonfiguration. As of this writting there are 3 approaches. The first two approaches require some changes in the fabfed workflow definition. These are the simpler approaches. There is also a third approach which allows one can use to provide a complete stitching policy. + Fabfed uses system defined stitching. Here we discuss the several ways one can override the stitch configuration. As of this writting there are 3 approaches. The first two approaches require some changes in the fabfed workflow definition. These are the simpler approaches. There is also a third approach which allows one can use to provide a complete stitching policy. ### Using The Network Resource The `stitch_option` can be used to tell fabfed to select a stitch point from sense to fabric that uses the agg4.ashb.net.internet2.edu. This stitch point returns a default profile needed by the sense provider. @@ -360,14 +360,14 @@ Instead of changing the policy files, one can simply use an attribute in the net profile: my-new-sense-profile stitch_with: '{{ network.fabric_network }}' stitch_option: - device_name: agg4.ashb.net.internet2.edu + device_name: agg3.ashb - fabric_network: provider: '{{ fabric.fabric_provider }}' layer3: "{{ layer3.fab_layer }}" ``` ### Using Policy Config -Using the `policy` config class, one can, for example, use a stich point with a new device that is not yet available in the fabfed stiching policy. Note how the cloudlab network points to the `policy` config using the attribute `policy` under the `stich_option`. +Using the `policy` config class, one can, for example, use a stitch point with a new device that is not yet available in the fabfed stitching policy. Note how the cloudlab network points to the `policy` config using the attribute `policy` under the `stitch_option`. ``` config: - policy: @@ -399,7 +399,7 @@ This approach can be useful when designing a new policy with several new stitch fabfed stitch-policy -providers "fabric,chi" -p my-policy.yaml # Test your workflow and verify the selected stitch point -fabfed workflow -s my-session -init -p my-policy.yaml -summary +fabfed workflow -s my-session -stitch-info -p my-policy.yaml -summary # Apply your workflow fabfed workflow -s my-session -apply -p my-policy.yaml From d64ef632be19f607ad0439e966e43a2d6928ade4 Mon Sep 17 00:00:00 2001 From: abessiari Date: Fri, 15 Mar 2024 18:45:03 -0700 Subject: [PATCH 56/75] Update workflow_design.md --- docs/workflow_design.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/workflow_design.md b/docs/workflow_design.md index fae8a513..c6595e55 100644 --- a/docs/workflow_design.md +++ b/docs/workflow_design.md @@ -375,6 +375,7 @@ config: consumer: fabric producer: cloudlab stitch_port: + name: my_policy profile: fabfed-stitch-v2 provider: cloudlab peer: From 86038d66685401df0bd4884979b34b6b168aa09f Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Fri, 15 Mar 2024 21:45:07 -0700 Subject: [PATCH 57/75] test fabric modify add + delete nodes --- .../ubuntu_22_and_test_modify_fabric_l2.yml | 89 +++++++++++++++++++ cicd/run-fabfed.sh | 8 ++ .../fabric_facility_port/config.fab | 7 +- fabfed/provider/fabric/fabric_network.py | 1 - fabfed/util/parser.py | 2 +- 5 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml diff --git a/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml b/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml new file mode 100644 index 00000000..1ec28bac --- /dev/null +++ b/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml @@ -0,0 +1,89 @@ +name: Ubuntu22.04 Test + +on: + push: + branches: + - knit8 + workflow_dispatch: + +env: + FABRIC_TOKEN: ${{ secrets.FABRIC_TOKEN }} + FABRIC_BASTION_KEY: ${{ secrets.FABRIC_BASTION_KEY }} + FABRIC_SLIVER_KEY: ${{ secrets.FABRIC_SLIVER_KEY }} + FABRIC_SLIVER_PUBKEY: ${{ secrets.FABRIC_SLIVER_PUBKEY }} + FABRIC_PROJECT: ${{ secrets.FABRIC_PROJECT }} + FABRIC_USER: ${{ secrets.FABRIC_USER }} + +jobs: + UbuntuTest: + runs-on: ubuntu-latest + steps: + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Fabfed And Requirements + run: | + pip install --upgrade pip + pip install setuptools --upgrade + pip install --no-cache-dir --ignore-requires-python neo4j==5.18.0 + pip list -v | grep neo4j + pip install --no-cache-dir --ignore-requires-python -r requirements.txt + python3 -m pip install --no-cache-dir . + pip install --no-cache-dir git+https://gitlab.flux.utah.edu/stoller/portal-tools.git + + - name: Save Credentials + run: | + mkdir -p ${{ github.workspace }}/creds + echo ${{ env.FABRIC_TOKEN }} | base64 --decode > ${{ github.workspace }}/creds/token.json + echo ${{ env.FABRIC_BASTION_KEY }} | base64 --decode > ${{ github.workspace }}/creds/bastion + echo ${{ env.FABRIC_SLIVER_KEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver + echo ${{ env.FABRIC_SLIVER_PUBKEY }} | base64 --decode > ${{ github.workspace }}/creds/sliver.pub + + - name: Test Fabric Modify with FacilityPort 0 nodes + run: | + export DO_NOT_DESTROY=1 + session=cicd-fabric-facility-port + echo "vlan: 3103" > $session-varfile.yml + echo "node_count: 0" >> $session-varfile.yml + ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml + + - name: Test Fabric Modify with FacilityPort Add One Node + run: | + export DO_NOT_DESTROY=1 + session=cicd-fabric-facility-port + echo "vlan: 3103" > $session-varfile.yml + echo "node_count: 1" >> $session-varfile.yml + ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml + + - name: Test Fabric Modify with FacilityPort Add Another Node + run: | + export DO_NOT_DESTROY=1 + session=cicd-fabric-facility-port + echo "vlan: 3103" > $session-varfile.yml + echo "node_count: 2" >> $session-varfile.yml + ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml + + - name: Test Fabric Modify with FacilityPort Remove One Node + run: | + export DO_NOT_DESTROY=1 + session=cicd-fabric-facility-port + echo "vlan: 3103" > $session-varfile.yml + echo "node_count: 1" >> $session-varfile.yml + ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml + + - name: Test Fabric Modify with FacilityPort Remove One Node + run: | + export DO_NOT_DESTROY=1 + session=cicd-fabric-facility-port + echo "vlan: 3103" > $session-varfile.yml + echo "node_count: 0" >> $session-varfile.yml + ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml + + - name: Test Fabric Modify with FacilityPort Destroy + run: | + export DO_NOT_DESTROY=0 + session=cicd-fabric-facility-port + echo "vlan: 3103" > $session-varfile.yml + echo "node_count: 0" >> $session-varfile.yml + ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_facility_port $session $session-varfile.yml diff --git a/cicd/run-fabfed.sh b/cicd/run-fabfed.sh index 985f75e1..5dc01525 100755 --- a/cicd/run-fabfed.sh +++ b/cicd/run-fabfed.sh @@ -41,6 +41,14 @@ fabfed workflow -c $conf_dir $options -s $session -show -summary > $session-appl fabfed workflow -c $conf_dir $options -s $session -show -summary fabfed sessions -show +DO_NOT_DESTROY="${DO_NOT_DESTROY:=0}" + +if [ $DO_NOT_DESTROY -ne 0 ]; then + echo "NOT_DESTROYING:APPLY RESULTS:" + cat $session-apply-state.yaml + exit $ret1 +fi + echo "***************** DESTOYING ****************" echo fabfed workflow -c $conf_dir $options -s $session -destroy fabfed workflow -c $conf_dir $options -s $session -destroy diff --git a/cicd/test_configs/fabric_facility_port/config.fab b/cicd/test_configs/fabric_facility_port/config.fab index f94736f0..b10ef228 100644 --- a/cicd/test_configs/fabric_facility_port/config.fab +++ b/cicd/test_configs/fabric_facility_port/config.fab @@ -1,6 +1,9 @@ variable: - vlan: default: 3101 + - node_count: + default: 0 + provider: - fabric: - fabric_provider: @@ -20,12 +23,12 @@ resource: site: UTAH # CLEM # UTAH image: default_rocky_8 nic_model: NIC_Basic - count: 0 + network: '{{ network.fabric_network }}' + count: '{{ var.node_count }}' - network: - fabric_network: provider: '{{ fabric.fabric_provider }}' layer3: "{{ layer3.my_layer }}" - # interface: '{{ node.fabric_node }}' device_name: Utah-Cloudlab-Powder site: UTAH interface: diff --git a/fabfed/provider/fabric/fabric_network.py b/fabfed/provider/fabric/fabric_network.py index fc7abde3..8a70855d 100644 --- a/fabfed/provider/fabric/fabric_network.py +++ b/fabfed/provider/fabric/fabric_network.py @@ -92,7 +92,6 @@ def __init__(self, label, provider: FabricProvider, slice_object: Slice, name, r self.layer3 = resource.get(Constants.RES_LAYER3) self.peering = resource.get(Constants.RES_PEERING) self.peer_layer3 = resource.get(Constants.RES_PEER_LAYER3) - self.device = None self.stitching_net = None self.label = label self.net = None diff --git a/fabfed/util/parser.py b/fabfed/util/parser.py index cec01b13..4acb727d 100644 --- a/fabfed/util/parser.py +++ b/fabfed/util/parser.py @@ -115,7 +115,7 @@ def _validate_variables(variables: List[Variable]): raise ParseConfigException(f'detected duplicate variables') for variable in variables: - if not variable.value: + if variable.value is None: raise ParseConfigException(f'variable {variable .name} is not bound') @staticmethod From 74d35b3835b7426b2547af30e94c32a320475968 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 15 Mar 2024 23:49:43 -0500 Subject: [PATCH 58/75] Update notebooks for workflows --- jupyter/chameleon-lib.ipynb | 173 ++++++++++++++++++++--------------- jupyter/chameleon.ipynb | 28 +++++- jupyter/cloudlab-lib.ipynb | 69 ++++++++++---- jupyter/cloudlab.ipynb | 30 ++++-- jupyter/native-aws-lib.ipynb | 153 ++++++++++++++++++------------- jupyter/native-aws.ipynb | 28 +++++- jupyter/native-gcp-lib.ipynb | 153 ++++++++++++++++++------------- jupyter/native-gcp.ipynb | 28 +++++- jupyter/sense-aws-lib.ipynb | 153 ++++++++++++++++++------------- jupyter/sense-aws.ipynb | 28 +++++- jupyter/sense-gcp-lib.ipynb | 153 ++++++++++++++++++------------- jupyter/sense-gcp.ipynb | 28 +++++- 12 files changed, 637 insertions(+), 387 deletions(-) diff --git a/jupyter/chameleon-lib.ipynb b/jupyter/chameleon-lib.ipynb index ec96314e..c8da1de1 100644 --- a/jupyter/chameleon-lib.ipynb +++ b/jupyter/chameleon-lib.ipynb @@ -26,7 +26,7 @@ "source": [ "import os\n", "\n", - "session_name = \"chameleon-lzhang9-lib\"\n", + "session_name = \"chameleon-demo-lib\"\n", "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", "fab_file_path = \"./examples/chameleon/star\"\n", "\n", @@ -49,10 +49,34 @@ "metadata": {}, "outputs": [], "source": [ - "from fabfed.util.config import WorkflowConfig\n", - "from fabfed.util import state as sutil\n", - "from fabfed.controller.controller import Controller\n", - "from fabfed.controller.provider_factory import default_provider_factory" + "try:\n", + " from fabfed.fabfed_manager import FabfedManager\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", + "else:\n", + " print(\"Succeed!\")" + ] + }, + { + "cell_type": "markdown", + "id": "f65f8bad-eadc-4df4-8d13-693ea129003a", + "metadata": {}, + "source": [ + "## Show availabe stitch ports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "315ed071-8f06-47e8-943d-e90748c0cb6b", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " fabfed_manager = FabfedManager(config_dir=fab_file_path)\n", + " fabfed_manager.show_available_stitch_ports(from_provider='chi', to_provider=\"fabric\")\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -60,7 +84,7 @@ "id": "73a535d1-4f1a-4606-b4a4-128c675255db", "metadata": {}, "source": [ - "## Read .fab files" + "## Validate .fab files" ] }, { @@ -70,197 +94,194 @@ "metadata": {}, "outputs": [], "source": [ - "def read_file_to_str(file_path):\n", - " try:\n", - " with open(file_path, 'r') as file:\n", - " file_content = file.read()\n", - " return file_content\n", - " except FileNotFoundError:\n", - " print(f\"File '{file_path}' not found.\")\n", - " return None\n", - "\n", - "# Read config.fab to config_str:\n", - "config_str = read_file_to_str(f\"{fab_file_path}/config.fab\")\n", - "if config_str:\n", - " print(\"File content:\")\n", - " print(config_str)\n", - "\n", - "print(config_str)" + "try:\n", + " config = fabfed_manager.validate()\n", + "except Exception as e:\n", + " print(f\"Error: {f}\")\n", + "else:\n", + " print(\"Succeed!\")" ] }, { "cell_type": "markdown", - "id": "f6106b4e-076f-4349-b215-c6db497c97a9", + "id": "e9ca58ea-dbc2-42b4-a0eb-a560f023a210", "metadata": {}, "source": [ - "## Create a workflow config object" + "## Show the order of handling resources" ] }, { "cell_type": "code", "execution_count": null, - "id": "ba75ff1a-7442-4e05-a4ca-b1128397983e", + "id": "7834d64f-44d1-4cb1-8708-4038d601153d", "metadata": {}, "outputs": [], "source": [ - "config = WorkflowConfig(content=config_str)" + "for resource_config in config.resource_configs:\n", + " dependencies = []\n", + " for dependency in resource_config.dependencies:\n", + " dependencies.append(dependency.resource)\n", + " print(resource_config.label, \"depends on \", dependencies)" ] }, { "cell_type": "markdown", - "id": "c87f8329-054e-45f3-9d69-3c4acc25252f", + "id": "f6106b4e-076f-4349-b215-c6db497c97a9", "metadata": {}, "source": [ - "## Create a controller" + "## Stitching information" ] }, { "cell_type": "code", "execution_count": null, - "id": "5d161b4f-4b6b-4993-80d3-26f59034c2cb", + "id": "ba75ff1a-7442-4e05-a4ca-b1128397983e", "metadata": {}, "outputs": [], "source": [ - "import logging\n", - "logger = logging.getLogger(__name__)\n", - "\n", - "controller = Controller(config=config, logger=logger)" + "try:\n", + " fabfed_manager.stitch_info(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { "cell_type": "markdown", - "id": "4e1aa491-7ddc-491b-82f6-ec9d41b08c3a", + "id": "c87f8329-054e-45f3-9d69-3c4acc25252f", "metadata": {}, "source": [ - "## Init stitching information" + "## List resources for creation and deletion" ] }, { "cell_type": "code", "execution_count": null, - "id": "7da5efbb-0c83-4842-b533-c81340028013", + "id": "5d161b4f-4b6b-4993-80d3-26f59034c2cb", "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session)\n", + "try:\n", + " to_be_created_count, to_be_deleted_count = fabfed_manager.plan(session=session_name)\n", + "except Exception as e:\n", + " print(\"Error: {e}\")\n", "\n", - "controller.init(session=session_name, provider_factory=default_provider_factory, provider_states=states)" + "print(\"\\n=== Total ===\")\n", + "print(f\"to_be_created: {to_be_created_count}\")\n", + "print(f\"to_be_deleted: {to_be_deleted_count}\")" ] }, { "cell_type": "markdown", - "id": "465ea762-1e62-4532-a0e3-aa5bef0e23c4", + "id": "4e1aa491-7ddc-491b-82f6-ec9d41b08c3a", "metadata": {}, "source": [ - "## Add resources to providers and validate resources" + "## Create resources" ] }, { "cell_type": "code", "execution_count": null, - "id": "f8ae4d51-a812-4d80-997c-11c7ef5548a2", + "id": "7da5efbb-0c83-4842-b533-c81340028013", "metadata": {}, "outputs": [], "source": [ - "controller.plan(provider_states=[])" + "try:\n", + " status_code = fabfed_manager.apply(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "beee11c5-5adc-4756-8133-a8eda0253969", + "metadata": {}, + "source": [ + "## Show session state" ] }, { "cell_type": "code", "execution_count": null, - "id": "7a1a864d-bdd9-4422-84a6-cade3a7c485c", + "id": "9ca3360b-c033-4af8-9f1d-0acf12b1c0d0", "metadata": {}, "outputs": [], "source": [ - "controller.add(provider_states=[])" + "fabfed_manager.show(session=session_name)" ] }, { "cell_type": "markdown", - "id": "78419199-2802-481e-b0c8-08b362f658e1", + "id": "8c2d5e44-4789-4254-8b7a-8cc656ac4a0f", "metadata": {}, "source": [ - "## Create resources" + "## Show all sessions" ] }, { "cell_type": "code", "execution_count": null, - "id": "4db644c5-b7d5-4092-8329-cca89a13aef6", + "id": "e0ad9df6-7cea-4a69-8342-978c92864f68", "metadata": {}, "outputs": [], "source": [ - "try:\n", - " controller.apply(provider_states=[])\n", - "except ControllerException as e:\n", - " assert isinstance(e.exceptions[0], DummyFailCreateException)" + "fabfed_manager.show_sessions()" ] }, { "cell_type": "markdown", - "id": "beee11c5-5adc-4756-8133-a8eda0253969", + "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", "metadata": {}, "source": [ - "## Save the workflow state" + "## Delete all resources" ] }, { "cell_type": "code", "execution_count": null, - "id": "9ca3360b-c033-4af8-9f1d-0acf12b1c0d0", + "id": "bc2c921a-4f27-470a-ac8b-a7c957559b2d", "metadata": {}, "outputs": [], "source": [ - "states = controller.get_states()\n", - "sutil.save_states(states, session_name)\n", - "\n", - "print(states)" + "try:\n", + " fabfed_manager.destroy(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { "cell_type": "markdown", - "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", + "id": "294f4277-e3c1-483a-8c5f-127d338fccc3", "metadata": {}, "source": [ - "## View the state of the workflow" + "## Show sesstion state" ] }, { "cell_type": "code", "execution_count": null, - "id": "1dfdcf82-03c0-42a7-8fd0-78a04ed1e0fe", + "id": "f590433f-abfe-413d-b119-01451752746a", "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session_name)\n", - "\n", - "print(states)" + "fabfed_manager.show(session=session_name)" ] }, { "cell_type": "markdown", - "id": "ab33e762-ad89-49ce-9b09-ceb79797ec78", + "id": "4650151a-0c2e-4f47-9c15-25efdf25bc78", "metadata": {}, "source": [ - "## Delete all resources" + "## Show all sessions" ] }, { "cell_type": "code", "execution_count": null, - "id": "bc2c921a-4f27-470a-ac8b-a7c957559b2d", + "id": "061f76ee-282c-4b6b-a430-16226b477716", "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session_name)\n", - "\n", - "controller.destroy(provider_states=states)\n", - "\n", - "sutil.save_states(states, session_name)\n", - "if not states:\n", - " sutil.destroy_session(session_name)\n", - "\n", - "print(states)" + "fabfed_manager.show_sessions()" ] } ], diff --git a/jupyter/chameleon.ipynb b/jupyter/chameleon.ipynb index 76f628ea..28753072 100644 --- a/jupyter/chameleon.ipynb +++ b/jupyter/chameleon.ipynb @@ -26,7 +26,7 @@ "source": [ "import os\n", "\n", - "session_name = \"chameleon-lzhang9\"\n", + "session_name = \"chameleon-demo\"\n", "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", "fab_file_path = \"./examples/chameleon/star\"\n", "\n", @@ -34,6 +34,24 @@ "print(fab_file_path)" ] }, + { + "cell_type": "markdown", + "id": "0ca5cb91-7fe7-4407-a68e-12119b5269a6", + "metadata": {}, + "source": [ + "## Show availabe stitch ports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8b8eddf-1d5e-4224-b107-fecd98d89bf9", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed stitch-policy -providers \"chi,fabric\"" + ] + }, { "cell_type": "markdown", "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", @@ -57,7 +75,7 @@ "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", "metadata": {}, "source": [ - "## Add resources to providers and validate resources" + "## List resources for creation and deletion" ] }, { @@ -67,7 +85,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -plan" + "!fabfed workflow -s $session_name -c $fab_file_path -plan -summary" ] }, { @@ -85,7 +103,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" + "!fabfed workflow -s $session_name -c $fab_file_path -stitch-info -summary" ] }, { @@ -123,7 +141,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -show" + "!fabfed workflow -s $session_name -c $fab_file_path -show -summary" ] }, { diff --git a/jupyter/cloudlab-lib.ipynb b/jupyter/cloudlab-lib.ipynb index 20065d7d..2296d9f8 100644 --- a/jupyter/cloudlab-lib.ipynb +++ b/jupyter/cloudlab-lib.ipynb @@ -44,7 +44,7 @@ "source": [ "import os\n", "\n", - "session_name = \"cloudlab-lzhang9-lib\"\n", + "session_name = \"cloudlab-demo-lib\"\n", "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/cloudlab\"\n", "fab_file_path = \"./examples/cloudlab\"\n", "\n", @@ -71,8 +71,30 @@ " from fabfed.fabfed_manager import FabfedManager\n", "except Exception as e:\n", " print(f\"Error: {e}\")\n", - " \n", - "print(\"Complete!\")" + "else:\n", + " print(\"Succeed!\")" + ] + }, + { + "cell_type": "markdown", + "id": "8e332b99-3385-44db-a3d7-2e77b0185487", + "metadata": {}, + "source": [ + "## Show availabe stitch ports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ce49fac-de21-41e6-8291-de9a2bac9d8f", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " fabfed_manager = FabfedManager(config_dir=fab_file_path)\n", + " fabfed_manager.show_available_stitch_ports(from_provider='cloudlab', to_provider=\"fabric\")\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -91,12 +113,33 @@ "outputs": [], "source": [ "try:\n", - " fabfed_manager = FabfedManager(config_dir=fab_file_path)\n", - " fabfed_manager.validate()\n", + " config = fabfed_manager.validate()\n", "except Exception as e:\n", " print(f\"Error: {e}\")\n", - " \n", - "print(\"Complete!\")" + "else:\n", + " print(\"Succeed!\")" + ] + }, + { + "cell_type": "markdown", + "id": "0f84ea1b-39fa-41d2-bc4b-01e346488146", + "metadata": {}, + "source": [ + "## Show the order of handling resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b24ca722-5310-4843-bef6-040546aea573", + "metadata": {}, + "outputs": [], + "source": [ + "for resource_config in config.resource_configs:\n", + " dependencies = []\n", + " for dependency in resource_config.dependencies:\n", + " dependencies.append(dependency.resource)\n", + " print(resource_config.label, \"depends on \", dependencies)" ] }, { @@ -117,9 +160,7 @@ "try:\n", " fabfed_manager.stitch_info(session=session_name)\n", "except Exception as e:\n", - " print(f\"Error: {e}\")\n", - "\n", - "print(\"Complete!\")" + " print(f\"Error: {e}\")" ] }, { @@ -165,9 +206,7 @@ "try:\n", " status_code = fabfed_manager.apply(session=session_name)\n", "except Exception as e:\n", - " print(f\"Error: {e}\")\n", - "\n", - "print(\"Complete!\")" + " print(f\"Error: {e}\")" ] }, { @@ -224,9 +263,7 @@ "try:\n", " fabfed_manager.destroy(session=session_name)\n", "except Exception as e:\n", - " print(f\"Error: {e}\")\n", - "\n", - "print(\"Complete!\")" + " print(f\"Error: {e}\")" ] }, { diff --git a/jupyter/cloudlab.ipynb b/jupyter/cloudlab.ipynb index 582bfd91..ee152b06 100644 --- a/jupyter/cloudlab.ipynb +++ b/jupyter/cloudlab.ipynb @@ -26,14 +26,32 @@ "source": [ "import os\n", "\n", - "session_name = \"cloudlab-lzhang9\"\n", + "session_name = \"cloudlab-demo\"\n", "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/cloudlab\"\n", "fab_file_path = \"./examples/cloudlab\"\n", - "\n", + "# more path with comments\n", "print(session_name)\n", "print(fab_file_path)" ] }, + { + "cell_type": "markdown", + "id": "745fc0cb-f4ee-42f3-8934-c0593871f8a6", + "metadata": {}, + "source": [ + "## Show availabe stitch ports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9e5a9b59-4349-473b-a029-de6b52a67023", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed stitch-policy -providers \"cloudlab,fabric\"" + ] + }, { "cell_type": "markdown", "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", @@ -57,7 +75,7 @@ "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", "metadata": {}, "source": [ - "## Add resources to providers and validate resources" + "## List resources for creation and deletion" ] }, { @@ -67,7 +85,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -plan" + "!fabfed workflow -s $session_name -c $fab_file_path -plan -summary" ] }, { @@ -85,7 +103,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" + "!fabfed workflow -s $session_name -c $fab_file_path -stitch-info -summary" ] }, { @@ -123,7 +141,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -show" + "!fabfed workflow -s $session_name -c $fab_file_path -show -summary" ] }, { diff --git a/jupyter/native-aws-lib.ipynb b/jupyter/native-aws-lib.ipynb index 184041d5..efecfb4c 100644 --- a/jupyter/native-aws-lib.ipynb +++ b/jupyter/native-aws-lib.ipynb @@ -26,7 +26,7 @@ "source": [ "import os\n", "\n", - "session_name = \"native-aws-lzhang9-lib\"\n", + "session_name = \"native-aws-demo-lib\"\n", "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", "fab_file_path = \"./examples/native-aws\"\n", "\n", @@ -49,10 +49,12 @@ "metadata": {}, "outputs": [], "source": [ - "from fabfed.util.config import WorkflowConfig\n", - "from fabfed.util import state as sutil\n", - "from fabfed.controller.controller import Controller\n", - "from fabfed.controller.provider_factory import default_provider_factory" + "try:\n", + " from fabfed.fabfed_manager import FabfedManager\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", + "else:\n", + " print(\"Succeed!\")" ] }, { @@ -60,7 +62,7 @@ "id": "73a535d1-4f1a-4606-b4a4-128c675255db", "metadata": {}, "source": [ - "## Read .fab files" + "## Show availabe stitch ports" ] }, { @@ -70,22 +72,11 @@ "metadata": {}, "outputs": [], "source": [ - "def read_file_to_str(file_path):\n", - " try:\n", - " with open(file_path, 'r') as file:\n", - " file_content = file.read()\n", - " return file_content\n", - " except FileNotFoundError:\n", - " print(f\"File '{file_path}' not found.\")\n", - " return None\n", - "\n", - "# Read config.fab to config_str:\n", - "config_str = read_file_to_str(f\"{fab_file_path}/config.fab\")\n", - "if config_str:\n", - " print(\"File content:\")\n", - " print(config_str)\n", - "\n", - "print(config_str)" + "try:\n", + " fabfed_manager = FabfedManager(config_dir=fab_file_path)\n", + " fabfed_manager.show_available_stitch_ports(from_provider='aws', to_provider=\"fabric\")\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -93,7 +84,7 @@ "id": "f6106b4e-076f-4349-b215-c6db497c97a9", "metadata": {}, "source": [ - "## Create a workflow config object" + "## Validate .fab files" ] }, { @@ -103,7 +94,12 @@ "metadata": {}, "outputs": [], "source": [ - "config = WorkflowConfig(content=config_str)" + "try:\n", + " config = fabfed_manager.validate()\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", + "else:\n", + " print(\"Succeed!\")" ] }, { @@ -111,7 +107,7 @@ "id": "c87f8329-054e-45f3-9d69-3c4acc25252f", "metadata": {}, "source": [ - "## Create a controller" + "## Show the order of handling resources" ] }, { @@ -121,10 +117,11 @@ "metadata": {}, "outputs": [], "source": [ - "import logging\n", - "logger = logging.getLogger(__name__)\n", - "\n", - "controller = Controller(config=config, logger=logger)" + "for resource_config in config.resource_configs:\n", + " dependencies = []\n", + " for dependency in resource_config.dependencies:\n", + " dependencies.append(dependency.resource)\n", + " print(resource_config.label, \"depends on \", dependencies)" ] }, { @@ -132,7 +129,7 @@ "id": "4e1aa491-7ddc-491b-82f6-ec9d41b08c3a", "metadata": {}, "source": [ - "## Init stitching information" + "## Stitching information" ] }, { @@ -142,9 +139,10 @@ "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session)\n", - "\n", - "controller.init(session=session_name, provider_factory=default_provider_factory, provider_states=states)" + "try:\n", + " fabfed_manager.stitch_info(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -152,7 +150,7 @@ "id": "465ea762-1e62-4532-a0e3-aa5bef0e23c4", "metadata": {}, "source": [ - "## Add resources to providers and validate resources" + "## List resources for creation and deletion" ] }, { @@ -162,17 +160,14 @@ "metadata": {}, "outputs": [], "source": [ - "controller.plan(provider_states=[])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7a1a864d-bdd9-4422-84a6-cade3a7c485c", - "metadata": {}, - "outputs": [], - "source": [ - "controller.add(provider_states=[])" + "try:\n", + " to_be_created_count, to_be_deleted_count = fabfed_manager.plan(session=session_name)\n", + "except Exception as e:\n", + " print(\"Error: {e}\")\n", + "\n", + "print(\"\\n=== Total ===\")\n", + "print(f\"to_be_created: {to_be_created_count}\")\n", + "print(f\"to_be_deleted: {to_be_deleted_count}\")" ] }, { @@ -191,9 +186,9 @@ "outputs": [], "source": [ "try:\n", - " controller.apply(provider_states=[])\n", - "except ControllerException as e:\n", - " assert isinstance(e.exceptions[0], DummyFailCreateException)" + " status_code = fabfed_manager.apply(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -201,7 +196,7 @@ "id": "beee11c5-5adc-4756-8133-a8eda0253969", "metadata": {}, "source": [ - "## Save the workflow state" + "## Show session state" ] }, { @@ -211,10 +206,7 @@ "metadata": {}, "outputs": [], "source": [ - "states = controller.get_states()\n", - "sutil.save_states(states, session_name)\n", - "\n", - "print(states)" + "fabfed_manager.show(session=session_name)" ] }, { @@ -222,7 +214,7 @@ "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", "metadata": {}, "source": [ - "## View the state of the workflow" + "## Show all sessions" ] }, { @@ -232,9 +224,7 @@ "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session_name)\n", - "\n", - "print(states)" + "fabfed_manager.show_sessions()" ] }, { @@ -252,15 +242,46 @@ "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session_name)\n", - "\n", - "controller.destroy(provider_states=states)\n", - "\n", - "sutil.save_states(states, session_name)\n", - "if not states:\n", - " sutil.destroy_session(session_name)\n", - "\n", - "print(states)" + "try:\n", + " fabfed_manager.destroy(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "08c51334-07ce-47d2-8089-060c0c9040f2", + "metadata": {}, + "source": [ + "## Show sesstion state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2d3adda-2350-48ac-be08-ff4bfc7e8175", + "metadata": {}, + "outputs": [], + "source": [ + "fabfed_manager.show(session=session_name)" + ] + }, + { + "cell_type": "markdown", + "id": "11f50603-626a-4d3c-87e2-b7a7e7d1861e", + "metadata": {}, + "source": [ + "## Show all sessions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d48c4ec-2e32-4402-a7ad-a2ab7b8f2c53", + "metadata": {}, + "outputs": [], + "source": [ + "fabfed_manager.show_sessions()" ] }, { diff --git a/jupyter/native-aws.ipynb b/jupyter/native-aws.ipynb index f4ab3c1f..39b7d005 100644 --- a/jupyter/native-aws.ipynb +++ b/jupyter/native-aws.ipynb @@ -26,7 +26,7 @@ "source": [ "import os\n", "\n", - "session_name = \"native-aws-lzhang9\"\n", + "session_name = \"native-aws-demo\"\n", "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", "fab_file_path = \"./examples/native-aws\"\n", "\n", @@ -34,6 +34,24 @@ "print(fab_file_path)" ] }, + { + "cell_type": "markdown", + "id": "7e3ad5ac-8894-455d-833e-ff8905242672", + "metadata": {}, + "source": [ + "## Show availabe stitch ports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4be384e-1108-42c5-aafe-a438b38d76b5", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed stitch-policy -providers \"aws,fabric\"" + ] + }, { "cell_type": "markdown", "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", @@ -57,7 +75,7 @@ "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", "metadata": {}, "source": [ - "## Add resources to providers and validate resources" + "## List resources for creation and deletion" ] }, { @@ -67,7 +85,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -plan" + "!fabfed workflow -s $session_name -c $fab_file_path -plan -summary" ] }, { @@ -85,7 +103,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" + "!fabfed workflow -s $session_name -c $fab_file_path -stitch-info -summary" ] }, { @@ -123,7 +141,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -show" + "!fabfed workflow -s $session_name -c $fab_file_path -show -summary" ] }, { diff --git a/jupyter/native-gcp-lib.ipynb b/jupyter/native-gcp-lib.ipynb index 291d643f..849216bb 100644 --- a/jupyter/native-gcp-lib.ipynb +++ b/jupyter/native-gcp-lib.ipynb @@ -26,7 +26,7 @@ "source": [ "import os\n", "\n", - "session_name = \"native-gcp-lzhang9-lib\"\n", + "session_name = \"native-gcp-demo-lib\"\n", "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/gcp\"\n", "fab_file_path = \"./examples/native-gcp\"\n", "\n", @@ -49,10 +49,12 @@ "metadata": {}, "outputs": [], "source": [ - "from fabfed.util.config import WorkflowConfig\n", - "from fabfed.util import state as sutil\n", - "from fabfed.controller.controller import Controller\n", - "from fabfed.controller.provider_factory import default_provider_factory" + "try:\n", + " from fabfed.fabfed_manager import FabfedManager\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", + "else:\n", + " print(\"Succeed!\")" ] }, { @@ -60,7 +62,7 @@ "id": "73a535d1-4f1a-4606-b4a4-128c675255db", "metadata": {}, "source": [ - "## Read .fab files" + "## Show availabe stitch ports" ] }, { @@ -70,22 +72,11 @@ "metadata": {}, "outputs": [], "source": [ - "def read_file_to_str(file_path):\n", - " try:\n", - " with open(file_path, 'r') as file:\n", - " file_content = file.read()\n", - " return file_content\n", - " except FileNotFoundError:\n", - " print(f\"File '{file_path}' not found.\")\n", - " return None\n", - "\n", - "# Read config.fab to config_str:\n", - "config_str = read_file_to_str(f\"{fab_file_path}/config.fab\")\n", - "if config_str:\n", - " print(\"File content:\")\n", - " print(config_str)\n", - "\n", - "print(config_str)" + "try:\n", + " fabfed_manager = FabfedManager(config_dir=fab_file_path)\n", + " fabfed_manager.show_available_stitch_ports(from_provider='cloudlab', to_provider=\"fabric\")\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -93,7 +84,7 @@ "id": "f6106b4e-076f-4349-b215-c6db497c97a9", "metadata": {}, "source": [ - "## Create a workflow config object" + "## Validate .fab files" ] }, { @@ -103,7 +94,12 @@ "metadata": {}, "outputs": [], "source": [ - "config = WorkflowConfig(content=config_str)" + "try:\n", + " config = fabfed_manager.validate()\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", + "else:\n", + " print(\"Succeed!\")" ] }, { @@ -111,7 +107,7 @@ "id": "c87f8329-054e-45f3-9d69-3c4acc25252f", "metadata": {}, "source": [ - "## Create a controller" + "## Show the order of handling resources" ] }, { @@ -121,10 +117,11 @@ "metadata": {}, "outputs": [], "source": [ - "import logging\n", - "logger = logging.getLogger(__name__)\n", - "\n", - "controller = Controller(config=config, logger=logger)" + "for resource_config in config.resource_configs:\n", + " dependencies = []\n", + " for dependency in resource_config.dependencies:\n", + " dependencies.append(dependency.resource)\n", + " print(resource_config.label, \"depends on \", dependencies)" ] }, { @@ -132,7 +129,7 @@ "id": "4e1aa491-7ddc-491b-82f6-ec9d41b08c3a", "metadata": {}, "source": [ - "## Init stitching information" + "## Stitching information" ] }, { @@ -142,9 +139,10 @@ "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session)\n", - "\n", - "controller.init(session=session_name, provider_factory=default_provider_factory, provider_states=states)" + "try:\n", + " fabfed_manager.stitch_info(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -152,7 +150,7 @@ "id": "465ea762-1e62-4532-a0e3-aa5bef0e23c4", "metadata": {}, "source": [ - "## Add resources to providers and validate resources" + "## List resources for creation and deletion" ] }, { @@ -162,17 +160,14 @@ "metadata": {}, "outputs": [], "source": [ - "controller.plan(provider_states=[])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7a1a864d-bdd9-4422-84a6-cade3a7c485c", - "metadata": {}, - "outputs": [], - "source": [ - "controller.add(provider_states=[])" + "try:\n", + " to_be_created_count, to_be_deleted_count = fabfed_manager.plan(session=session_name)\n", + "except Exception as e:\n", + " print(\"Error: {e}\")\n", + "\n", + "print(\"\\n=== Total ===\")\n", + "print(f\"to_be_created: {to_be_created_count}\")\n", + "print(f\"to_be_deleted: {to_be_deleted_count}\")" ] }, { @@ -191,9 +186,9 @@ "outputs": [], "source": [ "try:\n", - " controller.apply(provider_states=[])\n", - "except ControllerException as e:\n", - " assert isinstance(e.exceptions[0], DummyFailCreateException)" + " status_code = fabfed_manager.apply(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -201,7 +196,7 @@ "id": "beee11c5-5adc-4756-8133-a8eda0253969", "metadata": {}, "source": [ - "## Save the workflow state" + "## Show session state" ] }, { @@ -211,10 +206,7 @@ "metadata": {}, "outputs": [], "source": [ - "states = controller.get_states()\n", - "sutil.save_states(states, session_name)\n", - "\n", - "print(states)" + "fabfed_manager.show(session=session_name)" ] }, { @@ -222,7 +214,7 @@ "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", "metadata": {}, "source": [ - "## View the state of the workflow" + "## Show all sessions" ] }, { @@ -232,9 +224,7 @@ "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session_name)\n", - "\n", - "print(states)" + "fabfed_manager.show_sessions()" ] }, { @@ -252,15 +242,46 @@ "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session_name)\n", - "\n", - "controller.destroy(provider_states=states)\n", - "\n", - "sutil.save_states(states, session_name)\n", - "if not states:\n", - " sutil.destroy_session(session_name)\n", - "\n", - "print(states)" + "try:\n", + " fabfed_manager.destroy(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "54f66bf7-c426-42ef-8947-cf32de38936c", + "metadata": {}, + "source": [ + "## Show sesstion state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0ba999f-f0b2-4987-b7b2-2e95dd09411e", + "metadata": {}, + "outputs": [], + "source": [ + "fabfed_manager.show(session=session_name)" + ] + }, + { + "cell_type": "markdown", + "id": "17b5396d-0a8e-4d02-b49c-d32bcb7862b4", + "metadata": {}, + "source": [ + "## Show all sessions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31aa4003-7bb7-43ca-92f9-cc2fe7a650d7", + "metadata": {}, + "outputs": [], + "source": [ + "fabfed_manager.show_sessions()" ] }, { diff --git a/jupyter/native-gcp.ipynb b/jupyter/native-gcp.ipynb index f6cbf543..e8741bfc 100644 --- a/jupyter/native-gcp.ipynb +++ b/jupyter/native-gcp.ipynb @@ -26,7 +26,7 @@ "source": [ "import os\n", "\n", - "session_name = \"native-gcp-lzhang9\"\n", + "session_name = \"native-gcp-demo\"\n", "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/gcp\"\n", "fab_file_path = \"./examples/native-gcp\"\n", "\n", @@ -34,6 +34,24 @@ "print(fab_file_path)" ] }, + { + "cell_type": "markdown", + "id": "89613187-f264-4c89-9742-82ccd7f62f46", + "metadata": {}, + "source": [ + "## Show availabe stitch ports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "05f1cebe-f287-48c5-b096-1928ed020628", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed stitch-policy -providers \"gcp,fabric\"" + ] + }, { "cell_type": "markdown", "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", @@ -57,7 +75,7 @@ "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", "metadata": {}, "source": [ - "## Add resources to providers and validate resources" + "## List resources for creation and deletion" ] }, { @@ -67,7 +85,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -plan" + "!fabfed workflow -s $session_name -c $fab_file_path -plan -summary" ] }, { @@ -85,7 +103,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" + "!fabfed workflow -s $session_name -c $fab_file_path -stitch-info -summary" ] }, { @@ -121,7 +139,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -show" + "!fabfed workflow -s $session_name -c $fab_file_path -show -summary" ] }, { diff --git a/jupyter/sense-aws-lib.ipynb b/jupyter/sense-aws-lib.ipynb index 769f5a58..fa6e80b5 100644 --- a/jupyter/sense-aws-lib.ipynb +++ b/jupyter/sense-aws-lib.ipynb @@ -26,7 +26,7 @@ "source": [ "import os\n", "\n", - "session_name = \"sense-aws-lzhang9-lib\"\n", + "session_name = \"sense-aws-demo-lib\"\n", "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", "fab_file_path = \"./examples/sense-aws\"\n", "\n", @@ -49,10 +49,12 @@ "metadata": {}, "outputs": [], "source": [ - "from fabfed.util.config import WorkflowConfig\n", - "from fabfed.util import state as sutil\n", - "from fabfed.controller.controller import Controller\n", - "from fabfed.controller.provider_factory import default_provider_factory" + "try:\n", + " from fabfed.fabfed_manager import FabfedManager\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", + "else:\n", + " print(\"Succeed!\")" ] }, { @@ -60,7 +62,7 @@ "id": "73a535d1-4f1a-4606-b4a4-128c675255db", "metadata": {}, "source": [ - "## Read .fab files" + "## Show availabe stitch ports" ] }, { @@ -70,22 +72,11 @@ "metadata": {}, "outputs": [], "source": [ - "def read_file_to_str(file_path):\n", - " try:\n", - " with open(file_path, 'r') as file:\n", - " file_content = file.read()\n", - " return file_content\n", - " except FileNotFoundError:\n", - " print(f\"File '{file_path}' not found.\")\n", - " return None\n", - "\n", - "# Read config.fab to config_str:\n", - "config_str = read_file_to_str(f\"{fab_file_path}/config.fab\")\n", - "if config_str:\n", - " print(\"File content:\")\n", - " print(config_str)\n", - "\n", - "print(config_str)" + "try:\n", + " fabfed_manager = FabfedManager(config_dir=fab_file_path)\n", + " fabfed_manager.show_available_stitch_ports(from_provider='sense', to_provider=\"fabric\")\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -93,7 +84,7 @@ "id": "f6106b4e-076f-4349-b215-c6db497c97a9", "metadata": {}, "source": [ - "## Create a workflow config object" + "## Validate .fab files" ] }, { @@ -103,7 +94,12 @@ "metadata": {}, "outputs": [], "source": [ - "config = WorkflowConfig(content=config_str)" + "try:\n", + " config = fabfed_manager.validate()\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", + "else:\n", + " print(\"Succeed!\")" ] }, { @@ -111,7 +107,7 @@ "id": "c87f8329-054e-45f3-9d69-3c4acc25252f", "metadata": {}, "source": [ - "## Create a controller" + "## Show the order of handling resources" ] }, { @@ -121,10 +117,11 @@ "metadata": {}, "outputs": [], "source": [ - "import logging\n", - "logger = logging.getLogger(__name__)\n", - "\n", - "controller = Controller(config=config, logger=logger)" + "for resource_config in config.resource_configs:\n", + " dependencies = []\n", + " for dependency in resource_config.dependencies:\n", + " dependencies.append(dependency.resource)\n", + " print(resource_config.label, \"depends on \", dependencies)" ] }, { @@ -132,7 +129,7 @@ "id": "4e1aa491-7ddc-491b-82f6-ec9d41b08c3a", "metadata": {}, "source": [ - "## Init stitching information" + "## Stitching information" ] }, { @@ -142,9 +139,10 @@ "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session)\n", - "\n", - "controller.init(session=session_name, provider_factory=default_provider_factory, provider_states=states)" + "try:\n", + " fabfed_manager.stitch_info(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -152,7 +150,7 @@ "id": "465ea762-1e62-4532-a0e3-aa5bef0e23c4", "metadata": {}, "source": [ - "## Add resources to providers and validate resources" + "## List resources for creation and deletion" ] }, { @@ -162,17 +160,14 @@ "metadata": {}, "outputs": [], "source": [ - "controller.plan(provider_states=[])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7a1a864d-bdd9-4422-84a6-cade3a7c485c", - "metadata": {}, - "outputs": [], - "source": [ - "controller.add(provider_states=[])" + "try:\n", + " to_be_created_count, to_be_deleted_count = fabfed_manager.plan(session=session_name)\n", + "except Exception as e:\n", + " print(\"Error: {e}\")\n", + "\n", + "print(\"\\n=== Total ===\")\n", + "print(f\"to_be_created: {to_be_created_count}\")\n", + "print(f\"to_be_deleted: {to_be_deleted_count}\")" ] }, { @@ -191,9 +186,9 @@ "outputs": [], "source": [ "try:\n", - " controller.apply(provider_states=[])\n", - "except ControllerException as e:\n", - " assert isinstance(e.exceptions[0], DummyFailCreateException)" + " status_code = fabfed_manager.apply(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -201,7 +196,7 @@ "id": "beee11c5-5adc-4756-8133-a8eda0253969", "metadata": {}, "source": [ - "## Save the workflow state" + "## Show session state" ] }, { @@ -211,10 +206,7 @@ "metadata": {}, "outputs": [], "source": [ - "states = controller.get_states()\n", - "sutil.save_states(states, session_name)\n", - "\n", - "print(states)" + "fabfed_manager.show(session=session_name)" ] }, { @@ -222,7 +214,7 @@ "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", "metadata": {}, "source": [ - "## View the state of the workflow" + "## Show all sessions" ] }, { @@ -232,9 +224,7 @@ "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session_name)\n", - "\n", - "print(states)" + "fabfed_manager.show_sessions()" ] }, { @@ -252,15 +242,46 @@ "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session_name)\n", - "\n", - "controller.destroy(provider_states=states)\n", - "\n", - "sutil.save_states(states, session_name)\n", - "if not states:\n", - " sutil.destroy_session(session_name)\n", - "\n", - "print(states)" + "try:\n", + " fabfed_manager.destroy(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d3724202-8b2d-445e-bdd7-b8923b15b1a3", + "metadata": {}, + "source": [ + "## Show sesstion state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b31f04e-ea4d-422f-ab03-fe6bf63cbfc0", + "metadata": {}, + "outputs": [], + "source": [ + "fabfed_manager.show(session=session_name)" + ] + }, + { + "cell_type": "markdown", + "id": "673e6f8c-7a87-4b3c-84f5-5fbc8942e578", + "metadata": {}, + "source": [ + "## Show all sessions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0bbbe8f8-75fb-48d0-880c-7c584fee9c40", + "metadata": {}, + "outputs": [], + "source": [ + "fabfed_manager.show_sessions()" ] }, { diff --git a/jupyter/sense-aws.ipynb b/jupyter/sense-aws.ipynb index 5a532c81..96dfb539 100644 --- a/jupyter/sense-aws.ipynb +++ b/jupyter/sense-aws.ipynb @@ -26,7 +26,7 @@ "source": [ "import os\n", "\n", - "session_name = \"sense-aws-lzhang9\"\n", + "session_name = \"sense-aws-demo\"\n", "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/aws\"\n", "fab_file_path = \"./examples/sense-aws\"\n", "\n", @@ -34,6 +34,24 @@ "print(fab_file_path)" ] }, + { + "cell_type": "markdown", + "id": "a42f91b5-4c42-457f-945a-633ee4cbc88c", + "metadata": {}, + "source": [ + "## Show availabe stitch ports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7e446ac-65dc-4c5b-b821-f6dc5af6e24a", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed stitch-policy -providers \"sense,fabric\"" + ] + }, { "cell_type": "markdown", "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", @@ -57,7 +75,7 @@ "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", "metadata": {}, "source": [ - "## Add resources to providers and validate resources" + "## List resources for creation and deletion" ] }, { @@ -67,7 +85,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -plan" + "!fabfed workflow -s $session_name -c $fab_file_path -plan -summary" ] }, { @@ -85,7 +103,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" + "!fabfed workflow -s $session_name -c $fab_file_path -stitch-info -summary" ] }, { @@ -123,7 +141,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -show" + "!fabfed workflow -s $session_name -c $fab_file_path -show -summary" ] }, { diff --git a/jupyter/sense-gcp-lib.ipynb b/jupyter/sense-gcp-lib.ipynb index 99e088f9..28b6249d 100644 --- a/jupyter/sense-gcp-lib.ipynb +++ b/jupyter/sense-gcp-lib.ipynb @@ -26,7 +26,7 @@ "source": [ "import os\n", "\n", - "session_name = \"sense-gcp-lzhang9-lib\"\n", + "session_name = \"sense-gcp-demo-lib\"\n", "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/sense-gcp\"\n", "fab_file_path = \"./examples/sense-gcp\"\n", "\n", @@ -49,10 +49,12 @@ "metadata": {}, "outputs": [], "source": [ - "from fabfed.util.config import WorkflowConfig\n", - "from fabfed.util import state as sutil\n", - "from fabfed.controller.controller import Controller\n", - "from fabfed.controller.provider_factory import default_provider_factory" + "try:\n", + " from fabfed.fabfed_manager import FabfedManager\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", + "else:\n", + " print(\"Succeed!\")" ] }, { @@ -60,7 +62,7 @@ "id": "73a535d1-4f1a-4606-b4a4-128c675255db", "metadata": {}, "source": [ - "## Read .fab files" + "## Show availabe stitch ports" ] }, { @@ -70,22 +72,11 @@ "metadata": {}, "outputs": [], "source": [ - "def read_file_to_str(file_path):\n", - " try:\n", - " with open(file_path, 'r') as file:\n", - " file_content = file.read()\n", - " return file_content\n", - " except FileNotFoundError:\n", - " print(f\"File '{file_path}' not found.\")\n", - " return None\n", - "\n", - "# Read config.fab to config_str:\n", - "config_str = read_file_to_str(f\"{fab_file_path}/config.fab\")\n", - "if config_str:\n", - " print(\"File content:\")\n", - " print(config_str)\n", - "\n", - "print(config_str)" + "try:\n", + " fabfed_manager = FabfedManager(config_dir=fab_file_path)\n", + " fabfed_manager.show_available_stitch_ports(from_provider='cloudlab', to_provider=\"fabric\")\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -93,7 +84,7 @@ "id": "f6106b4e-076f-4349-b215-c6db497c97a9", "metadata": {}, "source": [ - "## Create a workflow config object" + "## Validate .fab files" ] }, { @@ -103,7 +94,12 @@ "metadata": {}, "outputs": [], "source": [ - "config = WorkflowConfig(content=config_str)" + "try:\n", + " config = fabfed_manager.validate()\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")\n", + "else:\n", + " print(\"Succeed!\")" ] }, { @@ -111,7 +107,7 @@ "id": "c87f8329-054e-45f3-9d69-3c4acc25252f", "metadata": {}, "source": [ - "## Create a controller" + "## Show the order of handling resources" ] }, { @@ -121,10 +117,11 @@ "metadata": {}, "outputs": [], "source": [ - "import logging\n", - "logger = logging.getLogger(__name__)\n", - "\n", - "controller = Controller(config=config, logger=logger)" + "for resource_config in config.resource_configs:\n", + " dependencies = []\n", + " for dependency in resource_config.dependencies:\n", + " dependencies.append(dependency.resource)\n", + " print(resource_config.label, \"depends on \", dependencies)" ] }, { @@ -132,7 +129,7 @@ "id": "4e1aa491-7ddc-491b-82f6-ec9d41b08c3a", "metadata": {}, "source": [ - "## Init stitching information" + "## Stitching information" ] }, { @@ -142,9 +139,10 @@ "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session)\n", - "\n", - "controller.init(session=session_name, provider_factory=default_provider_factory, provider_states=states)" + "try:\n", + " fabfed_manager.stitch_info(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -152,7 +150,7 @@ "id": "465ea762-1e62-4532-a0e3-aa5bef0e23c4", "metadata": {}, "source": [ - "## Add resources to providers and validate resources" + "## List resources for creation and deletion" ] }, { @@ -162,17 +160,14 @@ "metadata": {}, "outputs": [], "source": [ - "controller.plan(provider_states=[])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7a1a864d-bdd9-4422-84a6-cade3a7c485c", - "metadata": {}, - "outputs": [], - "source": [ - "controller.add(provider_states=[])" + "try:\n", + " to_be_created_count, to_be_deleted_count = fabfed_manager.plan(session=session_name)\n", + "except Exception as e:\n", + " print(\"Error: {e}\")\n", + "\n", + "print(\"\\n=== Total ===\")\n", + "print(f\"to_be_created: {to_be_created_count}\")\n", + "print(f\"to_be_deleted: {to_be_deleted_count}\")" ] }, { @@ -191,9 +186,9 @@ "outputs": [], "source": [ "try:\n", - " controller.apply(provider_states=[])\n", - "except ControllerException as e:\n", - " assert isinstance(e.exceptions[0], DummyFailCreateException)" + " status_code = fabfed_manager.apply(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" ] }, { @@ -201,7 +196,7 @@ "id": "beee11c5-5adc-4756-8133-a8eda0253969", "metadata": {}, "source": [ - "## Save the workflow state" + "## Show session state" ] }, { @@ -211,10 +206,7 @@ "metadata": {}, "outputs": [], "source": [ - "states = controller.get_states()\n", - "sutil.save_states(states, session_name)\n", - "\n", - "print(states)" + "fabfed_manager.show(session=session_name)" ] }, { @@ -222,7 +214,7 @@ "id": "f5a08986-10f1-478d-9460-c0c5c67f23d1", "metadata": {}, "source": [ - "## View the state of the workflow" + "## Show all sessions" ] }, { @@ -232,9 +224,7 @@ "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session_name)\n", - "\n", - "print(states)" + "fabfed_manager.show_sessions()" ] }, { @@ -252,15 +242,46 @@ "metadata": {}, "outputs": [], "source": [ - "states = sutil.load_states(session_name)\n", - "\n", - "controller.destroy(provider_states=states)\n", - "\n", - "sutil.save_states(states, session_name)\n", - "if not states:\n", - " sutil.destroy_session(session_name)\n", - "\n", - "print(states)" + "try:\n", + " fabfed_manager.destroy(session=session_name)\n", + "except Exception as e:\n", + " print(f\"Error: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cf809aa8-003e-4060-b6fc-b786471f9055", + "metadata": {}, + "source": [ + "## Show sesstion state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f6acfdf-8ab1-4312-9ed1-6f915fc601de", + "metadata": {}, + "outputs": [], + "source": [ + "fabfed_manager.show(session=session_name)" + ] + }, + { + "cell_type": "markdown", + "id": "b7880702-51ab-4e9a-b694-eaf5b4a454b2", + "metadata": {}, + "source": [ + "## Show all sessions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "506fc701-2a0c-4831-bcea-b27aea27e1ed", + "metadata": {}, + "outputs": [], + "source": [ + "fabfed_manager.show_sessions()" ] }, { diff --git a/jupyter/sense-gcp.ipynb b/jupyter/sense-gcp.ipynb index f0e54d7c..5b007da6 100644 --- a/jupyter/sense-gcp.ipynb +++ b/jupyter/sense-gcp.ipynb @@ -26,7 +26,7 @@ "source": [ "import os\n", "\n", - "session_name = \"sense-gcp-lzhang9\"\n", + "session_name = \"sense-gcp-demo\"\n", "#fab_file_path = os.path.dirname(os.getcwd()) + \"/examples/gcp\"\n", "fab_file_path = \"./examples/sense-gcp\"\n", "\n", @@ -34,6 +34,24 @@ "print(fab_file_path)" ] }, + { + "cell_type": "markdown", + "id": "e8d2a4cc-52af-4aa3-8a3f-60f0c5847121", + "metadata": {}, + "source": [ + "## Show availabe stitch ports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4b3c273-24d0-4639-b202-f2ce69ceacf6", + "metadata": {}, + "outputs": [], + "source": [ + "!fabfed stitch-policy -providers \"sense,fabric\"" + ] + }, { "cell_type": "markdown", "id": "1d63bdfb-177e-4dd0-bdfb-091c0f84f476", @@ -57,7 +75,7 @@ "id": "2df23e14-492c-44be-9cc8-e4e041f0bead", "metadata": {}, "source": [ - "## Add resources to providers and validate resources" + "## List resources for creation and deletion" ] }, { @@ -67,7 +85,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -plan" + "!fabfed workflow -s $session_name -c $fab_file_path -plan -summary" ] }, { @@ -85,7 +103,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -init -summary" + "!fabfed workflow -s $session_name -c $fab_file_path -stitch-info -summary" ] }, { @@ -121,7 +139,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fabfed workflow -s $session_name -c $fab_file_path -show" + "!fabfed workflow -s $session_name -c $fab_file_path -show -summary" ] }, { From 8e7521e1341c5a0c3a4a86aa10e1fbd94d0c6b5d Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 15 Mar 2024 23:51:14 -0500 Subject: [PATCH 59/75] Update fablib-tools.ipynb --- jupyter/fablib-tools.ipynb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jupyter/fablib-tools.ipynb b/jupyter/fablib-tools.ipynb index c9e36125..e6577cd5 100644 --- a/jupyter/fablib-tools.ipynb +++ b/jupyter/fablib-tools.ipynb @@ -213,7 +213,9 @@ "metadata": {}, "outputs": [], "source": [ - "fablib.list_slices()" + "fablib.list_slices()\n", + "\n", + "print(\"Complete!\")" ] }, { @@ -239,7 +241,7 @@ "metadata": {}, "outputs": [], "source": [ - "slice_name = \"native-gcp-lzhang9\"" + "slice_name = \"native-gcp-demo\"" ] }, { From bd837bc653673bfa3815301f4f0d4c5c21413dea Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 15 Mar 2024 23:54:48 -0500 Subject: [PATCH 60/75] Create config.fab --- .../examples/supply_stitch_policy/config.fab | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 jupyter/examples/supply_stitch_policy/config.fab diff --git a/jupyter/examples/supply_stitch_policy/config.fab b/jupyter/examples/supply_stitch_policy/config.fab new file mode 100644 index 00000000..730d97b2 --- /dev/null +++ b/jupyter/examples/supply_stitch_policy/config.fab @@ -0,0 +1,63 @@ +variable: + - node_count: + default: 1 + - vlan: + default: 3110 + +provider: + - cloudlab: + - cloudlab_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: cloudlab + - fabric: + - fabric_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: fabric + +config: + - policy: + - my_stitch_policy: + producer: cloudlab + consumer: fabric + stitch_port: + name: clab-mass + profile: OCT-MGHPCC + provider: fabric + device_name: OCT-MGHPCC + site: MASS + peer: + profile: fabfed-stitch-v2 + provider: cloudlab + option: + cluster: urn:publicid:IDN+cloudlab.umass.edu+authority+cm + interface: + - vlan: '{{ var.vlan }}' + + - layer3: + - my_layer: + subnet: 192.168.1.0/24 + gateway: 192.168.1.1 + ip_start: 192.168.1.2 + ip_end: 192.168.1.254 +resource: + - network: + - cnet: + provider: '{{cloudlab.cloudlab_provider }}' + layer3: "{{ layer3.my_layer }}" + - node: + - cloudlab_node: + provider: '{{ cloudlab.cloudlab_provider }}' + network: "{{ network.cnet }}" + count: '{{ var.node_count }}' + - node: + - fabric_node: + provider: '{{ fabric.fabric_provider }}' + network: "{{ network.fabric_network }}" + count: '{{ var.node_count }}' + - network: + - fabric_network: + provider: '{{ fabric.fabric_provider }}' + layer3: "{{ layer3.my_layer }}" + stitch_with: '{{ network.cnet }}' + stitch_option: + policy: '{{ policy.my_stitch_policy }}' From fdddf46f754fb2cb1bc1c79d0247fb4c18807d17 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Tue, 19 Mar 2024 10:38:20 -0700 Subject: [PATCH 61/75] changed name of cicd test --- .github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml b/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml index 1ec28bac..85bd22be 100644 --- a/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml +++ b/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml @@ -1,9 +1,9 @@ -name: Ubuntu22.04 Test +name: Ubuntu22.04 Modify Test on: - push: - branches: - - knit8 + # push: + # branches: + # - knit8 workflow_dispatch: env: From dd15bb60e254807465cdd3b733bead57b2f1b27e Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Tue, 19 Mar 2024 10:49:51 -0700 Subject: [PATCH 62/75] name of actual test --- .github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml b/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml index 85bd22be..71f8f6ab 100644 --- a/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml +++ b/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml @@ -15,7 +15,7 @@ env: FABRIC_USER: ${{ secrets.FABRIC_USER }} jobs: - UbuntuTest: + UbuntuModifyTest: runs-on: ubuntu-latest steps: From 90d45771aa9d8bc6ee42e05e4832d44d3e158d55 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Tue, 19 Mar 2024 10:58:34 -0700 Subject: [PATCH 63/75] name of actual test. push on knit8 --- .github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml b/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml index 71f8f6ab..2373ed26 100644 --- a/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml +++ b/.github/workflows/ubuntu_22_and_test_modify_fabric_l2.yml @@ -1,9 +1,9 @@ name: Ubuntu22.04 Modify Test on: - # push: - # branches: - # - knit8 + push: + branches: + - knit8 workflow_dispatch: env: From 78b0c49df4115bb0a9b3f6f3c1191746f3a13cb6 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Tue, 19 Mar 2024 11:32:30 -0700 Subject: [PATCH 64/75] Update config.fab --- jupyter/examples/chameleon/star/config.fab | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jupyter/examples/chameleon/star/config.fab b/jupyter/examples/chameleon/star/config.fab index 283c9d4f..af811080 100644 --- a/jupyter/examples/chameleon/star/config.fab +++ b/jupyter/examples/chameleon/star/config.fab @@ -1,3 +1,7 @@ +variable: + - node_count: + default: 1 + provider: - fabric: - fabric_provider: @@ -37,13 +41,13 @@ resource: - node: - fabric_node: provider: '{{ fabric.fabric_provider }}' - count: 1 + count: '{{ var.node_count }}' image: default_rocky_8 - chi_node: provider: '{{ chi.chi_provider }}' image: CC-Ubuntu20.04 network: '{{ network.chi_network }}' - count: 1 + count: '{{ var.node_count }}' flavor: m1.medium - service: - dtn_service: From 9b0227b882111ad46fb261f30e87326dd074ad29 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Wed, 17 Apr 2024 14:31:33 -0500 Subject: [PATCH 65/75] Update welcom.ipynb --- jupyter/welcom.ipynb | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/jupyter/welcom.ipynb b/jupyter/welcom.ipynb index a4f41e58..6df2a6be 100644 --- a/jupyter/welcom.ipynb +++ b/jupyter/welcom.ipynb @@ -63,8 +63,7 @@ "else:\n", " print(\"The file does not exist.\")\n", " import shutil\n", - " #source_file = 'fabfed_config/fabfed_credentials_template.yml'\n", - " source_file = '/users/lzhang9/.fabfed/fabfed_credentials.yml'\n", + " source_file = 'fabfed_config/fabfed_credentials_template.yml'\n", " destination_file = 'fabfed_config/fabfed_credentials.yml'\n", " shutil.copy(source_file, destination_file)\n", " print(\"File copied successfully from\", source_file, \"to\", destination_file)" @@ -78,16 +77,6 @@ "### Edit credentials in [credential file](./fabfed_config/fabfed_credentials.yml) if needed" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "779c81b2-3260-4222-be63-7084cc386133", - "metadata": {}, - "outputs": [], - "source": [ - "!cat $file_path" - ] - }, { "cell_type": "markdown", "id": "8127923d-a615-4472-981a-fc20d01aca35", @@ -95,13 +84,14 @@ "source": [ "## Examples of FabFed Workflows\n", "### Using FabFed CLI:\n", - "\n", + "- [Cloudlab](./cloudlab.ipynb): Stiching Cloudlab and Fabric (network only) around 2 minuates\n", + "- [Chameleon](./chameleon.ipynb): Stiching Chameleon and Fabric (network only) around 2 minuates\n", + "- [Cloudlab](./cloudlab.ipynb): Stiching Cloudlab and Fabric\n", + "- [Chameleon](./chameleon.ipynb): Stiching Chameleon and Fabric\n", "- [Native aws](./native-aws.ipynb): Stiching native AWS and Fabric\n", "- [Native gcp](./native-gcp.ipynb): Stiching native GCP and Fabric\n", "- [SENSE aws](./sense-aws.ipynb): Stiching SENSE AWS and Fabric\n", "- [SENSE gcp](./sense-gcp.ipynb): Stiching SENSE GCP and Fabric\n", - "- [Cloudlab](./cloudlab.ipynb): Stiching Cloudlab and Fabric\n", - "- [Chameleon](./chameleon.ipynb): Stiching Chameleon and Fabric\n", "\n", "### Using FabFed library:\n", "- [Native aws](./native-aws-lib.ipynb): Stiching native AWS and Fabric\n", @@ -149,7 +139,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.9.12" } }, "nbformat": 4, From ba2c98e7eef9f805f0b48638b443e24d58ef7cc7 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Wed, 17 Apr 2024 14:33:37 -0500 Subject: [PATCH 66/75] Update native-gcp.ipynb --- jupyter/native-gcp.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter/native-gcp.ipynb b/jupyter/native-gcp.ipynb index e8741bfc..f8abc8da 100644 --- a/jupyter/native-gcp.ipynb +++ b/jupyter/native-gcp.ipynb @@ -189,7 +189,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.9.12" } }, "nbformat": 4, From 7c648fed864e36a4a1c32ca6844e47bfb453c011 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Wed, 17 Apr 2024 14:38:34 -0500 Subject: [PATCH 67/75] Update welcom.ipynb --- jupyter/welcom.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter/welcom.ipynb b/jupyter/welcom.ipynb index 6df2a6be..19526dba 100644 --- a/jupyter/welcom.ipynb +++ b/jupyter/welcom.ipynb @@ -139,7 +139,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.9.18" } }, "nbformat": 4, From 48fdad67e315326c1b121fecb1cf51bc9643e8ee Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Wed, 17 Apr 2024 14:47:12 -0500 Subject: [PATCH 68/75] Update chameleon-lib.ipynb --- jupyter/chameleon-lib.ipynb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jupyter/chameleon-lib.ipynb b/jupyter/chameleon-lib.ipynb index c8da1de1..90606e57 100644 --- a/jupyter/chameleon-lib.ipynb +++ b/jupyter/chameleon-lib.ipynb @@ -73,6 +73,8 @@ "outputs": [], "source": [ "try:\n", + " #var_dict = dict(node_count=0)\n", + " #fabfed_manager = FabfedManager(config_dir=fab_file_path,var_dict=var_dict)\n", " fabfed_manager = FabfedManager(config_dir=fab_file_path)\n", " fabfed_manager.show_available_stitch_ports(from_provider='chi', to_provider=\"fabric\")\n", "except Exception as e:\n", From a5b11420734a449ff6d208af283a2186f65d547e Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Wed, 17 Apr 2024 14:50:51 -0500 Subject: [PATCH 69/75] Update config.fab --- jupyter/examples/cloudlab/config.fab | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jupyter/examples/cloudlab/config.fab b/jupyter/examples/cloudlab/config.fab index e5787d08..ecc785c4 100644 --- a/jupyter/examples/cloudlab/config.fab +++ b/jupyter/examples/cloudlab/config.fab @@ -31,7 +31,7 @@ resource: - fabric_network: provider: '{{ fabric.fabric_provider }}' layer3: "{{ layer3.my_layer }}" - interface: '{{ node.fabric_node }}' + #interface: '{{ node.fabric_node }}' stitch_with: '{{ network.cnet }}' - node: - fabric_node: @@ -39,11 +39,12 @@ resource: site: '{{ var.site }}' image: default_rocky_8 nic_model: NIC_Basic - count: 1 + network: "{{ network.fabric_network }}" + count: 0 - cnode: provider: '{{ cloudlab.cloudlab_provider }}' network: "{{ network.cnet }}" - count: 1 + count: 0 - service: - dtn_service: provider: '{{ janus.janus_provider }}' From 66e725d472b50673a6bce3655338b270accabd9c Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Wed, 17 Apr 2024 14:51:35 -0500 Subject: [PATCH 70/75] Update config.fab --- jupyter/examples/chameleon/star/config.fab | 23 +++------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/jupyter/examples/chameleon/star/config.fab b/jupyter/examples/chameleon/star/config.fab index af811080..4375eb88 100644 --- a/jupyter/examples/chameleon/star/config.fab +++ b/jupyter/examples/chameleon/star/config.fab @@ -1,7 +1,6 @@ variable: - node_count: - default: 1 - + default: 0 provider: - fabric: - fabric_provider: @@ -11,11 +10,6 @@ provider: - chi_provider: credential_file: ~/.fabfed/fabfed_credentials.yml profile: chi - - janus: - - janus_provider: - credential_file: ~/.fabfed/fabfed_credentials.yml - profile: janus - config: - layer3: - my_layer: @@ -29,12 +23,9 @@ resource: provider: '{{ chi.chi_provider }}' name: stitch_net layer3: "{{ layer3.my_layer }}" - count: 1 - fabric_network: provider: '{{ fabric.fabric_provider }}' - interface: '{{ node.fabric_node }}' layer3: "{{ layer3.my_layer }}" - count: 1 stitch_with: '{{ network.chi_network }}' stitch_option: site: STAR @@ -42,18 +33,10 @@ resource: - fabric_node: provider: '{{ fabric.fabric_provider }}' count: '{{ var.node_count }}' - image: default_rocky_8 + network: '{{ network.fabric_network }}' + - chi_node: provider: '{{ chi.chi_provider }}' - image: CC-Ubuntu20.04 network: '{{ network.chi_network }}' count: '{{ var.node_count }}' flavor: m1.medium - - service: - - dtn_service: - provider: '{{ janus.janus_provider }}' - node: [ '{{ node.chi_node }}', '{{ node.fabric_node }}' ] - controller: '{{ node.fabric_node }}' - image: dtnaas/tools - profile: fabfed - count: 0 From f29ac2cbd0fd93b4e6a9918cd49a413916e139ee Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Wed, 17 Apr 2024 15:03:03 -0500 Subject: [PATCH 71/75] Update gcp_network.py Patch GCP router with MD5 authentication. --- fabfed/provider/gcp/gcp_network.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fabfed/provider/gcp/gcp_network.py b/fabfed/provider/gcp/gcp_network.py index bb862cc5..9b5d6b35 100644 --- a/fabfed/provider/gcp/gcp_network.py +++ b/fabfed/provider/gcp/gcp_network.py @@ -66,6 +66,12 @@ def create(self): region=region, attachment_name=attachment_name) + # set the MD5 authentication + gcp_utils.patch_router(service_key_path=service_key_path, + project=project, + region=region, + router_name=router_name) + logger.info(f"attachment_details={attachment}") self.interface.append(dict(id=attachment.pairing_key, provider=self._provider.type)) From 3b154eff6947291e577f4885803aed3e5b43a9ee Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Wed, 17 Apr 2024 15:03:46 -0500 Subject: [PATCH 72/75] Update fabric_network.py Set GCP MTU to 1460. --- fabfed/provider/fabric/fabric_network.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fabfed/provider/fabric/fabric_network.py b/fabfed/provider/fabric/fabric_network.py index 8a70855d..a5e8a9b9 100644 --- a/fabfed/provider/fabric/fabric_network.py +++ b/fabfed/provider/fabric/fabric_network.py @@ -221,12 +221,17 @@ def handle_facility_port(self): logger.info(f"Creating Facility Port:Labels: {labels}") logger.info(f"Creating Facility Port:PeerLabels: {peer_labels}") + if cloud == "GCP": + mtu_size = 1460 + else: + mtu_size = 9001 + facility_port = self.slice_object.add_facility_port( name='Cloud-Facility-' + cloud, site=cloud, labels=labels, peer_labels=peer_labels, - capacities=Capacities(bw=50, mtu=9001)) + capacities=Capacities(bw=50, mtu=mtu_size)) logger.info("CreatedFacilityPort:" + facility_port.toJson()) else: From 1bce71634acee513d1507889117c86e3a0ca3793 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Wed, 17 Apr 2024 15:04:45 -0500 Subject: [PATCH 73/75] Update gcp_utils.py Added the method patch_router() to set MD5 authentication. --- fabfed/provider/gcp/gcp_utils.py | 36 +++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/fabfed/provider/gcp/gcp_utils.py b/fabfed/provider/gcp/gcp_utils.py index 5fd099de..7c0def26 100644 --- a/fabfed/provider/gcp/gcp_utils.py +++ b/fabfed/provider/gcp/gcp_utils.py @@ -4,7 +4,11 @@ from google.cloud.compute_v1.types import ( Router, RouterBgp, + RouterBgpPeer, + # RouterBgpPeerBfd, + RouterMd5AuthenticationKey, InsertRouterRequest, + PatchRouterRequest, InterconnectAttachment, ) from google.oauth2 import service_account @@ -80,6 +84,35 @@ def create_router(*, service_key_path, project, region, router_name, vpc, bgp_as check_operation_result(credentials=credentials, project=project, region=region, operation_name=response.name) logger.info(f'Router {router_name} created successfully.') +def patch_router(*, service_key_path, project, region, router_name): + credentials = service_account.Credentials.from_service_account_file(service_key_path) + compute_client = compute_v1.RoutersClient(credentials=credentials) + + # Get the router resource + router = compute_client.get(project=project, region=region, router=router_name) + + # Access the BGP peers associated with the router + bgp_peers = router.bgp_peers + + # Add md5 authentication + peer = bgp_peers[0] + peer.md5_authentication_key_name = 'md5-key-name-1' + + router_resource = Router( + name=router_name, + md5_authentication_keys = [RouterMd5AuthenticationKey(key='0xzsEwC7xk6c1fK_h.xHyAdx', name='md5-key-name-1')], + bgp_peers=[peer] + ) + request = PatchRouterRequest( + project=project, + region=region, + router=router_name, + router_resource=router_resource) + response = compute_client.patch(request=request) + logger.info(f'Response={response}') + check_operation_result(credentials=credentials, project=project, region=region, operation_name=response.name) + logger.info(f'Router {router_name} patched successfully.') + def delete_router(*, service_key_path, project, region, router_name): credentials = service_account.Credentials.from_service_account_file(service_key_path) @@ -122,7 +155,8 @@ def create_interconnect_attachment(*, service_key_path, project, region, router_ admin_enabled=True, encryption=None, router=f'projects/{project}/regions/{region}/routers/{router_name}', - type_='PARTNER' + type_='PARTNER', + mtu=1460 ) request = compute_v1.InsertInterconnectAttachmentRequest( From 9b3452d61e9e59ab43882c429f0ff81aa3f5352b Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Wed, 24 Apr 2024 10:31:58 -0700 Subject: [PATCH 74/75] we now support creation aws facility with no nodes --- .github/workflows/ubuntu_22_and_test.yml | 14 +++++++++++--- .gitignore | 2 +- fabfed/provider/fabric/fabric_network.py | 3 +++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ubuntu_22_and_test.yml b/.github/workflows/ubuntu_22_and_test.yml index 6ed57745..a6ad68fb 100644 --- a/.github/workflows/ubuntu_22_and_test.yml +++ b/.github/workflows/ubuntu_22_and_test.yml @@ -28,6 +28,7 @@ env: RUN_FABRIC_AWS_SENSE: ${{ vars.RUN_FABRIC_AWS_SENSE }} RUN_CLAB: ${{ vars.RUN_CLAB }} + RUN_FABRIC_AWS_WITH_NODES: ${{ vars.RUN_FABRIC_AWS_WITH_NODES }} jobs: UbuntuTest: @@ -102,10 +103,17 @@ jobs: run: | ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_l2_vpn cicd-fabric-l2-vpn - - name: Test Fabric AWS (Fabric Only) - if: ${{ env.RUN_FABRIC_AWS_SENSE == 'false' || env.RUN_FABRIC_AWS_SENSE == false }} + - name: Test Fabric AWS (Fabric Only With No Nodes) + run: | + session=aes-aws-native-no-nodes + echo "vlan: 4" > $session-varfile.yml + echo "node_count: 0" >> $session-varfile.yml + ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_native_aws $session $session-varfile.yml + + - name: Test Fabric AWS (Fabric Only With Nodes) + if: ${{ env.RUN_FABRIC_AWS_WITH_NODES == 'true' || env.RUN_FABRIC_AWS_WITH_NODES == true }} run: | - session=aes-aws-native + session=aes-aws-native-with-nodes echo "vlan: 4" > $session-varfile.yml ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/fabric_native_aws $session $session-varfile.yml diff --git a/.gitignore b/.gitignore index 761138ad..77458227 100644 --- a/.gitignore +++ b/.gitignore @@ -155,4 +155,4 @@ venv *.*~ # jupyter artifacts -*/.ipynb_checkpoints +.ipynb_checkpoints diff --git a/fabfed/provider/fabric/fabric_network.py b/fabfed/provider/fabric/fabric_network.py index 8a70855d..1f83b955 100644 --- a/fabfed/provider/fabric/fabric_network.py +++ b/fabfed/provider/fabric/fabric_network.py @@ -282,6 +282,9 @@ def handle_network(self, nodes): logger.warning(f"Node {node.name} has no available interface to stitch to network {self.net_name} ") # TODO DO WE NEED REALLY THIS? + if not interfaces: + return + for iface in interfaces: fim_iface1 = iface.get_fim_interface() From abfcdfe6e5cafe0bd43fb11fb4d4cd4e9f0018b8 Mon Sep 17 00:00:00 2001 From: Abdelilah Essiari Date: Wed, 24 Apr 2024 10:35:01 -0700 Subject: [PATCH 75/75] we now support creation aws facility with no nodes --- jupyter/examples/chameleon/star/config.fab | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter/examples/chameleon/star/config.fab b/jupyter/examples/chameleon/star/config.fab index af811080..591d6c86 100644 --- a/jupyter/examples/chameleon/star/config.fab +++ b/jupyter/examples/chameleon/star/config.fab @@ -32,7 +32,6 @@ resource: count: 1 - fabric_network: provider: '{{ fabric.fabric_provider }}' - interface: '{{ node.fabric_node }}' layer3: "{{ layer3.my_layer }}" count: 1 stitch_with: '{{ network.chi_network }}' @@ -42,6 +41,7 @@ resource: - fabric_node: provider: '{{ fabric.fabric_provider }}' count: '{{ var.node_count }}' + network: '{{ network.fabric_network }}' image: default_rocky_8 - chi_node: provider: '{{ chi.chi_provider }}'