From 1a919ba750b3301864923d74cea12d4497f74a32 Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Mon, 29 Jul 2024 12:40:25 -0500 Subject: [PATCH] chore: move sync-nb-server-to-ironic into python package Moved the sync-nb-server-to-ironic code into the our workflows python package. This required fixing up some places to craft the necessary entry points as well as setting dependencies. Removed the building of the container for the old directory and create a new container with these pieces. --- .github/workflows/build-container-images.yaml | 12 -- .github/workflows/containers.yaml | 47 ++++++++ .../code/port_configuration.py | 8 -- .../code/synchronize-obm-creds.py | 100 ---------------- .../code/synchronize-server.py | 89 -------------- .../containers/Dockerfile.ironic | 27 ----- .../containers/requirements.txt | 3 - .../synchronize-interfaces-to-ironic.yaml | 5 +- .../synchronize-obm-creds.yaml | 5 +- .../synchronize-server-to-ironic.yaml | 5 +- containers/Dockerfile.ironic-nautobot-client | 25 ++++ python/understack-workflows/pyproject.toml | 11 ++ .../understack_workflows}/create_node.py | 8 +- .../understack_workflows/ironic}/__init__.py | 0 .../understack_workflows}/ironic/client.py | 31 +++-- .../understack_workflows}/ironic/secrets.py | 11 +- .../understack_workflows/main/__init__.py | 0 .../main/synchronize_interfaces.py | 74 ++++++++---- .../main/synchronize_obm_creds.py | 111 ++++++++++++++++++ .../main/synchronize_server.py | 98 ++++++++++++++++ .../understack_workflows}/network_data.py | 0 .../network_data.schema.json | 0 .../node_configuration.py | 2 +- .../port_configuration.py | 12 ++ .../redfish_driver_info.py | 0 25 files changed, 395 insertions(+), 289 deletions(-) delete mode 100644 argo-workflows/sync-nb-server-to-ironic/code/port_configuration.py delete mode 100644 argo-workflows/sync-nb-server-to-ironic/code/synchronize-obm-creds.py delete mode 100644 argo-workflows/sync-nb-server-to-ironic/code/synchronize-server.py delete mode 100644 argo-workflows/sync-nb-server-to-ironic/containers/Dockerfile.ironic delete mode 100644 argo-workflows/sync-nb-server-to-ironic/containers/requirements.txt create mode 100644 containers/Dockerfile.ironic-nautobot-client rename {argo-workflows/sync-nb-server-to-ironic/code => python/understack-workflows/understack_workflows}/create_node.py (69%) rename {argo-workflows/sync-nb-server-to-ironic/code => python/understack-workflows/understack_workflows/ironic}/__init__.py (100%) rename {argo-workflows/sync-nb-server-to-ironic/code => python/understack-workflows/understack_workflows}/ironic/client.py (67%) rename {argo-workflows/sync-nb-server-to-ironic/code => python/understack-workflows/understack_workflows}/ironic/secrets.py (68%) create mode 100644 python/understack-workflows/understack_workflows/main/__init__.py rename argo-workflows/sync-nb-server-to-ironic/code/synchronize-interfaces.py => python/understack-workflows/understack_workflows/main/synchronize_interfaces.py (67%) create mode 100644 python/understack-workflows/understack_workflows/main/synchronize_obm_creds.py create mode 100644 python/understack-workflows/understack_workflows/main/synchronize_server.py rename {argo-workflows/sync-nb-server-to-ironic/code => python/understack-workflows/understack_workflows}/network_data.py (100%) rename {argo-workflows/sync-nb-server-to-ironic/code => python/understack-workflows/understack_workflows}/network_data.schema.json (100%) rename {argo-workflows/sync-nb-server-to-ironic/code => python/understack-workflows/understack_workflows}/node_configuration.py (98%) create mode 100644 python/understack-workflows/understack_workflows/port_configuration.py rename {argo-workflows/sync-nb-server-to-ironic/code => python/understack-workflows/understack_workflows}/redfish_driver_info.py (100%) diff --git a/.github/workflows/build-container-images.yaml b/.github/workflows/build-container-images.yaml index 3d63d5d5d..78c8003f1 100644 --- a/.github/workflows/build-container-images.yaml +++ b/.github/workflows/build-container-images.yaml @@ -20,7 +20,6 @@ on: env: VERSION_PYTHON311: 0.0.1 VERSION_PYTHON312: 0.0.1 - VERSION_PYTHON_IRONIC: 0.0.4 VERSION_ARGO_UTILS: 0.0.1 VERSION_OBM_UTILS: 0.0.1 VERSION_PYTHON_NAUTOBOT_INT_SYNC: 0.0.1 @@ -67,17 +66,6 @@ jobs: labels: | org.opencontainers.image.version=${{ env.VERSION_PYTHON312 }} - - name: Build and deploy Python 3.11 with Ironic client - uses: docker/build-push-action@v5 - with: - context: argo-workflows/sync-nb-server-to-ironic - file: argo-workflows/sync-nb-server-to-ironic/containers/Dockerfile.ironic - # push for all main branch commits - push: ${{ github.event_name != 'pull_request' }} - tags: ghcr.io/rackerlabs/understack/argo-ironic-client-python3.11.8:latest,ghcr.io/rackerlabs/understack/argo-ironic-client-python3.11.8:${{ env.VERSION_PYTHON_IRONIC }} - labels: | - org.opencontainers.image.version=${{ env.VERSION_PYTHON_IRONIC }} - - name: Build and deploy Argo Utils image uses: docker/build-push-action@v5 with: diff --git a/.github/workflows/containers.yaml b/.github/workflows/containers.yaml index c67e500d5..66d5ddbd0 100644 --- a/.github/workflows/containers.yaml +++ b/.github/workflows/containers.yaml @@ -100,3 +100,50 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + + workflows: + runs-on: ubuntu-latest + + strategy: + matrix: + container: + - name: ironic-nautobot-client + title: Understack Ironic Nautobot Clients + + steps: + - name: setup docker buildx + uses: docker/setup-buildx-action@v3 + - name: login to ghcr.io + if: ${{ github.event_name != 'pull_request' }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/rackerlabs/understack/${{ matrix.container.name }} + labels: | + org.opencontainers.image.title=${{ matrix.container.title }} + env: + # Create the annotations at the index as well since this + # defaults to manifest only and we have to manually merge + # the container is multi-arch because of provenance creating + # an 'unknown/unknown' arch with data. We've got no annotations + # that are arch specific so populate them at the index as well. + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + + - name: build and deploy container image to registry + uses: docker/build-push-action@v5 + with: + file: containers/Dockerfile.${{ matrix.container.name }} + pull: true # ensure we always have an up to date source + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + # prod is the target that has the code installed + target: prod diff --git a/argo-workflows/sync-nb-server-to-ironic/code/port_configuration.py b/argo-workflows/sync-nb-server-to-ironic/code/port_configuration.py deleted file mode 100644 index 8723d017f..000000000 --- a/argo-workflows/sync-nb-server-to-ironic/code/port_configuration.py +++ /dev/null @@ -1,8 +0,0 @@ -from pydantic import BaseModel, StringConstraints -from typing import Annotated - - -class PortConfiguration(BaseModel): - address: Annotated[str, StringConstraints(to_lower=True)] # ironicclient's Port class lowercases this attribute - uuid: str # using a str here to remain consistent with the ironicclient Port attribute - node_uuid: str # using a str here to remain consistent with the ironicclient Port attribute diff --git a/argo-workflows/sync-nb-server-to-ironic/code/synchronize-obm-creds.py b/argo-workflows/sync-nb-server-to-ironic/code/synchronize-obm-creds.py deleted file mode 100644 index 2f963b2bd..000000000 --- a/argo-workflows/sync-nb-server-to-ironic/code/synchronize-obm-creds.py +++ /dev/null @@ -1,100 +0,0 @@ -import json -import logging -import sys - -import ironicclient.common.apiclient.exceptions - - -from ironic.client import IronicClient -from node_configuration import IronicNodeConfiguration -from ironic.secrets import read_secret - -logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.DEBUG) -logger = logging.getLogger(__name__) - -if len(sys.argv) < 1: - raise ValueError("Please provide node configuration in JSON format as first argument.") - -logger.info("Pushing device new node to Ironic.") -client = IronicClient( - svc_url=read_secret("IRONIC_SVC_URL"), - username=read_secret("IRONIC_USERNAME"), - password=read_secret("IRONIC_PASSWORD"), - auth_url=read_secret("IRONIC_AUTH_URL"), - tenant_name=read_secret("IRONIC_TENANT"), -) - - -def event_to_node_configuration(event: dict) -> IronicNodeConfiguration: - node_config = IronicNodeConfiguration() - node_config.conductor_group = None - node_config.driver = "redfish" - - node_config.chassis_uuid = None - node_config.uuid = event["device"]["id"] - node_config.name = event["device"]["name"] - - return node_config - - -interface_update_event = json.loads(sys.argv[1]) -logger.debug(f"Received: {interface_update_event}") -update_data = interface_update_event["data"] - -node_id = update_data["device"]["id"] -logger.debug(f"Checking if node with UUID: {node_id} exists in Ironic.") - -try: - ironic_node = client.get_node(node_id) -except ironicclient.common.apiclient.exceptions.NotFound: - logger.debug(f"Node: {node_id} not found in Ironic.") - ironic_node = None - sys.exit(1) - -STATES_ALLOWING_UPDATES = ["enroll"] -if ironic_node.provision_state not in STATES_ALLOWING_UPDATES: - logger.info( - f"Device {node_id} is in a {ironic_node.provision_state} provisioning state, so the updates are not allowed." - ) - sys.exit(0) - - -def credential_secrets(): - """ - Returns name of the Kubernetes Secret used to store OBM credentials for server node_id - """ - username = None - password = None - with open("/etc/obm/username") as f: - # strip leading and trailing whitespace - username = f.read().strip() - - with open("/etc/obm/password") as f: - # strip leading and trailing whitespace - password = f.read().strip() - - return [username, password] - -def replace_or_add_field(path, current_val, expected_val): - if current_val == expected_val: - return None - if current_val is None: - return {"op": "add", "path": path, "value": expected_val} - else: - return {"op": "replace", "path": path, "value": expected_val} - -# Update OBM credentials -expected_username, expected_password = credential_secrets() - -current_username = ironic_node.driver_info.get("redfish_username", None) -current_password_is_set = ironic_node.driver_info.get("redfish_password", None) - -patches = [ - replace_or_add_field('/driver_info/redfish_username', current_username, expected_username), - replace_or_add_field('/driver_info/redfish_password', current_password_is_set, expected_password), -] -patches = [p for p in patches if p is not None] - -response = client.update_node(node_id, patches) -logger.info(f"Patching: {patches}") -logger.info(f"Updated: {response}") diff --git a/argo-workflows/sync-nb-server-to-ironic/code/synchronize-server.py b/argo-workflows/sync-nb-server-to-ironic/code/synchronize-server.py deleted file mode 100644 index 8d89cc489..000000000 --- a/argo-workflows/sync-nb-server-to-ironic/code/synchronize-server.py +++ /dev/null @@ -1,89 +0,0 @@ -from dataclasses import asdict -import json -import logging -import sys - -import ironicclient.common.apiclient.exceptions - - -from ironic.client import IronicClient -from node_configuration import IronicNodeConfiguration -from redfish_driver_info import RedfishDriverInfo -from ironic.secrets import read_secret - -logger = logging.getLogger(__name__) -handler = logging.StreamHandler() -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -handler.setFormatter(formatter) -logger.addHandler(handler) -logger.setLevel(logging.DEBUG) - -if len(sys.argv) < 1: - raise ValueError("Please provide node configuration in JSON format as first argument.") - -logger.info("Pushing device new node to Ironic.") -client = IronicClient( - svc_url=read_secret("IRONIC_SVC_URL"), - username=read_secret("IRONIC_USERNAME"), - password=read_secret("IRONIC_PASSWORD"), - auth_url=read_secret("IRONIC_AUTH_URL"), - tenant_name=read_secret("IRONIC_TENANT"), -) - -def event_to_node_configuration(event: dict) -> IronicNodeConfiguration: - node_config = IronicNodeConfiguration() - node_config.conductor_group = None - node_config.driver = 'redfish' - node_config.chassis_uuid = None - node_config.uuid = event['device']['id'] - node_config.name = event['device']['name'] - - return node_config - - -interface_update_event = json.loads(sys.argv[1]) -logger.debug(f"Received: {interface_update_event}") -update_data = interface_update_event['data'] - -node_id = update_data['device']['id'] -logger.debug(f"Checking if node with UUID: {node_id} exists in Ironic.") - -try: - ironic_node = client.get_node(node_id) -except ironicclient.common.apiclient.exceptions.NotFound: - logger.debug(f"Node: {node_id} not found in Ironic.") - ironic_node = None - -if not ironic_node: - node_config = event_to_node_configuration(update_data) - response = client.create_node(node_config.create_arguments()) - logger.debug(response) - ironic_node = client.get_node(node_id) - -STATES_ALLOWING_UPDATES=['enroll'] -if ironic_node.provision_state not in STATES_ALLOWING_UPDATES: - logger.info(f"Device {node_id} is in a {ironic_node.provision_state} provisioning state, so the updates are not allowed.") - sys.exit(0) - -def replace_or_add_field(path, current_val, expected_val): - if current_val == expected_val: - return None - if current_val is None: - return {"op": "add", "path": path, "value": expected_val} - else: - return {"op": "replace", "path": path, "value": expected_val} - -drac_ip = update_data['ip_addresses'][0]['host'] -expected_address = f"https://{drac_ip}" -current_address = ironic_node.driver_info.get('redfish_address', None) -current_verify_ca = ironic_node.driver_info.get('redfish_verify_ca', None) - -patches = [ - replace_or_add_field('/driver_info/redfish_address', current_address, expected_address), - replace_or_add_field('/driver_info/redfish_verify_ca', current_verify_ca, False) -] -patches = [p for p in patches if p is not None] - -response = client.update_node(node_id, patches) -logger.info(f"Patching: {patches}") -logger.info(f"Updated: {response}") diff --git a/argo-workflows/sync-nb-server-to-ironic/containers/Dockerfile.ironic b/argo-workflows/sync-nb-server-to-ironic/containers/Dockerfile.ironic deleted file mode 100644 index 22eab8233..000000000 --- a/argo-workflows/sync-nb-server-to-ironic/containers/Dockerfile.ironic +++ /dev/null @@ -1,27 +0,0 @@ -ARG BASE=ghcr.io/rackerlabs/understack/argo-python3.11.8-alpine3.19:latest - -FROM ${BASE} as builder -ARG APP_PATH=/app -ARG APP_USER=appuser -ARG APP_GROUP=appgroup -ARG APP_USER_UID=1000 -ARG APP_GROUP_GID=1000 -COPY --chown=${APP_USER}:${APP_GROUP} containers/requirements.txt /app -RUN --mount=type=cache,target=/var/cache/apk apk add --virtual build-deps gcc python3-dev musl-dev linux-headers -RUN --mount=type=cache,target=/root/.cache/.pip pip install --no-cache-dir -r /app/requirements.txt - -FROM ${BASE} as prod -LABEL org.opencontainers.image.title="Python 3.11 image with Ironic Client" -LABEL org.opencontainers.image.base.name="ghcr.io/rackerlabs/understack/argo-ironic-client-python3.11.8" -LABEL org.opencontainers.image.source=https://github.com/rackerlabs/understack - - -ENV PATH="/opt/venv/bin:$PATH" -COPY --from=builder /opt/venv /opt/venv - -WORKDIR /app - -USER $APP_USER - -COPY --chown=${APP_USER}:${APP_GROUP} code/ /app -CMD ["python", "/app/main.py"] diff --git a/argo-workflows/sync-nb-server-to-ironic/containers/requirements.txt b/argo-workflows/sync-nb-server-to-ironic/containers/requirements.txt deleted file mode 100644 index e5190ff80..000000000 --- a/argo-workflows/sync-nb-server-to-ironic/containers/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -python-ironicclient==5.5.0 -pydantic==2.8.0 -pynautobot==2.2.0 diff --git a/argo-workflows/sync-nb-server-to-ironic/workflowtemplates/synchronize-interfaces-to-ironic.yaml b/argo-workflows/sync-nb-server-to-ironic/workflowtemplates/synchronize-interfaces-to-ironic.yaml index e53f933ec..2a178967e 100644 --- a/argo-workflows/sync-nb-server-to-ironic/workflowtemplates/synchronize-interfaces-to-ironic.yaml +++ b/argo-workflows/sync-nb-server-to-ironic/workflowtemplates/synchronize-interfaces-to-ironic.yaml @@ -13,10 +13,9 @@ spec: parameters: - name: device_id container: - image: ghcr.io/rackerlabs/understack/argo-ironic-client-python3.11.8:latest + image: ghcr.io/rackerlabs/understack/ironic-nautobot-client:latest command: - - python - - /app/synchronize-interfaces.py + - synchronize-interfaces args: - --device-id - "{{inputs.parameters.device_id}}" diff --git a/argo-workflows/sync-nb-server-to-ironic/workflowtemplates/synchronize-obm-creds.yaml b/argo-workflows/sync-nb-server-to-ironic/workflowtemplates/synchronize-obm-creds.yaml index df49b9c2e..f36cf0dd3 100644 --- a/argo-workflows/sync-nb-server-to-ironic/workflowtemplates/synchronize-obm-creds.yaml +++ b/argo-workflows/sync-nb-server-to-ironic/workflowtemplates/synchronize-obm-creds.yaml @@ -31,10 +31,9 @@ spec: parameters: - name: obm container: - image: ghcr.io/rackerlabs/understack/argo-ironic-client-python3.11.8:latest + image: ghcr.io/rackerlabs/understack/ironic-nautobot-client:latest command: - - python - - /app/synchronize-obm-creds.py + - synchronize-obm-creds args: - "{{workflow.parameters.interface_update_event}}" volumeMounts: diff --git a/argo-workflows/sync-nb-server-to-ironic/workflowtemplates/synchronize-server-to-ironic.yaml b/argo-workflows/sync-nb-server-to-ironic/workflowtemplates/synchronize-server-to-ironic.yaml index 8946d9e13..a1a3e3c02 100644 --- a/argo-workflows/sync-nb-server-to-ironic/workflowtemplates/synchronize-server-to-ironic.yaml +++ b/argo-workflows/sync-nb-server-to-ironic/workflowtemplates/synchronize-server-to-ironic.yaml @@ -10,10 +10,9 @@ spec: templates: - name: synchronize-server container: - image: ghcr.io/rackerlabs/understack/argo-ironic-client-python3.11.8:0.0.3 + image: ghcr.io/rackerlabs/understack/ironic-nautobot-client:latest command: - - python - - /app/synchronize-server.py + - synchronize-server args: - "{{workflow.parameters.interface_update_event}}" volumeMounts: diff --git a/containers/Dockerfile.ironic-nautobot-client b/containers/Dockerfile.ironic-nautobot-client new file mode 100644 index 000000000..580dce7c9 --- /dev/null +++ b/containers/Dockerfile.ironic-nautobot-client @@ -0,0 +1,25 @@ +ARG BASE=ghcr.io/rackerlabs/understack/argo-python3.11.8-alpine3.19:latest + +FROM ${BASE} AS builder +ARG APP_PATH=/app +ARG APP_USER=appuser +ARG APP_GROUP=appgroup +ARG APP_USER_UID=1000 +ARG APP_GROUP_GID=1000 +RUN --mount=type=cache,target=/var/cache/apk apk add --virtual build-deps gcc python3-dev musl-dev linux-headers +RUN --mount=type=cache,target=/root/.cache/.pip pip install 'build==1.2.1' 'wheel==0.43.0' + +# copy in the code +COPY --chown=${APP_USER}:${APP_GROUP} python/understack-workflows /app +# build wheels +RUN --mount=type=cache,target=/root/.cache/.pip cd /app && python -m build + +FROM ${BASE} AS prod + +WORKDIR /app + +COPY --from=builder /app/dist/*.whl /opt/venv + +USER $APP_USER + +RUN /opt/venv/bin/pip install --no-cache-dir /opt/venv/*.whl diff --git a/python/understack-workflows/pyproject.toml b/python/understack-workflows/pyproject.toml index 5521ee7f1..1e9d0c5a2 100644 --- a/python/understack-workflows/pyproject.toml +++ b/python/understack-workflows/pyproject.toml @@ -24,6 +24,17 @@ dynamic = ["version"] # but need to adjust ruff rules too requires-python = ">= 3.11,<3.12" +dependencies = [ + "pydantic==2.8.0", + "pynautobot==2.2.0", + "python-ironicclient==5.5.0", +] + +[project.scripts] +synchronize-interfaces = "understack_workflows.main.synchronize_interfaces:main" +synchronize-obm-creds = "understack_workflows.main.synchronize_obm_creds:main" +synchronize-server = "understack_workflows.main.synchronize_server:main" + [build-system] requires = [ "setuptools>=61.0", diff --git a/argo-workflows/sync-nb-server-to-ironic/code/create_node.py b/python/understack-workflows/understack_workflows/create_node.py similarity index 69% rename from argo-workflows/sync-nb-server-to-ironic/code/create_node.py rename to python/understack-workflows/understack_workflows/create_node.py index d54a04251..8dff54e90 100644 --- a/argo-workflows/sync-nb-server-to-ironic/code/create_node.py +++ b/python/understack-workflows/understack_workflows/create_node.py @@ -2,14 +2,16 @@ import logging import sys -from ironic.client import IronicClient -from ironic.secrets import read_secret +from understack_workflows.ironic.client import IronicClient +from understack_workflows.ironic.secrets import read_secret logger = logging.getLogger(__name__) if len(sys.argv) < 1: - raise ValueError("Please provide node configuration in JSON format as first argument.") + raise ValueError( + "Please provide node configuration in JSON format as first argument." + ) logger.info("Pushing device new node to Ironic.") client = IronicClient( diff --git a/argo-workflows/sync-nb-server-to-ironic/code/__init__.py b/python/understack-workflows/understack_workflows/ironic/__init__.py similarity index 100% rename from argo-workflows/sync-nb-server-to-ironic/code/__init__.py rename to python/understack-workflows/understack_workflows/ironic/__init__.py diff --git a/argo-workflows/sync-nb-server-to-ironic/code/ironic/client.py b/python/understack-workflows/understack_workflows/ironic/client.py similarity index 67% rename from argo-workflows/sync-nb-server-to-ironic/code/ironic/client.py rename to python/understack-workflows/understack_workflows/ironic/client.py index 98900d3d3..140c44b03 100644 --- a/argo-workflows/sync-nb-server-to-ironic/code/ironic/client.py +++ b/python/understack-workflows/understack_workflows/ironic/client.py @@ -1,7 +1,6 @@ from ironicclient import client as iclient from keystoneauth1 import session from keystoneauth1.identity import v3 -from typing import List class IronicClient: @@ -31,7 +30,9 @@ def login(self): user_domain_name="Default", ) insecure_ssl = True - sess = session.Session(auth=auth, verify=(not insecure_ssl), app_name="nautobot") + sess = session.Session( + auth=auth, verify=(not insecure_ssl), app_name="nautobot" + ) self.client = iclient.Client( 1, endpoint_override=self.svc_url, @@ -44,7 +45,9 @@ def login(self): def create_node(self, node_data: dict): self._ensure_logged_in() - return self.client.node.create(os_ironic_api_version=self.os_ironic_api_version, **node_data) + return self.client.node.create( + os_ironic_api_version=self.os_ironic_api_version, **node_data + ) def list_nodes(self): self._ensure_logged_in() @@ -54,27 +57,37 @@ def list_nodes(self): def get_node(self, node_ident: str, fields: list[str] | None = None): self._ensure_logged_in() - return self.client.node.get(node_ident, fields, os_ironic_api_version=self.os_ironic_api_version) + return self.client.node.get( + node_ident, fields, os_ironic_api_version=self.os_ironic_api_version + ) def update_node(self, node_id, patch): self._ensure_logged_in() - return self.client.node.update(node_id, patch, os_ironic_api_version=self.os_ironic_api_version) + return self.client.node.update( + node_id, patch, os_ironic_api_version=self.os_ironic_api_version + ) def create_port(self, port_data: dict): self._ensure_logged_in() - return self.client.port.create(os_ironic_api_version=self.os_ironic_api_version, **port_data) + return self.client.port.create( + os_ironic_api_version=self.os_ironic_api_version, **port_data + ) - def update_port(self, port_id: str, patch: List): + def update_port(self, port_id: str, patch: list): self._ensure_logged_in() - return self.client.port.update(port_id, patch, os_ironic_api_version=self.os_ironic_api_version) + return self.client.port.update( + port_id, patch, os_ironic_api_version=self.os_ironic_api_version + ) def delete_port(self, port_id: str): self._ensure_logged_in() - return self.client.port.delete(port_id, os_ironic_api_version=self.os_ironic_api_version) + return self.client.port.delete( + port_id, os_ironic_api_version=self.os_ironic_api_version + ) def list_ports(self, node_id: dict): self._ensure_logged_in() diff --git a/argo-workflows/sync-nb-server-to-ironic/code/ironic/secrets.py b/python/understack-workflows/understack_workflows/ironic/secrets.py similarity index 68% rename from argo-workflows/sync-nb-server-to-ironic/code/ironic/secrets.py rename to python/understack-workflows/understack_workflows/ironic/secrets.py index 7eb44de16..a348cdd7d 100644 --- a/argo-workflows/sync-nb-server-to-ironic/code/ironic/secrets.py +++ b/python/understack-workflows/understack_workflows/ironic/secrets.py @@ -1,19 +1,20 @@ +import logging import os import re -import logging logger = logging.getLogger(__name__) + def read_secret(secret_name: str) -> str: """Retrieve value of Kubernetes secret""" + def normalized(name): - return re.sub(r'[^A-Za-z0-9-_]', '', name) + return re.sub(r"[^A-Za-z0-9-_]", "", name) - base_path = os.environ.get('SECRETS_BASE_PATH', '/etc/ironic-secrets/') + base_path = os.environ.get("SECRETS_BASE_PATH", "/etc/ironic-secrets/") secret_path = os.path.join(base_path, normalized(secret_name)) try: - return open(secret_path, "r").read() + return open(secret_path).read() except FileNotFoundError: logger.error(f"Secret {secret_name} is not defined.") return "" - diff --git a/python/understack-workflows/understack_workflows/main/__init__.py b/python/understack-workflows/understack_workflows/main/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/argo-workflows/sync-nb-server-to-ironic/code/synchronize-interfaces.py b/python/understack-workflows/understack_workflows/main/synchronize_interfaces.py similarity index 67% rename from argo-workflows/sync-nb-server-to-ironic/code/synchronize-interfaces.py rename to python/understack-workflows/understack_workflows/main/synchronize_interfaces.py index ffd71e4f1..6445749a3 100644 --- a/argo-workflows/sync-nb-server-to-ironic/code/synchronize-interfaces.py +++ b/python/understack-workflows/understack_workflows/main/synchronize_interfaces.py @@ -1,13 +1,13 @@ +import argparse import logging import os -from ironic.client import IronicClient -import pynautobot -import argparse from uuid import UUID -from typing import List -from port_configuration import PortConfiguration + +import pynautobot from ironicclient.v1.port import Port +from understack_workflows.ironic.client import IronicClient +from understack_workflows.port_configuration import PortConfiguration logger = logging.getLogger(__name__) handler = logging.StreamHandler() @@ -16,8 +16,11 @@ logger.addHandler(handler) -def get_nautobot_interfaces(device_id: UUID) -> List[PortConfiguration]: - """Return a List of Ironic Ports for all Nautobot Interfaceswith a MAC address, for the specified Device.""" +def get_nautobot_interfaces(device_id: UUID) -> list[PortConfiguration]: + """ + Return a List of Ironic Ports for all Nautobot Interfaceswith a + MAC address, for the specified Device. + """ nautobot_api = os.environ["NAUTOBOT_API"] nautobot_token = os.environ["NAUTOBOT_TOKEN"] @@ -27,12 +30,19 @@ def get_nautobot_interfaces(device_id: UUID) -> List[PortConfiguration]: ports = [] for i in interfaces: if i.mac_address: - ports.append(PortConfiguration(node_uuid=str(device_id), address=i.mac_address, uuid=i.id)) + ports.append( + PortConfiguration( + node_uuid=str(device_id), address=i.mac_address, uuid=i.id + ) + ) return ports -def get_patch(nautobot_port: PortConfiguration, port: Port) -> List: - """Compare attributes between Port objects and return a patch object containing any changes.""" +def get_patch(nautobot_port: PortConfiguration, port: Port) -> list: + """ + Compare attributes between Port objects and return a patch object + containing any changes. + """ patch = [] for a in nautobot_port.__fields__: new_value = getattr(nautobot_port, a) @@ -42,7 +52,26 @@ def get_patch(nautobot_port: PortConfiguration, port: Port) -> List: return patch -def main(args): +def main(): + parser = argparse.ArgumentParser( + description="Update Ironic ports from Nautobot Interfaces" + ) + parser.add_argument( + "--device-id", + required=True, + help="Ironic Node and Nautobot Device ID", + type=UUID, + ) + parser.add_argument( + "--debug", + action="store_const", + dest="loglevel", + const=logging.DEBUG, + default=logging.WARNING, + ) + parser.add_argument("--dry-run", action=argparse.BooleanOptionalAction) + args = parser.parse_args() + device_id = args.device_id dry_run = args.dry_run logger.setLevel(args.loglevel) @@ -68,7 +97,6 @@ def main(args): # Update existing Ironic Ports new_ports = [] for n in nautobot_ports: - # identify any matching Ironic Ports matching_port = None for i in ironic_ports: @@ -80,8 +108,9 @@ def main(args): new_ports.append(n) continue - # If a matching port was found, we will remove it from the ironic_ports list. Once this loop completes, any - # remaining ports in ironic_ports will be considered stale, and will be removed from Ironic + # If a matching port was found, we will remove it from the ironic_ports + # list. Once this loop completes, any remaining ports in ironic_ports will + # be considered stale, and will be removed from Ironic ironic_ports.remove(matching_port) # if any data has changed on this interface, patch the matching ironic Port @@ -93,7 +122,9 @@ def main(args): response = client.update_port(n.uuid, patch) logger.debug(f"Updated: {response}") else: - logger.debug(f"An existing Ironic Port was found for Nautobot Interface {n.uuid}") + logger.debug( + f"An existing Ironic Port was found for Nautobot Interface {n.uuid}" + ) # Create new Ironic Ports for p in new_ports: @@ -107,16 +138,13 @@ def main(args): for i in ironic_ports: logger.debug(f"[ - ] {i}") if not dry_run: - logger.info(f"Nautobot Interface {i.uuid} no longer exists, deleting corresponding Ironic Port") + logger.info( + f"Nautobot Interface {i.uuid} no longer exists, deleting " + f"corresponding Ironic Port" + ) response = client.delete_port(i.uuid) logger.debug(f"Deleted: {response}") if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Update Ironic ports from Nautobot Interfaces") - parser.add_argument("--device-id", required=True, help="Ironic Node and Nautobot Device ID", type=UUID) - parser.add_argument("--debug", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.WARNING) - parser.add_argument("--dry-run", action=argparse.BooleanOptionalAction) - args = parser.parse_args() - - main(args) + main() diff --git a/python/understack-workflows/understack_workflows/main/synchronize_obm_creds.py b/python/understack-workflows/understack_workflows/main/synchronize_obm_creds.py new file mode 100644 index 000000000..7353b30f8 --- /dev/null +++ b/python/understack-workflows/understack_workflows/main/synchronize_obm_creds.py @@ -0,0 +1,111 @@ +import json +import logging +import sys + +import ironicclient.common.apiclient.exceptions + +from understack_workflows.ironic.client import IronicClient +from understack_workflows.ironic.secrets import read_secret +from understack_workflows.node_configuration import IronicNodeConfiguration + +logging.basicConfig( + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.DEBUG +) +logger = logging.getLogger(__name__) + + +def event_to_node_configuration(event: dict) -> IronicNodeConfiguration: + node_config = IronicNodeConfiguration() + node_config.conductor_group = None + node_config.driver = "redfish" + + node_config.chassis_uuid = None + node_config.uuid = event["device"]["id"] + node_config.name = event["device"]["name"] + + return node_config + + +def credential_secrets(): + """ + Returns name of the Kubernetes Secret used to store OBM credentials + for server node_id + """ + username = None + password = None + with open("/etc/obm/username") as f: + # strip leading and trailing whitespace + username = f.read().strip() + + with open("/etc/obm/password") as f: + # strip leading and trailing whitespace + password = f.read().strip() + + return [username, password] + + +def replace_or_add_field(path, current_val, expected_val): + if current_val == expected_val: + return None + if current_val is None: + return {"op": "add", "path": path, "value": expected_val} + else: + return {"op": "replace", "path": path, "value": expected_val} + + +def main(): + if len(sys.argv) < 1: + raise ValueError( + "Please provide node configuration in JSON format as first argument." + ) + + logger.info("Pushing device new node to Ironic.") + client = IronicClient( + svc_url=read_secret("IRONIC_SVC_URL"), + username=read_secret("IRONIC_USERNAME"), + password=read_secret("IRONIC_PASSWORD"), + auth_url=read_secret("IRONIC_AUTH_URL"), + tenant_name=read_secret("IRONIC_TENANT"), + ) + + interface_update_event = json.loads(sys.argv[1]) + logger.debug(f"Received: {interface_update_event}") + update_data = interface_update_event["data"] + + node_id = update_data["device"]["id"] + logger.debug(f"Checking if node with UUID: {node_id} exists in Ironic.") + + try: + ironic_node = client.get_node(node_id) + except ironicclient.common.apiclient.exceptions.NotFound: + logger.debug(f"Node: {node_id} not found in Ironic.") + ironic_node = None + sys.exit(1) + + STATES_ALLOWING_UPDATES = ["enroll"] + if ironic_node.provision_state not in STATES_ALLOWING_UPDATES: + logger.info( + f"Device {node_id} is in a {ironic_node.provision_state} " + f"provisioning state, so the updates are not allowed." + ) + sys.exit(0) + + # Update OBM credentials + expected_username, expected_password = credential_secrets() + + current_username = ironic_node.driver_info.get("redfish_username", None) + current_password_is_set = ironic_node.driver_info.get("redfish_password", None) + + patches = [ + replace_or_add_field( + "/driver_info/redfish_username", current_username, expected_username + ), + replace_or_add_field( + "/driver_info/redfish_password", current_password_is_set, expected_password + ), + ] + patches = [p for p in patches if p is not None] + + response = client.update_node(node_id, patches) + logger.info(f"Patching: {patches}") + logger.info(f"Updated: {response}") diff --git a/python/understack-workflows/understack_workflows/main/synchronize_server.py b/python/understack-workflows/understack_workflows/main/synchronize_server.py new file mode 100644 index 000000000..0ef1657ec --- /dev/null +++ b/python/understack-workflows/understack_workflows/main/synchronize_server.py @@ -0,0 +1,98 @@ +import json +import logging +import sys + +import ironicclient.common.apiclient.exceptions + +from understack_workflows.ironic.client import IronicClient +from understack_workflows.ironic.secrets import read_secret +from understack_workflows.node_configuration import IronicNodeConfiguration + +logger = logging.getLogger(__name__) +handler = logging.StreamHandler() +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") +handler.setFormatter(formatter) +logger.addHandler(handler) +logger.setLevel(logging.DEBUG) + + +def replace_or_add_field(path, current_val, expected_val): + if current_val == expected_val: + return None + if current_val is None: + return {"op": "add", "path": path, "value": expected_val} + else: + return {"op": "replace", "path": path, "value": expected_val} + + +def event_to_node_configuration(event: dict) -> IronicNodeConfiguration: + node_config = IronicNodeConfiguration() + node_config.conductor_group = None + node_config.driver = "redfish" + node_config.chassis_uuid = None + node_config.uuid = event["device"]["id"] + node_config.name = event["device"]["name"] + + return node_config + + +def main(): + if len(sys.argv) < 1: + raise ValueError( + "Please provide node configuration in JSON format as first argument." + ) + + logger.info("Pushing device new node to Ironic.") + client = IronicClient( + svc_url=read_secret("IRONIC_SVC_URL"), + username=read_secret("IRONIC_USERNAME"), + password=read_secret("IRONIC_PASSWORD"), + auth_url=read_secret("IRONIC_AUTH_URL"), + tenant_name=read_secret("IRONIC_TENANT"), + ) + + interface_update_event = json.loads(sys.argv[1]) + logger.debug(f"Received: {interface_update_event}") + update_data = interface_update_event["data"] + + node_id = update_data["device"]["id"] + logger.debug(f"Checking if node with UUID: {node_id} exists in Ironic.") + + try: + ironic_node = client.get_node(node_id) + except ironicclient.common.apiclient.exceptions.NotFound: + logger.debug(f"Node: {node_id} not found in Ironic.") + ironic_node = None + + if not ironic_node: + node_config = event_to_node_configuration(update_data) + response = client.create_node(node_config.create_arguments()) + logger.debug(response) + ironic_node = client.get_node(node_id) + + STATES_ALLOWING_UPDATES = ["enroll"] + if ironic_node.provision_state not in STATES_ALLOWING_UPDATES: + logger.info( + f"Device {node_id} is in a {ironic_node.provision_state} " + f"provisioning state, so the updates are not allowed." + ) + sys.exit(0) + + drac_ip = update_data["ip_addresses"][0]["host"] + expected_address = f"https://{drac_ip}" + current_address = ironic_node.driver_info.get("redfish_address", None) + current_verify_ca = ironic_node.driver_info.get("redfish_verify_ca", None) + + patches = [ + replace_or_add_field( + "/driver_info/redfish_address", current_address, expected_address + ), + replace_or_add_field( + "/driver_info/redfish_verify_ca", current_verify_ca, False + ), + ] + patches = [p for p in patches if p is not None] + + response = client.update_node(node_id, patches) + logger.info(f"Patching: {patches}") + logger.info(f"Updated: {response}") diff --git a/argo-workflows/sync-nb-server-to-ironic/code/network_data.py b/python/understack-workflows/understack_workflows/network_data.py similarity index 100% rename from argo-workflows/sync-nb-server-to-ironic/code/network_data.py rename to python/understack-workflows/understack_workflows/network_data.py diff --git a/argo-workflows/sync-nb-server-to-ironic/code/network_data.schema.json b/python/understack-workflows/understack_workflows/network_data.schema.json similarity index 100% rename from argo-workflows/sync-nb-server-to-ironic/code/network_data.schema.json rename to python/understack-workflows/understack_workflows/network_data.schema.json diff --git a/argo-workflows/sync-nb-server-to-ironic/code/node_configuration.py b/python/understack-workflows/understack_workflows/node_configuration.py similarity index 98% rename from argo-workflows/sync-nb-server-to-ironic/code/node_configuration.py rename to python/understack-workflows/understack_workflows/node_configuration.py index a4725cc46..6b3d4482a 100644 --- a/argo-workflows/sync-nb-server-to-ironic/code/node_configuration.py +++ b/python/understack-workflows/understack_workflows/node_configuration.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from dataclasses import field -from redfish_driver_info import RedfishDriverInfo +from understack_workflows.redfish_driver_info import RedfishDriverInfo @dataclass diff --git a/python/understack-workflows/understack_workflows/port_configuration.py b/python/understack-workflows/understack_workflows/port_configuration.py new file mode 100644 index 000000000..46447205e --- /dev/null +++ b/python/understack-workflows/understack_workflows/port_configuration.py @@ -0,0 +1,12 @@ +from typing import Annotated + +from pydantic import BaseModel +from pydantic import StringConstraints + + +class PortConfiguration(BaseModel): + address: Annotated[ + str, StringConstraints(to_lower=True) + ] # ironicclient's Port class lowercases this attribute + uuid: str # using a str here to due to ironicclient Port attribute + node_uuid: str # using a str here due to ironicclient Port attribute diff --git a/argo-workflows/sync-nb-server-to-ironic/code/redfish_driver_info.py b/python/understack-workflows/understack_workflows/redfish_driver_info.py similarity index 100% rename from argo-workflows/sync-nb-server-to-ironic/code/redfish_driver_info.py rename to python/understack-workflows/understack_workflows/redfish_driver_info.py