feat: enable nova-ssh and tweak daemonsets (rackerlabs#259)

* feat: enable nova-ssh to support migrations * fix: enable debug logging for oslo.db OsloDB does not log very much at all, even at INFO. We want to force debug logging so we can confirm if and how often oslodb is retrying on DBDeadlock errors. * fix: Speed up daemonset upgrades We had max_unavailable set to 1 which means it can take up to n x 30+ seconds to perform rolling upgrades. That's a long time for big environments with hundreds of nodes.
LukeRepko · May 10, 2024 · aee052e · aee052e
1 parent 426c4c0
commit aee052e
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 10 deletions.
diff --git a/docs/openstack-compute-kit.md b/docs/openstack-compute-kit.md
@@ -6,16 +6,18 @@
 
 Part of running Nova is also running placement. Setup all credentials now so we can use them across the nova and placement services.
 
+### Shared
+
 ``` shell
-# Shared
 kubectl --namespace openstack \
         create secret generic metadata-shared-secret \
         --type Opaque \
         --from-literal=password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)"
 ```
 
+### Placement
+
 ``` shell
-# Placement
 kubectl --namespace openstack \
         create secret generic placement-db-password \
         --type Opaque \
@@ -26,8 +28,9 @@ kubectl --namespace openstack \
         --from-literal=password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)"
 ```
 
+### Nova
+
 ``` shell
-# Nova
 kubectl --namespace openstack \
         create secret generic nova-db-password \
         --type Opaque \
@@ -41,26 +44,36 @@ kubectl --namespace openstack \
         --type Opaque \
         --from-literal=username="nova" \
         --from-literal=password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-64};echo;)"
+ssh-keygen -qt ed25519 -N '' -C "nova_ssh" -f nova_ssh_key && \
+kubectl --namespace openstack \
+        create secret generic nova-ssh-keypair \
+        --type Opaque \
+        --from-literal=public_key="$(cat nova_ssh_key.pub)" \
+        --from-literal=private_key="$(cat nova_ssh_key)"
+rm nova_ssh_key nova_ssh_key.pub
 ```
 
+### Ironic (NOT IMPLEMENTED YET)
+
 ``` shell
-# Ironic (NOT IMPLEMENTED YET)
 kubectl --namespace openstack \
         create secret generic ironic-admin \
         --type Opaque \
         --from-literal=password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)"
 ```
 
+### Designate (NOT IMPLEMENTED YET)
+
 ``` shell
-# Designate (NOT IMPLEMENTED YET)
 kubectl --namespace openstack \
         create secret generic designate-admin \
         --type Opaque \
         --from-literal=password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)"
 ```
 
+### Neutron
+
 ``` shell
-# Neutron
 kubectl --namespace openstack \
         create secret generic neutron-rabbitmq-password \
         --type Opaque \
@@ -122,6 +135,8 @@ helm upgrade --install nova ./nova \
     --set conf.nova.cell0_database.slave_connection="mysql+pymysql://nova:$(kubectl --namespace openstack get secret nova-db-password -o jsonpath='{.data.password}' | base64 -d)@mariadb-galera-secondary.openstack.svc.cluster.local:3306/nova_cell0" \
     --set endpoints.oslo_messaging.auth.admin.password="$(kubectl --namespace openstack get secret rabbitmq-default-user -o jsonpath='{.data.password}' | base64 -d)" \
     --set endpoints.oslo_messaging.auth.nova.password="$(kubectl --namespace openstack get secret nova-rabbitmq-password -o jsonpath='{.data.password}' | base64 -d)" \
+    --set network.ssh.public_key="$(kubectl -n openstack get secret nova-ssh-keypair -o jsonpath='{.data.public_key}' | base64 -d)"$'\n' \
+    --set network.ssh.private_key="$(kubectl -n openstack get secret nova-ssh-keypair -o jsonpath='{.data.private_key}' | base64 -d)"$'\n' \
     --post-renderer /opt/genestack/kustomize/kustomize.sh \
     --post-renderer-args nova/base
 ```

diff --git a/helm-configs/nova/nova-helm-overrides.yaml b/helm-configs/nova/nova-helm-overrides.yaml
@@ -256,7 +256,7 @@ network:
       enabled: false
       port: 30682
   ssh:
-    enabled: false
+    enabled: true
     port: 8022
     from_subnet: 0.0.0.0/0
     key_types:
@@ -1468,6 +1468,7 @@ conf:
     loggers:
       keys:
         - root
+        - oslo.db
         - nova
         - os.brick
     handlers:
@@ -1487,6 +1488,11 @@ conf:
       handlers:
         - stdout
       qualname: nova
+    logger_oslo.db:
+      level: DEBUG
+      handlers:
+        - stdout
+      qualname: oslo_db.api
     logger_os.brick:
       level: INFO
       handlers:
@@ -2324,7 +2330,7 @@ pod:
         compute:
           enabled: true
           min_ready_seconds: 0
-          max_unavailable: 1
+          max_unavailable: 20%
     disruption_budget:
       metadata:
         min_available: 0

diff --git a/kustomize/libvirt/helm/libvirt-helm-overrides.yaml b/kustomize/libvirt/helm/libvirt-helm-overrides.yaml
@@ -94,7 +94,7 @@ conf:
     cert_file: "/etc/pki/libvirt/servercert.pem"
     key_file: "/etc/pki/libvirt/private/serverkey.pem"
     auth_unix_rw: "none"
-    listen_addr: 127.0.0.1
+    listen_addr: 0.0.0.0
     log_level: "3"
     log_outputs: "1:file:/var/log/libvirt/libvirtd.log"
   qemu:
@@ -222,7 +222,7 @@ pod:
         libvirt:
           enabled: true
           min_ready_seconds: 0
-          max_unavailable: 1
+          max_unavailable: 20%
   resources:
     enabled: false
     libvirt: