Strict patch

canonical · Nov 27, 2024 · 8f2f2e7 · 8f2f2e7
1 parent 3fc2241
commit 8f2f2e7
Show file tree

Hide file tree

Showing 4 changed files with 198 additions and 22 deletions.
diff --git a/k8s/hack/init.sh b/k8s/hack/init.sh
@@ -1,3 +1,7 @@
 #!/usr/bin/env bash
 
-# no-op for classic confinement
+DIR=`realpath $(dirname "${0}")`
+
+# Initialize node for integration tests
+"${DIR}/connect-interfaces.sh"
+"${DIR}/network-requirements.sh"
diff --git a/k8s/wrappers/services/containerd b/k8s/wrappers/services/containerd
@@ -21,9 +21,4 @@ You can try to apply the profile manually by running:
 "
 fi
 
-# Re-exec outside of apparmor confinement
-if [ -d /sys/kernel/security/apparmor ] && [ "$(cat /proc/self/attr/current)" != "unconfined" ]; then
-    exec aa-exec -p unconfined -- "$0" "$@"
-fi
-
 k8s::common::execute_service containerd
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
@@ -7,7 +7,7 @@ description: |-
   on any infrastructure
 license: GPL-3.0
 grade: stable
-confinement: classic
+confinement: strict
 base: core20
 environment:
   REAL_PATH: $PATH
@@ -218,6 +218,20 @@ parts:
 apps:
   k8s:
     command: k8s/wrappers/commands/k8s
+    plugs:
+      - firewall-control
+      - home-read-all
+      - home
+      - kernel-module-observe
+      - kubernetes-support
+      - login-session-observe
+      - log-observe
+      - mount-observe
+      - network
+      - network-control
+      - network-observe
+      - opengl
+      - system-observe
   containerd:
     command: k8s/wrappers/services/containerd
     daemon: notify
@@ -228,43 +242,198 @@ apps:
     restart-condition: always
     start-timeout: 5m
     before: [kubelet]
+    plugs:
+      - network-bind
+      - docker-privileged
+      - firewall-control
+      - network-control
+      - mount-observe
+      - kubernetes-support
+      - cilium-module-load
+      - opengl
+      - cifs-mount
+      - fuse-support
+      - kernel-crypto-api
   k8s-dqlite:
     command: k8s/wrappers/services/k8s-dqlite
     install-mode: disable
     daemon: simple
     before: [kube-apiserver]
+    plugs:
+      - network-bind
   k8sd:
     command: k8s/wrappers/services/k8sd
     install-mode: enable
     daemon: simple
+    # FIXME: we keep 'kubernetes-support' because 'mount-observe' is not sufficient for some reason, investigate
+    plugs:
+      - network
+      - network-bind
+      - mount-observe
+      - kubernetes-support
   kubelet:
     install-mode: disable
     command: k8s/wrappers/services/kubelet
     daemon: simple
     after: [containerd]
+    plugs:
+      - docker-privileged
+      - firewall-control
+      - hardware-observe
+      - kubernetes-support
+      - mount-observe
+      - network-bind
+      - network-observe
+      - network-control
+      - process-control
+      - system-observe
+      - opengl
+      - kernel-module-observe
   kube-apiserver:
     install-mode: disable
     command: k8s/wrappers/services/kube-apiserver
     daemon: simple
     before: [kubelet, kube-controller-manager, kube-proxy, kube-scheduler]
     stop-timeout: 5s
+    plugs:
+      - docker-privileged
+      - firewall-control
+      - hardware-observe
+      - kubernetes-support
+      - mount-observe
+      - network-bind
+      - network-observe
+      - network-control
+      - process-control
+      - system-observe
+      - opengl
+      - kernel-module-observe
   kube-controller-manager:
     install-mode: disable
     command: k8s/wrappers/services/kube-controller-manager
     daemon: simple
     after: [kube-apiserver]
+    plugs:
+      - docker-privileged
+      - firewall-control
+      - hardware-observe
+      - kubernetes-support
+      - mount-observe
+      - network-bind
+      - network-observe
+      - network-control
+      - process-control
+      - system-observe
+      - opengl
+      - kernel-module-observe
   kube-proxy:
     install-mode: disable
     command: k8s/wrappers/services/kube-proxy
     daemon: simple
     after: [kube-apiserver]
+    plugs:
+      - docker-privileged
+      - firewall-control
+      - hardware-observe
+      - kubernetes-support
+      - mount-observe
+      - network-bind
+      - network-observe
+      - network-control
+      - process-control
+      - system-observe
+      - opengl
+      - kernel-module-observe
   kube-scheduler:
     install-mode: disable
     command: k8s/wrappers/services/kube-scheduler
     daemon: simple
     after: [kube-apiserver]
+    plugs:
+      - docker-privileged
+      - firewall-control
+      - hardware-observe
+      - kubernetes-support
+      - mount-observe
+      - network-bind
+      - network-observe
+      - network-control
+      - process-control
+      - system-observe
+      - opengl
+      - kernel-module-observe
   k8s-apiserver-proxy:
     install-mode: disable
     command: k8s/wrappers/services/k8s-apiserver-proxy
     daemon: simple
     before: [kubelet, kube-proxy]
+    plugs:
+      - network-bind
+
+layout:
+  # Kubernetes paths
+  /etc/kubernetes:
+    bind: $SNAP_COMMON/etc/kubernetes
+  /etc/cni/net.d:
+    bind: $SNAP_COMMON/etc/cni/net.d
+  /opt/cni/bin:
+    bind: $SNAP_COMMON/opt/cni/bin
+  /var/lib/kubelet:
+    bind: $SNAP_COMMON/var/lib/kubelet
+  # Logs and temporary files
+  /var/log/pods:
+    bind: $SNAP_COMMON/var/log/pods
+  /var/log/containers:
+    bind: $SNAP_COMMON/var/log/containers
+  # CNI
+  /var/lib/cni:
+    bind: $SNAP_COMMON/var/lib/cni
+  /var/lib/calico:
+    bind: $SNAP_COMMON/var/lib/calico
+  # Extras
+  /usr/local:
+    bind: $SNAP_COMMON/usr/local
+  # TBD (maybe not required)
+  /usr/libexec:
+    bind: $SNAP_COMMON/usr/libexec
+  /var/lib/kube-proxy:
+    bind: $SNAP_COMMON/var/lib/kube-proxy
+  /etc/service/enabled:
+    bind: $SNAP_COMMON/etc/service/enabled
+  /etc/nanorc:
+    bind-file: $SNAP_COMMON/etc/nanorc
+
+plugs:
+  home-read-all:
+    interface: home
+    read: all
+  docker-privileged:
+    interface: docker-support
+    privileged-containers: true
+  docker-unprivileged:
+    interface: docker-support
+    privileged-containers: false
+  # Cilium Ingress requires and loads iptable_raw and xt_socket modules
+  # If these modules are not loaded, ingress responses return 503 Service Unavailable
+  # since datapath L7 redirection does not work correctly.
+  # https://github.com/cilium/cilium/blob/1ab043d546e52fb2428300e6c6ea35fa3bd7c711/install/kubernetes/cilium/values.yaml#L792-L796
+  # https://github.com/cilium/cilium/issues/25021#issuecomment-1699969830
+  cilium-module-load:
+    interface: kernel-module-load
+    modules:
+    - name: iptable_raw
+      load: "on-boot"
+    - name: xt_socket
+      load: "on-boot"
+
+hooks:
+  remove:
+    plugs:
+      - network
+      - network-bind
+      - network-control
+      - network-observe
+      - process-control
+      - firewall-control
+      - system-observe
+      - mount-observe
diff --git a/tests/integration/tests/test_util/util.py b/tests/integration/tests/test_util/util.py
@@ -197,21 +197,29 @@ def remove_k8s_snap(instance: harness.Instance):
         ["snap", "remove", config.SNAP_NAME, "--purge"]
     )
 
-    LOG.info("Waiting for shims to go away...")
-    stubbornly(retries=20, delay_s=5).on(instance).until(
-        lambda p: all(
-            x not in p.stdout.decode()
-            for x in ["containerd-shim", "cilium", "coredns", "/pause"]
-        )
-    ).exec(["ps", "-fea"])
-
-    LOG.info("Waiting for kubelet and containerd mounts to go away...")
-    stubbornly(retries=20, delay_s=5).on(instance).until(
-        lambda p: all(
-            x not in p.stdout.decode()
-            for x in ["/var/lib/kubelet/pods", "/run/containerd/io.containerd"]
-        )
-    ).exec(["mount"])
+    # NOTE(lpetrut): on "strict", the snap remove hook is unable to:
+    #  * terminate processes
+    #  * remove network namespaces
+    #  * list mounts
+    #
+    # https://paste.ubuntu.com/p/WscCCfnvGH/plain/
+    # https://paste.ubuntu.com/p/sSnJVvZkrr/plain/
+    #
+    # LOG.info("Waiting for shims to go away...")
+    # stubbornly(retries=20, delay_s=5).on(instance).until(
+    #     lambda p: all(
+    #         x not in p.stdout.decode()
+    #         for x in ["containerd-shim", "cilium", "coredns", "/pause"]
+    #     )
+    # ).exec(["ps", "-fea"])
+    #
+    # LOG.info("Waiting for kubelet and containerd mounts to go away...")
+    # stubbornly(retries=20, delay_s=5).on(instance).until(
+    #     lambda p: all(
+    #         x not in p.stdout.decode()
+    #         for x in ["/var/lib/kubelet/pods", "/run/containerd/io.containerd"]
+    #     )
+    # ).exec(["mount"])
 
     # NOTE(neoaggelos): Temporarily disable this as it fails on strict.
     # For details, `snap changes` then `snap change $remove_k8s_snap_change`.