diff --git a/k8s/hack/init.sh b/k8s/hack/init.sh index a0b57c7db..d53b528a0 100755 --- a/k8s/hack/init.sh +++ b/k8s/hack/init.sh @@ -1,3 +1,7 @@ #!/usr/bin/env bash -# no-op for classic confinement +DIR=`realpath $(dirname "${0}")` + +# Initialize node for integration tests +"${DIR}/connect-interfaces.sh" +"${DIR}/network-requirements.sh" diff --git a/k8s/wrappers/services/containerd b/k8s/wrappers/services/containerd index c3f71a012..a82e1c030 100755 --- a/k8s/wrappers/services/containerd +++ b/k8s/wrappers/services/containerd @@ -21,9 +21,4 @@ You can try to apply the profile manually by running: " fi -# Re-exec outside of apparmor confinement -if [ -d /sys/kernel/security/apparmor ] && [ "$(cat /proc/self/attr/current)" != "unconfined" ]; then - exec aa-exec -p unconfined -- "$0" "$@" -fi - k8s::common::execute_service containerd diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 435f40fb2..637d37532 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -7,7 +7,7 @@ description: |- on any infrastructure license: GPL-3.0 grade: stable -confinement: classic +confinement: strict base: core20 environment: REAL_PATH: $PATH @@ -218,6 +218,20 @@ parts: apps: k8s: command: k8s/wrappers/commands/k8s + plugs: + - firewall-control + - home-read-all + - home + - kernel-module-observe + - kubernetes-support + - login-session-observe + - log-observe + - mount-observe + - network + - network-control + - network-observe + - opengl + - system-observe containerd: command: k8s/wrappers/services/containerd daemon: notify @@ -228,43 +242,198 @@ apps: restart-condition: always start-timeout: 5m before: [kubelet] + plugs: + - network-bind + - docker-privileged + - firewall-control + - network-control + - mount-observe + - kubernetes-support + - cilium-module-load + - opengl + - cifs-mount + - fuse-support + - kernel-crypto-api k8s-dqlite: command: k8s/wrappers/services/k8s-dqlite install-mode: disable daemon: simple before: [kube-apiserver] + plugs: + - network-bind k8sd: command: k8s/wrappers/services/k8sd install-mode: enable daemon: simple + # FIXME: we keep 'kubernetes-support' because 'mount-observe' is not sufficient for some reason, investigate + plugs: + - network + - network-bind + - mount-observe + - kubernetes-support kubelet: install-mode: disable command: k8s/wrappers/services/kubelet daemon: simple after: [containerd] + plugs: + - docker-privileged + - firewall-control + - hardware-observe + - kubernetes-support + - mount-observe + - network-bind + - network-observe + - network-control + - process-control + - system-observe + - opengl + - kernel-module-observe kube-apiserver: install-mode: disable command: k8s/wrappers/services/kube-apiserver daemon: simple before: [kubelet, kube-controller-manager, kube-proxy, kube-scheduler] stop-timeout: 5s + plugs: + - docker-privileged + - firewall-control + - hardware-observe + - kubernetes-support + - mount-observe + - network-bind + - network-observe + - network-control + - process-control + - system-observe + - opengl + - kernel-module-observe kube-controller-manager: install-mode: disable command: k8s/wrappers/services/kube-controller-manager daemon: simple after: [kube-apiserver] + plugs: + - docker-privileged + - firewall-control + - hardware-observe + - kubernetes-support + - mount-observe + - network-bind + - network-observe + - network-control + - process-control + - system-observe + - opengl + - kernel-module-observe kube-proxy: install-mode: disable command: k8s/wrappers/services/kube-proxy daemon: simple after: [kube-apiserver] + plugs: + - docker-privileged + - firewall-control + - hardware-observe + - kubernetes-support + - mount-observe + - network-bind + - network-observe + - network-control + - process-control + - system-observe + - opengl + - kernel-module-observe kube-scheduler: install-mode: disable command: k8s/wrappers/services/kube-scheduler daemon: simple after: [kube-apiserver] + plugs: + - docker-privileged + - firewall-control + - hardware-observe + - kubernetes-support + - mount-observe + - network-bind + - network-observe + - network-control + - process-control + - system-observe + - opengl + - kernel-module-observe k8s-apiserver-proxy: install-mode: disable command: k8s/wrappers/services/k8s-apiserver-proxy daemon: simple before: [kubelet, kube-proxy] + plugs: + - network-bind + +layout: + # Kubernetes paths + /etc/kubernetes: + bind: $SNAP_COMMON/etc/kubernetes + /etc/cni/net.d: + bind: $SNAP_COMMON/etc/cni/net.d + /opt/cni/bin: + bind: $SNAP_COMMON/opt/cni/bin + /var/lib/kubelet: + bind: $SNAP_COMMON/var/lib/kubelet + # Logs and temporary files + /var/log/pods: + bind: $SNAP_COMMON/var/log/pods + /var/log/containers: + bind: $SNAP_COMMON/var/log/containers + # CNI + /var/lib/cni: + bind: $SNAP_COMMON/var/lib/cni + /var/lib/calico: + bind: $SNAP_COMMON/var/lib/calico + # Extras + /usr/local: + bind: $SNAP_COMMON/usr/local + # TBD (maybe not required) + /usr/libexec: + bind: $SNAP_COMMON/usr/libexec + /var/lib/kube-proxy: + bind: $SNAP_COMMON/var/lib/kube-proxy + /etc/service/enabled: + bind: $SNAP_COMMON/etc/service/enabled + /etc/nanorc: + bind-file: $SNAP_COMMON/etc/nanorc + +plugs: + home-read-all: + interface: home + read: all + docker-privileged: + interface: docker-support + privileged-containers: true + docker-unprivileged: + interface: docker-support + privileged-containers: false + # Cilium Ingress requires and loads iptable_raw and xt_socket modules + # If these modules are not loaded, ingress responses return 503 Service Unavailable + # since datapath L7 redirection does not work correctly. + # https://github.com/cilium/cilium/blob/1ab043d546e52fb2428300e6c6ea35fa3bd7c711/install/kubernetes/cilium/values.yaml#L792-L796 + # https://github.com/cilium/cilium/issues/25021#issuecomment-1699969830 + cilium-module-load: + interface: kernel-module-load + modules: + - name: iptable_raw + load: "on-boot" + - name: xt_socket + load: "on-boot" + +hooks: + remove: + plugs: + - network + - network-bind + - network-control + - network-observe + - process-control + - firewall-control + - system-observe + - mount-observe diff --git a/tests/integration/tests/test_util/util.py b/tests/integration/tests/test_util/util.py index 1c3106178..fb607d10f 100644 --- a/tests/integration/tests/test_util/util.py +++ b/tests/integration/tests/test_util/util.py @@ -197,21 +197,29 @@ def remove_k8s_snap(instance: harness.Instance): ["snap", "remove", config.SNAP_NAME, "--purge"] ) - LOG.info("Waiting for shims to go away...") - stubbornly(retries=20, delay_s=5).on(instance).until( - lambda p: all( - x not in p.stdout.decode() - for x in ["containerd-shim", "cilium", "coredns", "/pause"] - ) - ).exec(["ps", "-fea"]) - - LOG.info("Waiting for kubelet and containerd mounts to go away...") - stubbornly(retries=20, delay_s=5).on(instance).until( - lambda p: all( - x not in p.stdout.decode() - for x in ["/var/lib/kubelet/pods", "/run/containerd/io.containerd"] - ) - ).exec(["mount"]) + # NOTE(lpetrut): on "strict", the snap remove hook is unable to: + # * terminate processes + # * remove network namespaces + # * list mounts + # + # https://paste.ubuntu.com/p/WscCCfnvGH/plain/ + # https://paste.ubuntu.com/p/sSnJVvZkrr/plain/ + # + # LOG.info("Waiting for shims to go away...") + # stubbornly(retries=20, delay_s=5).on(instance).until( + # lambda p: all( + # x not in p.stdout.decode() + # for x in ["containerd-shim", "cilium", "coredns", "/pause"] + # ) + # ).exec(["ps", "-fea"]) + # + # LOG.info("Waiting for kubelet and containerd mounts to go away...") + # stubbornly(retries=20, delay_s=5).on(instance).until( + # lambda p: all( + # x not in p.stdout.decode() + # for x in ["/var/lib/kubelet/pods", "/run/containerd/io.containerd"] + # ) + # ).exec(["mount"]) # NOTE(neoaggelos): Temporarily disable this as it fails on strict. # For details, `snap changes` then `snap change $remove_k8s_snap_change`.