Cleans up containers and shims on microk8s stop and snap removal

.github/workflows/build-snap.yml

@@ -48,7 +48,7 @@ jobs:
           set -x
           sudo apt-get install python3-setuptools
           sudo pip3 install --upgrade pip
-          sudo pip3 install -U pytest sh
+          sudo pip3 install -U pytest sh psutil
           sudo apt-get -y install open-iscsi
           sudo systemctl enable iscsid
       - name: Fetch snap
@@ -73,7 +73,7 @@ jobs:
           set -x
           sudo apt-get install python3-setuptools
           sudo pip3 install --upgrade pip
-          sudo pip3 install -U pytest sh
+          sudo pip3 install -U pytest sh psutil
           sudo apt-get -y install open-iscsi
           sudo systemctl enable iscsid
       - name: Fetch snap
@@ -136,7 +136,7 @@ jobs:
           set -x
           sudo apt-get install python3-setuptools
           sudo pip3 install --upgrade pip
-          sudo pip3 install -U pytest sh
+          sudo pip3 install -U pytest sh psutil
           sudo apt-get -y install open-iscsi
           sudo systemctl enable iscsid
       - name: Fetch snap

microk8s-resources/actions/common/utils.sh

@@ -988,7 +988,7 @@ remove_all_containers() {
 
     for container in $("${SNAP}/microk8s-ctr.wrapper" containers ls | $SNAP/bin/sed -n '1!p' | $SNAP/usr/bin/gawk '{print $1}')
     do
-        "${SNAP}/microk8s-ctr.wrapper" container delete --force $container &>/dev/null || true
+        "${SNAP}/microk8s-ctr.wrapper" container delete $container &>/dev/null || true
     done
 }
 

microk8s-resources/default-hooks/remove.d/90-containers

@@ -2,9 +2,5 @@
 
 . "${SNAP}/actions/common/utils.sh"
 
-if is_strict
-then
-  remove_all_containers
-else
-  kill_all_container_shims
-fi
+remove_all_containers
+kill_all_container_shims

microk8s-resources/wrappers/apiservice-kicker

@@ -65,15 +65,9 @@ do
         echo "cert change detected. Reconfiguring the kube-apiserver"
         rm -rf .srl
         snapctl stop microk8s.daemon-kubelite
-        if is_strict
-        then
-          remove_all_containers
-          snapctl restart microk8s.daemon-containerd
-        else
-          snapctl stop microk8s.daemon-containerd
-          kill_all_container_shims
-          snapctl start microk8s.daemon-containerd
-        fi
+        remove_all_containers
+        kill_all_container_shims
+        snapctl restart microk8s.daemon-containerd
         snapctl start microk8s.daemon-kubelite
         start_all_containers
         restart_attempt=$[$restart_attempt+1]

microk8s-resources/wrappers/microk8s-stop.wrapper

@@ -40,26 +40,22 @@ while true; do
     esac
 done
 
-stopcmd="run_with_sudo snap stop ${SNAP_NAME} --disable"
+prefix_cmd="run_with_sudo snap"
 if is_strict
 then
-  stopcmd="snapctl stop microk8s.daemon-kubelite --disable"
+  prefix_cmd="snapctl"
 fi
 
-$stopcmd
+$prefix_cmd stop microk8s.daemon-kubelite --disable
 stop_status=$?
 
 if ! [ $stop_status -eq 0 ]
 then
     echo 'Failed to stop microk8s services. Check snapd logs with "journalctl -u snapd.service"'
     exit 1
 else
-    if is_strict
-    then
-      remove_all_containers
-      snapctl stop microk8s --disable
-    else
-      kill_all_container_shims
-    fi
+    remove_all_containers
+    kill_all_container_shims
+    $prefix_cmd stop microk8s --disable
     run_with_sudo touch ${SNAP_DATA}/var/lock/stopped.lock
 fi

snap/hooks/remove

@@ -7,6 +7,15 @@ use_snap_env
 
 snapctl stop ${SNAP_NAME}.daemon-kubelite 2>&1 || true
 
+# Temporarily start containerd so we can stop and kill all the microk8s containers.
+snapctl start ${SNAP_NAME}.daemon-containerd 2>&1 || true
+# wait for containerd to start.
+sleep 5
+
+# Remove any lingering containers and shims.
+remove_all_containers
+kill_all_container_shims
+
 # Try to symlink /var/lib/kubelet so that most kubelet device plugins work out of the box.
 if test -L /var/lib/kubelet; then
   unlink /var/lib/kubelet || true

tests/requirements.txt

@@ -7,5 +7,6 @@ pyyaml
 sh
 jsonschema==4.0.0
 pdbpp
+psutil
 netifaces
 requests

tests/test-simple.py

@@ -3,6 +3,8 @@
 import requests
 import os.path
 
+import utils
+
 
 class TestSimple(object):
     def test_microk8s_nodes_ready(self):
@@ -132,3 +134,17 @@ def test_microk8s_services_running(self):
 
         # Verify that all node services are running
         assert running_node_services == set(node_services), "Not all node services are running"
+
+    def test_microk8s_stop_start(self):
+        coredns_procs = utils._get_process("coredns")
+        assert len(coredns_procs) > 0, "Expected to find a coredns process running."
+
+        utils.run_until_success("/snap/bin/microk8s.stop", timeout_insec=180)
+
+        new_coredns_procs = utils._get_process("coredns")
+        assert len(new_coredns_procs) == 0, "coredns found still running after microk8s stop."
+
+        utils.run_until_success("/snap/bin/microk8s.start", timeout_insec=180)
+
+        new_coredns_procs = utils._get_process("coredns")
+        assert len(new_coredns_procs) > 0, "Expected to find a new coredns process running."

tests/test-upgrade.py

@@ -20,6 +20,7 @@
     is_container,
     is_ipv6_configured,
     kubectl,
+    _get_process,
 )
 
 upgrade_from = os.environ.get("UPGRADE_MICROK8S_FROM", "beta")
@@ -157,3 +158,5 @@ def test_upgrade(self):
         if not is_container():
             # On lxc umount docker overlay is not permitted.
             check_call("sudo snap remove microk8s".split())
+            coredns_procs = _get_process("coredns")
+            assert len(coredns_procs) == 0, "Expected to have 0 coredns processes running."

tests/utils.py

@@ -3,6 +3,7 @@
 import time
 import yaml
 import platform
+import psutil
 from subprocess import check_output, CalledProcessError, check_call
 
 
@@ -270,3 +271,7 @@ def is_ipv6_configured():
         return b"inet6" in output
     except CalledProcessError:
         return False
+
+
+def _get_process(name):
+    return [p for p in psutil.process_iter() if name == p.name()]