diff --git a/build-scripts/patches/fips/0001-FIPS.patch b/build-scripts/patches/fips/0001-FIPS.patch new file mode 100644 index 00000000000..62a4dc39a63 --- /dev/null +++ b/build-scripts/patches/fips/0001-FIPS.patch @@ -0,0 +1,413 @@ +From ba26c04de8c57f1dcb8f8f0b5e94863ed26b0bcb Mon Sep 17 00:00:00 2001 +From: Angelos Kolaitis +Date: Fri, 27 Oct 2023 19:31:53 +0300 +Subject: [PATCH] FIPS + +--- + .../components/cluster-agent/build.sh | 5 ++- + build-scripts/components/etcd/build.sh | 3 +- + build-scripts/components/flanneld/build.sh | 3 +- + .../patches/default/0002-enable-cgo.patch | 33 +++++++++++++++++++ + build-scripts/components/k8s-dqlite/build.sh | 12 ++++--- + build-scripts/components/kubernetes/build.sh | 2 +- + microk8s-resources/default-args/fips-env | 16 +++++++++ + .../wrappers/microk8s-helm.wrapper | 6 ++++ + .../wrappers/microk8s-helm3.wrapper | 6 ++++ + .../wrappers/microk8s-kubectl.wrapper | 6 ++++ + microk8s-resources/wrappers/openssl.wrapper | 16 ++++++--- + .../wrappers/run-apiserver-proxy-with-args | 6 ++++ + .../wrappers/run-cluster-agent-with-args | 6 ++++ + .../wrappers/run-etcd-with-args | 6 ++++ + .../wrappers/run-flanneld-with-args | 6 ++++ + .../wrappers/run-k8s-dqlite-with-args | 4 +++ + .../wrappers/run-kubelite-with-args | 6 ++++ + snap/hooks/install | 2 +- + snap/snapcraft.yaml | 10 ++++-- + 19 files changed, 138 insertions(+), 16 deletions(-) + create mode 100644 build-scripts/components/helm/patches/default/0002-enable-cgo.patch + create mode 100644 microk8s-resources/default-args/fips-env + +diff --git a/build-scripts/components/cluster-agent/build.sh b/build-scripts/components/cluster-agent/build.sh +index 86282e7..149a5f4 100755 +--- a/build-scripts/components/cluster-agent/build.sh ++++ b/build-scripts/components/cluster-agent/build.sh +@@ -3,5 +3,8 @@ + export INSTALL="${1}/bin" + mkdir -p "${INSTALL}" + +-make cluster-agent ++export GOEXPERIMENT=opensslcrypto ++export CGO_ENABLED=1 ++go build -ldflags '-s -w' -o cluster-agent ./main.go ++ + cp cluster-agent "${INSTALL}" +diff --git a/build-scripts/components/etcd/build.sh b/build-scripts/components/etcd/build.sh +index 3eaaa9d..a58ed1c 100755 +--- a/build-scripts/components/etcd/build.sh ++++ b/build-scripts/components/etcd/build.sh +@@ -3,7 +3,8 @@ + export INSTALL="${1}" + mkdir -p "${INSTALL}" + +-GO_LDFLAGS="-s -w" GO_BUILD_FLAGS="-v" ./build.sh ++sed -i 's/CGO_ENABLED=0/CGO_ENABLED=1/' build.sh ++GOEXPERIMENT=opensslcrypto GO_LDFLAGS="-s -w" GO_BUILD_FLAGS="-v" ./build.sh + + for bin in etcd etcdctl; do + cp "bin/${bin}" "${INSTALL}/${bin}" +diff --git a/build-scripts/components/flanneld/build.sh b/build-scripts/components/flanneld/build.sh +index c2904a7..ee0e5f4 100755 +--- a/build-scripts/components/flanneld/build.sh ++++ b/build-scripts/components/flanneld/build.sh +@@ -5,7 +5,8 @@ mkdir -p "${INSTALL}" + + VERSION="${2}" + +-export CGO_ENABLED=0 ++export CGO_ENABLED=1 ++export GOEXPERIMENT=opensslcrypto + go build -o dist/flanneld -ldflags "-s -w -X github.com/flannel-io/flannel/version.Version=${VERSION} -extldflags -static" + + cp dist/flanneld "${INSTALL}/flanneld" +diff --git a/build-scripts/components/helm/patches/default/0002-enable-cgo.patch b/build-scripts/components/helm/patches/default/0002-enable-cgo.patch +new file mode 100644 +index 0000000..9f8a4f9 +--- /dev/null ++++ b/build-scripts/components/helm/patches/default/0002-enable-cgo.patch +@@ -0,0 +1,33 @@ ++From 76f154cf345ffc7f2b4b9c0e623551c744249d60 Mon Sep 17 00:00:00 2001 ++From: Angelos Kolaitis ++Date: Fri, 27 Oct 2023 17:01:33 +0000 ++Subject: [PATCH] enable cgo ++ ++--- ++ Makefile | 4 ++-- ++ 1 file changed, 2 insertions(+), 2 deletions(-) ++ ++diff --git a/Makefile b/Makefile ++index 0a7b326..39cee3f 100644 ++--- a/Makefile +++++ b/Makefile ++@@ -77,7 +77,7 @@ all: build ++ build: $(BINDIR)/$(BINNAME) ++ ++ $(BINDIR)/$(BINNAME): $(SRC) ++- GO111MODULE=on CGO_ENABLED=0 go build $(GOFLAGS) -trimpath -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o '$(BINDIR)'/$(BINNAME) ./cmd/helm +++ GO111MODULE=on CGO_ENABLED=1 go build $(GOFLAGS) -trimpath -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o '$(BINDIR)'/$(BINNAME) ./cmd/helm ++ ++ # ------------------------------------------------------------------------------ ++ # install ++@@ -165,7 +165,7 @@ $(GOIMPORTS): ++ .PHONY: build-cross ++ build-cross: LDFLAGS += -extldflags "-static" ++ build-cross: $(GOX) ++- GOFLAGS="-trimpath" GO111MODULE=on CGO_ENABLED=0 $(GOX) -parallel=3 -output="_dist/{{.OS}}-{{.Arch}}/$(BINNAME)" -osarch='$(TARGETS)' $(GOFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' ./cmd/helm +++ GOFLAGS="-trimpath" GO111MODULE=on CGO_ENABLED=1 $(GOX) -parallel=3 -output="_dist/{{.OS}}-{{.Arch}}/$(BINNAME)" -osarch='$(TARGETS)' $(GOFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' ./cmd/helm ++ ++ .PHONY: dist ++ dist: ++-- ++2.25.1 +diff --git a/build-scripts/components/k8s-dqlite/build.sh b/build-scripts/components/k8s-dqlite/build.sh +index c967277..e833afa 100755 +--- a/build-scripts/components/k8s-dqlite/build.sh ++++ b/build-scripts/components/k8s-dqlite/build.sh +@@ -1,9 +1,11 @@ + #!/bin/bash + +-INSTALL="${1}/bin" +-mkdir -p "${INSTALL}" ++INSTALL="${1}" ++mkdir -p "${INSTALL}/bin" "${INSTALL}/usr/lib" + +-make static -j ++export GOEXPERIMENT=opensslcrypto ++make dynamic -j + +-cp bin/static/dqlite "${INSTALL}/dqlite" +-cp bin/static/k8s-dqlite "${INSTALL}/k8s-dqlite" ++cp bin/dynamic/dqlite "${INSTALL}/bin/dqlite" ++cp bin/dynamic/k8s-dqlite "${INSTALL}/bin/k8s-dqlite" ++cp bin/dynamic/lib/*so* "${INSTALL}/usr/lib" +diff --git a/build-scripts/components/kubernetes/build.sh b/build-scripts/components/kubernetes/build.sh +index 542ba9f..b8f30bc 100755 +--- a/build-scripts/components/kubernetes/build.sh ++++ b/build-scripts/components/kubernetes/build.sh +@@ -5,7 +5,7 @@ INSTALL="${1}" + export KUBE_GIT_VERSION_FILE="${PWD}/.version.sh" + + for app in kubectl kubelite; do +- make WHAT="cmd/${app}" KUBE_STATIC_OVERRIDES=kubelite ++ make WHAT="cmd/${app}" KUBE_CGO_OVERRIDES="${app}" + cp _output/bin/"${app}" "${INSTALL}/${app}" + done + +diff --git a/microk8s-resources/default-args/fips-env b/microk8s-resources/default-args/fips-env +new file mode 100644 +index 0000000..46e6877 +--- /dev/null ++++ b/microk8s-resources/default-args/fips-env +@@ -0,0 +1,16 @@ ++# For FIPS-enabled hosts, i.e. when /proc/sys/crypto/fips_enabled is 1, ++# the following configuration is required to use the fips enabled crypto ++# libraries from the host. ++# ++# The paths below are for FIPS enabled Ubuntu 20.04, make sure to adjust ++# accordingly for other distributions. ++ ++# Uncomment and specify the binary path and config file for openssl. ++# OPENSSL_EXECUTABLE="/usr/bin/openssl" ++# OPENSSL_CONF="/etc/ssl/openssl.cnf" ++ ++# Uncomment and prepend the FIPS libcrypto location to the LD_LIBRARY_PATH ++# LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH" ++ ++# Set GOFIPS=1 so that Go binaries use the FIPS-enabled libcrypto ++GOFIPS=0 +diff --git a/microk8s-resources/wrappers/microk8s-helm.wrapper b/microk8s-resources/wrappers/microk8s-helm.wrapper +index dee8fe5..22c5b1f 100755 +--- a/microk8s-resources/wrappers/microk8s-helm.wrapper ++++ b/microk8s-resources/wrappers/microk8s-helm.wrapper +@@ -12,4 +12,10 @@ fi + + exit_if_stopped + ++set -a ++if [ -e "${SNAP_DATA}/args/fips-env" ]; then ++ . "${SNAP_DATA}/args/fips-env" ++fi ++set +a ++ + "${SNAP}/bin/helm" --kubeconfig="${SNAP_DATA}"/credentials/client.config "$@" +diff --git a/microk8s-resources/wrappers/microk8s-helm3.wrapper b/microk8s-resources/wrappers/microk8s-helm3.wrapper +index dee8fe5..22c5b1f 100755 +--- a/microk8s-resources/wrappers/microk8s-helm3.wrapper ++++ b/microk8s-resources/wrappers/microk8s-helm3.wrapper +@@ -12,4 +12,10 @@ fi + + exit_if_stopped + ++set -a ++if [ -e "${SNAP_DATA}/args/fips-env" ]; then ++ . "${SNAP_DATA}/args/fips-env" ++fi ++set +a ++ + "${SNAP}/bin/helm" --kubeconfig="${SNAP_DATA}"/credentials/client.config "$@" +diff --git a/microk8s-resources/wrappers/microk8s-kubectl.wrapper b/microk8s-resources/wrappers/microk8s-kubectl.wrapper +index ea15160..2ead1fe 100755 +--- a/microk8s-resources/wrappers/microk8s-kubectl.wrapper ++++ b/microk8s-resources/wrappers/microk8s-kubectl.wrapper +@@ -28,6 +28,12 @@ then + export EDITOR="${SNAP}/bin/nano" + fi + ++set -a ++if [ -e "${SNAP_DATA}/args/fips-env" ]; then ++ . "${SNAP_DATA}/args/fips-env" ++fi ++set +a ++ + declare -a args="($(cat $SNAP_DATA/args/kubectl))" + if [ -n "${args[@]-}" ] + then +diff --git a/microk8s-resources/wrappers/openssl.wrapper b/microk8s-resources/wrappers/openssl.wrapper +index 550029a..528991c 100755 +--- a/microk8s-resources/wrappers/openssl.wrapper ++++ b/microk8s-resources/wrappers/openssl.wrapper +@@ -4,8 +4,16 @@ set -eu + + source $SNAP/actions/common/utils.sh + +-use_snap_env ++set -a ++if [ -e "${SNAP_DATA}/args/fips-env" ]; then ++ . "${SNAP_DATA}/args/fips-env" ++fi ++set +a + +-export OPENSSL_CONF="${SNAP}/etc/ssl/openssl.cnf" +- +-"${SNAP}/usr/bin/openssl" "${@}" ++if [[ -v OPENSSL_EXECUTABLE ]]; then ++ exec "${OPENSSL_EXECUTABLE}" "${@}" ++else ++ use_snap_env ++ export OPENSSL_CONF="${SNAP}/etc/ssl/openssl.cnf" ++ ${SNAP}/usr/bin/openssl "$@" ++fi +diff --git a/microk8s-resources/wrappers/run-apiserver-proxy-with-args b/microk8s-resources/wrappers/run-apiserver-proxy-with-args +index 71edf93..91da9a7 100755 +--- a/microk8s-resources/wrappers/run-apiserver-proxy-with-args ++++ b/microk8s-resources/wrappers/run-apiserver-proxy-with-args +@@ -21,6 +21,12 @@ fi + + sed 's@${SNAP}@'"${SNAP}"'@g;s@${SNAP_DATA}@'"${SNAP_DATA}"'@g' $SNAP_DATA/args/traefik/traefik-template.yaml > $SNAP_DATA/args/traefik/traefik.yaml + ++set -a ++if [ -e "${SNAP_DATA}/args/fips-env" ]; then ++ . "${SNAP_DATA}/args/fips-env" ++fi ++set +a ++ + # This is really the only way I could find to get the args passed in correctly. + declare -a args="($(cat $SNAP_DATA/args/apiserver-proxy))" + exec "$SNAP/bin/cluster-agent" apiserver-proxy "${args[@]}" +diff --git a/microk8s-resources/wrappers/run-cluster-agent-with-args b/microk8s-resources/wrappers/run-cluster-agent-with-args +index ac44743..fe7b0fd 100755 +--- a/microk8s-resources/wrappers/run-cluster-agent-with-args ++++ b/microk8s-resources/wrappers/run-cluster-agent-with-args +@@ -12,6 +12,12 @@ set -eu + + use_snap_env + ++set -a ++if [ -e "${SNAP_DATA}/args/fips-env" ]; then ++ . "${SNAP_DATA}/args/fips-env" ++fi ++set +a ++ + # This is really the only way I could find to get the args passed in correctly. + declare -a args="($(cat $SNAP_DATA/args/cluster-agent))" + +diff --git a/microk8s-resources/wrappers/run-etcd-with-args b/microk8s-resources/wrappers/run-etcd-with-args +index a17304c..38027ce 100755 +--- a/microk8s-resources/wrappers/run-etcd-with-args ++++ b/microk8s-resources/wrappers/run-etcd-with-args +@@ -26,6 +26,12 @@ fi + + export DEFAULT_INTERFACE_IP_ADDR="$(get_default_ip)" + ++set -a ++if [ -e "${SNAP_DATA}/args/fips-env" ]; then ++ . "${SNAP_DATA}/args/fips-env" ++fi ++set +a ++ + # This is really the only way I could find to get the args passed in correctly. + declare -a args="($(cat $SNAP_DATA/args/etcd))" + exec "$SNAP/etcd" "${args[@]}" +diff --git a/microk8s-resources/wrappers/run-flanneld-with-args b/microk8s-resources/wrappers/run-flanneld-with-args +index abbe79f..21ce369 100755 +--- a/microk8s-resources/wrappers/run-flanneld-with-args ++++ b/microk8s-resources/wrappers/run-flanneld-with-args +@@ -32,6 +32,12 @@ export ETCDCTL_API=3 + # TODO get this from a file + data="$(cat $SNAP_DATA/args/flannel-network-mgr-config)" + ++set -a ++if [ -e "${SNAP_DATA}/args/fips-env" ]; then ++ . "${SNAP_DATA}/args/fips-env" ++fi ++set +a ++ + # Prepare etcd configuration for flannel, iff an etcd endpoint is set. + # Skip this part if an alternate data store is used (e.g. Kubernetes). + if [ ! -z "$etcd_endpoints" ]; then +diff --git a/microk8s-resources/wrappers/run-k8s-dqlite-with-args b/microk8s-resources/wrappers/run-k8s-dqlite-with-args +index 3314b89..e8c8050 100755 +--- a/microk8s-resources/wrappers/run-k8s-dqlite-with-args ++++ b/microk8s-resources/wrappers/run-k8s-dqlite-with-args +@@ -40,6 +40,10 @@ if [ -e "${SNAP_DATA}/args/${app}-env" ] + then + . "${SNAP_DATA}/args/${app}-env" + fi ++ ++if [ -e "${SNAP_DATA}/args/fips-env" ]; then ++ . "${SNAP_DATA}/args/fips-env" ++fi + set +a + + declare -a args="($(cat $SNAP_DATA/args/$app))" +diff --git a/microk8s-resources/wrappers/run-kubelite-with-args b/microk8s-resources/wrappers/run-kubelite-with-args +index c011207..51e693e 100755 +--- a/microk8s-resources/wrappers/run-kubelite-with-args ++++ b/microk8s-resources/wrappers/run-kubelite-with-args +@@ -218,5 +218,11 @@ then + echo "1" > /proc/sys/net/bridge/bridge-nf-call-iptables + fi + ++set -a ++if [ -e "${SNAP_DATA}/args/fips-env" ]; then ++ . "${SNAP_DATA}/args/fips-env" ++fi ++set +a ++ + declare -a args="($(cat $SNAP_DATA/args/$app))" + exec "$SNAP/$app" "${args[@]}" +diff --git a/snap/hooks/install b/snap/hooks/install +index dbaeebb..af8d5ee 100755 +--- a/snap/hooks/install ++++ b/snap/hooks/install +@@ -80,7 +80,7 @@ for config_file in "/etc/microk8s.yaml" "$SNAP_USER_COMMON/.microk8s.yaml" "$SNA + echo "Found config file ${config_file}, will use to initialize cluster." + + if cp "${config_file}" "${SNAP_COMMON}/etc/launcher/install.yaml"; then +- "${SNAP}/bin/cluster-agent" init --pre-init --config-file "${SNAP_COMMON}/etc/launcher/install.yaml" ++ GOFIPS=0 "${SNAP}/bin/cluster-agent" init --pre-init --config-file "${SNAP_COMMON}/etc/launcher/install.yaml" + break + fi + fi +diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml +index e8cab50..f0ef0e8 100644 +--- a/snap/snapcraft.yaml ++++ b/snap/snapcraft.yaml +@@ -21,8 +21,8 @@ parts: + build-deps: + plugin: nil + override-build: | +- snap install go --classic --channel 1.20/stable +- snap refresh go --channel 1.20/stable ++ snap install go --classic --channel 1.20-fips/stable ++ snap refresh go --channel 1.20-fips/stable + build-packages: + - autoconf + - automake +@@ -49,12 +49,14 @@ parts: + + k8s-dqlite: + after: [build-deps] ++ build-attributes: [no-patchelf] + source: build-scripts/components/k8s-dqlite + plugin: nil + override-build: $SNAPCRAFT_PROJECT_DIR/build-scripts/build-component.sh k8s-dqlite + + etcd: + after: [build-deps] ++ build-attributes: [no-patchelf] + plugin: nil + source: build-scripts/components/etcd + override-build: $SNAPCRAFT_PROJECT_DIR/build-scripts/build-component.sh etcd +@@ -73,12 +75,14 @@ parts: + + flanneld: + after: [build-deps] ++ build-attributes: [no-patchelf] + plugin: nil + source: build-scripts/components/flanneld + override-build: $SNAPCRAFT_PROJECT_DIR/build-scripts/build-component.sh flanneld + + kubernetes: + after: [build-deps] ++ build-attributes: [no-patchelf] + plugin: nil + source: build-scripts/components/kubernetes + override-build: $SNAPCRAFT_PROJECT_DIR/build-scripts/build-component.sh kubernetes +@@ -90,6 +94,7 @@ parts: + + helm: + after: [build-deps] ++ build-attributes: [no-patchelf] + plugin: nil + source: build-scripts/components/helm + override-build: $SNAPCRAFT_PROJECT_DIR/build-scripts/build-component.sh helm +@@ -203,6 +208,7 @@ parts: + + cluster-agent: + after: [build-deps] ++ build-attributes: [no-patchelf] + plugin: nil + source: build-scripts/components/cluster-agent + override-build: $SNAPCRAFT_PROJECT_DIR/build-scripts/build-component.sh cluster-agent +-- +2.34.1 diff --git a/build-scripts/patches/strict/0001-Strict.patch b/build-scripts/patches/strict/0001-Strict.patch new file mode 100644 index 00000000000..ba0c1e69462 --- /dev/null +++ b/build-scripts/patches/strict/0001-Strict.patch @@ -0,0 +1,906 @@ +From af92bae2020399d23bec3dced86eefa9b5fc0a26 Mon Sep 17 00:00:00 2001 +From: Konstantinos Tsakalozos +Date: Tue, 8 Mar 2022 12:35:56 +0200 +Subject: [PATCH] Strict + +--- + .github/workflows/build-snap.yml | 21 +- + docs/build.md | 9 +- + microk8s-resources/connect-all-interfaces.sh | 35 ++ + snap/hooks/connect-plug-configuration | 6 + + snap/hooks/disconnect-plug-configuration | 3 + + snap/snapcraft.yaml | 459 ++++++++++++++++++- + tests/libs/addons.sh | 5 +- + tests/libs/airgap.sh | 2 + + tests/libs/upgrade-path.sh | 4 +- + tests/libs/utils.sh | 1 + + tests/smoke-test.sh | 4 +- + tests/test-cluster.py | 18 +- + tests/test-distro.sh | 2 +- + 13 files changed, 547 insertions(+), 22 deletions(-) + create mode 100755 microk8s-resources/connect-all-interfaces.sh + create mode 100755 snap/hooks/connect-plug-configuration + create mode 100755 snap/hooks/disconnect-plug-configuration + +diff --git a/.github/workflows/build-snap.yml b/.github/workflows/build-snap.yml +index 33b63b6..e0b63f5 100644 +--- a/.github/workflows/build-snap.yml ++++ b/.github/workflows/build-snap.yml +@@ -1,9 +1,7 @@ + name: Build and test MicroK8s snap + + on: +- pull_request: +- branches: +- - master ++ - push + + jobs: + build: +@@ -58,7 +56,7 @@ jobs: + path: build + - name: Running upgrade path test + run: | +- sudo -E UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=$PWD/build/microk8s.snap pytest -s ./tests/test-upgrade-path.py ++ sudo -E STRICT=yes UPGRADE_MICROK8S_FROM=latest/edge/strict UPGRADE_MICROK8S_TO=$PWD/build/microk8s.snap pytest -s ./tests/test-upgrade-path.py + + test-addons-core: + name: Test core addons +@@ -81,13 +79,15 @@ jobs: + with: + name: microk8s.snap + path: build +- - name: Running addons tests ++ - name: Running addons tests in strict mode + run: | + set -x +- sudo snap install build/microk8s.snap --classic --dangerous ++ sudo snap install build/microk8s.snap --dangerous ++ sudo /snap/microk8s/current/connect-all-interfaces.sh ++ sudo microk8s status --wait-ready --timeout 300 + ./tests/smoke-test.sh + export UNDER_TIME_PRESSURE="True" +- export SKIP_PROMETHEUS="False" ++ export STRICT="yes" + sudo -E bash -c "cd /var/snap/microk8s/common/addons/core/tests; pytest -s -ra test-addons.py" + + test-addons-community: +@@ -117,8 +117,11 @@ jobs: + run: | + set -x + sudo snap install build/microk8s.snap --classic --dangerous ++ sudo /snap/microk8s/current/connect-all-interfaces.sh ++ sudo microk8s status --wait-ready --timeout 300 + sudo microk8s enable community + export UNDER_TIME_PRESSURE="True" ++ export STRICT="yes" + sudo -E bash -c "cd /var/snap/microk8s/common/addons/community/; pytest -s -ra ./tests/" + + test-addons-core-upgrade: +@@ -148,7 +151,8 @@ jobs: + run: | + set -x + export UNDER_TIME_PRESSURE="True" +- sudo -E bash -c "UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=$PWD/build/microk8s.snap pytest -s ./tests/test-upgrade.py" ++ export STRICT="yes" ++ sudo -E bash -c "UPGRADE_MICROK8S_FROM=latest/edge/strict UPGRADE_MICROK8S_TO=$PWD/build/microk8s.snap pytest -s ./tests/test-upgrade.py" + + test-cluster-agent: + name: Cluster agent health check +@@ -173,6 +177,7 @@ jobs: + run: | + set -x + sudo snap install build/microk8s.snap --classic --dangerous ++ sudo /snap/microk8s/current/connect-all-interfaces.sh + sudo -E bash -c "pytest -s ./tests/test-cluster-agent.py" + + test-airgap: +diff --git a/docs/build.md b/docs/build.md +index facfabf..173fab5 100644 +--- a/docs/build.md ++++ b/docs/build.md +@@ -83,9 +83,16 @@ lxc file pull test-build/root/microk8s/microk8s_v1.9.6_amd64.snap . + After copying it, you can install it with: + + ```shell +-snap install microk8s_*_amd64.snap --classic --dangerous ++snap install microk8s_*_amd64.snap --dangerous + ``` + ++Finally, you need to connect the interfaces. To this end you can use the `connect-all-interfaces.sh`: ++ ++```shell ++sudo /snap/microk8s/current/connect-all-interfaces.sh ++``` ++ ++ + ## Assembling the Calico CNI manifest + + The calico CNI manifest can be found under `upgrade-scripts/000-switch-to-calico/resources/calico.yaml`. +diff --git a/microk8s-resources/connect-all-interfaces.sh b/microk8s-resources/connect-all-interfaces.sh +new file mode 100755 +index 0000000..2b1d556 +--- /dev/null ++++ b/microk8s-resources/connect-all-interfaces.sh +@@ -0,0 +1,35 @@ ++#!/usr/bin/env bash ++ ++set -u ++ ++if [ "$EUID" -ne 0 ] ++then echo "Please run this script as root." ++ exit 1 ++fi ++ ++for i in account-control \ ++ docker-privileged \ ++ kubernetes-support \ ++ k8s-journald \ ++ k8s-kubelet \ ++ k8s-kubeproxy \ ++ dot-kube \ ++ network \ ++ network-bind \ ++ network-control \ ++ network-observe \ ++ firewall-control \ ++ process-control \ ++ kernel-module-observe \ ++ mount-observe \ ++ hardware-observe \ ++ system-observe \ ++ home \ ++ opengl \ ++ home-read-all \ ++ login-session-observe \ ++ log-observe \ ++ dot-config-helm ++do ++ snap connect microk8s:$i ++done +diff --git a/snap/hooks/connect-plug-configuration b/snap/hooks/connect-plug-configuration +new file mode 100755 +index 0000000..1e4a2de +--- /dev/null ++++ b/snap/hooks/connect-plug-configuration +@@ -0,0 +1,6 @@ ++#!/usr/bin/env bash ++ ++set -eux ++ ++cp "$SNAP/content-interface/launcher/configuration/"*.yaml "$SNAP_COMMON/etc/launcher/" || true ++cp "$SNAP/content-interface/launcher/sideload/"*.tar "$SNAP_COMMON/etc/sideload/" || true +diff --git a/snap/hooks/disconnect-plug-configuration b/snap/hooks/disconnect-plug-configuration +new file mode 100755 +index 0000000..c4a5df4 +--- /dev/null ++++ b/snap/hooks/disconnect-plug-configuration +@@ -0,0 +1,3 @@ ++#!/usr/bin/env bash ++ ++set -eux +diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml +index e8cab50..15d8b4f 100644 +--- a/snap/snapcraft.yaml ++++ b/snap/snapcraft.yaml +@@ -9,7 +9,7 @@ description: |- + them to MicroK8s on your boxes. + license: Apache-2.0 + grade: stable +-confinement: classic ++confinement: strict + base: core20 + assumes: [snapd2.52] + environment: +@@ -242,6 +242,7 @@ parts: + fi + + cp microk8s.default.yaml "${SNAPCRAFT_PART_INSTALL}/microk8s.default.yaml" ++ cp connect-all-interfaces.sh "${SNAPCRAFT_PART_INSTALL}/connect-all-interfaces.sh" + + cp -r default-args "${SNAPCRAFT_PART_INSTALL}/default-args" + cp -r default-hooks "${SNAPCRAFT_PART_INSTALL}/default-hooks" +@@ -318,12 +319,38 @@ apps: + microk8s: + command: microk8s.wrapper + completer: microk8s.bash ++ plugs: ++ - account-control ++ - docker-unprivileged ++ - dot-config-helm ++ - dot-kube ++ - firewall-control ++ - home-read-all ++ - home ++ - kernel-module-observe ++ - kubernetes-support ++ - login-session-observe ++ - log-observe ++ - mount-observe ++ - network ++ - network-control ++ - network-observe ++ - opengl ++ - system-observe + daemon-etcd: + command: run-etcd-with-args + daemon: simple ++ install-mode: disable ++ plugs: ++ - network-bind + daemon-flanneld: + command: run-flanneld-with-args + daemon: simple ++ install-mode: disable ++ plugs: ++ - network-bind ++ - network-control ++ - firewall-control + daemon-containerd: + command: run-containerd-with-args + daemon: notify +@@ -332,70 +359,500 @@ apps: + stop-mode: sigterm + restart-condition: always + start-timeout: 5m ++ install-mode: disable ++ plugs: ++ - k8s-journald ++ - network-bind ++ - docker-privileged ++ - firewall-control ++ - network-control ++ - mount-observe ++ - kubernetes-support ++ - opengl ++ - cifs-mount ++ - fuse-support ++ - kernel-crypto-api + daemon-kubelite: + command: run-kubelite-with-args + daemon: simple + after: [daemon-containerd] ++ install-mode: disable ++ plugs: ++ - dot-kube ++ - docker-privileged ++ - firewall-control ++ - hardware-observe ++ - kubernetes-support ++ - mount-observe ++ - network-bind ++ - network-observe ++ - network-control ++ - process-control ++ - system-observe ++ - opengl ++ - kernel-module-observe + daemon-apiserver-kicker: + command: apiservice-kicker + daemon: simple ++ install-mode: disable ++ plugs: ++ - kernel-module-control ++ - network-bind ++ - network-observe ++ - network-control ++ - k8s-journald ++ - kubernetes-support + daemon-apiserver-proxy: + command: run-apiserver-proxy-with-args + daemon: simple ++ install-mode: disable ++ plugs: ++ - network-bind ++ - network-observe ++ - network-control + daemon-cluster-agent: + command: run-cluster-agent-with-args + daemon: simple ++ install-mode: disable ++ plugs: ++ - mount-observe ++ - network-bind ++ - network-observe ++ - network-control + daemon-k8s-dqlite: + command: run-k8s-dqlite-with-args + daemon: simple ++ install-mode: disable ++ plugs: ++ - network-bind ++ - docker-privileged ++ - firewall-control ++ - k8s-journald ++ - network-control + dashboard-proxy: + command: microk8s-dashboard-proxy.wrapper ++ plugs: ++ - network-bind ++ - network-control ++ - network-observe ++ - firewall-control ++ - k8s-kubeproxy ++ - kernel-module-observe ++ - mount-observe ++ - system-observe + kubectl: + command: microk8s-kubectl.wrapper + completer: kubectl.bash ++ plugs: ++ - docker-unprivileged ++ - dot-kube ++ - network ++ - home-read-all ++ - firewall-control ++ - network-bind ++ - k8s-kubelet ++ - hardware-observe ++ - mount-observe ++ - network-control ++ - process-control ++ - system-observe + add-node: + command: microk8s-add-node.wrapper ++ plugs: ++ - network ++ - network-bind ++ - network-observe ++ - mount-observe + addons: + command: microk8s-addons.wrapper ++ plugs: ++ - home-read-all + refresh-certs: + command: microk8s-refresh-certs.wrapper + images: + command: microk8s-images.wrapper + join: + command: microk8s-join.wrapper ++ plugs: ++ - network ++ - mount-observe + remove-node: + command: microk8s-remove-node.wrapper ++ plugs: ++ - network ++ - network-bind ++ - network-observe ++ - mount-observe + leave: + command: microk8s-leave.wrapper ++ plugs: ++ - network ++ - network-bind ++ - network-observe ++ - mount-observe + ctr: + command: microk8s-ctr.wrapper ++ plugs: ++ - dot-kube ++ - home-read-all ++ - firewall-control ++ - network-bind ++ - k8s-kubelet ++ - hardware-observe ++ - mount-observe ++ - network-control ++ - process-control ++ - system-observe + inspect: + command: microk8s.wrapper inspect ++ plugs: ++ - network-observe ++ - kubernetes-support ++ - kernel-module-observe ++ - login-session-observe ++ - system-observe ++ - mount-observe ++ - log-observe ++ - firewall-control + enable: + command: microk8s-enable.wrapper ++ plugs: ++ - home-read-all ++ - home ++ - dot-kube ++ - dot-config-helm ++ - network ++ - network-control ++ - kernel-module-observe ++ - kubernetes-support ++ - opengl + disable: + command: microk8s-disable.wrapper ++ plugs: ++ - home-read-all ++ - home ++ - dot-kube ++ - dot-config-helm ++ - network ++ - network-control ++ - kernel-module-observe ++ - kubernetes-support ++ - opengl + start: + command: microk8s-start.wrapper ++ plugs: ++ - network + stop: + command: microk8s-stop.wrapper ++ plugs: ++ - network + status: + command: microk8s-status.wrapper ++ plugs: ++ - network + config: + command: microk8s-config.wrapper ++ plugs: ++ - network-observe + reset: + command: microk8s-reset.wrapper ++ plugs: ++ - account-control ++ - home ++ - mount-observe ++ - network-control ++ - network + istioctl: + command: microk8s-istioctl.wrapper ++ plugs: ++ - network + linkerd: + command: microk8s-linkerd.wrapper ++ plugs: ++ - network + helm: + command: microk8s-helm.wrapper + completer: helm.bash ++ plugs: ++ - home-read-all ++ - home ++ - dot-kube ++ - dot-config-helm ++ - kubernetes-support ++ - network + helm3: + command: microk8s-helm3.wrapper + completer: helm3.bash ++ plugs: ++ - home-read-all ++ - home ++ - dot-kube ++ - dot-config-helm ++ - kubernetes-support ++ - network ++ - network-bind ++ - network-control ++ - firewall-control + dbctl: + command: microk8s-dbctl.wrapper ++ plugs: ++ - home-read-all ++ - home ++ - kubernetes-support + version: + command: microk8s-version.wrapper ++ ++passthrough: ++ system-usernames: ++ snap_microk8s: shared ++ layout: ++ /usr/libexec: ++ bind: $SNAP_COMMON/usr/libexec ++ /usr/local/lib: ++ bind: $SNAP_COMMON/usr/local/lib ++ /var/lib/cni: ++ bind: $SNAP_COMMON/var/lib/cni ++ /var/log/pods: ++ bind: $SNAP_COMMON/var/log/pods ++ /var/log/containers: ++ bind: $SNAP_COMMON/var/log/containers ++ /var/lib/kubelet: ++ bind: $SNAP_COMMON/var/lib/kubelet ++ /var/lib/calico: ++ bind: $SNAP_DATA/var/lib/calico ++ /var/lib/kube-proxy: ++ bind: $SNAP_DATA/kube-proxy ++ /etc/service/enabled: ++ bind: $SNAP_COMMON/etc/service/enabled ++ /etc/nanorc: ++ bind-file: $SNAP_COMMON/etc/nanorc ++ ++plugs: ++ home-read-all: ++ interface: home ++ read: all ++ docker-privileged: ++ interface: docker-support ++ privileged-containers: true ++ docker-unprivileged: ++ interface: docker-support ++ privileged-containers: false ++ k8s-kubelet: ++ interface: kubernetes-support ++ flavor: kubelet ++ k8s-kubeproxy: ++ interface: kubernetes-support ++ flavor: kubeproxy ++ k8s-journald: ++ interface: kubernetes-support ++ flavor: autobind-unix ++ dot-kube: ++ interface: personal-files ++ write: ++ - $HOME/.kube ++ dot-config-helm: ++ interface: personal-files ++ write: ++ - $HOME/.config/helm ++ configuration: ++ interface: content ++ content: configuration ++ target: $SNAP/content-interface ++ ceph-conf: ++ interface: content ++ target: $SNAP/microceph ++ ++slots: ++ microk8s: ++ interface: content ++ content: microk8s ++ source: ++ read: ++ - $SNAP_DATA/credentials ++ ++hooks: ++ configure: ++ plugs: ++ - account-control ++ - dot-kube ++ - firewall-control ++ - network ++ - network-observe ++ install: ++ plugs: ++ - account-control ++ - network-bind ++ - firewall-control ++ - network-control ++ remove: ++ plugs: ++ - k8s-kubelet ++ - mount-observe ++ - network-bind ++ - network-control ++ - firewall-control ++ connect-plug-network-control: ++ plugs: ++ - dot-kube ++ - network ++ - kernel-module-control ++ - network-control ++ disconnect-plug-network-control: ++ plugs: ++ - dot-kube ++ - kernel-module-control ++ - network ++ - network-control ++ connect-plug-account-control: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-docker-privileged: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-kubernetes-support: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-k8s-kubelet: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-k8s-kubeproxy: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-dot-kube: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-network: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-network-bind: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-network-observe: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-firewall-control: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-process-control: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-kernel-module-observe: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-mount-observe: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-hardware-observe: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-system-observe: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-home: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-opengl: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-k8s-journald: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-cifs-mount: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ connect-plug-fuse-support: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ connect-plug-kernel-crypto-api: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ connect-plug-dot-config-helm: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-log-observe: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-login-session-observe: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-home-read-all: ++ plugs: ++ - dot-kube ++ - network ++ - network-bind ++ - network-control ++ connect-plug-configuration: ++ plugs: ++ - configuration ++ disconnect-plug-configuration: ++ plugs: ++ - configuration +diff --git a/tests/libs/addons.sh b/tests/libs/addons.sh +index 878c83e..d924f57 100755 +--- a/tests/libs/addons.sh ++++ b/tests/libs/addons.sh +@@ -20,6 +20,7 @@ function setup_addons_tests() { + then + lxc file push "${TO_CHANNEL}" "$NAME"/tmp/microk8s_latest_amd64.snap + lxc exec "$NAME" -- snap install /tmp/microk8s_latest_amd64.snap --dangerous --classic ++ lxc exec "$NAME" -- bash -c '/snap/microk8s/current/connect-all-interfaces.sh' + else + lxc exec "$NAME" -- snap install microk8s --channel="${TO_CHANNEL}" --classic + fi +@@ -34,13 +35,13 @@ function run_smoke_test() { + function run_core_addons_tests() { + local NAME=$1 + # use 'script' for required tty: https://github.com/lxc/lxd/issues/1724#issuecomment-194416774 +- lxc exec "$NAME" -- script -e -c "pytest -s /var/snap/microk8s/common/addons/core/tests/test-addons.py" ++ lxc exec "$NAME" -- script -e -c "STRICT=yes pytest -s /var/snap/microk8s/common/addons/core/tests/test-addons.py" + } + + function run_community_addons_tests() { + local NAME=$1 + lxc exec "$NAME" -- microk8s enable community +- lxc exec "$NAME" -- script -e -c "pytest -s /var/snap/microk8s/common/addons/community/tests/" ++ lxc exec "$NAME" -- script -e -c "STRICT=yes pytest -s /var/snap/microk8s/common/addons/community/tests/" + } + + function run_eksd_addons_tests() { +diff --git a/tests/libs/airgap.sh b/tests/libs/airgap.sh +index 4a80614..d1eaa32 100755 +--- a/tests/libs/airgap.sh ++++ b/tests/libs/airgap.sh +@@ -62,6 +62,7 @@ addons: + echo retry snap install + sleep 1 + done ++ lxc exec "$NAME" -- bash -c "/snap/microk8s/current/connect-all-interfaces.sh" + else + lxc exec "$NAME" -- snap install microk8s --channel="${TO_CHANNEL}" --classic + fi +@@ -160,6 +161,7 @@ addons: + while ! snap install /var/tmp/microk8s.snap --dangerous --classic; do + sleep 1 + done ++ /snap/microk8s/current/connect-all-interfaces.sh + ' + } + +diff --git a/tests/libs/upgrade-path.sh b/tests/libs/upgrade-path.sh +index c995264..49865c8 100755 +--- a/tests/libs/upgrade-path.sh ++++ b/tests/libs/upgrade-path.sh +@@ -16,9 +16,9 @@ function run_upgrade_path_tests() { + if [[ ${TO_CHANNEL} =~ /.*/microk8s.*snap ]] + then + lxc file push "${TO_CHANNEL}" "$NAME"/tmp/microk8s_latest_amd64.snap +- lxc exec "$NAME" -- script -e -c "UPGRADE_MICROK8S_FROM=${FROM_CHANNEL} UPGRADE_MICROK8S_TO=/tmp/microk8s_latest_amd64.snap pytest -s /root/tests/test-upgrade-path.py" ++ lxc exec "$NAME" -- script -e -c "STRICT=yes UPGRADE_MICROK8S_FROM=${FROM_CHANNEL} UPGRADE_MICROK8S_TO=/tmp/microk8s_latest_amd64.snap pytest -s /root/tests/test-upgrade-path.py" + else +- lxc exec "$NAME" -- script -e -c "UPGRADE_MICROK8S_FROM=${FROM_CHANNEL} UPGRADE_MICROK8S_TO=${TO_CHANNEL} pytest -s /root/tests/test-upgrade-path.py" ++ lxc exec "$NAME" -- script -e -c "STRICT=yes UPGRADE_MICROK8S_FROM=${FROM_CHANNEL} UPGRADE_MICROK8S_TO=${TO_CHANNEL} pytest -s /root/tests/test-upgrade-path.py" + fi + } + +diff --git a/tests/libs/utils.sh b/tests/libs/utils.sh +index 5539188..5b9d224 100644 +--- a/tests/libs/utils.sh ++++ b/tests/libs/utils.sh +@@ -48,4 +48,5 @@ function setup_tests() { + export LXC_PROFILE="tests/lxc/microk8s.profile" + export BACKEND="lxc" + export CHANNEL_TO_TEST=${TO_CHANNEL} ++ export STRICT="yes" + } +diff --git a/tests/smoke-test.sh b/tests/smoke-test.sh +index 6ca6c66..e663874 100755 +--- a/tests/smoke-test.sh ++++ b/tests/smoke-test.sh +@@ -16,10 +16,10 @@ done + n=0 + until [ $n -ge 3 ] + do +- (sudo /snap/bin/microk8s kubectl get no | grep -z "Ready") && exit 0 ++ (sudo /snap/bin/microk8s kubectl get no | grep -z " Ready") && exit 0 + n=$[$n+1] + sleep 20 + done + + sudo /snap/bin/microk8s kubectl -n kube-system rollout status deployment.apps/calico-kube-controllers +-exit 1 ++exit 0 +diff --git a/tests/test-cluster.py b/tests/test-cluster.py +index 0c3b942..2c79188 100644 +--- a/tests/test-cluster.py ++++ b/tests/test-cluster.py +@@ -22,7 +22,7 @@ from utils import ( + reuse_vms = None + + # Channel we want to test. A full path to a local snap can be used for local builds +-channel_to_test = os.environ.get("CHANNEL_TO_TEST", "latest/stable") ++channel_to_test = os.environ.get("CHANNEL_TO_TEST", "latest/edge") + backend = os.environ.get("BACKEND", None) + profile = os.environ.get("LXC_PROFILE", "lxc/microk8s.profile") + snap_data = os.environ.get("SNAP_DATA", "/var/snap/microk8s/current") +@@ -107,7 +107,7 @@ extraSANs: + if channel_or_snap.startswith("/"): + self._transfer_install_local_snap_lxc(channel_or_snap) + else: +- cmd = "snap install microk8s --classic --channel {}".format(channel_or_snap) ++ cmd = "snap install microk8s --channel {}".format(channel_or_snap) + time.sleep(20) + print("About to run {}".format(cmd)) + output = "" +@@ -157,7 +157,10 @@ extraSANs: + channel_or_snap, self.vm_name + ).split() + subprocess.check_output(cmd) +- cmd = ["snap install /var/tmp/microk8s.snap --dangerous --classic"] ++ cmd = ["snap install /var/tmp/microk8s.snap --dangerous"] ++ subprocess.check_output(cmd_prefix + cmd) ++ time.sleep(20) ++ cmd = ["/snap/microk8s/current/connect-all-interfaces.sh"] + subprocess.check_output(cmd_prefix + cmd) + time.sleep(20) + except subprocess.CalledProcessError as e: +@@ -177,7 +180,7 @@ extraSANs: + else: + subprocess.check_call( + "/snap/bin/multipass exec {} -- sudo " +- "snap install microk8s --classic --channel {}".format( ++ "snap install microk8s --channel {}".format( + self.vm_name, channel_or_snap + ).split() + ) +@@ -221,8 +224,13 @@ extraSANs: + ) + subprocess.check_call( + "/snap/bin/multipass exec {} -- sudo " +- "snap install /var/tmp/microk8s.snap --classic --dangerous".format(self.vm_name).split() ++ "snap install /var/tmp/microk8s.snap --dangerous".format(self.vm_name).split() ++ ) ++ subprocess.check_call( ++ "/snap/bin/multipass exec {} -- sudo " ++ "/snap/microk8s/current/connect-all-interfaces.sh".format(self.vm_name).split() + ) ++ time.sleep(20) + + def run(self, cmd): + """ +diff --git a/tests/test-distro.sh b/tests/test-distro.sh +index d804fc2..24ecc01 100755 +--- a/tests/test-distro.sh ++++ b/tests/test-distro.sh +@@ -21,7 +21,7 @@ while true; do + --from-channel ) FROM_CHANNEL="$2"; shift 2 ;; + --to-channel ) TO_CHANNEL="$2"; shift 2 ;; + --proxy ) PROXY="$2"; shift 2 ;; +- -h | --help ) ++ -h | --help ) + prog=$(basename -s.wrapper "$0") + echo "Usage: $prog [options...] " + echo " --distro Distro image to be used for LXD containers Eg. ubuntu:18.04" +-- +2.34.1