From 836a07672da359c2f1c730c9a553b3899e22b226 Mon Sep 17 00:00:00 2001 From: Shingo Omura Date: Fri, 6 Sep 2024 01:17:09 +0900 Subject: [PATCH] Add subpath mount test with sidecar (restartable init containers) --- Makefile | 31 ++++++--- README.md | 46 +++++++++---- examples/check.sh | 15 +++-- examples/proxy/gcsfuse/Dockerfile | 2 +- examples/proxy/goofys/Dockerfile | 2 +- examples/proxy/goofys/deploy-subpath.yaml | 63 +++++++++++++++++ examples/proxy/mountpoint-s3/Dockerfile | 2 +- .../proxy/mountpoint-s3/deploy-subpath.yaml | 63 +++++++++++++++++ examples/proxy/ros3fs/Dockerfile | 2 +- examples/proxy/ros3fs/deploy-subpath.yaml | 67 +++++++++++++++++++ examples/proxy/s3fs/Dockerfile | 4 +- examples/proxy/s3fs/deploy-subpath.yaml | 64 ++++++++++++++++++ examples/proxy/sshfs/Dockerfile | 5 +- examples/proxy/sshfs/deploy-subpath.yaml | 52 ++++++++++++++ examples/starter/ros3fs/Dockerfile | 2 +- examples/starter/ros3fs/deploy-subpath.yaml | 65 ++++++++++++++++++ examples/starter/sshfs/Dockerfile | 4 +- examples/starter/sshfs/deploy-subpath.yaml | 49 ++++++++++++++ 18 files changed, 495 insertions(+), 43 deletions(-) create mode 100644 examples/proxy/goofys/deploy-subpath.yaml create mode 100644 examples/proxy/mountpoint-s3/deploy-subpath.yaml create mode 100644 examples/proxy/ros3fs/deploy-subpath.yaml create mode 100644 examples/proxy/s3fs/deploy-subpath.yaml create mode 100644 examples/proxy/sshfs/deploy-subpath.yaml create mode 100644 examples/starter/ros3fs/deploy-subpath.yaml create mode 100644 examples/starter/sshfs/deploy-subpath.yaml diff --git a/Makefile b/Makefile index 5d2b550..01d4107 100755 --- a/Makefile +++ b/Makefile @@ -130,23 +130,32 @@ push-examples: $(PUSH_EXAMPLES) define test-example-template ifneq ("$(EXAMPLE_TESTS)", "") -EXAMPLE_TESTS += test-example-$(1)-$(2) +EXAMPLE_TESTS += test-example-$(1)-$(2)-$(subst .,-,$(3)) else -EXAMPLE_TESTS := test-example-$(1)-$(2) +EXAMPLE_TESTS := test-example-$(1)-$(2)-$(subst .,-,$(3)) endif .PHONY: test-example-$1-$2 -test-example-$(1)-$(2): - ./examples/check.sh ./$1/$2 mfcp-example-$1-$2 $3 $4 $5 $6 +test-example-$(1)-$(2)-$(subst .,-,$(3)): + ./examples/check.sh ./$1/$2 $3 mfcp-example-$1-$2 $4 $5 $6 $7 endef -$(eval $(call test-example-template,proxy,mountpoint-s3,starter,/test.txt,busybox,/data/test.txt)) -$(eval $(call test-example-template,proxy,goofys,starter,/test.txt,busybox,/data/test.txt)) -$(eval $(call test-example-template,proxy,s3fs,starter,/test.txt,busybox,/data/test.txt)) -$(eval $(call test-example-template,proxy,ros3fs,starter,/test.txt,busybox,/data/test.txt)) -$(eval $(call test-example-template,proxy,sshfs,starter,/root/sshfs-example/test.txt,busybox,/data/test.txt)) -$(eval $(call test-example-template,starter,ros3fs,starter,/test.txt,busybox,/data/test.txt)) -$(eval $(call test-example-template,starter,sshfs,starter,/root/sshfs-example/test.txt,busybox,/data/test.txt)) +$(eval $(call test-example-template,proxy,mountpoint-s3,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt)) +$(eval $(call test-example-template,proxy,goofys,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt)) +$(eval $(call test-example-template,proxy,s3fs,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt)) +$(eval $(call test-example-template,proxy,ros3fs,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt)) +$(eval $(call test-example-template,proxy,sshfs,deploy.yaml,starter,/root/sshfs-example/subdir/test.txt,busybox,/data/subdir/test.txt)) +$(eval $(call test-example-template,starter,ros3fs,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt)) +$(eval $(call test-example-template,starter,sshfs,deploy.yaml,starter,/root/sshfs-example/subdir/test.txt,busybox,/data/subdir/test.txt)) +ifdef TEST_SUBPATH +$(eval $(call test-example-template,proxy,mountpoint-s3,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt)) +$(eval $(call test-example-template,proxy,goofys,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt)) +$(eval $(call test-example-template,proxy,s3fs,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt)) +$(eval $(call test-example-template,proxy,ros3fs,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt)) +$(eval $(call test-example-template,proxy,sshfs,deploy-subpath.yaml,starter,/root/sshfs-example/subdir/test.txt,busybox,/data-subpath/test.txt)) +$(eval $(call test-example-template,starter,ros3fs,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt)) +$(eval $(call test-example-template,starter,sshfs,deploy-subpath.yaml,starter,/root/sshfs-example/subdir/test.txt,busybox,/data-subpath/test.txt)) +endif .PHONY: test-examples test-examples: $(EXAMPLE_TESTS) diff --git a/README.md b/README.md index e4c19da..f9acdf5 100644 --- a/README.md +++ b/README.md @@ -96,22 +96,36 @@ pod "mfcp-example-proxy-mountpoint-s3" deleted ``` ## NOTICE + +### fuse volume is mounted lazily + meta-fuse-csi-plugin mounts FUSE implementations after the container started. Some applications may read the directory before mount. -To avoid such race condition, please wait for the FUSE impl is mounted. -`examples/proxy/mountpoint-s3/deploy.yaml` and `examples/check.sh` do such delaying. -```yaml - - image: busybox - name: busybox - command: ["/bin/ash"] - args: ["-c", "while [[ ! \"$(/bin/mount | grep fuse)\" ]]; do echo \"waiting for mount\" && sleep 1; done; sleep infinity"] -``` -or -```bash -function wait_for_fuse_mounted() { - while [[ ! $(kubectl exec $1 -c $2 -- /bin/mount | grep fuse) ]]; do echo "waiting for mount" && sleep 1; done -} -``` + +To avoid such race condition, there are two solutions. + +1. Wait for the FUSE impl is mounted. `examples/proxy/mountpoint-s3/deploy.yaml` and `examples/check.sh` do such delaying. + ```yaml + - image: busybox + name: busybox + command: ["/bin/ash"] + args: ["-c", "while [[ ! \"$(/bin/mount | grep fuse)\" ]]; do echo \"waiting for mount\" && sleep 1; done; sleep infinity"] + ``` + or + ```bash + function wait_for_fuse_mounted() { + while [[ ! $(kubectl exec $1 -c $2 -- /bin/mount | grep fuse) ]]; do echo "waiting for mount" && sleep 1; done + } + ``` +1. Use [sidecar](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/) container, a.k.a restartable init container (enabled by default since Kubernetes v1.30). + This can guarantees app containers can see the volume contents since the beggining. See `examples/proxy/mountpoint-s3/deploy-sidecar.yaml` for how to. + Please don't forget defining startup probe to make sure fuse volume is actually mounted before app containers are started. + +### `subPath` volume mount requires sidecar + +When fuse container is a normal container (i.e. not a sidecar), `subPath` volume mount creation by kubelet can race with actual fuse process startup. +This race might cause that mounted `subPath` volume could be empty. Thus, when you use `subPath` volume mount, you have to make fuse container be a sidecar container. +See `examples/proxy/mountpoint-s3/deploy-sidecar.yaml` for how to. ## Running E2E tests ### Tested Environment @@ -125,6 +139,10 @@ You can run E2E tests with kind. ```console $ make test-e2e + +# if you test subpath volume mount, you can set TEST_SUBPATH=true +# you will need kubernetes v1.30 or later because this tests needs sidecar containers +$ TEST_SUBPATH=true make test-e2e ``` ## How it works? diff --git a/examples/check.sh b/examples/check.sh index 4f2c4e5..314716a 100755 --- a/examples/check.sh +++ b/examples/check.sh @@ -13,15 +13,16 @@ function wait_for_fuse_mounted() { } MANIFEST_DIR=$1 # path to example manifest -POD_NAME=$2 -PROVIDER_CONTAINER=$3 -PROVIDED_FILENAME=$4 -MOUNTED_CONTAINER=$5 -MOUNTED_FILENAME=$6 +MAFNIFEST_FILENAME=$2 +POD_NAME=$3 +PROVIDER_CONTAINER=$4 +PROVIDED_FILENAME=$5 +MOUNTED_CONTAINER=$6 +MOUNTED_FILENAME=$7 clean_up () { ARG=$? - kubectl delete -f ./deploy.yaml + kubectl delete -f ./${MAFNIFEST_FILENAME} exit $ARG } trap clean_up EXIT @@ -30,7 +31,7 @@ cd $MANIFEST_DIR # Start to check the pod echo "Checking Pod \"$POD_NAME\"..." -kubectl apply -f ./deploy.yaml +kubectl apply -f ./${MAFNIFEST_FILENAME} # Waiting pod becomes ready wait_for_pod_becom_ready $POD_NAME diff --git a/examples/proxy/gcsfuse/Dockerfile b/examples/proxy/gcsfuse/Dockerfile index 08b9d21..1f4d7ef 100644 --- a/examples/proxy/gcsfuse/Dockerfile +++ b/examples/proxy/gcsfuse/Dockerfile @@ -22,7 +22,7 @@ COPY < /root/sshfs-example/test.txt +RUN mkdir -p /root/sshfs-example/subdir + +RUN echo "This file is an example for sshfs" > /root/sshfs-example/subdir/test.txt COPY < /root/sshfs-example/test.txt +RUN mkdir -p /root/sshfs-example/subdir +RUN echo "This file is an example for sshfs" > /root/sshfs-example/subdir/test.txt COPY <