Added examples of subPath volume mount with fuse sidecar

Makefile

@@ -130,23 +130,32 @@ push-examples: $(PUSH_EXAMPLES)
 
 define test-example-template
 ifneq ("$(EXAMPLE_TESTS)", "")
-EXAMPLE_TESTS += test-example-$(1)-$(2)
+EXAMPLE_TESTS += test-example-$(1)-$(2)-$(subst .,-,$(3))
 else
-EXAMPLE_TESTS := test-example-$(1)-$(2)
+EXAMPLE_TESTS := test-example-$(1)-$(2)-$(subst .,-,$(3))
 endif
 
 .PHONY: test-example-$1-$2
-test-example-$(1)-$(2):
-	./examples/check.sh ./$1/$2 mfcp-example-$1-$2 $3 $4 $5 $6
+test-example-$(1)-$(2)-$(subst .,-,$(3)):
+	./examples/check.sh ./$1/$2 $3 mfcp-example-$1-$2 $4 $5 $6 $7
 endef
 
-$(eval $(call test-example-template,proxy,mountpoint-s3,starter,/test.txt,busybox,/data/test.txt))
-$(eval $(call test-example-template,proxy,goofys,starter,/test.txt,busybox,/data/test.txt))
-$(eval $(call test-example-template,proxy,s3fs,starter,/test.txt,busybox,/data/test.txt))
-$(eval $(call test-example-template,proxy,ros3fs,starter,/test.txt,busybox,/data/test.txt))
-$(eval $(call test-example-template,proxy,sshfs,starter,/root/sshfs-example/test.txt,busybox,/data/test.txt))
-$(eval $(call test-example-template,starter,ros3fs,starter,/test.txt,busybox,/data/test.txt))
-$(eval $(call test-example-template,starter,sshfs,starter,/root/sshfs-example/test.txt,busybox,/data/test.txt))
+$(eval $(call test-example-template,proxy,mountpoint-s3,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt))
+$(eval $(call test-example-template,proxy,goofys,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt))
+$(eval $(call test-example-template,proxy,s3fs,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt))
+$(eval $(call test-example-template,proxy,ros3fs,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt))
+$(eval $(call test-example-template,proxy,sshfs,deploy.yaml,starter,/root/sshfs-example/subdir/test.txt,busybox,/data/subdir/test.txt))
+$(eval $(call test-example-template,starter,ros3fs,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt))
+$(eval $(call test-example-template,starter,sshfs,deploy.yaml,starter,/root/sshfs-example/subdir/test.txt,busybox,/data/subdir/test.txt))
+ifdef TEST_SUBPATH
+$(eval $(call test-example-template,proxy,mountpoint-s3,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt))
+$(eval $(call test-example-template,proxy,goofys,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt))
+$(eval $(call test-example-template,proxy,s3fs,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt))
+$(eval $(call test-example-template,proxy,ros3fs,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt))
+$(eval $(call test-example-template,proxy,sshfs,deploy-subpath.yaml,starter,/root/sshfs-example/subdir/test.txt,busybox,/data-subpath/test.txt))
+$(eval $(call test-example-template,starter,ros3fs,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt))
+$(eval $(call test-example-template,starter,sshfs,deploy-subpath.yaml,starter,/root/sshfs-example/subdir/test.txt,busybox,/data-subpath/test.txt))
+endif
 
 .PHONY: test-examples
 test-examples: $(EXAMPLE_TESTS)

README.md

@@ -96,22 +96,36 @@ pod "mfcp-example-proxy-mountpoint-s3" deleted
 ```
 
 ## NOTICE
+
+### fuse volume is mounted lazily
+
 meta-fuse-csi-plugin mounts FUSE implementations after the container started.
 Some applications may read the directory before mount.
-To avoid such race condition, please wait for the FUSE impl is mounted.
-`examples/proxy/mountpoint-s3/deploy.yaml` and `examples/check.sh` do such delaying.
-```yaml
-  - image: busybox
-    name: busybox
-    command: ["/bin/ash"]
-    args: ["-c", "while [[ ! \"$(/bin/mount | grep fuse)\" ]]; do echo \"waiting for mount\" && sleep 1; done; sleep infinity"]
-```
-or
-```bash
-function wait_for_fuse_mounted() {
-    while [[ ! $(kubectl exec $1 -c $2 -- /bin/mount | grep fuse) ]]; do echo "waiting for mount" && sleep 1; done
-}
-```
+
+To avoid such race condition, there are two solutions.
+
+1. Wait for the FUSE impl is mounted. `examples/proxy/mountpoint-s3/deploy.yaml` and `examples/check.sh` do such delaying.
+  ```yaml
+    - image: busybox
+      name: busybox
+      command: ["/bin/ash"]
+      args: ["-c", "while [[ ! \"$(/bin/mount | grep fuse)\" ]]; do echo \"waiting for mount\" && sleep 1; done; sleep infinity"]
+  ```
+  or
+  ```bash
+  function wait_for_fuse_mounted() {
+      while [[ ! $(kubectl exec $1 -c $2 -- /bin/mount | grep fuse) ]]; do echo "waiting for mount" && sleep 1; done
+  }
+  ```
+1. Use [sidecar](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/) container, a.k.a restartable init container (enabled by default since Kubernetes v1.30).
+   This can guarantees app containers can see the volume contents since the beggining. See `examples/proxy/mountpoint-s3/deploy-sidecar.yaml` for how to.
+   Please don't forget defining startup probe to make sure fuse volume is actually mounted before app containers are started.
+
+### `subPath` volume mount requires sidecar
+
+When fuse container is a normal container (i.e. not a sidecar), `subPath` volume mount creation by kubelet can race with actual fuse process startup.
+This race might cause that mounted `subPath` volume could be empty. Thus, when you use `subPath` volume mount, you have to make fuse container be a sidecar container.
+See `examples/proxy/mountpoint-s3/deploy-sidecar.yaml` for how to.
 
 ## Running E2E tests
 ### Tested Environment
@@ -125,6 +139,10 @@ You can run E2E tests with kind.
 
 ```console
 $ make test-e2e
+
+# if you test subpath volume mount, you can set TEST_SUBPATH=true
+# you will need kubernetes v1.30 or later because this tests needs sidecar containers
+$ TEST_SUBPATH=true make test-e2e
 ```
 
 ## How it works?

examples/check.sh

@@ -13,15 +13,16 @@ function wait_for_fuse_mounted() {
 }
 
 MANIFEST_DIR=$1 # path to example manifest
-POD_NAME=$2
-PROVIDER_CONTAINER=$3
-PROVIDED_FILENAME=$4
-MOUNTED_CONTAINER=$5
-MOUNTED_FILENAME=$6
+MAFNIFEST_FILENAME=$2
+POD_NAME=$3
+PROVIDER_CONTAINER=$4
+PROVIDED_FILENAME=$5
+MOUNTED_CONTAINER=$6
+MOUNTED_FILENAME=$7
 
 clean_up () {
     ARG=$?
-    kubectl delete -f ./deploy.yaml
+    kubectl delete -f ./${MAFNIFEST_FILENAME}
     exit $ARG
 }
 trap clean_up EXIT
@@ -30,7 +31,7 @@ cd $MANIFEST_DIR
 
 # Start to check the pod
 echo "Checking Pod \"$POD_NAME\"..."
-kubectl apply -f ./deploy.yaml
+kubectl apply -f ./${MAFNIFEST_FILENAME}
 
 # Waiting pod becomes ready
 wait_for_pod_becom_ready $POD_NAME

examples/proxy/gcsfuse/Dockerfile

@@ -22,7 +22,7 @@ COPY <<EOF /configure_minio.sh
 set -eux
 /usr/bin/mc alias set k8s-minio-dev http://localhost:9000 minioadmin minioadmin
 /usr/bin/mc mb k8s-minio-dev/test-bucket
-/usr/bin/mc cp /test.txt k8s-minio-dev/test-bucket
+/usr/bin/mc cp /test.txt k8s-minio-dev/test-bucket/subdir/
 EOF
 RUN chmod +x /configure_minio.sh
 

examples/proxy/goofys/Dockerfile

@@ -22,7 +22,7 @@ COPY <<EOF /configure_minio.sh
 set -eux
 /usr/bin/mc alias set k8s-minio-dev http://localhost:9000 minioadmin minioadmin
 /usr/bin/mc mb k8s-minio-dev/test-bucket
-/usr/bin/mc cp /test.txt k8s-minio-dev/test-bucket
+/usr/bin/mc cp /test.txt k8s-minio-dev/test-bucket/subdir/
 EOF
 RUN chmod +x /configure_minio.sh
 

examples/proxy/goofys/deploy-subpath.yaml

@@ -0,0 +1,63 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: mfcp-example-proxy-goofys
+  namespace: default
+spec:
+  terminationGracePeriodSeconds: 10
+  initContainers:
+  - name: minio
+    restartPolicy: Always
+    image: quay.io/minio/minio:latest
+    command: ["/bin/bash"]
+    args: ["-c", "minio server /data --console-address :9090"]
+  - name: starter
+    restartPolicy: Always
+    image: ghcr.io/pfnet-research/meta-fuse-csi-plugin/mfcp-example-proxy-goofys:latest
+    imagePullPolicy: IfNotPresent
+    command: ["/bin/bash"]
+    args: ["-c", "./configure_minio.sh && /goofys --endpoint http://localhost:9000 -f test-bucket /tmp"]
+    env:
+    - name: FUSERMOUNT3PROXY_FDPASSING_SOCKPATH
+      value: "/fusermount3-proxy/fuse-csi-ephemeral.sock"
+    - name: AWS_ACCESS_KEY_ID
+      value: "minioadmin"
+    - name: AWS_SECRET_ACCESS_KEY
+      value: "minioadmin"
+    volumeMounts:
+    - name: fuse-fd-passing
+      mountPath: /fusermount3-proxy
+    - name: fuse-csi-ephemeral
+      mountPath: /data
+      readOnly: true
+      mountPropagation: HostToContainer
+    startupProbe:
+      exec:
+        command: ['sh', '-c', 'mount | grep /data | grep fuse']
+      failureThreshold: 300
+      periodSeconds: 1
+  containers:
+  - image: busybox
+    name: busybox
+    command: ["sleep"]
+    args: ["infinity"]
+    volumeMounts:
+    - name: fuse-csi-ephemeral
+      mountPath: /data
+      readOnly: true
+      mountPropagation: HostToContainer
+    - name: fuse-csi-ephemeral
+      mountPath: /data-subpath
+      readOnly: true
+      subPath: subdir
+      mountPropagation: HostToContainer
+  volumes:
+  - name: fuse-fd-passing
+    emptyDir: {}
+  - name: fuse-csi-ephemeral
+    csi:
+      driver: meta-fuse-csi-plugin.csi.storage.pfn.io
+      readOnly: true
+      volumeAttributes:
+        fdPassingEmptyDirName: fuse-fd-passing
+        fdPassingSocketName: fuse-csi-ephemeral.sock

examples/proxy/mountpoint-s3/Dockerfile

@@ -22,7 +22,7 @@ COPY <<EOF /configure_minio.sh
 set -eux
 /usr/bin/mc alias set k8s-minio-dev http://localhost:9000 minioadmin minioadmin
 /usr/bin/mc mb k8s-minio-dev/test-bucket
-/usr/bin/mc cp /test.txt k8s-minio-dev/test-bucket
+/usr/bin/mc cp /test.txt k8s-minio-dev/test-bucket/subdir/
 EOF
 RUN chmod +x /configure_minio.sh
 

examples/proxy/mountpoint-s3/deploy-subpath.yaml

@@ -0,0 +1,63 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: mfcp-example-proxy-mountpoint-s3
+  namespace: default
+spec:
+  terminationGracePeriodSeconds: 10
+  initContainers:
+  - name: minio
+    restartPolicy: Always
+    image: quay.io/minio/minio:latest
+    command: ["/bin/bash"]
+    args: ["-c", "minio server /data --console-address :9090"]
+  - name: starter
+    restartPolicy: Always
+    image: ghcr.io/pfnet-research/meta-fuse-csi-plugin/mfcp-example-proxy-mountpoint-s3:latest
+    imagePullPolicy: IfNotPresent
+    command: ["/bin/bash"]
+    args: ["-c", "./configure_minio.sh && mount-s3 test-bucket /tmp --endpoint-url \"http://localhost:9000\" -d --allow-other --auto-unmount --foreground --force-path-style"] # "--auto-unmount" forces mountpoint-s3 to use fusermount3
+    env:
+    - name: FUSERMOUNT3PROXY_FDPASSING_SOCKPATH # UDS path to connect to csi driver
+      value: "/fusermount3-proxy/fuse-csi-ephemeral.sock"
+    - name: AWS_ACCESS_KEY_ID
+      value: "minioadmin"
+    - name: AWS_SECRET_ACCESS_KEY
+      value: "minioadmin"
+    volumeMounts:
+    - name: fuse-fd-passing # dir for UDS
+      mountPath: /fusermount3-proxy
+    - name: fuse-csi-ephemeral
+      mountPath: /data
+      readOnly: true
+      mountPropagation: HostToContainer
+    startupProbe:
+      exec:
+        command: ['sh', '-c', 'mount | grep /data | grep fuse']
+      failureThreshold: 300
+      periodSeconds: 1
+  containers:
+  - image: busybox
+    name: busybox
+    command: ["/bin/ash"]
+    args: ["-c", "while [[ ! \"$(/bin/mount | grep fuse)\" ]]; do echo \"waiting for mount\" && sleep 1; done; sleep infinity"]
+    volumeMounts:
+    - name: fuse-csi-ephemeral
+      mountPath: /data
+      readOnly: true
+      mountPropagation: HostToContainer
+    - name: fuse-csi-ephemeral
+      mountPath: /data-subpath
+      readOnly: true
+      subPath: subdir
+      mountPropagation: HostToContainer
+  volumes:
+  - name: fuse-fd-passing # dir for UDS
+    emptyDir: {}
+  - name: fuse-csi-ephemeral # volume with meta-fuse-csi-plugin
+    csi:
+      driver: meta-fuse-csi-plugin.csi.storage.pfn.io
+      readOnly: true
+      volumeAttributes:
+        fdPassingEmptyDirName: fuse-fd-passing
+        fdPassingSocketName: fuse-csi-ephemeral.sock

examples/proxy/ros3fs/Dockerfile

@@ -33,7 +33,7 @@ COPY <<EOF /configure_minio.sh
 set -eux
 /usr/bin/mc alias set k8s-minio-dev http://localhost:9000 minioadmin minioadmin
 /usr/bin/mc mb k8s-minio-dev/test-bucket
-/usr/bin/mc cp /test.txt k8s-minio-dev/test-bucket
+/usr/bin/mc cp /test.txt k8s-minio-dev/test-bucket/subdir/
 EOF
 RUN chmod +x /configure_minio.sh
 

examples/proxy/ros3fs/deploy-subpath.yaml

@@ -0,0 +1,67 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: mfcp-example-proxy-ros3fs
+  namespace: default
+spec:
+  terminationGracePeriodSeconds: 10
+  initContainers:
+  - name: minio
+    restartPolicy: Always
+    image: quay.io/minio/minio:latest
+    command: ["/bin/bash"]
+    args: ["-c", "minio server /data --console-address :9090"]
+  - name: starter
+    restartPolicy: Always
+    image: ghcr.io/pfnet-research/meta-fuse-csi-plugin/mfcp-example-proxy-ros3fs:latest
+    imagePullPolicy: IfNotPresent
+    command: ["/bin/bash"]
+    args: ["-c", "./configure_minio.sh && touch /dev/fuse && /ros3fs /tmp --endpoint=http://localhost:9000 --bucket_name=test-bucket/ --cache_dir=/ro3fs-temp -f"]
+    env:
+    - name: FUSERMOUNT3PROXY_FDPASSING_SOCKPATH
+      value: "/fusermount3-proxy/fuse-csi-ephemeral.sock"
+    - name: AWS_ACCESS_KEY_ID
+      value: "minioadmin"
+    - name: AWS_SECRET_ACCESS_KEY
+      value: "minioadmin"
+    volumeMounts:
+    - name: fuse-fd-passing
+      mountPath: /fusermount3-proxy
+    - name: ros3fs-temp
+      mountPath: /ros3fs-temp
+    - name: fuse-csi-ephemeral
+      mountPath: /data
+      readOnly: true
+      mountPropagation: HostToContainer
+    startupProbe:
+      exec:
+        command: ['sh', '-c', 'mount | grep /data | grep fuse']
+      failureThreshold: 300
+      periodSeconds: 1
+  containers:
+  - image: busybox
+    name: busybox
+    command: ["sleep"]
+    args: ["infinity"]
+    volumeMounts:
+    - name: fuse-csi-ephemeral
+      mountPath: /data
+      readOnly: true
+      mountPropagation: HostToContainer
+    - name: fuse-csi-ephemeral
+      mountPath: /data-subpath
+      readOnly: true
+      subPath: subdir
+      mountPropagation: HostToContainer
+  volumes:
+  - name: fuse-fd-passing
+    emptyDir: {}
+  - name: ros3fs-temp
+    emptyDir: {}
+  - name: fuse-csi-ephemeral
+    csi:
+      driver: meta-fuse-csi-plugin.csi.storage.pfn.io
+      readOnly: true
+      volumeAttributes:
+        fdPassingEmptyDirName: fuse-fd-passing
+        fdPassingSocketName: fuse-csi-ephemeral.sock

examples/proxy/s3fs/Dockerfile

@@ -5,8 +5,8 @@ ADD . .
 RUN make fusermount3-proxy BINDIR=/bin
 
 FROM ubuntu:22.04
-
 ARG TARGETARCH
+
 RUN apt update && apt upgrade -y
 RUN apt install -y ca-certificates wget libfuse2 fuse3
 
@@ -22,7 +22,7 @@ COPY <<EOF /configure_minio.sh
 set -eux
 /usr/bin/mc alias set k8s-minio-dev http://localhost:9000 minioadmin minioadmin
 /usr/bin/mc mb k8s-minio-dev/test-bucket
-/usr/bin/mc cp /test.txt k8s-minio-dev/test-bucket
+/usr/bin/mc cp /test.txt k8s-minio-dev/test-bucket/subdir/
 EOF
 RUN chmod +x /configure_minio.sh