From 7bd1475a8744978b705ba53290ffd26d78970c5a Mon Sep 17 00:00:00 2001 From: daemon1024 Date: Thu, 29 Feb 2024 16:40:16 +0530 Subject: [PATCH 1/8] chore(dockerfile): keep CORE compiled system monitor in the kubearmor container This will help us avoid running init container if the kernel has BTF Signed-off-by: daemon1024 --- Dockerfile | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 28fde5ef8a..f467d9d181 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,9 +18,29 @@ RUN go install github.com/golang/protobuf/protoc-gen-go@latest RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest RUN make + +WORKDIR /usr/src/KubeArmor/BPF + +RUN apk add curl +# install bpftool +RUN arch=$(uname -m) bpftool_version=v7.3.0 && \ + if [[ "$arch" == "aarch64" ]]; then \ + arch=arm64; \ + elif [[ "$arch" == "x86_64" ]]; then \ + arch=amd64; \ + fi && \ + curl -LO https://github.com/libbpf/bpftool/releases/download/$bpftool_version/bpftool-$bpftool_version-$arch.tar.gz && \ + tar -xzf bpftool-$bpftool_version-$arch.tar.gz -C /usr/local/bin && \ + chmod +x /usr/local/bin/bpftool + + +COPY ./KubeArmor/BPF . + +RUN make + ### Make executable image -FROM alpine:3.18 as kubearmor +FROM alpine:3.19 as kubearmor RUN echo "@community http://dl-cdn.alpinelinux.org/alpine/edge/community" | tee -a /etc/apk/repositories @@ -28,6 +48,7 @@ RUN apk --no-cache update RUN apk add apparmor@community apparmor-utils@community bash COPY --from=builder /usr/src/KubeArmor/KubeArmor/kubearmor /KubeArmor/kubearmor +COPY --from=builder /usr/src/KubeArmor/BPF/*.o /opt/kubearmor/BPF/ COPY --from=builder /usr/src/KubeArmor/KubeArmor/templates/* /KubeArmor/templates/ ENTRYPOINT ["/KubeArmor/kubearmor"] From 4d33888a797af1e6b26ee95c90c0569d29f50671 Mon Sep 17 00:00:00 2001 From: daemon1024 Date: Thu, 29 Feb 2024 16:50:48 +0530 Subject: [PATCH 2/8] enhance(operator): remove init container if btf is present We can work with system monitor object file in the kubearmor container if btf is present on the node Signed-off-by: daemon1024 --- pkg/KubeArmorOperator/common/defaults.go | 44 +++++++++++-------- .../internal/controller/resources.go | 15 +++++-- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/pkg/KubeArmorOperator/common/defaults.go b/pkg/KubeArmorOperator/common/defaults.go index 1f1e4605a6..283903024f 100644 --- a/pkg/KubeArmorOperator/common/defaults.go +++ b/pkg/KubeArmorOperator/common/defaults.go @@ -218,13 +218,23 @@ func ShortSHA(s string) string { return hex.EncodeToString(res)[:5] } -var CommonVolumes = []corev1.Volume{ +var BPFVolumes = []corev1.Volume{ { Name: "bpf", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, +} + +var BPFVolumesMount = []corev1.VolumeMount{ + { + Name: "bpf", + MountPath: "/opt/kubearmor/BPF", + }, +} + +var CommonVolumes = []corev1.Volume{ { Name: "sys-kernel-debug-path", VolumeSource: corev1.VolumeSource{ @@ -234,31 +244,13 @@ var CommonVolumes = []corev1.Volume{ }, }, }, - { - Name: "os-release-path", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc/os-release", - Type: &HostPathFile, - }, - }, - }, } var CommonVolumesMount = []corev1.VolumeMount{ - { - Name: "bpf", - MountPath: "/opt/kubearmor/BPF", - }, { Name: "sys-kernel-debug-path", MountPath: "/sys/kernel/debug", }, - { - Name: "os-release-path", - MountPath: "/media/root/etc/os-release", - ReadOnly: true, - }, } var KubeArmorCaVolume = []corev1.Volume{ @@ -369,6 +361,15 @@ var KernelHeaderVolumes = []corev1.Volume{ }, }, }, + { + Name: "os-release-path", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/os-release", + Type: &HostPathFile, + }, + }, + }, } var KernelHeaderVolumesMount = []corev1.VolumeMount{ @@ -382,6 +383,11 @@ var KernelHeaderVolumesMount = []corev1.VolumeMount{ MountPath: "/lib/modules", ReadOnly: true, }, + { + Name: "os-release-path", + MountPath: "/media/root/etc/os-release", + ReadOnly: true, + }, } func GetFreeRandSuffix(c *kubernetes.Clientset, namespace string) (suffix string, err error) { diff --git a/pkg/KubeArmorOperator/internal/controller/resources.go b/pkg/KubeArmorOperator/internal/controller/resources.go index 5c339c3e48..3beb43abbe 100644 --- a/pkg/KubeArmorOperator/internal/controller/resources.go +++ b/pkg/KubeArmorOperator/internal/controller/resources.go @@ -41,11 +41,17 @@ func generateDaemonset(name, enforcer, runtime, socket, btfPresent, apparmorfs, commonVolMnts := common.CommonVolumesMount if btfPresent == "no" { commonVols = append(commonVols, common.KernelHeaderVolumes...) + commonVols = append(commonVols, common.BPFVolumes...) commonVolMnts = append(commonVolMnts, common.KernelHeaderVolumesMount...) + commonVolMnts = append(commonVolMnts, common.BPFVolumesMount...) } vols = append(vols, commonVols...) volMnts = append(volMnts, commonVolMnts...) daemonset := deployments.GenerateDaemonSet("generic", common.Namespace) + + if btfPresent != "no" { + daemonset.Spec.Template.Spec.InitContainers = []corev1.Container{} + } daemonset.Name = name labels := map[string]string{ common.EnforcerLabel: enforcer, @@ -80,7 +86,6 @@ func generateDaemonset(name, enforcer, runtime, socket, btfPresent, apparmorfs, common.AddOrReplaceArg("-tlsEnabled=false", "-tlsEnabled=true", &daemonset.Spec.Template.Spec.Containers[0].Args) } daemonset.Spec.Template.Spec.Volumes = vols - daemonset.Spec.Template.Spec.InitContainers[0].VolumeMounts = commonVolMnts daemonset.Spec.Template.Spec.Containers[0].VolumeMounts = volMnts // update images @@ -98,8 +103,12 @@ func generateDaemonset(name, enforcer, runtime, socket, btfPresent, apparmorfs, daemonset.Spec.Template.Spec.Containers[0].Image = common.GetApplicationImage(common.KubeArmorName) daemonset.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullPolicy(common.KubeArmorImagePullPolicy) - daemonset.Spec.Template.Spec.InitContainers[0].Image = common.GetApplicationImage(common.KubeArmorInitName) - daemonset.Spec.Template.Spec.InitContainers[0].ImagePullPolicy = corev1.PullPolicy(common.KubeArmorInitImagePullPolicy) + + if btfPresent == "no" { + daemonset.Spec.Template.Spec.InitContainers[0].VolumeMounts = commonVolMnts + daemonset.Spec.Template.Spec.InitContainers[0].Image = common.GetApplicationImage(common.KubeArmorInitName) + daemonset.Spec.Template.Spec.InitContainers[0].ImagePullPolicy = corev1.PullPolicy(common.KubeArmorInitImagePullPolicy) + } daemonset = addOwnership(daemonset).(*appsv1.DaemonSet) fmt.Printf("generated daemonset: %v", daemonset) From de2a1db96fbd15b66090df2b0054f02bfd11aec8 Mon Sep 17 00:00:00 2001 From: daemon1024 Date: Fri, 1 Mar 2024 12:08:24 +0530 Subject: [PATCH 3/8] chore(ci): build operator in ci-test-controllers Signed-off-by: daemon1024 --- .github/workflows/ci-test-controllers.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci-test-controllers.yml b/.github/workflows/ci-test-controllers.yml index e2e6fa4516..34f5533eeb 100644 --- a/.github/workflows/ci-test-controllers.yml +++ b/.github/workflows/ci-test-controllers.yml @@ -53,6 +53,11 @@ jobs: if: steps.filter.outputs.kubearmor == 'true' run: GITHUB_SHA=$GITHUB_SHA ./KubeArmor/build/build_kubearmor.sh + - name: Build Kubearmor-Operator + working-directory: pkg/KubeArmorOperator + run: | + make docker-build + - name: Build KubeArmorController run: make -C pkg/KubeArmorController/ docker-build TAG=latest From 852a799981c8adf9a177a504a9d07e91128579f0 Mon Sep 17 00:00:00 2001 From: Aryan-sharma11 Date: Tue, 14 May 2024 09:17:12 +0530 Subject: [PATCH 4/8] remove hostpath /sys/fs/bpf Signed-off-by: Aryan-sharma11 --- .github/workflows/ci-test-controllers.yml | 5 -- Dockerfile | 3 +- deployments/get/objects.go | 13 ---- deployments/helm/KubeArmor/values.yaml | 62 ------------------- pkg/KubeArmorOperator/common/defaults.go | 18 ------ .../internal/controller/resources.go | 5 +- 6 files changed, 5 insertions(+), 101 deletions(-) diff --git a/.github/workflows/ci-test-controllers.yml b/.github/workflows/ci-test-controllers.yml index 34f5533eeb..e2e6fa4516 100644 --- a/.github/workflows/ci-test-controllers.yml +++ b/.github/workflows/ci-test-controllers.yml @@ -53,11 +53,6 @@ jobs: if: steps.filter.outputs.kubearmor == 'true' run: GITHUB_SHA=$GITHUB_SHA ./KubeArmor/build/build_kubearmor.sh - - name: Build Kubearmor-Operator - working-directory: pkg/KubeArmorOperator - run: | - make docker-build - - name: Build KubeArmorController run: make -C pkg/KubeArmorController/ docker-build TAG=latest diff --git a/Dockerfile b/Dockerfile index f467d9d181..a15279fd98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ RUN apk --no-cache update RUN apk add apparmor@community apparmor-utils@community bash COPY --from=builder /usr/src/KubeArmor/KubeArmor/kubearmor /KubeArmor/kubearmor -COPY --from=builder /usr/src/KubeArmor/BPF/*.o /opt/kubearmor/BPF/ +COPY --from=builder /usr/src/KubeArmor/BPF/*.o /KubeArmor/BPF/ COPY --from=builder /usr/src/KubeArmor/KubeArmor/templates/* /KubeArmor/templates/ ENTRYPOINT ["/KubeArmor/kubearmor"] @@ -88,6 +88,7 @@ RUN groupadd --gid 1000 default \ COPY LICENSE /licenses/license.txt COPY --from=builder --chown=default:default /usr/src/KubeArmor/KubeArmor/kubearmor /KubeArmor/kubearmor +COPY --from=builder --chown=default:default /usr/src/KubeArmor/BPF/*.o /KubeArmor/BPF/ COPY --from=builder --chown=default:default /usr/src/KubeArmor/KubeArmor/templates/* /KubeArmor/templates/ # TODO diff --git a/deployments/get/objects.go b/deployments/get/objects.go index 41fe81e37a..692796dd2a 100644 --- a/deployments/get/objects.go +++ b/deployments/get/objects.go @@ -270,10 +270,6 @@ func GenerateDaemonSet(env, namespace string) *appsv1.DaemonSet { MountPath: "/lib/modules", ReadOnly: true, }, - { - Name: "sys-fs-bpf-path", //BPF (read-write) - MountPath: "/sys/fs/bpf", - }, { Name: "sys-kernel-security-path", //LSM (read-only) MountPath: "/sys/kernel/security", @@ -305,15 +301,6 @@ func GenerateDaemonSet(env, namespace string) *appsv1.DaemonSet { }, }, }, - { - Name: "sys-fs-bpf-path", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/sys/fs/bpf", - Type: &hostPathDirectory, - }, - }, - }, { Name: "sys-kernel-security-path", VolumeSource: corev1.VolumeSource{ diff --git a/deployments/helm/KubeArmor/values.yaml b/deployments/helm/KubeArmor/values.yaml index 409befa0ef..e9f75c5fb0 100644 --- a/deployments/helm/KubeArmor/values.yaml +++ b/deployments/helm/KubeArmor/values.yaml @@ -163,8 +163,6 @@ kubearmor: - mountPath: /lib/modules name: lib-modules-path readOnly: true - - mountPath: /sys/fs/bpf - name: sys-fs-bpf-path - mountPath: /sys/kernel/security name: sys-kernel-security-path - mountPath: /sys/kernel/debug @@ -183,8 +181,6 @@ kubearmor: - mountPath: /lib/modules name: lib-modules-path readOnly: true - - mountPath: /sys/fs/bpf - name: sys-fs-bpf-path - mountPath: /sys/kernel/security name: sys-kernel-security-path - mountPath: /sys/kernel/debug @@ -205,8 +201,6 @@ kubearmor: - mountPath: /lib/modules name: lib-modules-path readOnly: true - - mountPath: /sys/fs/bpf - name: sys-fs-bpf-path - mountPath: /sys/kernel/security name: sys-kernel-security-path - mountPath: /sys/kernel/debug @@ -227,8 +221,6 @@ kubearmor: - mountPath: /lib/modules name: lib-modules-path readOnly: true - - mountPath: /sys/fs/bpf - name: sys-fs-bpf-path - mountPath: /sys/kernel/security name: sys-kernel-security-path - mountPath: /sys/kernel/debug @@ -249,8 +241,6 @@ kubearmor: - mountPath: /lib/modules name: lib-modules-path readOnly: true - - mountPath: /sys/fs/bpf - name: sys-fs-bpf-path - mountPath: /sys/kernel/security name: sys-kernel-security-path - mountPath: /sys/kernel/debug @@ -271,8 +261,6 @@ kubearmor: - mountPath: /lib/modules name: lib-modules-path readOnly: true - - mountPath: /sys/fs/bpf - name: sys-fs-bpf-path - mountPath: /sys/kernel/security name: sys-kernel-security-path - mountPath: /sys/kernel/debug @@ -293,8 +281,6 @@ kubearmor: - mountPath: /lib/modules name: lib-modules-path readOnly: true - - mountPath: /sys/fs/bpf - name: sys-fs-bpf-path - mountPath: /sys/kernel/security name: sys-kernel-security-path - mountPath: /sys/kernel/debug @@ -315,8 +301,6 @@ kubearmor: - mountPath: /lib/modules name: lib-modules-path readOnly: true - - mountPath: /sys/fs/bpf - name: sys-fs-bpf-path - mountPath: /sys/kernel/security name: sys-kernel-security-path - mountPath: /sys/kernel/debug @@ -333,8 +317,6 @@ kubearmor: - mountPath: /lib/modules name: lib-modules-path readOnly: true - - mountPath: /sys/fs/bpf - name: sys-fs-bpf-path - mountPath: /sys/kernel/security name: sys-kernel-security-path - mountPath: /sys/kernel/debug @@ -355,8 +337,6 @@ kubearmor: - mountPath: /lib/modules name: lib-modules-path readOnly: true - - mountPath: /sys/fs/bpf - name: sys-fs-bpf-path - mountPath: /sys/kernel/security name: sys-kernel-security-path - mountPath: /sys/kernel/debug @@ -377,8 +357,6 @@ kubearmor: - mountPath: /lib/modules name: lib-modules-path readOnly: true - - mountPath: /sys/fs/bpf - name: sys-fs-bpf-path - mountPath: /sys/kernel/security name: sys-kernel-security-path - mountPath: /sys/kernel/debug @@ -404,10 +382,6 @@ kubearmor: path: /lib/modules type: DirectoryOrCreate name: lib-modules-path - - hostPath: - path: /sys/fs/bpf - type: Directory - name: sys-fs-bpf-path - hostPath: path: /sys/kernel/security type: Directory @@ -438,10 +412,6 @@ kubearmor: path: /lib/modules type: DirectoryOrCreate name: lib-modules-path - - hostPath: - path: /sys/fs/bpf - type: Directory - name: sys-fs-bpf-path - hostPath: path: /sys/kernel/security type: Directory @@ -472,10 +442,6 @@ kubearmor: path: /lib/modules type: DirectoryOrCreate name: lib-modules-path - - hostPath: - path: /sys/fs/bpf - type: Directory - name: sys-fs-bpf-path - hostPath: path: /sys/kernel/security type: Directory @@ -506,10 +472,6 @@ kubearmor: path: /lib/modules type: DirectoryOrCreate name: lib-modules-path - - hostPath: - path: /sys/fs/bpf - type: Directory - name: sys-fs-bpf-path - hostPath: path: /sys/kernel/security type: Directory @@ -540,10 +502,6 @@ kubearmor: path: /lib/modules type: DirectoryOrCreate name: lib-modules-path - - hostPath: - path: /sys/fs/bpf - type: Directory - name: sys-fs-bpf-path - hostPath: path: /sys/kernel/security type: Directory @@ -574,10 +532,6 @@ kubearmor: path: /lib/modules type: Directory name: lib-modules-path - - hostPath: - path: /sys/fs/bpf - type: Directory - name: sys-fs-bpf-path - hostPath: path: /sys/kernel/security type: Directory @@ -608,10 +562,6 @@ kubearmor: path: /lib/modules type: DirectoryOrCreate name: lib-modules-path - - hostPath: - path: /sys/fs/bpf - type: Directory - name: sys-fs-bpf-path - hostPath: path: /sys/kernel/security type: Directory @@ -642,10 +592,6 @@ kubearmor: path: /lib/modules type: DirectoryOrCreate name: lib-modules-path - - hostPath: - path: /sys/fs/bpf - type: Directory - name: sys-fs-bpf-path - hostPath: path: /sys/kernel/security type: Directory @@ -672,10 +618,6 @@ kubearmor: path: /lib/modules type: DirectoryOrCreate name: lib-modules-path - - hostPath: - path: /sys/fs/bpf - type: Directory - name: sys-fs-bpf-path - hostPath: path: /sys/kernel/security type: Directory @@ -706,10 +648,6 @@ kubearmor: path: /lib/modules type: DirectoryOrCreate name: lib-modules-path - - hostPath: - path: /sys/fs/bpf - type: Directory - name: sys-fs-bpf-path - hostPath: path: /sys/kernel/security type: Directory diff --git a/pkg/KubeArmorOperator/common/defaults.go b/pkg/KubeArmorOperator/common/defaults.go index 283903024f..7908487684 100644 --- a/pkg/KubeArmorOperator/common/defaults.go +++ b/pkg/KubeArmorOperator/common/defaults.go @@ -169,12 +169,6 @@ var EnforcerVolumesMounts = map[string][]corev1.VolumeMount{ MountPath: "/etc/apparmor.d", }, }, - "bpf": { - { - Name: "sys-fs-bpf-path", - MountPath: "/sys/fs/bpf", - }, - }, } var EnforcerVolumes = map[string][]corev1.Volume{ @@ -189,18 +183,6 @@ var EnforcerVolumes = map[string][]corev1.Volume{ }, }, }, - "bpf": { - - { - Name: "sys-fs-bpf-path", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/sys/fs/bpf", - Type: &HostPathDirectory, - }, - }, - }, - }, } var RuntimeSocketLocation = map[string]string{ diff --git a/pkg/KubeArmorOperator/internal/controller/resources.go b/pkg/KubeArmorOperator/internal/controller/resources.go index 3beb43abbe..0e48841861 100644 --- a/pkg/KubeArmorOperator/internal/controller/resources.go +++ b/pkg/KubeArmorOperator/internal/controller/resources.go @@ -39,11 +39,12 @@ func generateDaemonset(name, enforcer, runtime, socket, btfPresent, apparmorfs, volMnts = append(volMnts, runtimeVolumeMounts...) commonVols := common.CommonVolumes commonVolMnts := common.CommonVolumesMount + if btfPresent == "no" { - commonVols = append(commonVols, common.KernelHeaderVolumes...) commonVols = append(commonVols, common.BPFVolumes...) - commonVolMnts = append(commonVolMnts, common.KernelHeaderVolumesMount...) commonVolMnts = append(commonVolMnts, common.BPFVolumesMount...) + commonVols = append(commonVols, common.KernelHeaderVolumes...) + commonVolMnts = append(commonVolMnts, common.KernelHeaderVolumesMount...) } vols = append(vols, commonVols...) volMnts = append(volMnts, commonVolMnts...) From 2326d957feadefaa0d4e6ef799be8799f9d6307a Mon Sep 17 00:00:00 2001 From: Aryan-sharma11 Date: Thu, 23 May 2024 01:06:28 +0530 Subject: [PATCH 5/8] add specific hostpaths instead of mounting "/" for kubearmor snitch Signed-off-by: Aryan-sharma11 --- .github/workflows/ci-test-ubi-image.yml | 6 +- .github/workflows/cleanup.sh | 4 +- Dockerfile | 8 +-- pkg/KubeArmorOperator/cmd/operator/main.go | 5 +- .../internal/controller/cluster.go | 12 ++-- .../internal/controller/resources.go | 61 ++++++++++++++++--- 6 files changed, 77 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci-test-ubi-image.yml b/.github/workflows/ci-test-ubi-image.yml index 39261863d4..83bc4d81eb 100644 --- a/.github/workflows/ci-test-ubi-image.yml +++ b/.github/workflows/ci-test-ubi-image.yml @@ -78,7 +78,11 @@ jobs: kubectl wait --timeout=7m --for=condition=ready pod -l kubearmor-app,kubearmor-app!=kubearmor-snitch,kubearmor-app!=kubearmor-controller -n kubearmor kubectl wait --timeout=1m --for=condition=ready pod -l kubearmor-app=kubearmor-controller -n kubearmor kubectl get pods -A - + + - name: Operator may take upto 10 sec to enable TLS, Sleep for 15Sec + run: | + sleep 15 + - name: Test KubeArmor using Ginkgo run: | go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo diff --git a/.github/workflows/cleanup.sh b/.github/workflows/cleanup.sh index aaf2bdafcd..3c539903a7 100755 --- a/.github/workflows/cleanup.sh +++ b/.github/workflows/cleanup.sh @@ -6,12 +6,14 @@ cleanup() { echo "Performing cleanup..." - ./usr/local/bin/k3s-killall.sh + /usr/local/bin/k3s-killall.sh /usr/local/bin/k3s-uninstall.sh docker system prune -a -f + sudo podman system prune -a -f + # rm -rf /home/vagrant/actions-runner/_work/KubeArmor echo "Cleanup complete." diff --git a/Dockerfile b/Dockerfile index a15279fd98..04729c768c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ### Builder -FROM golang:1.22-alpine3.19 as builder +FROM golang:1.22-alpine3.20 as builder RUN apk --no-cache update RUN apk add --no-cache git clang llvm make gcc protobuf @@ -40,7 +40,7 @@ RUN make ### Make executable image -FROM alpine:3.19 as kubearmor +FROM alpine:3.20 as kubearmor RUN echo "@community http://dl-cdn.alpinelinux.org/alpine/edge/community" | tee -a /etc/apk/repositories @@ -48,7 +48,7 @@ RUN apk --no-cache update RUN apk add apparmor@community apparmor-utils@community bash COPY --from=builder /usr/src/KubeArmor/KubeArmor/kubearmor /KubeArmor/kubearmor -COPY --from=builder /usr/src/KubeArmor/BPF/*.o /KubeArmor/BPF/ +COPY --from=builder /usr/src/KubeArmor/BPF/*.o /opt/kubearmor/BPF/ COPY --from=builder /usr/src/KubeArmor/KubeArmor/templates/* /KubeArmor/templates/ ENTRYPOINT ["/KubeArmor/kubearmor"] @@ -88,7 +88,7 @@ RUN groupadd --gid 1000 default \ COPY LICENSE /licenses/license.txt COPY --from=builder --chown=default:default /usr/src/KubeArmor/KubeArmor/kubearmor /KubeArmor/kubearmor -COPY --from=builder --chown=default:default /usr/src/KubeArmor/BPF/*.o /KubeArmor/BPF/ +COPY --from=builder --chown=default:default /usr/src/KubeArmor/BPF/*.o /opt/kubearmor/BPF/ COPY --from=builder --chown=default:default /usr/src/KubeArmor/KubeArmor/templates/* /KubeArmor/templates/ # TODO diff --git a/pkg/KubeArmorOperator/cmd/operator/main.go b/pkg/KubeArmorOperator/cmd/operator/main.go index 8b5801ce2e..b455935db8 100644 --- a/pkg/KubeArmorOperator/cmd/operator/main.go +++ b/pkg/KubeArmorOperator/cmd/operator/main.go @@ -27,6 +27,7 @@ var PathPrefix string var DeploymentName string var ExtClient *apiextensionsclientset.Clientset var Opv1Client *opv1client.Clientset +var InitDeploy bool // Cmd represents the base command when called without any subcommands var Cmd = &cobra.Command{ @@ -43,7 +44,7 @@ var Cmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { - nodeWatcher := controllers.NewClusterWatcher(K8sClient, Logger, ExtClient, Opv1Client, PathPrefix, DeploymentName) + nodeWatcher := controllers.NewClusterWatcher(K8sClient, Logger, ExtClient, Opv1Client, PathPrefix, DeploymentName, InitDeploy) go nodeWatcher.WatchConfigCrd() nodeWatcher.WatchNodes() @@ -69,6 +70,8 @@ func init() { Cmd.PersistentFlags().StringVar(&LsmOrder, "lsm", "bpf,apparmor,selinux", "lsm preference order to use") Cmd.PersistentFlags().StringVar(&PathPrefix, "pathprefix", "/rootfs/", "path prefix for runtime search") Cmd.PersistentFlags().StringVar(&DeploymentName, "deploymentName", "kubearmor-operator", "operator deployment name") + // TODO:- set initDeploy to false by default once this change is added to stable + Cmd.PersistentFlags().BoolVar(&InitDeploy, "initDeploy", true, "Init container deployment") } // Execute adds all child commands to the root command and sets flags appropriately. diff --git a/pkg/KubeArmorOperator/internal/controller/cluster.go b/pkg/KubeArmorOperator/internal/controller/cluster.go index b1fb2082be..811bcdd9a9 100644 --- a/pkg/KubeArmorOperator/internal/controller/cluster.go +++ b/pkg/KubeArmorOperator/internal/controller/cluster.go @@ -37,6 +37,7 @@ var informer informers.SharedInformerFactory var deployment_uuid types.UID var deployment_name string = "kubearmor-operator" var PathPrefix string +var initDeploy bool type ClusterWatcher struct { Nodes []Node @@ -59,7 +60,7 @@ type Node struct { Seccomp string } -func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, extClient *apiextensionsclientset.Clientset, opv1Client *opv1client.Clientset, pathPrefix, deploy_name string) *ClusterWatcher { +func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, extClient *apiextensionsclientset.Clientset, opv1Client *opv1client.Clientset, pathPrefix, deploy_name string, initdeploy bool) *ClusterWatcher { if informer == nil { informer = informers.NewSharedInformerFactory(client, 0) } @@ -75,6 +76,7 @@ func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, ext } PathPrefix = pathPrefix deployment_name = deploy_name + initDeploy = initdeploy return &ClusterWatcher{ Nodes: []Node{}, Daemonsets: make(map[string]int), @@ -226,7 +228,7 @@ func (clusterWatcher *ClusterWatcher) UpdateDaemonsets(action, enforcer, runtime } } if newDaemonSet { - daemonset := generateDaemonset(daemonsetName, enforcer, runtime, socket, btfPresent, apparmorfs, seccompPresent) + daemonset := generateDaemonset(daemonsetName, enforcer, runtime, socket, btfPresent, apparmorfs, seccompPresent, initDeploy) _, err := clusterWatcher.Client.AppsV1().DaemonSets(common.Namespace).Create(context.Background(), daemonset, v1.CreateOptions{}) if err != nil { clusterWatcher.Log.Warnf("Cannot Create daemonset %s, error=%s", daemonsetName, err.Error()) @@ -363,8 +365,10 @@ func (clusterWatcher *ClusterWatcher) UpdateKubeArmorImages(images []string) err for _, ds := range dsList.Items { ds.Spec.Template.Spec.Containers[0].Image = common.GetApplicationImage(common.KubeArmorName) ds.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullPolicy(common.KubeArmorInitImagePullPolicy) - ds.Spec.Template.Spec.InitContainers[0].Image = common.GetApplicationImage(common.KubeArmorInitName) - ds.Spec.Template.Spec.InitContainers[0].ImagePullPolicy = corev1.PullPolicy(common.KubeArmorInitImagePullPolicy) + if len(ds.Spec.Template.Spec.InitContainers) != 0 { + ds.Spec.Template.Spec.InitContainers[0].Image = common.GetApplicationImage(common.KubeArmorInitName) + ds.Spec.Template.Spec.InitContainers[0].ImagePullPolicy = corev1.PullPolicy(common.KubeArmorInitImagePullPolicy) + } _, err = clusterWatcher.Client.AppsV1().DaemonSets(common.Namespace).Update(context.Background(), &ds, v1.UpdateOptions{}) if err != nil { clusterWatcher.Log.Warnf("Cannot update daemonset=%s error=%s", ds.Name, err.Error()) diff --git a/pkg/KubeArmorOperator/internal/controller/resources.go b/pkg/KubeArmorOperator/internal/controller/resources.go index 0e48841861..ce3824bbab 100644 --- a/pkg/KubeArmorOperator/internal/controller/resources.go +++ b/pkg/KubeArmorOperator/internal/controller/resources.go @@ -24,7 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func generateDaemonset(name, enforcer, runtime, socket, btfPresent, apparmorfs, seccompPresent string) *appsv1.DaemonSet { +func generateDaemonset(name, enforcer, runtime, socket, btfPresent, apparmorfs, seccompPresent string, initDeploy bool) *appsv1.DaemonSet { enforcerVolumes := []corev1.Volume{} enforcerVolumeMounts := []corev1.VolumeMount{} if !(enforcer == "apparmor" && apparmorfs == "no") { @@ -40,7 +40,7 @@ func generateDaemonset(name, enforcer, runtime, socket, btfPresent, apparmorfs, commonVols := common.CommonVolumes commonVolMnts := common.CommonVolumesMount - if btfPresent == "no" { + if btfPresent == "no" || initDeploy { commonVols = append(commonVols, common.BPFVolumes...) commonVolMnts = append(commonVolMnts, common.BPFVolumesMount...) commonVols = append(commonVols, common.KernelHeaderVolumes...) @@ -50,7 +50,7 @@ func generateDaemonset(name, enforcer, runtime, socket, btfPresent, apparmorfs, volMnts = append(volMnts, commonVolMnts...) daemonset := deployments.GenerateDaemonSet("generic", common.Namespace) - if btfPresent != "no" { + if btfPresent != "no" && !initDeploy { daemonset.Spec.Template.Spec.InitContainers = []corev1.Container{} } daemonset.Name = name @@ -105,7 +105,7 @@ func generateDaemonset(name, enforcer, runtime, socket, btfPresent, apparmorfs, daemonset.Spec.Template.Spec.Containers[0].Image = common.GetApplicationImage(common.KubeArmorName) daemonset.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullPolicy(common.KubeArmorImagePullPolicy) - if btfPresent == "no" { + if btfPresent == "no" || initDeploy { daemonset.Spec.Template.Spec.InitContainers[0].VolumeMounts = commonVolMnts daemonset.Spec.Template.Spec.InitContainers[0].Image = common.GetApplicationImage(common.KubeArmorInitName) daemonset.Spec.Template.Spec.InitContainers[0].ImagePullPolicy = corev1.PullPolicy(common.KubeArmorInitImagePullPolicy) @@ -238,9 +238,25 @@ func deploySnitch(nodename string, runtime string) *batchv1.Job { }, ImagePullPolicy: corev1.PullIfNotPresent, VolumeMounts: []corev1.VolumeMount{ + + { + Name: "var-path", + MountPath: "/rootfs/var/", + ReadOnly: true, + }, + { + Name: "run-path", + MountPath: "/rootfs/run/", + ReadOnly: true, + }, + { + Name: "sys-path", + MountPath: "/rootfs/sys/", + ReadOnly: true, + }, { - Name: "rootfs", - MountPath: PathPrefix, + Name: "apparmor-path", + MountPath: "/rootfs/etc/apparmor.d/", ReadOnly: true, }, { @@ -268,16 +284,45 @@ func deploySnitch(nodename string, runtime string) *batchv1.Job { // For Unknown Reasons hostPID will be true if snitch gets deployed on OpenShift // for some reasons github.com/kubearmor/KubeArmor/KubeArmor/utils/bpflsmprobe will // not work if hostPID is set false. + + // change for snitch host path HostPID: common.HostPID, NodeName: nodename, RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: common.KubeArmorSnitchRoleName, Volumes: []corev1.Volume{ { - Name: "rootfs", + Name: "sys-path", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/sys/", + Type: &common.HostPathDirectory, + }, + }, + }, + { + Name: "apparmor-path", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/apparmor.d/", + Type: &common.HostPathDirectory, + }, + }, + }, + { + Name: "var-path", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/", + Type: &common.HostPathDirectory, + }, + }, + }, + { + Name: "run-path", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ - Path: "/", + Path: "/run/", Type: &common.HostPathDirectory, }, }, From b1e51a13eec9e9bed8ff54e85309c9cbd57cd0f9 Mon Sep 17 00:00:00 2001 From: Aryan-sharma11 Date: Fri, 31 May 2024 10:57:42 +0530 Subject: [PATCH 6/8] fix node info not available for older kernels Signed-off-by: Aryan-sharma11 --- .../templates/clusterrole-rbac.yaml | 7 ++++++ .../internal/controller/resources.go | 22 +++++++++---------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/deployments/helm/KubeArmorOperator/templates/clusterrole-rbac.yaml b/deployments/helm/KubeArmorOperator/templates/clusterrole-rbac.yaml index a16899dec2..4b06441df0 100644 --- a/deployments/helm/KubeArmorOperator/templates/clusterrole-rbac.yaml +++ b/deployments/helm/KubeArmorOperator/templates/clusterrole-rbac.yaml @@ -90,6 +90,13 @@ rules: - customresourcedefinitions verbs: - create +- apiGroups: + - batch + resources: + - jobs + - cronjobs + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/pkg/KubeArmorOperator/internal/controller/resources.go b/pkg/KubeArmorOperator/internal/controller/resources.go index ce3824bbab..94029195d1 100644 --- a/pkg/KubeArmorOperator/internal/controller/resources.go +++ b/pkg/KubeArmorOperator/internal/controller/resources.go @@ -88,29 +88,29 @@ func generateDaemonset(name, enforcer, runtime, socket, btfPresent, apparmorfs, } daemonset.Spec.Template.Spec.Volumes = vols daemonset.Spec.Template.Spec.Containers[0].VolumeMounts = volMnts - // update images + if btfPresent == "no" || initDeploy { + daemonset.Spec.Template.Spec.InitContainers[0].VolumeMounts = commonVolMnts + daemonset.Spec.Template.Spec.InitContainers[0].Image = common.GetApplicationImage(common.KubeArmorInitName) + daemonset.Spec.Template.Spec.InitContainers[0].ImagePullPolicy = corev1.PullPolicy(common.KubeArmorInitImagePullPolicy) + } + // update images if seccompPresent == "yes" && common.ConfigDefaultSeccompEnabled == "true" { daemonset.Spec.Template.Spec.Containers[0].SecurityContext.SeccompProfile = &corev1.SeccompProfile{ Type: corev1.SeccompProfileTypeLocalhost, LocalhostProfile: &common.SeccompProfile, } - daemonset.Spec.Template.Spec.InitContainers[0].SecurityContext.SeccompProfile = &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeLocalhost, - LocalhostProfile: &common.SeccompInitProfile, + if len(daemonset.Spec.Template.Spec.InitContainers) != 0 { + daemonset.Spec.Template.Spec.InitContainers[0].SecurityContext.SeccompProfile = &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeLocalhost, + LocalhostProfile: &common.SeccompInitProfile, + } } } daemonset.Spec.Template.Spec.Containers[0].Image = common.GetApplicationImage(common.KubeArmorName) daemonset.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullPolicy(common.KubeArmorImagePullPolicy) - - if btfPresent == "no" || initDeploy { - daemonset.Spec.Template.Spec.InitContainers[0].VolumeMounts = commonVolMnts - daemonset.Spec.Template.Spec.InitContainers[0].Image = common.GetApplicationImage(common.KubeArmorInitName) - daemonset.Spec.Template.Spec.InitContainers[0].ImagePullPolicy = corev1.PullPolicy(common.KubeArmorInitImagePullPolicy) - } - daemonset = addOwnership(daemonset).(*appsv1.DaemonSet) fmt.Printf("generated daemonset: %v", daemonset) return daemonset From 1307fa5532a477dae6c7d5381317b3927cbe3bdf Mon Sep 17 00:00:00 2001 From: Aryan-sharma11 Date: Thu, 6 Jun 2024 11:51:20 +0530 Subject: [PATCH 7/8] fix go vuln Signed-off-by: Aryan-sharma11 --- KubeArmor/go.mod | 2 +- deployments/go.mod | 2 +- pkg/KubeArmorController/go.mod | 2 +- pkg/KubeArmorOperator/go.mod | 2 +- protobuf/go.mod | 2 +- tests/go.mod | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/KubeArmor/go.mod b/KubeArmor/go.mod index 04ec7b2003..a8d2ffc814 100644 --- a/KubeArmor/go.mod +++ b/KubeArmor/go.mod @@ -2,7 +2,7 @@ module github.com/kubearmor/KubeArmor/KubeArmor go 1.21.0 -toolchain go1.21.9 +toolchain go1.21.11 replace ( github.com/kubearmor/KubeArmor => ../../ diff --git a/deployments/go.mod b/deployments/go.mod index 3f32515769..0d34e3e3ef 100644 --- a/deployments/go.mod +++ b/deployments/go.mod @@ -2,7 +2,7 @@ module github.com/kubearmor/KubeArmor/deployments go 1.21.0 -toolchain go1.21.9 +toolchain go1.21.11 replace ( github.com/kubearmor/KubeArmor => ../ diff --git a/pkg/KubeArmorController/go.mod b/pkg/KubeArmorController/go.mod index d56a3abaca..b2fa4c0fb9 100644 --- a/pkg/KubeArmorController/go.mod +++ b/pkg/KubeArmorController/go.mod @@ -2,7 +2,7 @@ module github.com/kubearmor/KubeArmor/pkg/KubeArmorController go 1.21.0 -toolchain go1.21.9 +toolchain go1.21.11 require ( github.com/go-logr/logr v1.4.1 diff --git a/pkg/KubeArmorOperator/go.mod b/pkg/KubeArmorOperator/go.mod index 32e1b02351..41cc5a1a09 100644 --- a/pkg/KubeArmorOperator/go.mod +++ b/pkg/KubeArmorOperator/go.mod @@ -2,7 +2,7 @@ module github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator go 1.21.0 -toolchain go1.21.9 +toolchain go1.21.11 replace ( github.com/kubearmor/KubeArmor/KubeArmor => ../../KubeArmor diff --git a/protobuf/go.mod b/protobuf/go.mod index 01fcac08b0..f0b7476b52 100644 --- a/protobuf/go.mod +++ b/protobuf/go.mod @@ -2,7 +2,7 @@ module github.com/kubearmor/KubeArmor/protobuf go 1.21.0 -toolchain go1.21.9 +toolchain go1.21.11 replace ( github.com/kubearmor/KubeArmor => ../ diff --git a/tests/go.mod b/tests/go.mod index 1751136f70..6d2feeaa2e 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -2,7 +2,7 @@ module github.com/kubearmor/KubeArmor/tests go 1.21.0 -toolchain go1.21.9 +toolchain go1.21.11 replace ( github.com/cilium/cilium => github.com/cilium/cilium v1.14.8 From 215260da914bbcdb7aed06f429104a969b439b9e Mon Sep 17 00:00:00 2001 From: Aryan-sharma11 Date: Fri, 7 Jun 2024 13:48:58 +0530 Subject: [PATCH 8/8] expose initdeploy to helm deployment Signed-off-by: Aryan-sharma11 --- .../helm/KubeArmorOperator/templates/deployment.yaml | 9 +++++++++ deployments/helm/KubeArmorOperator/values.yaml | 1 + 2 files changed, 10 insertions(+) diff --git a/deployments/helm/KubeArmorOperator/templates/deployment.yaml b/deployments/helm/KubeArmorOperator/templates/deployment.yaml index e0a4689a55..24f397ba8d 100644 --- a/deployments/helm/KubeArmorOperator/templates/deployment.yaml +++ b/deployments/helm/KubeArmorOperator/templates/deployment.yaml @@ -26,4 +26,13 @@ spec: {{- end }} image: {{ include "operatorImage" . }} imagePullPolicy: {{ .Values.kubearmorOperator.imagePullPolicy }} + + {{- $tag := (split ":" (include "operatorImage" .))._1 -}} + {{- if or (eq $tag "latest") (and (hasPrefix "v" $tag) (semverCompare "^1.4.0" $tag)) }} + # initDeploy flag is only supported from v1.4.0 + args: + - --initDeploy={{.Values.kubearmorOperator.initDeploy }} + {{- end }} + serviceAccountName: {{ .Values.kubearmorOperator.name }} + diff --git a/deployments/helm/KubeArmorOperator/values.yaml b/deployments/helm/KubeArmorOperator/values.yaml index ad47707ba2..a6efab57b6 100644 --- a/deployments/helm/KubeArmorOperator/values.yaml +++ b/deployments/helm/KubeArmorOperator/values.yaml @@ -36,6 +36,7 @@ kubearmorOperator: repository: kubearmor/kubearmor-operator tag: "" imagePullPolicy: IfNotPresent + initDeploy: true kubearmorConfig: defaultCapabilitiesPosture: audit