Merge pull request #4 from metal-stack/rework-proxy

Add the ability to use a konnectivity server for connection into the cluster if this is used in the cluster.

Dockerfile

@@ -3,9 +3,10 @@ FROM golang:1.15 AS builder
 WORKDIR /work
 
 COPY .git Makefile go.* *.go /work/
+COPY pkg/ /work/pkg/
 RUN make bin/audit-forwarder
 
-FROM fluent/fluent-bit:1.7.0-debug
+FROM fluent/fluent-bit:1.7.3-debug
 
 COPY --from=builder /work/bin/audit-forwarder /fluent-bit/bin/
 COPY *.conf /fluent-bit/etc/

go.mod

@@ -1,4 +1,4 @@
-module github.com/mreiger/audit-forwarder
+module github.com/metal-stack/audit-forwarder
 
 go 1.15
 

kind/konnectivity-agent.yaml

@@ -0,0 +1,95 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: konnectivity-agent
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: konnectivity-agent
+    k8s-app: konnectivity-agent
+  name: konnectivity-agent
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: konnectivity-agent
+  template:
+    metadata:
+      labels:
+        app: konnectivity-agent
+        k8s-app: konnectivity-agent
+        type: tunnel
+    spec:
+      containers:
+      - args:
+        # - --log-file=/var/log/konnectivity-agent/info.log
+        - --logtostderr=true
+        # - --log-file-max-size=12
+        - --ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        - --proxy-server-host=APISERVERIP
+        - --proxy-server-port=8132
+        - --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token
+        command:
+        - /proxy-agent
+        image: k8s.gcr.io/kas-network-proxy/proxy-agent:v0.0.12
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            host: 127.0.0.1
+            path: /healthz
+            port: 8093
+            scheme: HTTP
+          initialDelaySeconds: 15
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 15
+        name: konnectivity-agent
+        resources:
+          limits:
+            cpu: "1"
+            memory: 1Gi
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /var/log/konnectivity-agent
+          name: konnectivity-agent-log
+        - mountPath: /var/run/secrets/tokens
+          name: konnectivity-agent-token
+      dnsPolicy: ClusterFirst
+      hostNetwork: true
+      priorityClassName: system-cluster-critical
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      serviceAccount: konnectivity-agent
+      serviceAccountName: konnectivity-agent
+      terminationGracePeriodSeconds: 30
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      - effect: NoSchedule
+        operator: Exists
+      volumes:
+      - emptyDir: {}
+        name: konnectivity-agent-log
+      - name: konnectivity-agent-token
+        projected:
+          defaultMode: 420
+          sources:
+          - serviceAccountToken:
+              audience: system:konnectivity-server
+              expirationSeconds: 3600
+              path: konnectivity-agent-token
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

kind/konnectivity/egress-selector-configuration.yaml

@@ -0,0 +1,15 @@
+apiVersion: apiserver.k8s.io/v1alpha1
+kind: EgressSelectorConfiguration
+egressSelections:
+- name: cluster
+  connection:
+    proxyProtocol: HTTPConnect
+    transport:
+      uds:
+        udsName: /etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket
+- name: master
+  connection:
+    proxyProtocol: Direct
+- name: etcd
+  connection:
+    proxyProtocol: Direct

kind/konnectivity/known_tokens.csv

@@ -0,0 +1 @@
+RN7qiASqKso/qaxFw9jA6lsaGGOAp6UdlVcFDw7l/jc=,system:konnectivity-server,uid:system:konnectivity-server

kind/konnectivity/kubeconfig

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Config
+users:
+- name: konnectivity-server
+  user:
+    token: RN7qiASqKso/qaxFw9jA6lsaGGOAp6UdlVcFDw7l/jc=
+clusters:
+- name: local
+  cluster:
+    insecure-skip-tls-verify: true
+    server: https://localhost:6443
+contexts:
+- context:
+    cluster: local
+    user: konnectivity-server
+  name: konnectivity-server
+current-context: konnectivity-server

kind/kustomize-auditforwarder-konnectivity/kube-apiserver_patch.yaml

@@ -0,0 +1,37 @@
+# This file adds the auditforwarder.
+- op: add
+  path: /spec/containers/1
+  value:
+    image: ghcr.io/metal-stack/audit-forwarder:pr-rework-proxy
+    imagePullPolicy: Always
+    name: audit-forwarder
+    env:
+    - name: AUDIT_KUBECFG
+      value: "/kube.config"
+    - name: AUDIT_LOG_LEVEL
+      value: "info"
+    - name: AUDIT_KONNECTIVITY_UDS_SOCKET
+      value: "/konnectivity-uds/konnectivity-server.socket"
+    volumeMounts:
+    - mountPath: /auditlog
+      name: auditlog
+    - mountPath: /kube.config
+      name: kubeconfig
+    - mountPath: /konnectivity-uds
+      name: konnectivity-uds
+    # - mountPath: /fluent-bit/etc/ssl
+    #   name: forwarder-certs
+- op: add
+  path: /spec/volumes/0
+  value:
+    hostPath:
+      path: /etc/kubernetes/audit/kube.config
+      type: File
+    name: kubeconfig
+# - op: add
+#   path: /spec/volumes/0
+#   value:
+#     hostPath:
+#       path: /etc/kubernetes/audit/ssl
+#       type: Directory
+#     name: forwarder-certs

kind/kube-apiserver_patch.yaml → kind/kustomize-auditforwarder/kube-apiserver_patch.yaml

@@ -1,7 +1,7 @@
 - op: add
   path: /spec/containers/1
   value:
-    image: mreiger/audit-forwarder:latest
+    image: ghcr.io/metal-stack/audit-forwarder:pr-rework-proxy
     imagePullPolicy: Always
     name: audit-forwarder
     env:

kind/kustomize-auditforwarder/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+- kube-apiserver.yaml
+patches:
+- path: kube-apiserver_patch.yaml
+  target:
+    kind: Pod
+    name: kube-apiserver

kind/kustomize-konnectivity/kube-apiserver_konnectivity_patch.yaml

@@ -0,0 +1,87 @@
+# This file adds the konnectivity-server sidecar.
+- op: add
+  path: /spec/containers/1
+  value:
+    args:
+    - --uds-name=/etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket
+    - --logtostderr=true
+    - --cluster-cert=/etc/kubernetes/pki/apiserver.crt
+    - --cluster-key=/etc/kubernetes/pki/apiserver.key
+    - --agent-namespace=kube-system
+    - --agent-service-account=konnectivity-agent
+    - --kubeconfig=/etc/kubernetes/konnectivity/kubeconfig
+    - --authentication-audience=system:konnectivity-server
+    - --mode=http-connect
+    - --server-count=2
+    - --server-port=0
+    - --agent-port=8132
+    - --admin-port=8133
+    - --health-port=8134
+    - --delete-existing-uds-file=true
+    command:
+    - /proxy-server
+    image: k8s.gcr.io/kas-network-proxy/proxy-server:v0.0.12
+    imagePullPolicy: IfNotPresent
+    livenessProbe:
+      failureThreshold: 3
+      httpGet:
+        path: /healthz
+        port: 8134
+        scheme: HTTP
+      initialDelaySeconds: 10
+      periodSeconds: 10
+      successThreshold: 1
+      timeoutSeconds: 60
+    name: konnectivity-server
+    ports:
+    - containerPort: 8132
+      name: agentport
+      protocol: TCP
+    - containerPort: 8133
+      name: adminport
+      protocol: TCP
+    - containerPort: 8134
+      name: healthport
+      protocol: TCP
+    resources:
+      limits:
+        cpu: 200m
+        memory: 500M
+      requests:
+        cpu: 50m
+        memory: 128Mi
+    terminationMessagePath: /dev/termination-log
+    terminationMessagePolicy: File
+    volumeMounts:
+    - mountPath: /etc/srv/kubernetes/konnectivity-server
+      name: konnectivity-uds
+    - mountPath: /etc/kubernetes/konnectivity
+      name: konnectivity
+      readOnly: true
+    - mountPath: /etc/kubernetes/pki
+      name: k8s-certs
+      readOnly: true
+
+
+# - op: add
+#   path: /spec/volumes/0
+#   value:
+#     name: konnectivity-server-kubeconfig
+#     secret:
+#       defaultMode: 420
+#       secretName: konnectivity-server-kubeconfig
+# - op: add
+#   path: /spec/volumes/0
+#   value:
+#     name: konnectivity
+#     hostPath:
+#       path: /etc/kubernetes/konnectivity
+#       type: Directory
+
+# - op: add
+#   path: /spec/volumes/0
+#   value:
+#     hostPath:
+#       path: /etc/kubernetes/audit/ssl
+#       type: Directory
+#     name: forwarder-certs

kind/kustomize-konnectivity/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+- kube-apiserver.yaml
+patches:
+- path: kube-apiserver_konnectivity_patch.yaml
+  target:
+    kind: Pod
+    name: kube-apiserver

kind/make-audit-forwarder

@@ -3,42 +3,29 @@
 # First check if a forwarder has already been applied and get the kube-apiserver manifest if it isn't.
 if grep forwarder kind-etc-kubernetes/manifests/kube-apiserver.yaml >/dev/null; then
     echo "Forwarder config already applied."
-    if [ ! -f kube-apiserver.yaml ]; then
+    if [ ! -f kustomize-auditforwarder/kube-apiserver.yaml ]; then
         echo "No saved kube-apiserver manifest exists, exiting."
         exit
     else
-        if grep forwarder kube-apiserver.yaml >/dev/null; then
+        if grep forwarder kustomize-auditforwarder/kube-apiserver.yaml >/dev/null; then
             echo "Saved config contains forwarder too, can not patch. Exiting."
             exit
         fi
     fi
 else
     echo "Getting kube-apiserver manifest."
-    cp kind-etc-kubernetes/manifests/kube-apiserver.yaml .
+    cp kind-etc-kubernetes/manifests/kube-apiserver.yaml kustomize-auditforwarder/
 fi
 
 # Patch the generated kind kubeconfig with the apiserver URL valid from within the cluster.
 
 echo "Generating the in-cluster kubeconfig:"
 
 # Get the IP and port from the apiserver manifest:
-line=`grep kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint kube-apiserver.yaml`
+line=`grep kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint kustomize-auditforwarder/kube-apiserver.yaml`
 apiserver=${line##*kubeadm.kubernetes.io\/kube-apiserver.advertise-address.endpoint:?}
 
 sed "s+https://.*$+https://$apiserver+" kube.config >kind-etc-kubernetes/audit/kube.config
 
-# Copy the certs file in place
-# echo "Copying certs"
-# if [ ! -d kind-etc-kubernetes/audit/ssl ]; then
-#     mkdir kind-etc-kubernetes/audit/ssl
-# fi
-# cp certs/ca.crt kind-etc-kubernetes/audit/ssl/
-# cp certs/forwarder.crt kind-etc-kubernetes/audit/ssl/
-# cp certs/forwarder.key kind-etc-kubernetes/audit/ssl/
-
-# Wait a little so there's no timing problem
-echo "Waiting a bit"
-sleep 5
-
 echo "Patching and applying the kube-apiserver manifest:"
-kustomize build >kind-etc-kubernetes/manifests/kube-apiserver.yaml
+kustomize build kustomize-auditforwarder >kind-etc-kubernetes/manifests/kube-apiserver.yaml

kind/make-audit-forwarder-konnectivity

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# First check if a forwarder has already been applied and get the kube-apiserver manifest if it isn't.
+if grep forwarder kind-etc-kubernetes/manifests/kube-apiserver.yaml >/dev/null; then
+    echo "Forwarder config already applied."
+    if [ ! -f kustomize-auditforwarder-konnectivity/kube-apiserver.yaml ]; then
+        echo "No saved kube-apiserver manifest exists, exiting."
+        exit
+    else
+        if grep forwarder kustomize-auditforwarder-konnectivity/kube-apiserver.yaml >/dev/null; then
+            echo "Saved config contains forwarder too, can not patch. Exiting."
+            exit
+        fi
+    fi
+else
+    echo "Getting kube-apiserver manifest."
+    cp kind-etc-kubernetes/manifests/kube-apiserver.yaml kustomize-auditforwarder-konnectivity/
+fi
+
+# Patch the generated kind kubeconfig with the apiserver URL valid from within the cluster.
+
+echo "Generating the in-cluster kubeconfig:"
+
+# Get the IP and port from the apiserver manifest:
+line=`grep kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint kustomize-auditforwarder-konnectivity/kube-apiserver.yaml`
+apiserver=${line##*kubeadm.kubernetes.io\/kube-apiserver.advertise-address.endpoint:?}
+
+sed "s+https://.*$+https://$apiserver+" kube.config >kind-etc-kubernetes/audit/kube.config
+
+echo "Patching and applying the kube-apiserver manifest:"
+kustomize build kustomize-auditforwarder-konnectivity >kind-etc-kubernetes/manifests/kube-apiserver.yaml