From 8dd01575f4058b181bb84f231e6cce081b71057a Mon Sep 17 00:00:00 2001 From: Leonid Podolinskiy Date: Thu, 30 May 2024 11:42:01 +0300 Subject: [PATCH 1/4] fix eks/gke semver restriction by `kubeVersion` --- helm-chart/Chart.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml index 922d9d2..53e130e 100755 --- a/helm-chart/Chart.yaml +++ b/helm-chart/Chart.yaml @@ -13,15 +13,16 @@ type: application icon: https://lightrun-public.s3.amazonaws.com/img/lightrun-logo.png ## Kubeversion due to "seccompProfile" in the controller deployment -kubeVersion: ">= 1.19.0" +## -0 is used to allow any patch version +## it is resolving GKE and EKS versioning +## Example of EKS version: v1.28.9-eks-036c24b +kubeVersion: ">= 1.19.0-0" # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) version: 0.1.0 # Will be updated by CI pipeline - - # ArtifactHub.io annotations annotations: artifacthub.io/crds: | @@ -43,7 +44,7 @@ annotations: serverHostname: app.lightrun.com agentEnvVarName: JAVA_TOOL_OPTIONS initContainer: - image: "lightruncom/k8s-operator-init-java-agent-linux:1.8.5-init.1" + image: "lightruncom/k8s-operator-init-java-agent-linux:latest" sharedVolumeName: lightrun-agent-init sharedVolumeMountPath: "/lightrun" agentTags: @@ -51,8 +52,6 @@ annotations: - example - 1.8.3 - - artifacthub.io/license: Apache-2.0 artifacthub.io/links: | - name: Operator repo @@ -63,8 +62,7 @@ annotations: - name: Lightrun devops team email: devopsdevops@lightrun.com - name: LeonidP - email: leonidp@lightrun.com + email: leonidp@lightrun.com artifacthub.io/operator: "true" artifacthub.io/operatorCapabilities: Basic Install artifacthub.io/prerelease: "false" - From 4e057a9d524b477b95fa3dcc03f0b4df13e567de Mon Sep 17 00:00:00 2001 From: Leonid Podolinskiy Date: Thu, 30 May 2024 11:47:58 +0300 Subject: [PATCH 2/4] add pprof optional endpoint --- cmd/main.go | 5 ++++- helm-chart/templates/deployment.yaml | 7 +++++-- helm-chart/values.yaml | 20 ++++++++++---------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 91df6d3..c48c4c2 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -72,10 +72,12 @@ func getWatchNamespaces() ([]string, error) { func main() { var metricsAddr string - var enableLeaderElection bool var probeAddr string + var pprofAddr string + var enableLeaderElection bool flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.StringVar(&pprofAddr, "pprof-bind-address", "0", "The address the pprof endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -98,6 +100,7 @@ func main() { HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "5b425f09.lightrun.com", + PprofBindAddress: pprofAddr, // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily // when the Manager ends. This requires the binary to immediately end when the diff --git a/helm-chart/templates/deployment.yaml b/helm-chart/templates/deployment.yaml index 18eea8f..a98dc32 100644 --- a/helm-chart/templates/deployment.yaml +++ b/helm-chart/templates/deployment.yaml @@ -45,10 +45,13 @@ spec: drop: - "ALL" - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --health-probe-bind-address={{ .Values.managerConfig.healthProbe.bindAddress }} + - --metrics-bind-address={{ .Values.managerConfig.metrics.bindAddress }} - --leader-elect - --zap-log-level={{ .Values.managerConfig.logLevel }} + {{- if .Values.managerConfig.profiler.bindAddress }} + - --pprof-bind-address={{ .Values.managerConfig.profiler.bindAddress }} + {{- end }} command: - /manager image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag | default .Chart.AppVersion }} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 395ca8c..d4f90c0 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -60,16 +60,16 @@ managerConfig: logLevel: info ## Default values of the container inside pod. In most cases you don't need to change those - controllerManagerConfigYaml: - health: - healthProbeBindAddress: :8081 - leaderElection: - leaderElect: true - resourceName: 5b425f09.lightrun.com - metrics: - bindAddress: 127.0.0.1:8080 - webhook: - port: 9443 + healthProbe: + bindAddress: ":8081" + metrics: + bindAddress: ":8080" + # -- Profiler is used for debugging and performance analysis + # It is disabled by default + # To enable it, specify the bindAddress, similar to the metrics or the health probes + # Make sure to protect this endpoint as it is containing sensitive information + profiler: + bindAddress: "" # -- Operator may work in 2 scopes: cluster and namespaced # Cluster scope will give permissions to operator to watch and patch deployment in the whole cluster # With namespaced scope you need to provide list of namespaces that operator will be able to watch. From a66cd85396b75ba87c36a4e5038f8daade6a4854 Mon Sep 17 00:00:00 2001 From: Leonid Podolinskiy Date: Thu, 30 May 2024 11:50:31 +0300 Subject: [PATCH 3/4] deprecate kube-proxy due to [PR](https://github.com/kubernetes-sigs/kubebuilder/pull/3899) --- config/default/kustomization.yaml | 13 +--- config/default/manager_auth_proxy_patch.yaml | 40 ------------ config/manager/manager.yaml | 69 ++++++++++---------- config/samples/operator.yaml | 31 +-------- examples/operator.yaml | 25 +------ helm-chart/templates/deployment.yaml | 17 ----- helm-chart/values.yaml | 14 ---- 7 files changed, 41 insertions(+), 168 deletions(-) delete mode 100644 config/default/manager_auth_proxy_patch.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 44b647b..6507036 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -15,9 +15,9 @@ namePrefix: lightrun-k8s-operator- # someName: someValue resources: -- ../crd -- ../rbac -- ../manager + - ../crd + - ../rbac + - ../manager # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml #- ../webhook @@ -26,10 +26,3 @@ resources: # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -patches: -- path: manager_auth_proxy_patch.yaml diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index c287b7b..0000000 --- a/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" - - "--zap-log-level=0" diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 9f1d36d..9344e86 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -64,40 +64,39 @@ spec: seccompProfile: type: RuntimeDefault containers: - - command: - - /manager - env: - - name: WATCH_NAMESPACE - value: "" - args: - - --leader-elect - image: controller:latest - name: manager - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - # TODO(user): Configure the resources accordingly based on the project requirements. - # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi + - command: + - /manager + env: + - name: WATCH_NAMESPACE + value: "" + args: + - --leader-elect + - --zap-log-level=0 + image: controller:latest + name: manager + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 50m + memory: 128Mi serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/config/samples/operator.yaml b/config/samples/operator.yaml index 46605cf..0ec60d1 100644 --- a/config/samples/operator.yaml +++ b/config/samples/operator.yaml @@ -481,31 +481,6 @@ spec: spec: containers: - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - --leader-elect - --zap-log-level=0 command: @@ -530,10 +505,10 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 512Mi requests: - cpu: 10m - memory: 64Mi + cpu: 50m + memory: 128Mi securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/examples/operator.yaml b/examples/operator.yaml index 45f0f4a..6b249e9 100644 --- a/examples/operator.yaml +++ b/examples/operator.yaml @@ -480,32 +480,9 @@ spec: kubectl.kubernetes.io/default-container: manager spec: containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - args: - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=:8080 - --leader-elect - --zap-log-level=info command: diff --git a/helm-chart/templates/deployment.yaml b/helm-chart/templates/deployment.yaml index a98dc32..200f37f 100644 --- a/helm-chart/templates/deployment.yaml +++ b/helm-chart/templates/deployment.yaml @@ -27,23 +27,6 @@ spec: kubectl.kubernetes.io/default-container: manager spec: containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: {{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag }} - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: {{- toYaml .Values.controllerManager.kubeRbacProxy.resources | nindent 10 }} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - args: - --health-probe-bind-address={{ .Values.managerConfig.healthProbe.bindAddress }} - --metrics-bind-address={{ .Values.managerConfig.metrics.bindAddress }} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index d4f90c0..be40aba 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -2,20 +2,6 @@ controllerManager: replicas: 1 - ## Kube proxy config - ## In most cases you don't need to change those - kubeRbacProxy: - image: - repository: gcr.io/kubebuilder/kube-rbac-proxy - tag: v0.15.0 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - ## Controller image manager: image: From 8bf535b57f24bcb2766b2f188a96bca61fb47b5a Mon Sep 17 00:00:00 2001 From: Leonid Podolinskiy Date: Thu, 30 May 2024 13:11:47 +0300 Subject: [PATCH 4/4] Debug pipeline (#27) fix e2e pipeline --- .github/workflows/e2e.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 7f737c7..559bded 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -148,14 +148,13 @@ jobs: - printf "\n\nAgent INFO log\n" - kubectl exec -t deploy/sample-deployment -c app -- cat /tmp/lightrun_java_agent.INFO - printf "\n\nAgent ERROR log\n" - kubectl exec -t deploy/sample-deployment -c app -- cat /tmp/lightrun_java_agent.ERROR || true + printf "\n\nAgent log\n" + log_file_name=/tmp/$(kubectl exec -t deploy/sample-deployment - c app -- ls -t /tmp/ | grep lightrun_java_agent | head -n 1) + kubectl exec -t deploy/sample-deployment -c app -- cat $log_file_name - printf "\nSearching for "registered" in INFO log\n" - if kubectl exec -t deploy/sample-deployment -c app -- cat /tmp/lightrun_java_agent.INFO | grep Debuggee |grep registered > /dev/null; then + printf "\nSearching for "registered" in log\n" + if kubectl exec -t deploy/sample-deployment -c app -- cat $log_file_name | grep Debuggee |grep registered > /dev/null; then printf "\n----------------\nAgent registered succesfully!\n----------------\n" else printf "\n----------------\nAgent failed to register!\n----------------\n"