From b3e2188abbeca066f2ae6d36040af4f43a0bf410 Mon Sep 17 00:00:00 2001 From: ricolin Date: Thu, 9 Nov 2023 21:27:20 +0800 Subject: [PATCH] chore(cilium): bump charts --- charts/cilium/.helmignore | 1 + charts/cilium/Chart.yaml | 27 +- charts/cilium/README.md | 266 +- charts/cilium/README.md.gotmpl | 2 +- .../dashboards/cilium-dashboard.json | 8245 +++++++++++++++++ .../configmap/bootstrap-config.json | 323 + .../dashboards/cilium-operator-dashboard.json | 998 ++ .../hubble/dashboards/hubble-dashboard.json | 3232 +++++++ .../hubble-l7-http-metrics-by-workload.json | 239 +- charts/cilium/files/nodeinit/prestop.bash | 4 + charts/cilium/files/nodeinit/startup.bash | 8 +- charts/cilium/files/spire/init.bash | 48 + charts/cilium/files/spire/wait-for-spire.bash | 35 + .../templates/cilium-agent/clusterrole.yaml | 16 + .../templates/cilium-agent/daemonset.yaml | 113 +- .../cilium-agent/dashboards-configmap.yaml | 25 + .../cilium/templates/cilium-agent/role.yaml | 20 + .../templates/cilium-agent/rolebinding.yaml | 18 + .../templates/cilium-agent/service.yaml | 10 +- .../templates/cilium-ca-bundle-configmap.yaml | 11 + charts/cilium/templates/cilium-ca-secret.yaml | 2 +- charts/cilium/templates/cilium-configmap.yaml | 247 +- .../templates/cilium-envoy/configmap.yaml | 12 + .../templates/cilium-envoy/daemonset.yaml | 254 + .../templates/cilium-envoy/service.yaml | 27 + .../cilium-envoy/serviceaccount.yaml | 11 + .../cilium-envoy/servicemonitor.yaml | 39 + .../templates/cilium-ingress-class.yaml | 4 + .../templates/cilium-ingress-service.yaml | 11 + .../templates/cilium-nodeinit/daemonset.yaml | 13 + .../cilium-nodeinit/serviceaccount.yaml | 11 + .../cilium-operator/clusterrole.yaml | 15 +- .../cilium-operator/dashboards-configmap.yaml | 25 + .../templates/cilium-operator/deployment.yaml | 33 + .../cilium-preflight/clusterrole.yaml | 16 + .../templates/cilium-preflight/daemonset.yaml | 6 +- .../templates/cilium-secrets-namespace.yaml | 14 + .../clustermesh-apiserver/_helpers.tpl | 11 + .../clustermesh-apiserver/deployment.yaml | 198 +- .../metrics-service.yaml | 38 + .../clustermesh-apiserver/service.yaml | 6 + .../clustermesh-apiserver/servicemonitor.yaml | 71 + .../tls-certmanager/admin-secret.yaml | 2 +- .../tls-certmanager/remote-secret.yaml | 2 +- .../tls-certmanager/server-secret.yaml | 1 + .../tls-cronjob/_job-spec.tpl | 8 +- .../tls-cronjob/job.yaml | 18 +- .../tls-helm/admin-secret.yaml | 2 +- .../tls-helm/remote-secret.yaml | 2 +- .../tls-helm/server-secret.yaml | 2 +- .../tls-provided/admin-secret.yaml | 2 +- .../tls-provided/ca-secret.yaml | 6 +- .../tls-provided/client-secret.yaml | 2 +- .../tls-provided/remote-secret.yaml | 2 +- .../tls-provided/server-secret.yaml | 2 +- .../users-configmap.yaml | 21 + .../templates/clustermesh-config/_helpers.tpl | 21 +- .../clustermesh-secret.yaml | 11 +- .../kvstoremesh-secret.yaml | 21 + .../templates/hubble-relay/configmap.yaml | 17 +- .../templates/hubble-relay/deployment.yaml | 30 +- .../templates/hubble-ui/deployment.yaml | 14 +- .../cilium/templates/hubble-ui/ingress.yaml | 3 + .../cilium/templates/hubble/peer-service.yaml | 2 +- .../hubble/tls-cronjob/_job-spec.tpl | 6 +- .../hubble/tls-cronjob/ca-secret.yaml | 15 - .../hubble/tls-cronjob/clusterrole.yaml | 1 - .../templates/hubble/tls-cronjob/cronjob.yaml | 4 + .../templates/hubble/tls-cronjob/job.yaml | 18 +- .../templates/hubble/tls-helm/_helpers.tpl | 16 +- .../templates/hubble/tls-helm/ca-secret.yaml | 12 - .../hubble/tls-provided/ca-secret.yaml | 12 - .../tls-provided/relay-client-secret.yaml | 2 +- .../tls-provided/relay-server-secret.yaml | 2 +- .../hubble/tls-provided/server-secret.yaml | 2 +- .../hubble/tls-provided/ui-client-certs.yaml | 2 +- .../templates/spire/agent/clusterrole.yaml | 11 + .../spire/agent/clusterrolebinding.yaml | 15 + .../templates/spire/agent/configmap.yaml | 49 + .../templates/spire/agent/daemonset.yaml | 90 + .../templates/spire/agent/serviceaccount.yaml | 7 + .../templates/spire/bundle-configmap.yaml | 7 + charts/cilium/templates/spire/namespace.yaml | 6 + .../templates/spire/server/clusterrole.yaml | 22 + .../spire/server/clusterrolebinding.yaml | 14 + .../templates/spire/server/configmap.yaml | 68 + .../cilium/templates/spire/server/role.yaml | 12 + .../templates/spire/server/rolebinding.yaml | 29 + .../templates/spire/server/service.yaml | 24 + .../spire/server/serviceaccount.yaml | 7 + .../templates/spire/server/statefulset.yaml | 95 + charts/cilium/templates/validate.yaml | 35 +- charts/cilium/values.yaml | 847 +- charts/cilium/values.yaml.tmpl | 809 +- hack/sync-charts.sh | 2 +- 95 files changed, 16666 insertions(+), 471 deletions(-) create mode 100644 charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json create mode 100644 charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json create mode 100644 charts/cilium/files/cilium-operator/dashboards/cilium-operator-dashboard.json create mode 100644 charts/cilium/files/hubble/dashboards/hubble-dashboard.json create mode 100644 charts/cilium/files/spire/init.bash create mode 100644 charts/cilium/files/spire/wait-for-spire.bash create mode 100644 charts/cilium/templates/cilium-agent/dashboards-configmap.yaml create mode 100644 charts/cilium/templates/cilium-ca-bundle-configmap.yaml create mode 100644 charts/cilium/templates/cilium-envoy/configmap.yaml create mode 100644 charts/cilium/templates/cilium-envoy/daemonset.yaml create mode 100644 charts/cilium/templates/cilium-envoy/service.yaml create mode 100644 charts/cilium/templates/cilium-envoy/serviceaccount.yaml create mode 100644 charts/cilium/templates/cilium-envoy/servicemonitor.yaml create mode 100644 charts/cilium/templates/cilium-nodeinit/serviceaccount.yaml create mode 100644 charts/cilium/templates/cilium-operator/dashboards-configmap.yaml create mode 100644 charts/cilium/templates/clustermesh-apiserver/_helpers.tpl create mode 100644 charts/cilium/templates/clustermesh-apiserver/metrics-service.yaml create mode 100644 charts/cilium/templates/clustermesh-apiserver/servicemonitor.yaml create mode 100644 charts/cilium/templates/clustermesh-apiserver/users-configmap.yaml create mode 100644 charts/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml delete mode 100644 charts/cilium/templates/hubble/tls-cronjob/ca-secret.yaml delete mode 100644 charts/cilium/templates/hubble/tls-helm/ca-secret.yaml delete mode 100644 charts/cilium/templates/hubble/tls-provided/ca-secret.yaml create mode 100644 charts/cilium/templates/spire/agent/clusterrole.yaml create mode 100644 charts/cilium/templates/spire/agent/clusterrolebinding.yaml create mode 100644 charts/cilium/templates/spire/agent/configmap.yaml create mode 100644 charts/cilium/templates/spire/agent/daemonset.yaml create mode 100644 charts/cilium/templates/spire/agent/serviceaccount.yaml create mode 100644 charts/cilium/templates/spire/bundle-configmap.yaml create mode 100644 charts/cilium/templates/spire/namespace.yaml create mode 100644 charts/cilium/templates/spire/server/clusterrole.yaml create mode 100644 charts/cilium/templates/spire/server/clusterrolebinding.yaml create mode 100644 charts/cilium/templates/spire/server/configmap.yaml create mode 100644 charts/cilium/templates/spire/server/role.yaml create mode 100644 charts/cilium/templates/spire/server/rolebinding.yaml create mode 100644 charts/cilium/templates/spire/server/service.yaml create mode 100644 charts/cilium/templates/spire/server/serviceaccount.yaml create mode 100644 charts/cilium/templates/spire/server/statefulset.yaml diff --git a/charts/cilium/.helmignore b/charts/cilium/.helmignore index 0e8a0eb3..f9dcfa99 100644 --- a/charts/cilium/.helmignore +++ b/charts/cilium/.helmignore @@ -4,6 +4,7 @@ .DS_Store # Common VCS dirs .git/ +.github/ .gitignore .bzr/ .bzrignore diff --git a/charts/cilium/Chart.yaml b/charts/cilium/Chart.yaml index 795a45bb..6ce6541b 100644 --- a/charts/cilium/Chart.yaml +++ b/charts/cilium/Chart.yaml @@ -1,5 +1,5 @@ annotations: - artifacthub.io/crds: |- + artifacthub.io/crds: | - kind: CiliumNetworkPolicy version: v2 name: ciliumnetworkpolicies.cilium.io @@ -102,11 +102,30 @@ annotations: description: | CiliumNodeConfig is a list of configuration key-value pairs. It is applied to nodes indicated by a label selector. + - kind: CiliumCIDRGroup + version: v2alpha1 + name: ciliumcidrgroups.cilium.io + displayName: Cilium CIDR Group + description: | + CiliumCIDRGroup is a list of CIDRs that can be referenced as a single entity from CiliumNetworkPolicies. + - kind: CiliumL2AnnouncementPolicy + version: v2alpha1 + name: ciliuml2announcementpolicies.cilium.io + displayName: Cilium L2 Announcement Policy + description: | + CiliumL2AnnouncementPolicy is a policy which determines which service IPs will be announced to + the local area network, by which nodes, and via which interfaces. + - kind: CiliumPodIPPool + version: v2alpha1 + name: ciliumpodippools.cilium.io + displayName: Cilium Pod IP Pool + description: | + CiliumPodIPPool defines an IP pool that can be used for pooled IPAM (i.e. the multi-pool IPAM mode). apiVersion: v2 -appVersion: 1.13.3 +appVersion: 1.14.3 description: eBPF-based Networking, Security, and Observability home: https://cilium.io/ -icon: https://cdn.jsdelivr.net/gh/cilium/cilium@v1.13/Documentation/images/logo-solo.svg +icon: https://cdn.jsdelivr.net/gh/cilium/cilium@v1.14/Documentation/images/logo-solo.svg keywords: - BPF - eBPF @@ -119,4 +138,4 @@ kubeVersion: '>= 1.16.0-0' name: cilium sources: - https://github.com/cilium/cilium -version: 1.13.3 +version: 1.14.3 diff --git a/charts/cilium/README.md b/charts/cilium/README.md index f2e4aecd..a3e83a1f 100644 --- a/charts/cilium/README.md +++ b/charts/cilium/README.md @@ -1,6 +1,6 @@ # cilium -![Version: 1.13.3](https://img.shields.io/badge/Version-1.13.3-informational?style=flat-square) ![AppVersion: 1.13.3](https://img.shields.io/badge/AppVersion-1.13.3-informational?style=flat-square) +![Version: 1.14.3](https://img.shields.io/badge/Version-1.14.3-informational?style=flat-square) ![AppVersion: 1.14.3](https://img.shields.io/badge/AppVersion-1.14.3-informational?style=flat-square) Cilium is open source software for providing and transparently securing network connectivity and loadbalancing between application workloads such as @@ -14,7 +14,7 @@ integration points such as: network IO, application sockets, and tracepoints to implement security, networking and visibility logic. eBPF is highly efficient and flexible. -![Cilium feature overview](https://raw.githubusercontent.com/cilium/cilium/master/Documentation/images/cilium_overview.png) +![Cilium feature overview](https://raw.githubusercontent.com/cilium/cilium/main/Documentation/images/cilium_overview.png) ## Prerequisites @@ -60,6 +60,38 @@ contributors across the globe, there is almost always someone available to help. | aksbyocni.enabled | bool | `false` | Enable AKS BYOCNI integration. Note that this is incompatible with AKS clusters not created in BYOCNI mode: use Azure integration (`azure.enabled`) instead. | | alibabacloud.enabled | bool | `false` | Enable AlibabaCloud ENI integration | | annotateK8sNode | bool | `false` | Annotate k8s node upon initialization with Cilium's metadata. | +| authentication.enabled | bool | `true` | Enable authentication processing and garbage collection. Note that if disabled, policy enforcement will still block requests that require authentication. But the resulting authentication requests for these requests will not be processed, therefore the requests not be allowed. | +| authentication.gcInterval | string | `"5m0s"` | Interval for garbage collection of auth map entries. | +| authentication.mutual.port | int | `4250` | Port on the agent where mutual authentication handshakes between agents will be performed | +| authentication.mutual.spire.adminSocketPath | string | `"/run/spire/sockets/admin.sock"` | SPIRE socket path where the SPIRE delegated api agent is listening | +| authentication.mutual.spire.agentSocketPath | string | `"/run/spire/sockets/agent/agent.sock"` | SPIRE socket path where the SPIRE workload agent is listening. Applies to both the Cilium Agent and Operator | +| authentication.mutual.spire.connectionTimeout | string | `"30s"` | SPIRE connection timeout | +| authentication.mutual.spire.enabled | bool | `false` | Enable SPIRE integration (beta) | +| authentication.mutual.spire.install.agent.annotations | object | `{}` | SPIRE agent annotations | +| authentication.mutual.spire.install.agent.image | string | `"ghcr.io/spiffe/spire-agent:1.6.3@sha256:8eef9857bf223181ecef10d9bbcd2f7838f3689e9bd2445bede35066a732e823"` | SPIRE agent image | +| authentication.mutual.spire.install.agent.labels | object | `{}` | SPIRE agent labels | +| authentication.mutual.spire.install.agent.serviceAccount | object | `{"create":true,"name":"spire-agent"}` | SPIRE agent service account | +| authentication.mutual.spire.install.agent.skipKubeletVerification | bool | `true` | SPIRE Workload Attestor kubelet verification. | +| authentication.mutual.spire.install.enabled | bool | `true` | Enable SPIRE installation. This will only take effect only if authentication.mutual.spire.enabled is true | +| authentication.mutual.spire.install.namespace | string | `"cilium-spire"` | SPIRE namespace to install into | +| authentication.mutual.spire.install.server.annotations | object | `{}` | SPIRE server annotations | +| authentication.mutual.spire.install.server.ca.keyType | string | `"rsa-4096"` | SPIRE CA key type AWS requires the use of RSA. EC cryptography is not supported | +| authentication.mutual.spire.install.server.ca.subject | object | `{"commonName":"Cilium SPIRE CA","country":"US","organization":"SPIRE"}` | SPIRE CA Subject | +| authentication.mutual.spire.install.server.dataStorage.accessMode | string | `"ReadWriteOnce"` | Access mode of the SPIRE server data storage | +| authentication.mutual.spire.install.server.dataStorage.enabled | bool | `true` | Enable SPIRE server data storage | +| authentication.mutual.spire.install.server.dataStorage.size | string | `"1Gi"` | Size of the SPIRE server data storage | +| authentication.mutual.spire.install.server.dataStorage.storageClass | string | `nil` | StorageClass of the SPIRE server data storage | +| authentication.mutual.spire.install.server.image | string | `"ghcr.io/spiffe/spire-server:1.6.3@sha256:f4bc49fb0bd1d817a6c46204cc7ce943c73fb0a5496a78e0e4dc20c9a816ad7f"` | SPIRE server image | +| authentication.mutual.spire.install.server.initContainers | list | `[]` | SPIRE server init containers | +| authentication.mutual.spire.install.server.labels | object | `{}` | SPIRE server labels | +| authentication.mutual.spire.install.server.service.annotations | object | `{}` | Annotations to be added to the SPIRE server service | +| authentication.mutual.spire.install.server.service.labels | object | `{}` | Labels to be added to the SPIRE server service | +| authentication.mutual.spire.install.server.service.type | string | `"ClusterIP"` | Service type for the SPIRE server service | +| authentication.mutual.spire.install.server.serviceAccount | object | `{"create":true,"name":"spire-server"}` | SPIRE server service account | +| authentication.mutual.spire.serverAddress | string | `nil` | SPIRE server address used by Cilium Operator If k8s Service DNS along with port number is used (e.g. ..svc(.*): format), Cilium Operator will resolve its address by looking up the clusterIP from Service resource. Example values: 10.0.0.1:8081, spire-server.cilium-spire.svc:8081 | +| authentication.mutual.spire.trustDomain | string | `"spiffe.cilium"` | SPIFFE trust domain to use for fetching certificates | +| authentication.queueSize | int | `1024` | Buffer size of the channel Cilium uses to receive authentication events from the signal map. | +| authentication.rotatedIdentitiesQueueSize | int | `1024` | Buffer size of the channel Cilium uses to receive certificate expiration events from auth handlers. | | autoDirectNodeRoutes | bool | `false` | Enable installation of PodCIDR routes between worker nodes if worker nodes share a common L2 network segment. | | azure.enabled | bool | `false` | Enable Azure integration. Note that this is incompatible with AKS clusters created in BYOCNI mode: use AKS BYOCNI integration (`aksbyocni.enabled`) instead. | | bandwidthManager | object | `{"bbr":false,"enabled":false}` | Enable bandwidth manager to optimize TCP and UDP workloads and allow for rate-limiting traffic from individual Pods with EDT (Earliest Departure Time) through the "kubernetes.io/egress-bandwidth" Pod annotation. | @@ -71,7 +103,8 @@ contributors across the globe, there is almost always someone available to help. | bgp.enabled | bool | `false` | Enable BGP support inside Cilium; embeds a new ConfigMap for BGP inside cilium-agent and cilium-operator | | bgpControlPlane | object | `{"enabled":false}` | This feature set enables virtual BGP routers to be created via CiliumBGPPeeringPolicy CRDs. | | bgpControlPlane.enabled | bool | `false` | Enables the BGP control plane. | -| bpf.clockProbe | bool | `false` | Enable BPF clock source probing for more efficient tick retrieval. | +| bpf.authMapMax | int | `524288` | Configure the maximum number of entries in auth map. | +| bpf.autoMount.enabled | bool | `true` | Enable automatic mount of BPF filesystem When `autoMount` is enabled, the BPF filesystem is mounted at `bpf.root` path on the underlying host and inside the cilium agent pod. If users disable `autoMount`, it's expected that users have mounted bpffs filesystem at the specified `bpf.root` volume, and then the volume will be mounted inside the cilium agent pod at the same path. | | bpf.ctAnyMax | int | `262144` | Configure the maximum number of entries for the non-TCP connection tracking table. | | bpf.ctTcpMax | int | `524288` | Configure the maximum number of entries in the TCP connection tracking table. | | bpf.hostLegacyRouting | bool | `false` | Configure whether direct routing mode should route traffic via host stack (true) or directly and more efficiently out of BPF (false) if the kernel supports it. The latter has the implication that it will also bypass netfilter in the host namespace. | @@ -89,7 +122,9 @@ contributors across the globe, there is almost always someone available to help. | bpf.root | string | `"/sys/fs/bpf"` | Configure the mount point for the BPF filesystem | | bpf.tproxy | bool | `false` | Configure the eBPF-based TPROXY to reduce reliance on iptables rules for implementing Layer 7 policy. | | bpf.vlanBypass | list | `[]` | Configure explicitly allowed VLAN id's for bpf logic bypass. [0] will allow all VLAN id's without any filtering. | -| certgen | object | `{"extraVolumeMounts":[],"extraVolumes":[],"image":{"override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/certgen","tag":"v0.1.8@sha256:4a456552a5f192992a6edcec2febb1c54870d665173a33dc7d876129b199ddbd"},"podLabels":{},"tolerations":[],"ttlSecondsAfterFinished":1800}` | Configure certificate generation for Hubble integration. If hubble.tls.auto.method=cronJob, these values are used for the Kubernetes CronJob which will be scheduled regularly to (re)generate any certificates not provided manually. | +| bpfClockProbe | bool | `false` | Enable BPF clock source probing for more efficient tick retrieval. | +| certgen | object | `{"annotations":{"cronJob":{},"job":{}},"extraVolumeMounts":[],"extraVolumes":[],"image":{"digest":"sha256:89a0847753686444daabde9474b48340993bd19c7bea66a46e45b2974b82041f","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/certgen","tag":"v0.1.9","useDigest":true},"podLabels":{},"tolerations":[],"ttlSecondsAfterFinished":1800}` | Configure certificate generation for Hubble integration. If hubble.tls.auto.method=cronJob, these values are used for the Kubernetes CronJob which will be scheduled regularly to (re)generate any certificates not provided manually. | +| certgen.annotations | object | `{"cronJob":{},"job":{}}` | Annotations to be added to the hubble-certgen initial Job and CronJob | | certgen.extraVolumeMounts | list | `[]` | Additional certgen volumeMounts. | | certgen.extraVolumes | list | `[]` | Additional certgen volumes. | | certgen.podLabels | object | `{}` | Labels to be added to hubble-certgen pods | @@ -102,16 +137,43 @@ contributors across the globe, there is almost always someone available to help. | cleanBpfState | bool | `false` | Clean all eBPF datapath state from the initContainer of the cilium-agent DaemonSet. WARNING: Use with care! | | cleanState | bool | `false` | Clean all local Cilium state from the initContainer of the cilium-agent DaemonSet. Implies cleanBpfState: true. WARNING: Use with care! | | cluster.id | int | `0` | Unique ID of the cluster. Must be unique across all connected clusters and in the range of 1 to 255. Only required for Cluster Mesh, may be 0 if Cluster Mesh is not used. | -| cluster.name | string | `"default"` | Name of the cluster. Only required for Cluster Mesh. | +| cluster.name | string | `"default"` | Name of the cluster. Only required for Cluster Mesh and mutual authentication with SPIRE. | | clustermesh.apiserver.affinity | object | `{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"clustermesh-apiserver"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for clustermesh.apiserver | -| clustermesh.apiserver.etcd.image | object | `{"override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/coreos/etcd","tag":"v3.5.4@sha256:795d8660c48c439a7c3764c2330ed9222ab5db5bb524d8d0607cac76f7ba82a3"}` | Clustermesh API server etcd image. | +| clustermesh.apiserver.etcd.image | object | `{"digest":"sha256:795d8660c48c439a7c3764c2330ed9222ab5db5bb524d8d0607cac76f7ba82a3","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/coreos/etcd","tag":"v3.5.4","useDigest":true}` | Clustermesh API server etcd image. | | clustermesh.apiserver.etcd.init.resources | object | `{}` | Specifies the resources for etcd init container in the apiserver | | clustermesh.apiserver.etcd.resources | object | `{}` | Specifies the resources for etcd container in the apiserver | | clustermesh.apiserver.etcd.securityContext | object | `{}` | Security context to be added to clustermesh-apiserver etcd containers | +| clustermesh.apiserver.extraArgs | list | `[]` | Additional clustermesh-apiserver arguments. | | clustermesh.apiserver.extraEnv | list | `[]` | Additional clustermesh-apiserver environment variables. | | clustermesh.apiserver.extraVolumeMounts | list | `[]` | Additional clustermesh-apiserver volumeMounts. | | clustermesh.apiserver.extraVolumes | list | `[]` | Additional clustermesh-apiserver volumes. | -| clustermesh.apiserver.image | object | `{"digest":"sha256:5ad8e9dc17f5677d1d75b53a4e80ec2e5c4fcf4973ced8b30f8ad53933c6969a","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/clustermesh-apiserver","tag":"v1.13.3","useDigest":true}` | Clustermesh API server image. | +| clustermesh.apiserver.image | object | `{"digest":"sha256:a148485acda60f5aa9fc217ed2376ed93b7de9dd90706addeb361c6df928c192","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/clustermesh-apiserver","tag":"v1.14.3","useDigest":true}` | Clustermesh API server image. | +| clustermesh.apiserver.kvstoremesh.enabled | bool | `false` | Enable KVStoreMesh. KVStoreMesh caches the information retrieved from the remote clusters in the local etcd instance. | +| clustermesh.apiserver.kvstoremesh.extraArgs | list | `[]` | Additional KVStoreMesh arguments. | +| clustermesh.apiserver.kvstoremesh.extraEnv | list | `[]` | Additional KVStoreMesh environment variables. | +| clustermesh.apiserver.kvstoremesh.extraVolumeMounts | list | `[]` | Additional KVStoreMesh volumeMounts. | +| clustermesh.apiserver.kvstoremesh.image | object | `{"digest":"sha256:36d0b45c3f65ff3f391d5446b6434c5650d33576ba215a6e3159f0f8dc2ec693","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/kvstoremesh","tag":"v1.14.3","useDigest":true}` | KVStoreMesh image. | +| clustermesh.apiserver.kvstoremesh.resources | object | `{}` | Resource requests and limits for the KVStoreMesh container | +| clustermesh.apiserver.kvstoremesh.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}` | KVStoreMesh Security context | +| clustermesh.apiserver.metrics.enabled | bool | `true` | Enables exporting apiserver metrics in OpenMetrics format. | +| clustermesh.apiserver.metrics.etcd.enabled | bool | `false` | Enables exporting etcd metrics in OpenMetrics format. | +| clustermesh.apiserver.metrics.etcd.mode | string | `"basic"` | Set level of detail for etcd metrics; specify 'extensive' to include server side gRPC histogram metrics. | +| clustermesh.apiserver.metrics.etcd.port | int | `9963` | Configure the port the etcd metric server listens on. | +| clustermesh.apiserver.metrics.kvstoremesh.enabled | bool | `true` | Enables exporting KVStoreMesh metrics in OpenMetrics format. | +| clustermesh.apiserver.metrics.kvstoremesh.port | int | `9964` | Configure the port the KVStoreMesh metric server listens on. | +| clustermesh.apiserver.metrics.port | int | `9962` | Configure the port the apiserver metric server listens on. | +| clustermesh.apiserver.metrics.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor clustermesh-apiserver | +| clustermesh.apiserver.metrics.serviceMonitor.enabled | bool | `false` | Enable service monitor. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) | +| clustermesh.apiserver.metrics.serviceMonitor.etcd.interval | string | `"10s"` | Interval for scrape metrics (etcd metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.etcd.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (etcd metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.etcd.relabelings | string | `nil` | Relabeling configs for the ServiceMonitor clustermesh-apiserver (etcd metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics (apiserver metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.interval | string | `"10s"` | Interval for scrape metrics (KVStoreMesh metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (KVStoreMesh metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.relabelings | string | `nil` | Relabeling configs for the ServiceMonitor clustermesh-apiserver (KVStoreMesh metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor clustermesh-apiserver | +| clustermesh.apiserver.metrics.serviceMonitor.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (apiserver metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.relabelings | string | `nil` | Relabeling configs for the ServiceMonitor clustermesh-apiserver (apiserver metrics) | | clustermesh.apiserver.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | clustermesh.apiserver.podAnnotations | object | `{}` | Annotations to be added to clustermesh-apiserver pods | | clustermesh.apiserver.podDisruptionBudget.enabled | bool | `false` | enable PodDisruptionBudget ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ | @@ -124,16 +186,19 @@ contributors across the globe, there is almost always someone available to help. | clustermesh.apiserver.resources | object | `{}` | Resource requests and limits for the clustermesh-apiserver | | clustermesh.apiserver.securityContext | object | `{}` | Security context to be added to clustermesh-apiserver containers | | clustermesh.apiserver.service.annotations | object | `{}` | Annotations for the clustermesh-apiserver For GKE LoadBalancer, use annotation cloud.google.com/load-balancer-type: "Internal" For EKS LoadBalancer, use annotation service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 | +| clustermesh.apiserver.service.externalTrafficPolicy | string | `nil` | The externalTrafficPolicy of service used for apiserver access. | +| clustermesh.apiserver.service.internalTrafficPolicy | string | `nil` | The internalTrafficPolicy of service used for apiserver access. | | clustermesh.apiserver.service.nodePort | int | `32379` | Optional port to use as the node port for apiserver access. WARNING: make sure to configure a different NodePort in each cluster if kube-proxy replacement is enabled, as Cilium is currently affected by a known bug (#24692) when NodePorts are handled by the KPR implementation. If a service with the same NodePort exists both in the local and the remote cluster, all traffic originating from inside the cluster and targeting the corresponding NodePort will be redirected to a local backend, regardless of whether the destination node belongs to the local or the remote cluster. | | clustermesh.apiserver.service.type | string | `"NodePort"` | The type of service used for apiserver access. | | clustermesh.apiserver.tls.admin | object | `{"cert":"","key":""}` | base64 encoded PEM values for the clustermesh-apiserver admin certificate and private key. Used if 'auto' is not enabled. | +| clustermesh.apiserver.tls.authMode | string | `"legacy"` | Configure the clustermesh authentication mode. Supported values: - legacy: All clusters access remote clustermesh instances with the same username (i.e., remote). The "remote" certificate must be generated with CN=remote if provided manually. - migration: Intermediate mode required to upgrade from legacy to cluster (and vice versa) with no disruption. Specifically, it enables the creation of the per-cluster usernames, while still using the common one for authentication. The "remote" certificate must be generated with CN=remote if provided manually (same as legacy). - cluster: Each cluster accesses remote etcd instances with a username depending on the local cluster name (i.e., remote-). The "remote" certificate must be generated with CN=remote- if provided manually. Cluster mode is meaningful only when the same CA is shared across all clusters part of the mesh. | | clustermesh.apiserver.tls.auto | object | `{"certManagerIssuerRef":{},"certValidityDuration":1095,"enabled":true,"method":"helm"}` | Configure automatic TLS certificates generation. A Kubernetes CronJob is used the generate any certificates not provided by the user at installation time. | | clustermesh.apiserver.tls.auto.certManagerIssuerRef | object | `{}` | certmanager issuer used when clustermesh.apiserver.tls.auto.method=certmanager. | | clustermesh.apiserver.tls.auto.certValidityDuration | int | `1095` | Generated certificates validity duration in days. | | clustermesh.apiserver.tls.auto.enabled | bool | `true` | When set to true, automatically generate a CA and certificates to enable mTLS between clustermesh-apiserver and external workload instances. If set to false, the certs to be provided by setting appropriate values below. | -| clustermesh.apiserver.tls.ca | object | `{"cert":"","key":""}` | base64 encoded PEM values for the ExternalWorkload CA certificate and private key. | -| clustermesh.apiserver.tls.ca.cert | string | `""` | Optional CA cert. If it is provided, it will be used by the 'cronJob' method to generate all other certificates. Otherwise, an ephemeral CA is generated. | -| clustermesh.apiserver.tls.ca.key | string | `""` | Optional CA private key. If it is provided, it will be used by the 'cronJob' method to generate all other certificates. Otherwise, an ephemeral CA is generated. | +| clustermesh.apiserver.tls.ca | object | `{"cert":"","key":""}` | Deprecated in favor of tls.ca. To be removed in 1.15. base64 encoded PEM values for the ExternalWorkload CA certificate and private key. | +| clustermesh.apiserver.tls.ca.cert | string | `""` | Deprecated in favor of tls.ca.cert. To be removed in 1.15. Optional CA cert. If it is provided, it will be used by the 'cronJob' method to generate all other certificates. Otherwise, an ephemeral CA is generated. | +| clustermesh.apiserver.tls.ca.key | string | `""` | Deprecated in favor of tls.ca.key. To be removed in 1.15. Optional CA private key. If it is provided, it will be used by the 'cronJob' method to generate all other certificates. Otherwise, an ephemeral CA is generated. | | clustermesh.apiserver.tls.client | object | `{"cert":"","key":""}` | base64 encoded PEM values for the clustermesh-apiserver client certificate and private key. Used if 'auto' is not enabled. | | clustermesh.apiserver.tls.remote | object | `{"cert":"","key":""}` | base64 encoded PEM values for the clustermesh-apiserver remote cluster certificate and private key. Used if 'auto' is not enabled. | | clustermesh.apiserver.tls.server | object | `{"cert":"","extraDnsNames":[],"extraIpAddresses":[],"key":""}` | base64 encoded PEM values for the clustermesh-apiserver server certificate and private key. Used if 'auto' is not enabled. | @@ -148,7 +213,8 @@ contributors across the globe, there is almost always someone available to help. | clustermesh.config.enabled | bool | `false` | Enable the Clustermesh explicit configuration. | | clustermesh.useAPIServer | bool | `false` | Deploy clustermesh-apiserver for clustermesh | | cni.binPath | string | `"/opt/cni/bin"` | Configure the path to the CNI binary directory on the host. | -| cni.chainingMode | string | `"none"` | Configure chaining on top of other CNI plugins. Possible values: - none - aws-cni - flannel - generic-veth - portmap | +| cni.chainingMode | string | `nil` | Configure chaining on top of other CNI plugins. Possible values: - none - aws-cni - flannel - generic-veth - portmap | +| cni.chainingTarget | string | `nil` | A CNI network name in to which the Cilium plugin should be added as a chained plugin. This will cause the agent to watch for a CNI network with this network name. When it is found, this will be used as the basis for Cilium's CNI configuration file. If this is set, it assumes a chaining mode of generic-veth. As a special case, a chaining mode of aws-cni implies a chainingTarget of aws-cni. | | cni.confFileMountPath | string | `"/tmp/cni-configuration"` | Configure the path to where to mount the ConfigMap inside the agent pod. | | cni.confPath | string | `"/etc/cni/net.d"` | Configure the path to the CNI configuration directory on the host. | | cni.configMapKey | string | `"cni-config"` | Configure the key in the CNI ConfigMap to read the contents of the CNI configuration from. | @@ -157,10 +223,11 @@ contributors across the globe, there is almost always someone available to help. | cni.hostConfDirMountPath | string | `"/host/etc/cni/net.d"` | Configure the path to where the CNI configuration directory is mounted inside the agent pod. | | cni.install | bool | `true` | Install the CNI configuration and binary files into the filesystem. | | cni.logFile | string | `"/var/run/cilium/cilium-cni.log"` | Configure the log file for CNI logging with retention policy of 7 days. Disable CNI file logging by setting this field to empty explicitly. | -| cni.uninstall | bool | `true` | Remove the CNI configuration and binary files on agent shutdown. Enable this if you're removing Cilium from the cluster. Disable this to prevent the CNI configuration file from being removed during agent upgrade, which can cause nodes to go unmanageable. | +| cni.uninstall | bool | `false` | Remove the CNI configuration and binary files on agent shutdown. Enable this if you're removing Cilium from the cluster. Disable this to prevent the CNI configuration file from being removed during agent upgrade, which can cause nodes to go unmanageable. | | conntrackGCInterval | string | `"0s"` | Configure how frequently garbage collection should occur for the datapath connection tracking table. | -| containerRuntime | object | `{"integration":"none"}` | Configure container runtime specific integration. | -| containerRuntime.integration | string | `"none"` | Enables specific integrations for container runtimes. Supported values: - containerd - crio - docker - none - auto (automatically detect the container runtime) | +| conntrackGCMaxInterval | string | `""` | Configure the maximum frequency for the garbage collection of the connection tracking table. Only affects the automatic computation for the frequency and has no effect when 'conntrackGCInterval' is set. This can be set to more frequently clean up unused identities created from ToFQDN policies. | +| containerRuntime | object | `{"integration":"none"}` | Configure container runtime specific integration. Deprecated in favor of bpf.autoMount.enabled. To be removed in 1.15. | +| containerRuntime.integration | string | `"none"` | Enables specific integrations for container runtimes. Supported values: - crio - none | | crdWaitTimeout | string | `"5m"` | Configure timeout in which Cilium will exit if CRDs are not available | | customCalls | object | `{"enabled":false}` | Tail call hooks for custom eBPF programs. | | customCalls.enabled | bool | `false` | Enable tail call hooks for custom eBPF programs. | @@ -168,41 +235,46 @@ contributors across the globe, there is almost always someone available to help. | daemon.blockedConfigOverrides | string | `nil` | blockedConfigOverrides is a list of config-map keys that may not be overridden. In other words, if any of these keys appear in a configuration source excepting the first one, they will be ignored This is ignored if allowedConfigOverrides is set. By default, all keys may be overridden. | | daemon.configSources | string | `nil` | Configure a custom list of possible configuration override sources The default is "config-map:cilium-config,cilium-node-config". For supported values, see the help text for the build-config subcommand. Note that this value should be a comma-separated string. | | daemon.runPath | string | `"/var/run/cilium"` | Configure where Cilium runtime state should be stored. | +| dashboards | object | `{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":null}` | Grafana dashboards for cilium-agent grafana can import dashboards based on the label and value ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards | | debug.enabled | bool | `false` | Enable debug logging | | debug.verbose | string | `nil` | Configure verbosity levels for debug logging This option is used to enable debug messages for operations related to such sub-system such as (e.g. kvstore, envoy, datapath or policy), and flow is for enabling debug messages emitted per request, message and connection. Applicable values: - flow - kvstore - envoy - datapath - policy | -| disableEndpointCRD | string | `"false"` | Disable the usage of CiliumEndpoint CRD. | +| disableEndpointCRD | bool | `false` | Disable the usage of CiliumEndpoint CRD. | | dnsPolicy | string | `""` | DNS policy for Cilium agent pods. Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy | | dnsProxy.dnsRejectResponseCode | string | `"refused"` | DNS response code for rejecting DNS requests, available options are '[nameError refused]'. | | dnsProxy.enableDnsCompression | bool | `true` | Allow the DNS proxy to compress responses to endpoints that are larger than 512 Bytes or the EDNS0 option, if present. | | dnsProxy.endpointMaxIpPerHostname | int | `50` | Maximum number of IPs to maintain per FQDN name for each endpoint. | | dnsProxy.idleConnectionGracePeriod | string | `"0s"` | Time during which idle but previously active connections with expired DNS lookups are still considered alive. | | dnsProxy.maxDeferredConnectionDeletes | int | `10000` | Maximum number of IPs to retain for expired DNS lookups with still-active connections. | -| dnsProxy.minTtl | int | `3600` | The minimum time, in seconds, to use DNS data for toFQDNs policies. | +| dnsProxy.minTtl | int | `0` | The minimum time, in seconds, to use DNS data for toFQDNs policies. If the upstream DNS server returns a DNS record with a shorter TTL, Cilium overwrites the TTL with this value. Setting this value to zero means that Cilium will honor the TTLs returned by the upstream DNS server. | | dnsProxy.preCache | string | `""` | DNS cache data at this path is preloaded on agent startup. | | dnsProxy.proxyPort | int | `0` | Global port on which the in-agent DNS proxy should listen. Default 0 is a OS-assigned port. | | dnsProxy.proxyResponseMaxDelay | string | `"100ms"` | The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information. | -| egressGateway | object | `{"enabled":false,"installRoutes":false}` | Enables egress gateway to redirect and SNAT the traffic that leaves the cluster. | +| egressGateway | object | `{"enabled":false,"installRoutes":false,"reconciliationTriggerInterval":"1s"}` | Enables egress gateway to redirect and SNAT the traffic that leaves the cluster. | | egressGateway.installRoutes | bool | `false` | Install egress gateway IP rules and routes in order to properly steer egress gateway traffic to the correct ENI interface | +| egressGateway.reconciliationTriggerInterval | string | `"1s"` | Time between triggers of egress gateway state reconciliations | | enableCiliumEndpointSlice | bool | `false` | Enable CiliumEndpointSlice feature. | | enableCnpStatusUpdates | bool | `false` | Whether to enable CNP status updates. | | enableCriticalPriorityClass | bool | `true` | Explicitly enable or disable priority class. .Capabilities.KubeVersion is unsettable in `helm template` calls, it depends on k8s libraries version that Helm was compiled against. This option allows to explicitly disable setting the priority class, which is useful for rendering charts for gke clusters in advance. | +| enableIPv4BIGTCP | bool | `false` | Enables IPv4 BIG TCP support which increases maximum IPv4 GSO/GRO limits for nodes and pods | | enableIPv4Masquerade | bool | `true` | Enables masquerading of IPv4 traffic leaving the node from endpoints. | -| enableIPv6BIGTCP | bool | `false` | Enables IPv6 BIG TCP support which increases maximum GSO/GRO limits for nodes and pods | +| enableIPv6BIGTCP | bool | `false` | Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods | | enableIPv6Masquerade | bool | `true` | Enables masquerading of IPv6 traffic leaving the node from endpoints. | | enableK8sEventHandover | bool | `false` | Configures the use of the KVStore to optimize Kubernetes event handling by mirroring it into the KVstore for reduced overhead in large clusters. | | enableK8sTerminatingEndpoint | bool | `true` | Configure whether to enable auto detect of terminating state for endpoints in order to support graceful termination. | | enableRuntimeDeviceDetection | bool | `false` | Enables experimental support for the detection of new and removed datapath devices. When devices change the eBPF datapath is reloaded and services updated. If "devices" is set then only those devices, or devices matching a wildcard will be considered. | | enableXTSocketFallback | bool | `true` | Enables the fallback compatibility solution for when the xt_socket kernel module is missing and it is needed for the datapath L7 redirection to work properly. See documentation for details on when this can be disabled: https://docs.cilium.io/en/stable/operations/system_requirements/#linux-kernel. | | encryption.enabled | bool | `false` | Enable transparent network encryption. | -| encryption.interface | string | `""` | Deprecated in favor of encryption.ipsec.interface. The interface to use for encrypted traffic. This option is only effective when encryption.type is set to ipsec. | +| encryption.interface | string | `""` | Deprecated in favor of encryption.ipsec.interface. To be removed in 1.15. The interface to use for encrypted traffic. This option is only effective when encryption.type is set to ipsec. | | encryption.ipsec.interface | string | `""` | The interface to use for encrypted traffic. | | encryption.ipsec.keyFile | string | `""` | Name of the key file inside the Kubernetes secret configured via secretName. | +| encryption.ipsec.keyRotationDuration | string | `"5m"` | Maximum duration of the IPsec key rotation. The previous key will be removed after that delay. | +| encryption.ipsec.keyWatcher | bool | `true` | Enable the key watcher. If disabled, a restart of the agent will be necessary on key rotations. | | encryption.ipsec.mountPath | string | `""` | Path to mount the secret inside the Cilium pod. | | encryption.ipsec.secretName | string | `""` | Name of the Kubernetes secret containing the encryption keys. | -| encryption.keyFile | string | `"keys"` | Deprecated in favor of encryption.ipsec.keyFile. Name of the key file inside the Kubernetes secret configured via secretName. This option is only effective when encryption.type is set to ipsec. | -| encryption.mountPath | string | `"/etc/ipsec"` | Deprecated in favor of encryption.ipsec.mountPath. Path to mount the secret inside the Cilium pod. This option is only effective when encryption.type is set to ipsec. | -| encryption.nodeEncryption | bool | `false` | Enable encryption for pure node to node traffic. This option is only effective when encryption.type is set to ipsec. | -| encryption.secretName | string | `"cilium-ipsec-keys"` | Deprecated in favor of encryption.ipsec.secretName. Name of the Kubernetes secret containing the encryption keys. This option is only effective when encryption.type is set to ipsec. | +| encryption.keyFile | string | `"keys"` | Deprecated in favor of encryption.ipsec.keyFile. To be removed in 1.15. Name of the key file inside the Kubernetes secret configured via secretName. This option is only effective when encryption.type is set to ipsec. | +| encryption.mountPath | string | `"/etc/ipsec"` | Deprecated in favor of encryption.ipsec.mountPath. To be removed in 1.15. Path to mount the secret inside the Cilium pod. This option is only effective when encryption.type is set to ipsec. | +| encryption.nodeEncryption | bool | `false` | Enable encryption for pure node to node traffic. This option is only effective when encryption.type is set to "wireguard". | +| encryption.secretName | string | `"cilium-ipsec-keys"` | Deprecated in favor of encryption.ipsec.secretName. To be removed in 1.15. Name of the Kubernetes secret containing the encryption keys. This option is only effective when encryption.type is set to ipsec. | | encryption.type | string | `"ipsec"` | Encryption method. Can be either ipsec or wireguard. | | encryption.wireguard.userspaceFallback | bool | `false` | Enables the fallback to the user-space implementation. | | endpointHealthChecking.enabled | bool | `true` | Enable connectivity health checking between virtual endpoints. | @@ -219,14 +291,62 @@ contributors across the globe, there is almost always someone available to help. | eni.instanceTagsFilter | list | `[]` | Filter via AWS EC2 Instance tags (k=v) which will dictate which AWS EC2 Instances are going to be used to create new ENIs | | eni.subnetIDsFilter | list | `[]` | Filter via subnet IDs which will dictate which subnets are going to be used to create new ENIs Important note: This requires that each instance has an ENI with a matching subnet attached when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, use the CNI configuration file settings (cni.customConf) instead. | | eni.subnetTagsFilter | list | `[]` | Filter via tags (k=v) which will dictate which subnets are going to be used to create new ENIs Important note: This requires that each instance has an ENI with a matching subnet attached when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, use the CNI configuration file settings (cni.customConf) instead. | -| eni.updateEC2AdapterLimitViaAPI | bool | `false` | Update ENI Adapter limits from the EC2 API | +| eni.updateEC2AdapterLimitViaAPI | bool | `true` | Update ENI Adapter limits from the EC2 API | +| envoy.affinity | object | `{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium-envoy"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for cilium-envoy. | +| envoy.connectTimeoutSeconds | int | `2` | Time in seconds after which a TCP connection attempt times out | +| envoy.dnsPolicy | string | `nil` | DNS policy for Cilium envoy pods. Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy | +| envoy.enabled | bool | `false` | Enable Envoy Proxy in standalone DaemonSet. | +| envoy.extraArgs | list | `[]` | Additional envoy container arguments. | +| envoy.extraContainers | list | `[]` | Additional containers added to the cilium Envoy DaemonSet. | +| envoy.extraEnv | list | `[]` | Additional envoy container environment variables. | +| envoy.extraHostPathMounts | list | `[]` | Additional envoy hostPath mounts. | +| envoy.extraVolumeMounts | list | `[]` | Additional envoy volumeMounts. | +| envoy.extraVolumes | list | `[]` | Additional envoy volumes. | +| envoy.healthPort | int | `9878` | TCP port for the health API. | +| envoy.idleTimeoutDurationSeconds | int | `60` | Set Envoy upstream HTTP idle connection timeout seconds. Does not apply to connections with pending requests. Default 60s | +| envoy.image | object | `{"digest":"sha256:bfa1e919ed02afc66e9ff36c1fd9148237fc8b8560a0b44d89acf144b0ffb08c","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-envoy","tag":"v1.25.10-f71a313bd0daee41470af31ce6ea20c750fe35dd","useDigest":true}` | Envoy container image. | +| envoy.livenessProbe.failureThreshold | int | `10` | failure threshold of liveness probe | +| envoy.livenessProbe.periodSeconds | int | `30` | interval between checks of the liveness probe | +| envoy.log.format | string | `"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"` | The format string to use for laying out the log message metadata of Envoy. | +| envoy.log.path | string | `""` | Path to a separate Envoy log file, if any. Defaults to /dev/stdout. | +| envoy.maxConnectionDurationSeconds | int | `0` | Set Envoy HTTP option max_connection_duration seconds. Default 0 (disable) | +| envoy.maxRequestsPerConnection | int | `0` | ProxyMaxRequestsPerConnection specifies the max_requests_per_connection setting for Envoy | +| envoy.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selector for cilium-envoy. | +| envoy.podAnnotations | object | `{}` | Annotations to be added to envoy pods | +| envoy.podLabels | object | `{}` | Labels to be added to envoy pods | +| envoy.podSecurityContext | object | `{}` | Security Context for cilium-envoy pods. | +| envoy.priorityClassName | string | `nil` | The priority class to use for cilium-envoy. | +| envoy.prometheus.enabled | bool | `true` | Enable prometheus metrics for cilium-envoy | +| envoy.prometheus.port | string | `"9964"` | Serve prometheus metrics for cilium-envoy on the configured port | +| envoy.prometheus.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor cilium-envoy | +| envoy.prometheus.serviceMonitor.enabled | bool | `false` | Enable service monitors. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) | +| envoy.prometheus.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics. | +| envoy.prometheus.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor cilium-envoy | +| envoy.prometheus.serviceMonitor.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor cilium-envoy | +| envoy.prometheus.serviceMonitor.relabelings | list | `[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]` | Relabeling configs for the ServiceMonitor cilium-envoy | +| envoy.readinessProbe.failureThreshold | int | `3` | failure threshold of readiness probe | +| envoy.readinessProbe.periodSeconds | int | `30` | interval between checks of the readiness probe | +| envoy.resources | object | `{}` | Envoy resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | +| envoy.rollOutPods | bool | `false` | Roll out cilium envoy pods automatically when configmap is updated. | +| envoy.securityContext.capabilities.envoy | list | `["NET_ADMIN","SYS_ADMIN"]` | Capabilities for the `cilium-envoy` container | +| envoy.securityContext.privileged | bool | `false` | Run the pod with elevated privileges | +| envoy.securityContext.seLinuxOptions | object | `{"level":"s0","type":"spc_t"}` | SELinux options for the `cilium-envoy` container | +| envoy.startupProbe.failureThreshold | int | `105` | failure threshold of startup probe. 105 x 2s translates to the old behaviour of the readiness probe (120s delay + 30 x 3s) | +| envoy.startupProbe.periodSeconds | int | `2` | interval between checks of the startup probe | +| envoy.terminationGracePeriodSeconds | int | `1` | Configure termination grace period for cilium-envoy DaemonSet. | +| envoy.tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for envoy scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| envoy.updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":2},"type":"RollingUpdate"}` | cilium-envoy update strategy ref: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/#updating-a-daemonset | +| envoyConfig.enabled | bool | `false` | Enable CiliumEnvoyConfig CRD CiliumEnvoyConfig CRD can also be implicitly enabled by other options. | +| envoyConfig.secretsNamespace | object | `{"create":true,"name":"cilium-secrets"}` | SecretsNamespace is the namespace in which envoy SDS will retrieve secrets from. | +| envoyConfig.secretsNamespace.create | bool | `true` | Create secrets namespace for CiliumEnvoyConfig CRDs. | +| envoyConfig.secretsNamespace.name | string | `"cilium-secrets"` | The name of the secret namespace to which Cilium agents are given read access. | | etcd.clusterDomain | string | `"cluster.local"` | Cluster domain for cilium-etcd-operator. | | etcd.enabled | bool | `false` | Enable etcd mode for the agent. | | etcd.endpoints | list | `["https://CHANGE-ME:2379"]` | List of etcd endpoints (not needed when using managed=true). | | etcd.extraArgs | list | `[]` | Additional cilium-etcd-operator container arguments. | | etcd.extraVolumeMounts | list | `[]` | Additional cilium-etcd-operator volumeMounts. | | etcd.extraVolumes | list | `[]` | Additional cilium-etcd-operator volumes. | -| etcd.image | object | `{"override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-etcd-operator","tag":"v2.0.7@sha256:04b8327f7f992693c2cb483b999041ed8f92efc8e14f2a5f3ab95574a65ea2dc"}` | cilium-etcd-operator image. | +| etcd.image | object | `{"digest":"sha256:04b8327f7f992693c2cb483b999041ed8f92efc8e14f2a5f3ab95574a65ea2dc","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-etcd-operator","tag":"v2.0.7","useDigest":true}` | cilium-etcd-operator image. | | etcd.k8sService | bool | `false` | If etcd is behind a k8s service set this option to true so that Cilium does the service translation automatically without requiring a DNS to be running. | | etcd.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for cilium-etcd-operator pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | etcd.podAnnotations | object | `{}` | Annotations to be added to cilium-etcd-operator pods | @@ -260,12 +380,15 @@ contributors across the globe, there is almost always someone available to help. | gke.enabled | bool | `false` | Enable Google Kubernetes Engine integration | | healthChecking | bool | `true` | Enable connectivity health checking. | | healthPort | int | `9879` | TCP port for the agent health API. This is not the port for cilium-health. | +| highScaleIPcache | object | `{"enabled":false}` | EnableHighScaleIPcache enables the special ipcache mode for high scale clusters. The ipcache content will be reduced to the strict minimum and traffic will be encapsulated to carry security identities. | +| highScaleIPcache.enabled | bool | `false` | Enable the high scale mode for the ipcache. | | hostFirewall | object | `{"enabled":false}` | Configure the host firewall. | | hostFirewall.enabled | bool | `false` | Enables the enforcement of host policies in the eBPF datapath. | | hostPort.enabled | bool | `false` | Enable hostPort service support. | | hubble.enabled | bool | `true` | Enable Hubble (true by default). | | hubble.listenAddress | string | `":4244"` | An additional address for Hubble to listen to. Set this field ":4244" if you are enabling Hubble Relay, as it assumes that Hubble is listening on port 4244. | | hubble.metrics | object | `{"dashboards":{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":null},"enableOpenMetrics":false,"enabled":null,"port":9965,"serviceAnnotations":{},"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","labels":{},"metricRelabelings":null,"relabelings":[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]}}` | Hubble metrics configuration. See https://docs.cilium.io/en/stable/observability/metrics/#hubble-metrics for more comprehensive documentation about Hubble metrics. | +| hubble.metrics.dashboards | object | `{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":null}` | Grafana dashboards for hubble grafana can import dashboards based on the label and value ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards | | hubble.metrics.enableOpenMetrics | bool | `false` | Enables exporting hubble metrics in OpenMetrics format. | | hubble.metrics.enabled | string | `nil` | Configures the list of metrics to collect. If empty or null, metrics are disabled. Example: enabled: - dns:query;ignoreAAAA - drop - tcp - flow - icmp - http You can specify the list of metrics from the helm CLI: --set metrics.enabled="{dns:query;ignoreAAAA,drop,tcp,flow,icmp,http}" | | hubble.metrics.port | int | `9965` | Configure the port the hubble metric server listens on. | @@ -277,14 +400,15 @@ contributors across the globe, there is almost always someone available to help. | hubble.metrics.serviceMonitor.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor hubble | | hubble.metrics.serviceMonitor.relabelings | list | `[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]` | Relabeling configs for the ServiceMonitor hubble | | hubble.peerService.clusterDomain | string | `"cluster.local"` | The cluster domain to use to query the Hubble Peer service. It should be the local cluster. | -| hubble.peerService.enabled | bool | `true` | Enable a K8s Service for the Peer service, so that it can be accessed by a non-local client. This configuration option is deprecated, the peer service will be non-optional starting Cilium v1.14. | | hubble.peerService.targetPort | int | `4244` | Target Port for the Peer service, must match the hubble.listenAddress' port. | | hubble.preferIpv6 | bool | `false` | Whether Hubble should prefer to announce IPv6 or IPv4 addresses if both are available. | | hubble.relay.affinity | object | `{"podAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for hubble-replay | | hubble.relay.dialTimeout | string | `nil` | Dial timeout to connect to the local hubble instance to receive peer information (e.g. "30s"). | | hubble.relay.enabled | bool | `false` | Enable Hubble Relay (requires hubble.enabled=true) | | hubble.relay.extraEnv | list | `[]` | Additional hubble-relay environment variables. | -| hubble.relay.image | object | `{"digest":"sha256:19e4aae5ff72cd9fbcb7d2d16a1570533320a478acc015fc91a4d41a177cadf6","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-relay","tag":"v1.13.3","useDigest":true}` | Hubble-relay container image. | +| hubble.relay.gops.enabled | bool | `true` | Enable gops for hubble-relay | +| hubble.relay.gops.port | int | `9893` | Configure gops listen port for hubble-relay | +| hubble.relay.image | object | `{"digest":"sha256:3f5c425faca4f8a38e29d4f86db2aadd8f33c9f05debd78a2bb2a24dd9e565d8","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-relay","tag":"v1.14.3","useDigest":true}` | Hubble-relay container image. | | hubble.relay.listenHost | string | `""` | Host to listen to. Specify an empty string to bind to all the interfaces. | | hubble.relay.listenPort | string | `"4245"` | Port to listen to. | | hubble.relay.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | @@ -293,6 +417,7 @@ contributors across the globe, there is almost always someone available to help. | hubble.relay.podDisruptionBudget.maxUnavailable | int | `1` | Maximum number/percentage of pods that may be made unavailable | | hubble.relay.podDisruptionBudget.minAvailable | string | `nil` | Minimum number/percentage of pods that should remain scheduled. When it's set, maxUnavailable must be disabled by `maxUnavailable: null` | | hubble.relay.podLabels | object | `{}` | Labels to be added to hubble-relay pods | +| hubble.relay.podSecurityContext | object | `{"fsGroup":65532}` | hubble-relay pod security context | | hubble.relay.pprof.address | string | `"localhost"` | Configure pprof listen address for hubble-relay | | hubble.relay.pprof.enabled | bool | `false` | Enable pprof for hubble-relay | | hubble.relay.pprof.port | int | `6062` | Configure pprof listen port for hubble-relay | @@ -308,16 +433,16 @@ contributors across the globe, there is almost always someone available to help. | hubble.relay.resources | object | `{}` | Specifies the resources for the hubble-relay pods | | hubble.relay.retryTimeout | string | `nil` | Backoff duration to retry connecting to the local hubble instance in case of failure (e.g. "30s"). | | hubble.relay.rollOutPods | bool | `false` | Roll out Hubble Relay pods automatically when configmap is updated. | -| hubble.relay.securityContext | object | `{}` | hubble-relay security context | +| hubble.relay.securityContext | object | `{"capabilities":{"drop":["ALL"]},"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | hubble-relay container security context | | hubble.relay.service | object | `{"nodePort":31234,"type":"ClusterIP"}` | hubble-relay service configuration. | | hubble.relay.service.nodePort | int | `31234` | - The port to use when the service type is set to NodePort. | | hubble.relay.service.type | string | `"ClusterIP"` | - The type of service used for Hubble Relay access, either ClusterIP or NodePort. | | hubble.relay.sortBufferDrainTimeout | string | `nil` | When the per-request flows sort buffer is not full, a flow is drained every time this timeout is reached (only affects requests in follow-mode) (e.g. "1s"). | | hubble.relay.sortBufferLenMax | string | `nil` | Max number of flows that can be buffered for sorting before being sent to the client (per request) (e.g. 100). | | hubble.relay.terminationGracePeriodSeconds | int | `1` | Configure termination grace period for hubble relay Deployment. | -| hubble.relay.tls | object | `{"client":{"cert":"","key":""},"server":{"cert":"","enabled":false,"extraDnsNames":[],"extraIpAddresses":[],"key":""}}` | TLS configuration for Hubble Relay | +| hubble.relay.tls | object | `{"client":{"cert":"","key":""},"server":{"cert":"","enabled":false,"extraDnsNames":[],"extraIpAddresses":[],"key":"","mtls":false}}` | TLS configuration for Hubble Relay | | hubble.relay.tls.client | object | `{"cert":"","key":""}` | base64 encoded PEM values for the hubble-relay client certificate and private key This keypair is presented to Hubble server instances for mTLS authentication and is required when hubble.tls.enabled is true. These values need to be set manually if hubble.tls.auto.enabled is false. | -| hubble.relay.tls.server | object | `{"cert":"","enabled":false,"extraDnsNames":[],"extraIpAddresses":[],"key":""}` | base64 encoded PEM values for the hubble-relay server certificate and private key | +| hubble.relay.tls.server | object | `{"cert":"","enabled":false,"extraDnsNames":[],"extraIpAddresses":[],"key":"","mtls":false}` | base64 encoded PEM values for the hubble-relay server certificate and private key | | hubble.relay.tls.server.extraDnsNames | list | `[]` | extra DNS names added to certificate when its auto gen | | hubble.relay.tls.server.extraIpAddresses | list | `[]` | extra IP addresses added to certificate when its auto gen | | hubble.relay.tolerations | list | `[]` | Node tolerations for pod assignment on nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | @@ -325,16 +450,13 @@ contributors across the globe, there is almost always someone available to help. | hubble.relay.updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | hubble-relay update strategy | | hubble.skipUnknownCGroupIDs | bool | `true` | Skip Hubble events with unknown cgroup ids | | hubble.socketPath | string | `"/var/run/cilium/hubble.sock"` | Unix domain socket path to listen to when Hubble is enabled. | -| hubble.tls | object | `{"auto":{"certManagerIssuerRef":{},"certValidityDuration":1095,"enabled":true,"method":"helm","schedule":"0 0 1 */4 *"},"ca":{"cert":"","key":""},"enabled":true,"server":{"cert":"","extraDnsNames":[],"extraIpAddresses":[],"key":""}}` | TLS configuration for Hubble | +| hubble.tls | object | `{"auto":{"certManagerIssuerRef":{},"certValidityDuration":1095,"enabled":true,"method":"helm","schedule":"0 0 1 */4 *"},"enabled":true,"server":{"cert":"","extraDnsNames":[],"extraIpAddresses":[],"key":""}}` | TLS configuration for Hubble | | hubble.tls.auto | object | `{"certManagerIssuerRef":{},"certValidityDuration":1095,"enabled":true,"method":"helm","schedule":"0 0 1 */4 *"}` | Configure automatic TLS certificates generation. | | hubble.tls.auto.certManagerIssuerRef | object | `{}` | certmanager issuer used when hubble.tls.auto.method=certmanager. | | hubble.tls.auto.certValidityDuration | int | `1095` | Generated certificates validity duration in days. | | hubble.tls.auto.enabled | bool | `true` | Auto-generate certificates. When set to true, automatically generate a CA and certificates to enable mTLS between Hubble server and Hubble Relay instances. If set to false, the certs for Hubble server need to be provided by setting appropriate values below. | | hubble.tls.auto.method | string | `"helm"` | Set the method to auto-generate certificates. Supported values: - helm: This method uses Helm to generate all certificates. - cronJob: This method uses a Kubernetes CronJob the generate any certificates not provided by the user at installation time. - certmanager: This method use cert-manager to generate & rotate certificates. | | hubble.tls.auto.schedule | string | `"0 0 1 */4 *"` | Schedule for certificates regeneration (regardless of their expiration date). Only used if method is "cronJob". If nil, then no recurring job will be created. Instead, only the one-shot job is deployed to generate the certificates at installation time. Defaults to midnight of the first day of every fourth month. For syntax, see https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#schedule-syntax | -| hubble.tls.ca | object | `{"cert":"","key":""}` | Deprecated in favor of tls.ca. To be removed in 1.13. base64 encoded PEM values for the Hubble CA certificate and private key. | -| hubble.tls.ca.cert | string | `""` | Deprecated in favor of tls.ca.cert. To be removed in 1.13. | -| hubble.tls.ca.key | string | `""` | Deprecated in favor of tls.ca.key. To be removed in 1.13. The CA private key (optional). If it is provided, then it will be used by hubble.tls.auto.method=cronJob to generate all other certificates. Otherwise, a ephemeral CA is generated if hubble.tls.auto.enabled=true. | | hubble.tls.enabled | bool | `true` | Enable mutual TLS for listenAddress. Setting this value to false is highly discouraged as the Hubble API provides access to potentially sensitive network flow metadata and is exposed on the host network. | | hubble.tls.server | object | `{"cert":"","extraDnsNames":[],"extraIpAddresses":[],"key":""}` | base64 encoded PEM values for the Hubble server certificate and private key | | hubble.tls.server.extraDnsNames | list | `[]` | Extra DNS names added to certificate when it's auto generated | @@ -343,7 +465,7 @@ contributors across the globe, there is almost always someone available to help. | hubble.ui.backend.extraEnv | list | `[]` | Additional hubble-ui backend environment variables. | | hubble.ui.backend.extraVolumeMounts | list | `[]` | Additional hubble-ui backend volumeMounts. | | hubble.ui.backend.extraVolumes | list | `[]` | Additional hubble-ui backend volumes. | -| hubble.ui.backend.image | object | `{"override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-ui-backend","tag":"v0.11.0@sha256:14c04d11f78da5c363f88592abae8d2ecee3cbe009f443ef11df6ac5f692d839"}` | Hubble-ui backend image. | +| hubble.ui.backend.image | object | `{"digest":"sha256:1f86f3400827a0451e6332262467f894eeb7caf0eb8779bd951e2caa9d027cbe","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-ui-backend","tag":"v0.12.1","useDigest":true}` | Hubble-ui backend image. | | hubble.ui.backend.resources | object | `{}` | Resource requests and limits for the 'backend' container of the 'hubble-ui' deployment. | | hubble.ui.backend.securityContext | object | `{}` | Hubble-ui backend security context. | | hubble.ui.baseUrl | string | `"/"` | Defines base url prefix for all hubble-ui http requests. It needs to be changed in case if ingress for hubble-ui is configured under some sub-path. Trailing `/` is required for custom path, ex. `/service-map/` | @@ -351,11 +473,11 @@ contributors across the globe, there is almost always someone available to help. | hubble.ui.frontend.extraEnv | list | `[]` | Additional hubble-ui frontend environment variables. | | hubble.ui.frontend.extraVolumeMounts | list | `[]` | Additional hubble-ui frontend volumeMounts. | | hubble.ui.frontend.extraVolumes | list | `[]` | Additional hubble-ui frontend volumes. | -| hubble.ui.frontend.image | object | `{"override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-ui","tag":"v0.11.0@sha256:bcb369c47cada2d4257d63d3749f7f87c91dde32e010b223597306de95d1ecc8"}` | Hubble-ui frontend image. | +| hubble.ui.frontend.image | object | `{"digest":"sha256:9e5f81ee747866480ea1ac4630eb6975ff9227f9782b7c93919c081c33f38267","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-ui","tag":"v0.12.1","useDigest":true}` | Hubble-ui frontend image. | | hubble.ui.frontend.resources | object | `{}` | Resource requests and limits for the 'frontend' container of the 'hubble-ui' deployment. | | hubble.ui.frontend.securityContext | object | `{}` | Hubble-ui frontend security context. | | hubble.ui.frontend.server.ipv6 | object | `{"enabled":true}` | Controls server listener for ipv6 | -| hubble.ui.ingress | object | `{"annotations":{},"className":"","enabled":false,"hosts":["chart-example.local"],"tls":[]}` | hubble-ui ingress configuration. | +| hubble.ui.ingress | object | `{"annotations":{},"className":"","enabled":false,"hosts":["chart-example.local"],"labels":{},"tls":[]}` | hubble-ui ingress configuration. | | hubble.ui.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | hubble.ui.podAnnotations | object | `{}` | Annotations to be added to hubble-ui pods | | hubble.ui.podDisruptionBudget.enabled | bool | `false` | enable PodDisruptionBudget ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ | @@ -365,8 +487,7 @@ contributors across the globe, there is almost always someone available to help. | hubble.ui.priorityClassName | string | `""` | The priority class to use for hubble-ui | | hubble.ui.replicas | int | `1` | The number of replicas of Hubble UI to deploy. | | hubble.ui.rollOutPods | bool | `false` | Roll out Hubble-ui pods automatically when configmap is updated. | -| hubble.ui.securityContext | object | `{"enabled":true,"fsGroup":1001,"runAsGroup":1001,"runAsUser":1001}` | Security context to be added to Hubble UI pods | -| hubble.ui.securityContext.enabled | bool | `true` | Deprecated in favor of hubble.ui.securityContext. Whether to set the security context on the Hubble UI pods. | +| hubble.ui.securityContext | object | `{"fsGroup":1001,"runAsGroup":1001,"runAsUser":1001}` | Security context to be added to Hubble UI pods | | hubble.ui.service | object | `{"annotations":{},"nodePort":31235,"type":"ClusterIP"}` | hubble-ui service configuration. | | hubble.ui.service.annotations | object | `{}` | Annotations to be added for the Hubble UI service | | hubble.ui.service.nodePort | int | `31235` | - The port to use when the service type is set to NodePort. | @@ -379,8 +500,11 @@ contributors across the globe, there is almost always someone available to help. | hubble.ui.updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | hubble-ui update strategy. | | identityAllocationMode | string | `"crd"` | Method to use for identity allocation (`crd` or `kvstore`). | | identityChangeGracePeriod | string | `"5s"` | Time to wait before using new identity on endpoint identity change. | -| image | object | `{"digest":"sha256:77176464a1e11ea7e89e984ac7db365e7af39851507e94f137dcf56c87746314","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.13.3","useDigest":true}` | Agent container image. | +| image | object | `{"digest":"sha256:e5ca22526e01469f8d10c14e2339a82a13ad70d9a359b879024715540eef4ace","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.14.3","useDigest":true}` | Agent container image. | | imagePullSecrets | string | `nil` | Configure image pull secrets for pulling container images | +| ingressController.default | bool | `false` | Set cilium ingress controller to be the default ingress controller This will let cilium ingress controller route entries without ingress class set | +| ingressController.defaultSecretName | string | `nil` | Default secret name for ingresses without .spec.tls[].secretName set. | +| ingressController.defaultSecretNamespace | string | `nil` | Default secret namespace for ingresses without .spec.tls[].secretName set. | | ingressController.enabled | bool | `false` | Enable cilium ingress controller This will automatically set enable-envoy-config as well. | | ingressController.enforceHttps | bool | `true` | Enforce https for host having matching TLS host in Ingress. Incoming traffic to http listener will return 308 http error code with respective location in header. | | ingressController.ingressLBAnnotationPrefixes | list | `["service.beta.kubernetes.io","service.kubernetes.io","cloud.google.com"]` | IngressLBAnnotations are the annotation prefixes, which are used to filter annotations to propagate from Ingress to the Load Balancer service | @@ -389,22 +513,25 @@ contributors across the globe, there is almost always someone available to help. | ingressController.secretsNamespace.create | bool | `true` | Create secrets namespace for Ingress. | | ingressController.secretsNamespace.name | string | `"cilium-secrets"` | Name of Ingress secret namespace. | | ingressController.secretsNamespace.sync | bool | `true` | Enable secret sync, which will make sure all TLS secrets used by Ingress are synced to secretsNamespace.name. If disabled, TLS secrets must be maintained externally. | -| ingressController.service | object | `{"annotations":{},"insecureNodePort":null,"labels":{},"name":"cilium-ingress","secureNodePort":null,"type":"LoadBalancer"}` | Load-balancer service in shared mode. This is a single load-balancer service for all Ingress resources. | +| ingressController.service | object | `{"allocateLoadBalancerNodePorts":null,"annotations":{},"insecureNodePort":null,"labels":{},"loadBalancerClass":null,"loadBalancerIP":null,"name":"cilium-ingress","secureNodePort":null,"type":"LoadBalancer"}` | Load-balancer service in shared mode. This is a single load-balancer service for all Ingress resources. | +| ingressController.service.allocateLoadBalancerNodePorts | string | `nil` | Configure if node port allocation is required for LB service ref: https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-nodeport-allocation | | ingressController.service.annotations | object | `{}` | Annotations to be added for the shared LB service | | ingressController.service.insecureNodePort | string | `nil` | Configure a specific nodePort for insecure HTTP traffic on the shared LB service | | ingressController.service.labels | object | `{}` | Labels to be added for the shared LB service | +| ingressController.service.loadBalancerClass | string | `nil` | Configure a specific loadBalancerClass on the shared LB service (requires Kubernetes 1.24+) | +| ingressController.service.loadBalancerIP | string | `nil` | Configure a specific loadBalancerIP on the shared LB service | | ingressController.service.name | string | `"cilium-ingress"` | Service name | | ingressController.service.secureNodePort | string | `nil` | Configure a specific nodePort for secure HTTPS traffic on the shared LB service | | ingressController.service.type | string | `"LoadBalancer"` | Service type for the shared LB service | | installNoConntrackIptablesRules | bool | `false` | Install Iptables rules to skip netfilter connection tracking on all pod traffic. This option is only effective when Cilium is running in direct routing and full KPR mode. Moreover, this option cannot be enabled when Cilium is running in a managed Kubernetes environment or in a chained CNI setup. | | ipMasqAgent | object | `{"enabled":false}` | Configure the eBPF-based ip-masq-agent | +| ipam.ciliumNodeUpdateRate | string | `"15s"` | Maximum rate at which the CiliumNode custom resource is updated. | | ipam.mode | string | `"cluster-pool"` | Configure IP Address Management mode. ref: https://docs.cilium.io/en/stable/network/concepts/ipam/ | +| ipam.operator.autoCreateCiliumPodIPPools | object | `{}` | IP pools to auto-create in multi-pool IPAM mode. | | ipam.operator.clusterPoolIPv4MaskSize | int | `24` | IPv4 CIDR mask size to delegate to individual nodes for IPAM. | -| ipam.operator.clusterPoolIPv4PodCIDR | string | `"10.0.0.0/8"` | Deprecated in favor of ipam.operator.clusterPoolIPv4PodCIDRList. IPv4 CIDR range to delegate to individual nodes for IPAM. | -| ipam.operator.clusterPoolIPv4PodCIDRList | list | `[]` | IPv4 CIDR list range to delegate to individual nodes for IPAM. | +| ipam.operator.clusterPoolIPv4PodCIDRList | list | `["10.0.0.0/8"]` | IPv4 CIDR list range to delegate to individual nodes for IPAM. | | ipam.operator.clusterPoolIPv6MaskSize | int | `120` | IPv6 CIDR mask size to delegate to individual nodes for IPAM. | -| ipam.operator.clusterPoolIPv6PodCIDR | string | `"fd00::/104"` | Deprecated in favor of ipam.operator.clusterPoolIPv6PodCIDRList. IPv6 CIDR range to delegate to individual nodes for IPAM. | -| ipam.operator.clusterPoolIPv6PodCIDRList | list | `[]` | IPv6 CIDR list range to delegate to individual nodes for IPAM. | +| ipam.operator.clusterPoolIPv6PodCIDRList | list | `["fd00::/104"]` | IPv6 CIDR list range to delegate to individual nodes for IPAM. | | ipam.operator.externalAPILimitBurstSize | string | `20` | The maximum burst size when rate limiting access to external APIs. Also known as the token bucket capacity. | | ipam.operator.externalAPILimitQPS | string | `4.0` | The maximum queries per second when rate limiting access to external APIs. Also known as the bucket refill rate, which is used to refill the bucket up to the burst size capacity. | | ipv4.enabled | bool | `true` | Enable IPv4 support. | @@ -412,6 +539,10 @@ contributors across the globe, there is almost always someone available to help. | ipv6.enabled | bool | `false` | Enable IPv6 support. | | ipv6NativeRoutingCIDR | string | `""` | Allows to explicitly specify the IPv6 CIDR for native routing. When specified, Cilium assumes networking for this CIDR is preconfigured and hands traffic destined for that range to the Linux network stack without applying any SNAT. Generally speaking, specifying a native routing CIDR implies that Cilium can depend on the underlying networking stack to route packets to their destination. To offer a concrete example, if Cilium is configured to use direct routing and the Kubernetes CIDR is included in the native routing CIDR, the user must configure the routes to reach pods, either manually or by setting the auto-direct-node-routes flag. | | k8s | object | `{}` | Configure Kubernetes specific configuration | +| k8sClientRateLimit | object | `{"burst":10,"qps":5}` | Configure the client side rate limit for the agent and operator If the amount of requests to the Kubernetes API server exceeds the configured rate limit, the agent and operator will start to throttle requests by delaying them until there is budget or the request times out. | +| k8sClientRateLimit.burst | int | `10` | The burst request rate in requests per second. The rate limiter will allow short bursts with a higher rate. | +| k8sClientRateLimit.qps | int | `5` | The sustained request rate in requests per second. | +| k8sNetworkPolicy.enabled | bool | `true` | Enable support for K8s NetworkPolicy | | k8sServiceHost | string | `""` | Kubernetes service host | | k8sServicePort | string | `""` | Kubernetes service port | | keepDeprecatedLabels | bool | `false` | Keep the deprecated selector labels when deploying Cilium DaemonSet. | @@ -420,13 +551,18 @@ contributors across the globe, there is almost always someone available to help. | kubeProxyReplacementHealthzBindAddr | string | `""` | healthz server bind address for the kube-proxy replacement. To enable set the value to '0.0.0.0:10256' for all ipv4 addresses and this '[::]:10256' for all ipv6 addresses. By default it is disabled. | | l2NeighDiscovery.enabled | bool | `true` | Enable L2 neighbor discovery in the agent | | l2NeighDiscovery.refreshPeriod | string | `"30s"` | Override the agent's default neighbor resolution refresh period. | +| l2announcements | object | `{"enabled":false}` | Configure L2 announcements | +| l2announcements.enabled | bool | `false` | Enable L2 announcements | +| l2podAnnouncements | object | `{"enabled":false,"interface":"eth0"}` | Configure L2 pod announcements | +| l2podAnnouncements.enabled | bool | `false` | Enable L2 pod announcements | +| l2podAnnouncements.interface | string | `"eth0"` | Interface used for sending Gratuitous ARP pod announcements | | l7Proxy | bool | `true` | Enable Layer 7 network policy. | | livenessProbe.failureThreshold | int | `10` | failure threshold of liveness probe | | livenessProbe.periodSeconds | int | `30` | interval between checks of the liveness probe | | loadBalancer | object | `{"l7":{"algorithm":"round_robin","backend":"disabled","ports":[]}}` | Configure service load balancing | | loadBalancer.l7 | object | `{"algorithm":"round_robin","backend":"disabled","ports":[]}` | L7 LoadBalancer | | loadBalancer.l7.algorithm | string | `"round_robin"` | Default LB algorithm The default LB algorithm to be used for services, which can be overridden by the service annotation (e.g. service.cilium.io/lb-l7-algorithm) Applicable values: round_robin, least_request, random | -| loadBalancer.l7.backend | string | `"disabled"` | Enable L7 service load balancing via envoy proxy. The request to a k8s service, which has specific annotation e.g. service.cilium.io/lb-l7, will be forwarded to the local backend proxy to be load balanced to the service endpoints. Please refer to docs for supported annotations for more configuration. Applicable values: - envoy: Enable L7 load balancing via envoy proxy. This will automatically set enable-envoy-config as well. - disabled: Disable L7 load balancing. | +| loadBalancer.l7.backend | string | `"disabled"` | Enable L7 service load balancing via envoy proxy. The request to a k8s service, which has specific annotation e.g. service.cilium.io/lb-l7, will be forwarded to the local backend proxy to be load balanced to the service endpoints. Please refer to docs for supported annotations for more configuration. Applicable values: - envoy: Enable L7 load balancing via envoy proxy. This will automatically set enable-envoy-config as well. - disabled: Disable L7 load balancing by way of service annotation. | | loadBalancer.l7.ports | list | `[]` | List of ports from service to be automatically redirected to above backend. Any service exposing one of these ports will be automatically redirected. Fine-grained control can be achieved by using the service annotation. | | localRedirectPolicy | bool | `false` | Enable Local Redirect Policy. | | logSystemLoad | bool | `false` | Enables periodic logging of system load | @@ -446,16 +582,21 @@ contributors across the globe, there is almost always someone available to help. | nodeinit.bootstrapFile | string | `"/tmp/cilium-bootstrap.d/cilium-bootstrap-time"` | bootstrapFile is the location of the file where the bootstrap timestamp is written by the node-init DaemonSet | | nodeinit.enabled | bool | `false` | Enable the node initialization DaemonSet | | nodeinit.extraEnv | list | `[]` | Additional nodeinit environment variables. | -| nodeinit.image | object | `{"override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/startup-script","tag":"d69851597ea019af980891a4628fb36b7880ec26"}` | node-init image. | +| nodeinit.extraVolumeMounts | list | `[]` | Additional nodeinit volumeMounts. | +| nodeinit.extraVolumes | list | `[]` | Additional nodeinit volumes. | +| nodeinit.image | object | `{"override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/startup-script","tag":"62093c5c233ea914bfa26a10ba41f8780d9b737f"}` | node-init image. | | nodeinit.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for nodeinit pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | nodeinit.podAnnotations | object | `{}` | Annotations to be added to node-init pods. | | nodeinit.podLabels | object | `{}` | Labels to be added to node-init pods. | +| nodeinit.prestop | object | `{"postScript":"","preScript":""}` | prestop offers way to customize prestop nodeinit script (pre and post position) | | nodeinit.priorityClassName | string | `""` | The priority class to use for the nodeinit pod. | | nodeinit.resources | object | `{"requests":{"cpu":"100m","memory":"100Mi"}}` | nodeinit resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | | nodeinit.securityContext | object | `{"capabilities":{"add":["SYS_MODULE","NET_ADMIN","SYS_ADMIN","SYS_CHROOT","SYS_PTRACE"]},"privileged":false,"seLinuxOptions":{"level":"s0","type":"spc_t"}}` | Security context to be added to nodeinit pods. | +| nodeinit.startup | object | `{"postScript":"","preScript":""}` | startup offers way to customize startup nodeinit script (pre and post position) | | nodeinit.tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for nodeinit scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | | nodeinit.updateStrategy | object | `{"type":"RollingUpdate"}` | node-init update strategy | | operator.affinity | object | `{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"io.cilium/app":"operator"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for cilium-operator | +| operator.dashboards | object | `{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":null}` | Grafana dashboards for cilium-operator grafana can import dashboards based on the label and value ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards | | operator.dnsPolicy | string | `""` | DNS policy for Cilium operator pods. Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy | | operator.enabled | bool | `true` | Enable the cilium-operator component (required). | | operator.endpointGCInterval | string | `"5m0s"` | Interval for endpoint garbage collection. | @@ -466,7 +607,7 @@ contributors across the globe, there is almost always someone available to help. | operator.extraVolumes | list | `[]` | Additional cilium-operator volumes. | | operator.identityGCInterval | string | `"15m0s"` | Interval for identity garbage collection. | | operator.identityHeartbeatTimeout | string | `"30m0s"` | Timeout for identity heartbeats. | -| operator.image | object | `{"alibabacloudDigest":"sha256:8dba4795cb38200746a2236623f5b84742ee2c56a8afda724c85f5027ea854eb","awsDigest":"sha256:394c40d156235d3c2004f77bb73402457092351cc6debdbc5727ba36fbd863ae","azureDigest":"sha256:7749b732d510954d9fb74f7e675b31b49100fd773e588c6fbbf42529acfb1be8","genericDigest":"sha256:fa7003cbfdf8358cb71786afebc711b26e5e44a2ed99bd4944930bba915b8910","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/operator","suffix":"","tag":"v1.13.3","useDigest":true}` | cilium-operator image. | +| operator.image | object | `{"alibabacloudDigest":"sha256:64fceb06c305f524a96138e7b165bdd868a4454af55210ddc5fc13de2b36f979","awsDigest":"sha256:120a437f75bc9b06e969e5183ac36e346e5ad02e7eb144d5d14c2918f0455c63","azureDigest":"sha256:b5f6b37d7c90fec6d066686d59924fb03eb39288234492526e50b46f8d1e7c28","genericDigest":"sha256:c9613277b72103ed36e9c0d16b9a17cafd507461d59340e432e3e9c23468b5e2","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/operator","suffix":"","tag":"v1.14.3","useDigest":true}` | cilium-operator image. | | operator.nodeGCInterval | string | `"5m0s"` | Interval for cilium node garbage collection. | | operator.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for cilium-operator pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | operator.podAnnotations | object | `{}` | Annotations to be added to cilium-operator pods | @@ -492,13 +633,14 @@ contributors across the globe, there is almost always someone available to help. | operator.rollOutPods | bool | `false` | Roll out cilium-operator pods automatically when configmap is updated. | | operator.securityContext | object | `{}` | Security context to be added to cilium-operator pods | | operator.setNodeNetworkStatus | bool | `true` | Set Node condition NetworkUnavailable to 'false' with the reason 'CiliumIsUp' for nodes that have a healthy Cilium pod. | +| operator.setNodeTaints | string | same as removeNodeTaints | Taint nodes where Cilium is scheduled but not running. This prevents pods from being scheduled to nodes where Cilium is not the default CNI provider. | | operator.skipCNPStatusStartupClean | bool | `false` | Skip CNP node status clean up at operator startup. | | operator.skipCRDCreation | bool | `false` | Skip CRDs creation for cilium-operator | | operator.tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for cilium-operator scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | | operator.topologySpreadConstraints | list | `[]` | Pod topology spread constraints for cilium-operator | | operator.unmanagedPodWatcher.intervalSeconds | int | `15` | Interval, in seconds, to check if there are any pods that are not managed by Cilium. | | operator.unmanagedPodWatcher.restart | bool | `true` | Restart any pod that are not managed by Cilium. | -| operator.updateStrategy | object | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":1},"type":"RollingUpdate"}` | cilium-operator update strategy | +| operator.updateStrategy | object | `{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"50%"},"type":"RollingUpdate"}` | cilium-operator update strategy | | pmtuDiscovery.enabled | bool | `false` | Enable path MTU discovery to send ICMP fragmentation-needed replies to the client. | | podAnnotations | object | `{}` | Annotations to be added to agent pods | | podLabels | object | `{}` | Labels to be added to agent pods | @@ -512,7 +654,7 @@ contributors across the globe, there is almost always someone available to help. | preflight.extraEnv | list | `[]` | Additional preflight environment variables. | | preflight.extraVolumeMounts | list | `[]` | Additional preflight volumeMounts. | | preflight.extraVolumes | list | `[]` | Additional preflight volumes. | -| preflight.image | object | `{"digest":"sha256:77176464a1e11ea7e89e984ac7db365e7af39851507e94f137dcf56c87746314","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.13.3","useDigest":true}` | Cilium pre-flight image. | +| preflight.image | object | `{"digest":"sha256:e5ca22526e01469f8d10c14e2339a82a13ad70d9a359b879024715540eef4ace","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.14.3","useDigest":true}` | Cilium pre-flight image. | | preflight.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for preflight pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | preflight.podAnnotations | object | `{}` | Annotations to be added to preflight pods | | preflight.podDisruptionBudget.enabled | bool | `false` | enable PodDisruptionBudget ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ | @@ -529,15 +671,18 @@ contributors across the globe, there is almost always someone available to help. | preflight.updateStrategy | object | `{"type":"RollingUpdate"}` | preflight update strategy | | preflight.validateCNPs | bool | `true` | By default we should always validate the installed CNPs before upgrading Cilium. This will make sure the user will have the policies deployed in the cluster with the right schema. | | priorityClassName | string | `""` | The priority class to use for cilium-agent. | -| prometheus | object | `{"enabled":false,"metrics":null,"port":9962,"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","labels":{},"metricRelabelings":null,"relabelings":[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]}}` | Configure prometheus metrics on the configured port at /metrics | -| prometheus.metrics | string | `nil` | Metrics that should be enabled or disabled from the default metric list. (+metric_foo to enable metric_foo , -metric_bar to disable metric_bar). ref: https://docs.cilium.io/en/stable/observability/metrics/ | +| prometheus | object | `{"enabled":false,"metrics":null,"port":9962,"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","labels":{},"metricRelabelings":null,"relabelings":[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}],"trustCRDsExist":false}}` | Configure prometheus metrics on the configured port at /metrics | +| prometheus.metrics | string | `nil` | Metrics that should be enabled or disabled from the default metric list. The list is expected to be separated by a space. (+metric_foo to enable metric_foo , -metric_bar to disable metric_bar). ref: https://docs.cilium.io/en/stable/observability/metrics/ | | prometheus.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor cilium-agent | | prometheus.serviceMonitor.enabled | bool | `false` | Enable service monitors. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) | | prometheus.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics. | | prometheus.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor cilium-agent | | prometheus.serviceMonitor.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor cilium-agent | | prometheus.serviceMonitor.relabelings | list | `[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]` | Relabeling configs for the ServiceMonitor cilium-agent | -| proxy | object | `{"prometheus":{"enabled":true,"port":"9964"},"sidecarImageRegex":"cilium/istio_proxy"}` | Configure Istio proxy options. | +| prometheus.serviceMonitor.trustCRDsExist | bool | `false` | Set to `true` and helm will not check for monitoring.coreos.com/v1 CRDs before deploying | +| proxy | object | `{"prometheus":{"enabled":true,"port":null},"sidecarImageRegex":"cilium/istio_proxy"}` | Configure Istio proxy options. | +| proxy.prometheus.enabled | bool | `true` | Deprecated in favor of envoy.prometheus.enabled | +| proxy.prometheus.port | string | `nil` | Deprecated in favor of envoy.prometheus.port | | proxy.sidecarImageRegex | string | `"cilium/istio_proxy"` | Regular expression matching compatible Istio sidecar istio-proxy container image names | | rbac.create | bool | `true` | Enable creation of Resource-Based Access Control configuration. | | readinessProbe.failureThreshold | int | `3` | failure threshold of readiness probe | @@ -546,6 +691,7 @@ contributors across the globe, there is almost always someone available to help. | resourceQuotas | object | `{"cilium":{"hard":{"pods":"10k"}},"enabled":false,"operator":{"hard":{"pods":"15"}}}` | Enable resource quotas for priority classes used in the cluster. | | resources | object | `{}` | Agent resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | | rollOutCiliumPods | bool | `false` | Roll out cilium agent pods automatically when configmap is updated. | +| routingMode | string | `"tunnel"` | Enable native-routing mode or tunneling mode. Possible values: - "" - native - tunnel | | sctp | object | `{"enabled":false}` | SCTP Configuration Values | | sctp.enabled | bool | `false` | Enable SCTP support. NOTE: Currently, SCTP support does not support rewriting ports or multihoming. | | securityContext.capabilities.applySysctlOverwrites | list | `["SYS_ADMIN","SYS_CHROOT","SYS_PTRACE"]` | capabilities for the `apply-sysctl-overwrites` init container | @@ -557,24 +703,30 @@ contributors across the globe, there is almost always someone available to help. | serviceAccounts | object | Component's fully qualified name. | Define serviceAccount names for components. | | serviceAccounts.clustermeshcertgen | object | `{"annotations":{},"automount":true,"create":true,"name":"clustermesh-apiserver-generate-certs"}` | Clustermeshcertgen is used if clustermesh.apiserver.tls.auto.method=cronJob | | serviceAccounts.hubblecertgen | object | `{"annotations":{},"automount":true,"create":true,"name":"hubble-generate-certs"}` | Hubblecertgen is used if hubble.tls.auto.method=cronJob | +| serviceAccounts.nodeinit.enabled | bool | `false` | Enabled is temporary until https://github.com/cilium/cilium-cli/issues/1396 is implemented. Cilium CLI doesn't create the SAs for node-init, thus the workaround. Helm is not affected by this issue. Name and automount can be configured, if enabled is set to true. Otherwise, they are ignored. Enabled can be removed once the issue is fixed. Cilium-nodeinit DS must also be fixed. | | sleepAfterInit | bool | `false` | Do not run Cilium agent when running with clean mode. Useful to completely uninstall Cilium as it will stop Cilium from starting and create artifacts in the node. | | socketLB | object | `{"enabled":false}` | Configure socket LB | | socketLB.enabled | bool | `false` | Enable socket LB | -| sockops | object | `{"enabled":false}` | Configure BPF socket operations configuration | | startupProbe.failureThreshold | int | `105` | failure threshold of startup probe. 105 x 2s translates to the old behaviour of the readiness probe (120s delay + 30 x 3s) | | startupProbe.periodSeconds | int | `2` | interval between checks of the startup probe | | svcSourceRangeCheck | bool | `true` | Enable check of service source ranges (currently, only for LoadBalancer). | | synchronizeK8sNodes | bool | `true` | Synchronize Kubernetes nodes to kvstore and perform CNP GC. | | terminationGracePeriodSeconds | int | `1` | Configure termination grace period for cilium-agent DaemonSet. | -| tls | object | `{"ca":{"cert":"","certValidityDuration":1095,"key":""},"secretsBackend":"local"}` | Configure TLS configuration in the agent. | -| tls.ca | object | `{"cert":"","certValidityDuration":1095,"key":""}` | Base64 encoded PEM values for the CA certificate and private key. This can be used as common CA to generate certificates used by hubble and clustermesh components | +| tls | object | `{"ca":{"cert":"","certValidityDuration":1095,"key":""},"caBundle":{"enabled":false,"key":"ca.crt","name":"cilium-root-ca.crt","useSecret":false},"secretsBackend":"local"}` | Configure TLS configuration in the agent. | +| tls.ca | object | `{"cert":"","certValidityDuration":1095,"key":""}` | Base64 encoded PEM values for the CA certificate and private key. This can be used as common CA to generate certificates used by hubble and clustermesh components. It is neither required nor used when cert-manager is used to generate the certificates. | | tls.ca.cert | string | `""` | Optional CA cert. If it is provided, it will be used by cilium to generate all other certificates. Otherwise, an ephemeral CA is generated. | | tls.ca.certValidityDuration | int | `1095` | Generated certificates validity duration in days. This will be used for auto generated CA. | | tls.ca.key | string | `""` | Optional CA private key. If it is provided, it will be used by cilium to generate all other certificates. Otherwise, an ephemeral CA is generated. | +| tls.caBundle | object | `{"enabled":false,"key":"ca.crt","name":"cilium-root-ca.crt","useSecret":false}` | Configure the CA trust bundle used for the validation of the certificates leveraged by hubble and clustermesh. When enabled, it overrides the content of the 'ca.crt' field of the respective certificates, allowing for CA rotation with no down-time. | +| tls.caBundle.enabled | bool | `false` | Enable the use of the CA trust bundle. | +| tls.caBundle.key | string | `"ca.crt"` | Entry of the ConfigMap containing the CA trust bundle. | +| tls.caBundle.name | string | `"cilium-root-ca.crt"` | Name of the ConfigMap containing the CA trust bundle. | +| tls.caBundle.useSecret | bool | `false` | Use a Secret instead of a ConfigMap. | | tls.secretsBackend | string | `"local"` | This configures how the Cilium agent loads the secrets used TLS-aware CiliumNetworkPolicies (namely the secrets referenced by terminatingTLS and originatingTLS). Possible values: - local - k8s | | tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for agent scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | -| tunnel | string | `"vxlan"` | Configure the encapsulation configuration for communication between nodes. Possible values: - disabled - vxlan (default) - geneve | +| tunnel | string | `"vxlan"` | Configure the encapsulation configuration for communication between nodes. Deprecated in favor of tunnelProtocol and routingMode. To be removed in 1.15. Possible values: - disabled - vxlan - geneve | | tunnelPort | int | Port 8472 for VXLAN, Port 6081 for Geneve | Configure VXLAN and Geneve tunnel port. | +| tunnelProtocol | string | `"vxlan"` | Tunneling protocol to use in tunneling mode and for ad-hoc tunnels. Possible values: - "" - vxlan - geneve | | updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":2},"type":"RollingUpdate"}` | Cilium agent update strategy | | vtep.cidr | string | `""` | A space separated list of VTEP device CIDRs, for example "1.1.1.0/24 1.1.2.0/24" | | vtep.enabled | bool | `false` | Enables VXLAN Tunnel Endpoint (VTEP) Integration (beta) to allow Cilium-managed pods to talk to third party VTEP devices over Cilium tunnel. | diff --git a/charts/cilium/README.md.gotmpl b/charts/cilium/README.md.gotmpl index 225a0f81..db2d81b7 100644 --- a/charts/cilium/README.md.gotmpl +++ b/charts/cilium/README.md.gotmpl @@ -16,7 +16,7 @@ integration points such as: network IO, application sockets, and tracepoints to implement security, networking and visibility logic. eBPF is highly efficient and flexible. -![Cilium feature overview](https://raw.githubusercontent.com/cilium/cilium/master/Documentation/images/cilium_overview.png) +![Cilium feature overview](https://raw.githubusercontent.com/cilium/cilium/main/Documentation/images/cilium_overview.png) ## Prerequisites diff --git a/charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json b/charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json new file mode 100644 index 00000000..997608ee --- /dev/null +++ b/charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json @@ -0,0 +1,8245 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Dashboard for Cilium (https://cilium.io/) metrics", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "iteration": 1606309591568, + "links": [], + "panels": [ + { + "aliasColors": { + "error": "#890f02", + "warning": "#c15c17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 76, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "error", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_errors_warnings_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, level) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{level}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Errors & Warnings", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "avg": "#cffaff" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(irate(cilium_process_cpu_seconds_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(irate(cilium_process_cpu_seconds_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(irate(cilium_process_cpu_seconds_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage per node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 161, + "panels": [], + "title": "Generic", + "type": "row" + }, + { + "aliasColors": { + "AVG_virtual_memory_bytes": "#508642", + "Average Virtual Memory": "#f9d9f9", + "MAX_virtual_memory_bytes": "#e5ac0e", + "Max Virtual Memory": "#584477" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max Virtual Memory", + "fillBelowTo": "Min Virtual Memory", + "lines": false + }, + { + "alias": "Min Virtual Memory", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_process_virtual_memory_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Min Virtual Memory", + "refId": "A" + }, + { + "expr": "avg(cilium_process_virtual_memory_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Average Virtual Memory", + "refId": "B" + }, + { + "expr": "max(cilium_process_virtual_memory_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max Virtual Memory", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Virtual Memory Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "MAX_resident_memory_bytes_max": "#e5ac0e" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 6 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(cilium_process_resident_memory_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "AVG_resident_memory_bytes", + "refId": "C" + }, + { + "expr": "max(cilium_process_resident_memory_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "MAX_resident_memory_bytes_max", + "refId": "D" + }, + { + "expr": "min(cilium_process_resident_memory_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "MIN_resident_memory_bytes_min", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Resident memory status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "all nodes": "#e5a8e2" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 6 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "all nodes", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cilium_process_open_fds{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "all nodes", + "refId": "A" + }, + { + "expr": "min(cilium_process_open_fds{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min/node", + "refId": "B" + }, + { + "expr": "avg(cilium_process_open_fds{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg/node", + "refId": "C" + }, + { + "expr": "max(cilium_process_open_fds{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max/node", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Open file descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "MAX_resident_memory_bytes_max": "#e5ac0e" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "BPF memory usage in the entire system including components not managed by Cilium.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 178, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(cilium_bpf_maps_virtual_memory_max_bytes{k8s_app=\"cilium\", pod=~\"$pod\"} + cilium_bpf_progs_virtual_memory_max_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "AVG_bpf_memory_bytes_avg", + "refId": "C" + }, + { + "expr": "max(cilium_bpf_maps_virtual_memory_max_bytes{k8s_app=\"cilium\", pod=~\"$pod\"} + cilium_bpf_progs_virtual_memory_max_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "MAX_bpf_memory_bytes_max", + "refId": "D" + }, + { + "expr": "min(cilium_bpf_maps_virtual_memory_max_bytes{k8s_app=\"cilium\", pod=~\"$pod\"} + cilium_bpf_progs_virtual_memory_max_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "MIN_bpf_memory_bytes_min", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "System-wide BPF memory usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:136", + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:137", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Fill percentage of BPF maps, tagged by map name", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "hiddenSeries": false, + "id": 194, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cilium_bpf_map_pressure{k8s_app=\"cilium\", pod=~\"$pod\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "BPF map pressure", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:230", + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.0", + "min": null, + "show": true + }, + { + "$$hashKey": "object:231", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 155, + "panels": [], + "title": "API", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 152, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_agent_api_process_time_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])/rate(cilium_agent_api_process_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API call latency (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 153, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(cilium_agent_api_process_time_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])/rate(cilium_agent_api_process_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API call latency (max node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 156, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_agent_api_process_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}} ", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# API calls (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 157, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(cilium_agent_api_process_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}} ", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# API calls (max node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 159, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_agent_api_process_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path, return_code)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{return_code}} ({{method}} {{path}} )", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API return codes (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 30 + }, + "hiddenSeries": false, + "id": 158, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_agent_api_process_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path, return_code)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{return_code}} ({{method}} {{path}} )", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API return codes (sum all nodes)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 72, + "panels": [], + "title": "Cilium", + "type": "row" + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 144, + "links": [], + "mode": "markdown", + "title": "BPF", + "type": "text" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "hiddenSeries": false, + "id": 146, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_bpf_syscall_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, operation)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# system calls (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, + "hiddenSeries": false, + "id": 145, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(cilium_bpf_syscall_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, operation)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# system calls (max node)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 46 + }, + "hiddenSeries": false, + "id": 140, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_bpf_syscall_duration_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])/ rate(cilium_bpf_syscall_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, operation)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "system call latency (avg node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 46 + }, + "hiddenSeries": false, + "id": 148, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(cilium_bpf_syscall_duration_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])/ rate(cilium_bpf_syscall_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, operation)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "system call latency (max node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 52 + }, + "hiddenSeries": false, + "id": 142, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app=\"cilium\", pod=~\"$pod\"}[5m])) by (pod, map_name, operation))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{map_name}} {{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "map ops (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 52 + }, + "hiddenSeries": false, + "id": 147, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(5, max(rate(cilium_bpf_map_ops_total{k8s_app=\"cilium\", pod=~\"$pod\"}[5m])) by (pod, map_name, operation))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{map_name}} {{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "map ops (max node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 52 + }, + "hiddenSeries": false, + "id": 143, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_bpf_map_ops_total{k8s_app=\"cilium\",outcome=\"fail\", pod=~\"$pod\"}[5m])) by (pod, map_name, operation)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{map_name}} {{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "map ops (sum failures)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 182, + "links": [], + "mode": "markdown", + "title": "kvstore", + "type": "text" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 59 + }, + "hiddenSeries": false, + "id": 184, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kvstore_operations_total{pod=~\"$pod\"}[1m])) by (pod, scope, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{scope}} {{action}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# operations (sum all nodes)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 59 + }, + "hiddenSeries": false, + "id": 186, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(kvstore_operations_total{pod=~\"$pod\"}[1m])) by (pod, scope, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{scope}} {{action}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# operations (max node)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 64 + }, + "hiddenSeries": false, + "id": 188, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(5, avg(rate(cilium_kvstore_operations_duration_seconds_sum{pod=~\"$pod\"}[1m])) by (pod, action, scope) / avg(rate(cilium_kvstore_operations_duration_seconds_count{pod=~\"$pod\"}[1m])) by (pod, action, scope))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "latency (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 64 + }, + "hiddenSeries": false, + "id": 190, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(5, max(rate(cilium_kvstore_operations_duration_seconds_sum{pod=~\"$pod\"}[1m])) by (pod, action, scope) / avg(rate(cilium_kvstore_operations_duration_seconds_count{pod=~\"$pod\"}[1m])) by (pod, action, scope))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "latency (max node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 69 + }, + "hiddenSeries": false, + "id": 192, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kvstore_events_queue_seconds_count{pod=~\"$pod\"}[1m])) by (pod, scope, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Events received (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 47, + "links": [], + "mode": "markdown", + "title": "Cilium network information", + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 76 + }, + "hiddenSeries": false, + "id": 81, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_forward_count_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, direction)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{direction}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Forwarded Packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 76 + }, + "hiddenSeries": false, + "id": 111, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "EGRESS", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_forward_bytes_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, direction) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{direction}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Forwarded Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Alive ipv4": "#0a50a1", + "Alive ipv4 non-TCP": "#f9d9f9", + "Alive ipv6": "#614d93", + "Alive ipv6 TCP": "#806eb7", + "Alive ipv6 non-TCP": "#614d93", + "Alive CT entries ipv6": "#badff4", + "Deleted CT entries ipv4": "#bf1b00", + "Deleted ipv4": "#890f02", + "Deleted ipv4 non-TCP": "#890f02", + "Deleted ipv6": "#bf1b00", + "L7 denied request": "#890f02", + "L7 forwarded request": "#7eb26d", + "avg": "#e0f9d7", + "deleted": "#6ed0e0", + "deleted max": "#447ebc", + "max": "#629e51", + "min": "#629e51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 82 + }, + "hiddenSeries": false, + "id": 56, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "deleted", + "yaxis": 2 + }, + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + }, + { + "alias": "deleted max", + "yaxis": 2 + }, + { + "alias": "deleted min", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv4\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv4\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv4\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv4\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted", + "refId": "D" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv4\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted max", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IPv4 Conntrack TCP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Alive ipv4": "#0a50a1", + "Alive ipv4 non-TCP": "#f9d9f9", + "Alive ipv6": "#614d93", + "Alive ipv6 TCP": "#806eb7", + "Alive ipv6 non-TCP": "#614d93", + "Alive CT entries ipv6": "#badff4", + "Deleted CT entries ipv4": "#bf1b00", + "Deleted ipv4": "#890f02", + "Deleted ipv4 non-TCP": "#890f02", + "Deleted ipv6": "#bf1b00", + "L7 denied request": "#890f02", + "L7 forwarded request": "#7eb26d", + "avg": "#e0f9d7", + "deleted": "#6ed0e0", + "deleted max": "#447ebc", + "max": "#629e51", + "min": "#629e51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 82 + }, + "hiddenSeries": false, + "id": 128, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "deleted", + "yaxis": 2 + }, + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + }, + { + "alias": "deleted max", + "yaxis": 2 + }, + { + "alias": "deleted min", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv6\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv6\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv6\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv6\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted", + "refId": "D" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv6\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted max", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IPv6 Conntrack TCP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Alive ipv4": "#0a50a1", + "Alive ipv4 non-TCP": "#f9d9f9", + "Alive ipv6": "#614d93", + "Alive ipv6 TCP": "#806eb7", + "Alive ipv6 non-TCP": "#614d93", + "Alive CT entries ipv6": "#badff4", + "Deleted CT entries ipv4": "#bf1b00", + "Deleted ipv4": "#890f02", + "Deleted ipv4 non-TCP": "#890f02", + "Deleted ipv6": "#bf1b00", + "L7 denied request": "#890f02", + "L7 forwarded request": "#7eb26d", + "avg": "#e0f9d7", + "deleted": "#6ed0e0", + "deleted max": "#447ebc", + "max": "#629e51", + "min": "#629e51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 88 + }, + "hiddenSeries": false, + "id": 129, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "deleted", + "yaxis": 2 + }, + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + }, + { + "alias": "deleted max", + "yaxis": 2 + }, + { + "alias": "deleted min", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv4\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv4\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv4\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv4\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted", + "refId": "D" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv4\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted max", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IPv4 Conntrack Non-TCP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Alive ipv4": "#0a50a1", + "Alive ipv4 non-TCP": "#f9d9f9", + "Alive ipv6": "#614d93", + "Alive ipv6 TCP": "#806eb7", + "Alive ipv6 non-TCP": "#614d93", + "Alive CT entries ipv6": "#badff4", + "Deleted CT entries ipv4": "#bf1b00", + "Deleted ipv4": "#890f02", + "Deleted ipv4 non-TCP": "#890f02", + "Deleted ipv6": "#bf1b00", + "L7 denied request": "#890f02", + "L7 forwarded request": "#7eb26d", + "avg": "#e0f9d7", + "deleted": "#6ed0e0", + "deleted max": "#447ebc", + "max": "#629e51", + "min": "#629e51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 88 + }, + "hiddenSeries": false, + "id": 130, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "deleted", + "yaxis": 2 + }, + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + }, + { + "alias": "deleted max", + "yaxis": 2 + }, + { + "alias": "deleted min", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv6\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv6\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv6\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv6\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted", + "refId": "D" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv6\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted max", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IPv6 Conntrack Non-TCP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "ipv4": "#5195ce", + "ipv6": "#6d1f62" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 87, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cilium_ip_addresses{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod, family)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{family}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Allocated Addresses", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "dump_interrupts conntrack ipv4": "#ea6460", + "dump_interrupts conntrack ipv6": "#58140c" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 94 + }, + "hiddenSeries": false, + "id": 79, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cilium_datapath_conntrack_dump_resets_total{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod, area, family, name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{name}} {{area}} {{family}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Datapath Conntrack Dump Resets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 99 + }, + "hiddenSeries": false, + "id": 106, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_services_events_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Service Updates", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 99 + }, + "hiddenSeries": false, + "id": 89, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "avg(cilium_unreachable_health_endpoints) by (pod)", + "yaxis": 2 + }, + { + "alias": "average unreachable health endpoints", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cilium_unreachable_nodes{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "unreachable nodes", + "refId": "A" + }, + { + "expr": "sum(cilium_unreachable_health_endpoints{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "unreachable health endpoints", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connectivity Health", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 104 + }, + "hiddenSeries": false, + "id": 39, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_drop_count_total{direction=\"EGRESS\", k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (reason)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Dropped Egress Packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Avg": "#cca300", + "Max": "rgb(167, 150, 111)" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 104 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max", + "fillBelowTo": "Min", + "lines": false + }, + { + "alias": "Min", + "lines": false + }, + { + "alias": "add k8s", + "yaxis": 2 + }, + { + "alias": "delete k8s", + "yaxis": 2 + }, + { + "alias": "update k8s", + "yaxis": 2 + }, + { + "alias": "add local-node", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_nodes_all_events_received_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, event_type, source) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{eventType}} {{source}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Node Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 109 + }, + "hiddenSeries": false, + "id": 113, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_drop_bytes_total{direction=\"EGRESS\", k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (reason) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Dropped Egress Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Average Nodes": "#eab839", + "Max Nodes": "#c15c17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 109 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max Nodes", + "fillBelowTo": "Min Nodes", + "lines": false + }, + { + "alias": "Min Nodes", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(cilium_nodes_all_num{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Average Nodes", + "refId": "A" + }, + { + "expr": "min(cilium_nodes_all_num{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Min Nodes", + "refId": "B" + }, + { + "expr": "max(cilium_nodes_all_num{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max Nodes", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nodes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 114 + }, + "id": 28, + "links": [], + "mode": "markdown", + "title": "Policy", + "type": "text" + }, + { + "aliasColors": { + "L7 denied request": "#ea6460", + "L7 forwarded request": "#7eb26d", + "denied": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 115 + }, + "hiddenSeries": false, + "id": 53, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "L7 denied request", + "yaxis": 2 + }, + { + "alias": "denied", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_policy_l7_denied_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "denied", + "refId": "A" + }, + { + "expr": "sum(rate(cilium_policy_l7_forwarded_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "forwarded", + "refId": "B" + }, + { + "expr": "sum(rate(cilium_policy_l7_received_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "received", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "L7 forwarded request", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 115 + }, + "hiddenSeries": false, + "id": 37, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_drop_count_total{direction=\"INGRESS\", k8s_app=\"cilium\", pod=~\"$pod\"}[5m])) by (reason)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cilium drops Ingress", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Max per node processingTime": "#e24d42", + "Max per node upstreamTime": "#58140c", + "avg(cilium_policy_l7_parse_errors_total{pod=~\"cilium.*\"})": "#bf1b00", + "parse errors": "#bf1b00" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 120 + }, + "hiddenSeries": false, + "id": 94, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max per node processingTime", + "yaxis": 2 + }, + { + "alias": "Max per node upstreamTime", + "yaxis": 2 + }, + { + "alias": "avg(cilium_policy_l7_parse_errors_total{pod=~\"cilium.*\"})", + "yaxis": 2 + }, + { + "alias": "parse errors", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_proxy_upstream_reply_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope) / sum(rate(cilium_proxy_upstream_reply_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{scope}}", + "refId": "A" + }, + { + "expr": "avg(cilium_policy_l7_parse_errors_total{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "parse errors", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Proxy response time (Avg)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 120 + }, + "hiddenSeries": false, + "id": 114, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_drop_bytes_total{direction=\"INGRESS\", k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (reason) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Dropped Ingress Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "avg": "#64b0c8", + "count": "#9ac48a", + "max": "#5195ce", + "min": "#6ed0e0" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 125 + }, + "hiddenSeries": false, + "id": 104, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + }, + { + "alias": "avg count", + "yaxis": 2 + }, + { + "alias": "max count", + "yaxis": 2 + }, + { + "alias": "avg count" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(rate(cilium_triggers_policy_update_call_duration_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope) / sum(rate(cilium_triggers_policy_update_call_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(rate(cilium_triggers_policy_update_call_duration_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope) / sum(rate(cilium_triggers_policy_update_call_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(rate(cilium_triggers_policy_update_call_duration_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope) / sum(rate(cilium_triggers_policy_update_call_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Policy Trigger Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Max per node processingTime": "#e24d42", + "Max per node upstreamTime": "#58140c", + "parse errors": "#bf1b00" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 125 + }, + "hiddenSeries": false, + "id": 66, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "parse errors", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(cilium_proxy_upstream_reply_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope) / sum(rate(cilium_proxy_upstream_reply_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max {{scope}}", + "refId": "B" + }, + { + "expr": "max(rate(cilium_policy_l7_parse_errors_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "parse errors", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Proxy response time (Max)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "both": "#7eb26d", + "egress": "#e5ac0e", + "ingress": "#e0752d", + "none": "#bf1b00" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 130 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cilium_policy_endpoint_enforcement_status{k8s_app=\"cilium\", pod=~\"$pod\"}) by (enforcement)", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1s", + "intervalFactor": 1, + "legendFormat": "{{enforcement}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Endpoints policy enforcement status", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "avg": "#b7dbab", + "max": "rgba(89, 132, 76, 0.54)", + "min": "#2f575e" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 130 + }, + "hiddenSeries": false, + "id": 100, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_proxy_redirects{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_proxy_redirects{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_proxy_redirects{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Proxy Redirects", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "average duration": "#d683ce", + "folds": "#614d93", + "max duration": "#614d93", + "max trigger": "#967302", + "min duration": "#584477", + "min trigger": "#fceaca" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 130 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min trigger", + "lines": false + }, + { + "alias": "min trigger", + "lines": false + }, + { + "alias": "folds", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(rate(cilium_triggers_policy_update_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min trigger", + "refId": "A" + }, + { + "expr": "avg(rate(cilium_triggers_policy_update_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "average trigger", + "refId": "B" + }, + { + "expr": "max(rate(cilium_triggers_policy_update_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max trigger", + "refId": "C" + }, + { + "expr": "max(rate(cilium_triggers_policy_update_folds{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "folds", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Policy Trigger Runs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max": "#f2c96d", + "policy errors": "#bf1b00", + "policy change errors": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 135 + }, + "hiddenSeries": false, + "id": 85, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "policy errors", + "yaxis": 2 + }, + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + }, + { + "alias": "policy change errors", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_policy{k8s_app=\"cilium\", pod=~\"$pod\"}) by(pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_policy{k8s_app=\"cilium\", pod=~\"$pod\"}) by(pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_policy{k8s_app=\"cilium\", pod=~\"$pod\"}) by(pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + }, + { + "expr": "sum(cilium_policy_change_total{k8s_app=\"cilium\", pod=~\"$pod\"}, outcome=\"fail\") by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "policy change errors", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Policies Per Node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Max per node processingTime": "#e24d42", + "Max per node upstreamTime": "#58140c", + "parse errors": "#bf1b00" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 135 + }, + "hiddenSeries": false, + "id": 123, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "parse errors", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_proxy_upstream_reply_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{scope}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DNS proxy requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "avg": "#f9d9f9", + "max": "#806eb7", + "min": "#806eb7" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 140 + }, + "hiddenSeries": false, + "id": 117, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_policy_max_revision{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_policy_max_revision{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_policy_max_revision{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Policy Revision", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 145 + }, + "id": 73, + "links": [], + "mode": "markdown", + "title": "Endpoints", + "type": "text" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 146 + }, + "hiddenSeries": false, + "id": 55, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(histogram_quantile(0.90, rate(cilium_endpoint_regeneration_time_stats_seconds_bucket{k8s_app=\"cilium\", scope!=\"total\", pod=~\"$pod\"}[5m]))) by (scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Endpoint regeneration time (90th percentile)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 146 + }, + "hiddenSeries": false, + "id": 115, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(histogram_quantile(0.99, rate(cilium_endpoint_regeneration_time_stats_seconds_bucket{k8s_app=\"cilium\", scope!=\"total\", pod=~\"$pod\"}[5m]))) by (scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Endpoint regeneration time (99th percentile)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "fail": "#bf1b00", + "fail/min": "#890f02", + "success": "#447ebc", + "success/min": "#3f6833" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 155 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "avg": true, + "current": false, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "fail", + "yaxis": 2 + }, + { + "alias": "success" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_endpoint_regenerations_total{k8s_app=\"cilium\", pod=~\"$pod\"}[30s])) by(outcome)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{outcome}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Endpoint regenerations", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "disconnecting": "#614d93", + "ready": "rgba(81, 220, 95, 0.52)", + "waiting-to-regenerate": "#0a50a1" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 155 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cilium_endpoint_state{k8s_app=\"cilium\", pod=~\"$pod\"}) by (endpoint_state)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{endpoint_state}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cilium endpoint state", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 160 + }, + "id": 74, + "links": [], + "mode": "markdown", + "title": "Controllers", + "type": "text" + }, + { + "aliasColors": { + "Failed": "#bf1b00", + "Failing": "#890f02", + "Runs": "#5195ce" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 161 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Failing", + "yaxis": 1 + }, + { + "alias": "Failed", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_controllers_runs_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Runs", + "refId": "A" + }, + { + "expr": "sum(cilium_controllers_failing{k8s_app=\"cilium\", pod=~\"$pod\"}) by(pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Failed", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controllers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "duration failure": "#890f02", + "duration success": "#508642", + "failure": "#890f02", + "runs failure": "#890f02", + "runs success": "#7eb26d", + "success": "#508642" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 161 + }, + "hiddenSeries": false, + "id": 68, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [ + { + "alias": "success", + "yaxis": 1 + }, + { + "alias": "failure", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_controllers_runs_duration_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, status) / sum(rate(cilium_controllers_runs_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Durations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 166 + }, + "id": 60, + "links": [], + "mode": "markdown", + "title": "Kubernetes integration", + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 167 + }, + "hiddenSeries": false, + "id": 163, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_k8s_client_api_latency_time_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])/rate(cilium_k8s_client_api_latency_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "apiserver latency (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 167 + }, + "hiddenSeries": false, + "id": 165, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(cilium_k8s_client_api_latency_time_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])/rate(cilium_k8s_client_api_latency_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "apiserver latency (max node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 174 + }, + "hiddenSeries": false, + "id": 168, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_k8s_client_api_latency_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "apiserver #calls (sum all nodes)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 174 + }, + "hiddenSeries": false, + "id": 166, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_k8s_client_api_calls_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, return_code)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{return_code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "apiserver calls (sum all nodes)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 182 + }, + "hiddenSeries": false, + "id": 172, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_kubernetes_events_received_total{k8s_app=\"cilium\", equal=\"true\", valid=\"true\", pod=~\"$pod\"}[5m])) by (pod, scope, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Valid, Unnecessary K8s Events Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 182 + }, + "hiddenSeries": false, + "id": 174, + "legend": { + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_kubernetes_events_received_total{k8s_app=\"cilium\", equal=\"true\", valid=\"false\", pod=~\"$pod\"}[5m])) by (pod, scope, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Invalid, Unnecessary K8s Events Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 188 + }, + "hiddenSeries": false, + "id": 175, + "legend": { + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_kubernetes_events_received_total{k8s_app=\"cilium\", equal=\"false\", valid=\"true\", pod=~\"$pod\"}[5m])) by (pod, scope, action, valid)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Valid, Necessary K8s Events Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 188 + }, + "hiddenSeries": false, + "id": 173, + "legend": { + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_kubernetes_events_received_total{k8s_app=\"cilium\", equal=\"false\", valid=\"false\", pod=~\"$pod\"}[5m])) by (pod, scope, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Invalid, Necessary K8s Events Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 196 + }, + "hiddenSeries": false, + "id": 108, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"CiliumNetworkPolicy\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} avg", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CiliumNetworkPolicy Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "create avg": "#70dbed", + "delete avg": "#e24d42", + "update avg": "#e0f9d7" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 196 + }, + "hiddenSeries": false, + "id": 119, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"NetworkPolicy\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} avg", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "NetworkPolicy Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "create avg": "#70dbed", + "delete avg": "#e24d42", + "update avg": "#e0f9d7" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 203 + }, + "hiddenSeries": false, + "id": 109, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"Pod\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} avg", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "create avg": "#70dbed", + "delete avg": "#e24d42", + "update avg": "#e0f9d7" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 203 + }, + "hiddenSeries": false, + "id": 122, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"Node\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} avg", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Node Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 210 + }, + "hiddenSeries": false, + "id": 118, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"Service\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Service Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 210 + }, + "hiddenSeries": false, + "id": 120, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"Endpoint\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Endpoints Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 217 + }, + "hiddenSeries": false, + "id": 121, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"Namespace\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Namespace Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 25, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "cilium.*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "prometheus", + "definition": "label_values(cilium_version, pod)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "pod", + "options": [], + "query": "label_values(cilium_version, pod)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Cilium Metrics", + "uid": "vtuWtdumz", + "version": 1 +} diff --git a/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json b/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json new file mode 100644 index 00000000..3d8656c3 --- /dev/null +++ b/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json @@ -0,0 +1,323 @@ +{ + "node": { + "id": "host~127.0.0.1~no-id~localdomain", + "cluster": "ingress-cluster" + }, + "staticResources": { + "listeners": [ + {{- if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled }} + { + "name": "envoy-prometheus-metrics-listener", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} + } + }, + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "envoy-prometheus-metrics-listener", + "route_config": { + "virtual_hosts": [ + { + "name": "prometheus_metrics_route", + "domains": [ + "*" + ], + "routes": [ + { + "name": "prometheus_metrics_route", + "match": { + "prefix": "/metrics" + }, + "route": { + "cluster": "envoy-admin", + "prefix_rewrite": "/stats/prometheus" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "stream_idle_timeout": "0s" + } + } + ] + } + ] + }, + {{- end }} + { + "name": "envoy-health-listener", + "address": { + "socket_address": { + "address": {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }}, + "port_value": {{ .Values.envoy.healthPort }} + } + }, + {{- if and .Values.ipv4.enabled .Values.ipv6.enabled }} + "additional_addresses": [ + { + "address": { + "socket_address": { + "address": "::1", + "port_value": {{ .Values.envoy.healthPort }} + } + } + } + ], + {{- end }} + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "envoy-health-listener", + "route_config": { + "virtual_hosts": [ + { + "name": "health", + "domains": [ + "*" + ], + "routes": [ + { + "name": "health", + "match": { + "prefix": "/healthz" + }, + "route": { + "cluster": "envoy-admin", + "prefix_rewrite": "/ready" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "stream_idle_timeout": "0s" + } + } + ] + } + ] + } + ], + "clusters": [ + { + "name": "ingress-cluster", + "type": "ORIGINAL_DST", + "connectTimeout": "{{ .Values.envoy.connectTimeoutSeconds }}s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "{{ .Values.envoy.idleTimeoutDurationSeconds }}s", + "maxConnectionDuration": "{{ .Values.envoy.maxConnectionDurationSeconds }}s", + "maxRequestsPerConnection": {{ .Values.envoy.maxRequestsPerConnection }} + }, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "{{ .Values.envoy.connectTimeoutSeconds }}.500s" + }, + { + "name": "egress-cluster-tls", + "type": "ORIGINAL_DST", + "connectTimeout": "{{ .Values.envoy.connectTimeoutSeconds }}s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "{{ .Values.envoy.idleTimeoutDurationSeconds }}s", + "maxConnectionDuration": "{{ .Values.envoy.maxConnectionDurationSeconds }}s", + "maxRequestsPerConnection": {{ .Values.envoy.maxRequestsPerConnection }} + }, + "upstreamHttpProtocolOptions": {}, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "{{ .Values.envoy.connectTimeoutSeconds }}.500s", + "transportSocket": { + "name": "cilium.tls_wrapper", + "typedConfig": { + "@type": "type.googleapis.com/cilium.UpstreamTlsWrapperContext" + } + } + }, + { + "name": "egress-cluster", + "type": "ORIGINAL_DST", + "connectTimeout": "{{ .Values.envoy.connectTimeoutSeconds }}s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "{{ .Values.envoy.idleTimeoutDurationSeconds }}s", + "maxConnectionDuration": "{{ .Values.envoy.maxConnectionDurationSeconds }}s", + "maxRequestsPerConnection": {{ .Values.envoy.maxRequestsPerConnection }} + }, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "{{ .Values.envoy.connectTimeoutSeconds }}.500s" + }, + { + "name": "ingress-cluster-tls", + "type": "ORIGINAL_DST", + "connectTimeout": "{{ .Values.envoy.connectTimeoutSeconds }}s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "{{ .Values.envoy.idleTimeoutDurationSeconds }}s", + "maxConnectionDuration": "{{ .Values.envoy.maxConnectionDurationSeconds }}s", + "maxRequestsPerConnection": {{ .Values.envoy.maxRequestsPerConnection }} + }, + "upstreamHttpProtocolOptions": {}, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "{{ .Values.envoy.connectTimeoutSeconds }}.500s", + "transportSocket": { + "name": "cilium.tls_wrapper", + "typedConfig": { + "@type": "type.googleapis.com/cilium.UpstreamTlsWrapperContext" + } + } + }, + { + "name": "xds-grpc-cilium", + "type": "STATIC", + "connectTimeout": "{{ .Values.envoy.connectTimeoutSeconds }}s", + "loadAssignment": { + "clusterName": "xds-grpc-cilium", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/var/run/cilium/envoy/sockets/xds.sock" + } + } + } + } + ] + } + ] + }, + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "explicitHttpConfig": { + "http2ProtocolOptions": {} + } + } + } + }, + { + "name": "envoy-admin", + "type": "STATIC", + "connectTimeout": "{{ .Values.envoy.connectTimeoutSeconds }}s", + "loadAssignment": { + "clusterName": "envoy-admin", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/var/run/cilium/envoy/sockets/admin.sock" + } + } + } + } + ] + } + ] + } + } + ] + }, + "dynamicResources": { + "ldsConfig": { + "apiConfigSource": { + "apiType": "GRPC", + "transportApiVersion": "V3", + "grpcServices": [ + { + "envoyGrpc": { + "clusterName": "xds-grpc-cilium" + } + } + ], + "setNodeOnFirstMessageOnly": true + }, + "resourceApiVersion": "V3" + }, + "cdsConfig": { + "apiConfigSource": { + "apiType": "GRPC", + "transportApiVersion": "V3", + "grpcServices": [ + { + "envoyGrpc": { + "clusterName": "xds-grpc-cilium" + } + } + ], + "setNodeOnFirstMessageOnly": true + }, + "resourceApiVersion": "V3" + } + }, + "layeredRuntime": { + "layers": [ + { + "name": "static_layer_0", + "staticLayer": { + "overload": { + "global_downstream_max_connections": 50000 + } + } + } + ] + }, + "admin": { + "address": { + "pipe": { + "path": "/var/run/cilium/envoy/sockets/admin.sock" + } + } + } +} diff --git a/charts/cilium/files/cilium-operator/dashboards/cilium-operator-dashboard.json b/charts/cilium/files/cilium-operator/dashboards/cilium-operator-dashboard.json new file mode 100644 index 00000000..e677deae --- /dev/null +++ b/charts/cilium/files/cilium-operator/dashboards/cilium-operator-dashboard.json @@ -0,0 +1,998 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 8, + "links": [], + "panels": [ + { + "aliasColors": { + "avg": "#cffaff" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(irate(cilium_operator_process_cpu_seconds_total{io_cilium_app=\"operator\"}[1m])) by (pod) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(irate(cilium_operator_process_cpu_seconds_total{io_cilium_app=\"operator\"}[1m])) by (pod) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(irate(cilium_operator_process_cpu_seconds_total{io_cilium_app=\"operator\"}[1m])) by (pod) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage per node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "MAX_resident_memory_bytes_max": "#e5ac0e" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(cilium_operator_process_resident_memory_bytes{io_cilium_app=\"operator\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "AVG_resident_memory_bytes", + "refId": "C" + }, + { + "expr": "max(cilium_operator_process_resident_memory_bytes{io_cilium_app=\"operator\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "MAX_resident_memory_bytes_max", + "refId": "D" + }, + { + "expr": "min(cilium_operator_process_resident_memory_bytes{io_cilium_app=\"operator\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "MIN_resident_memory_bytes_min", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Resident memory status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 6, + "panels": [], + "title": "IPAM", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(cilium_operator_ipam_ips) by (type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IP Addresses", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(cilium_operator_ec2_api_duration_seconds_sum[1m])/rate(cilium_operator_ec2_api_duration_seconds_count[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}} {{response_code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "EC2 API Interactions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dtdurations", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cilium_operator_ipam_nodes", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{category}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of nodes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 14 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cilium_operator_ipam_available", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "interfaces", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# interfaces with addresses available", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 14 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(cilium_operator_ipam_resync_total[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "operations", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Metadata Resync Operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(cilium_operator_ec2_api_rate_limit_duration_seconds_sum[1m])/rate(cilium_operator_ec2_api_rate_limit_duration_seconds_count[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "EC2 client side rate limiting", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_operator_ipam_interface_creation_ops[1m])) by (subnetId, status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{status}} ({{subnetId}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Interface Creation", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 25, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Cilium Operator", + "uid": "1GC0TT4Wz", + "version": 2 +} diff --git a/charts/cilium/files/hubble/dashboards/hubble-dashboard.json b/charts/cilium/files/hubble/dashboards/hubble-dashboard.json new file mode 100644 index 00000000..12de657d --- /dev/null +++ b/charts/cilium/files/hubble/dashboards/hubble-dashboard.json @@ -0,0 +1,3232 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "panels": [], + "title": "General Processing", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "avg", + "lines": false + }, + { + "alias": "avg", + "fill": 0, + "fillBelowTo": "min" + }, + { + "alias": "min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(sum(rate(hubble_flows_processed_total[1m])) by (pod))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "A" + }, + { + "expr": "min(sum(rate(hubble_flows_processed_total[1m])) by (pod))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "B" + }, + { + "expr": "max(sum(rate(hubble_flows_processed_total[1m])) by (pod))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Flows processed Per Node", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_flows_processed_total[1m])) by (pod, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Flows Types", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 59, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_flows_processed_total{type=\"L7\"}[1m])) by (pod, subtype)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{subtype}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "L7 Flow Distribution", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 60, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_flows_processed_total{type=\"Trace\"}[1m])) by (pod, subtype)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{subtype}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Trace Flow Distribution", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 16, + "panels": [], + "title": "Network", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 33, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_flows_processed_total[1m])) by (pod, verdict)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{verdict}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Forwarded vs Dropped", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_drop_total[1m])) by (pod, reason)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Drop Reason", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": true, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate(hubble_port_distribution_total[1m])) by (pod, protocol)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{protocol}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Protocol Usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, sum (rate(hubble_port_distribution_total{port!=\"0\"}[1m])) by (pod, port, protocol))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{port}}/{{protocol}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top 10 Port Distribution", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "fin", + "yaxis": 1 + }, + { + "alias": "FIN", + "yaxis": 2 + }, + { + "alias": "RST", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_tcp_flags_total{family=\"IPv4\"}[1m])) by (pod, flag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{flag}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TCPv4", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 0.2 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "Missing TCP SYN-ACK", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 62, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "fin", + "yaxis": 1 + }, + { + "alias": "FIN", + "yaxis": 2 + }, + { + "alias": "RST", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_tcp_flags_total{family=\"IPv4\", flag=\"SYN\"}[1m])) by (pod) - sum(rate(hubble_tcp_flags_total{family=\"IPv4\", flag=\"SYN-ACK\"}[1m])) by (pod)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Missing SYN-ACK", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.2 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Missing TCPv4 SYN-ACKs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "fin", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_tcp_flags_total{family=\"IPv6\"}[1m])) by (pod, flag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{flag}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TCPv6", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 0.2 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "Missing TCPv6 SYN-ACKs alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 63, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "fin", + "yaxis": 1 + }, + { + "alias": "FIN", + "yaxis": 2 + }, + { + "alias": "RST", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_tcp_flags_total{family=\"IPv6\", flag=\"SYN\"}[1m])) by (pod) - sum(rate(hubble_tcp_flags_total{family=\"IPv6\", flag=\"SYN-ACK\"}[1m])) by (pod)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Missing SYN-ACK", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.2 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Missing TCPv6 SYN-ACKs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_icmp_total{family=\"IPv4\"}[1m])) by (pod, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ICMPv4", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 0.1 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "Missing ICMPv4 Echo-Reply alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 64, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_icmp_total{family=\"IPv4\", type=\"EchoRequest\"}[1m])) by (pod) - sum(rate(hubble_icmp_total{family=\"IPv4\", type=\"EchoReply\"}[1m])) by (pod)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Missing ICMP Echo-Reply", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.1 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Missing ICMPv4 Echo-Reply", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 36, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_icmp_total{family=\"IPv6\"}[1m])) by (pod, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ICMPv6", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 65, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_icmp_total{family=\"IPv6\", type=\"EchoRequest\"}[1m])) by (pod) - sum(rate(hubble_icmp_total{family=\"IPv6\", type=\"EchoReply\"}[1m])) by (pod)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Missing ICMP Echo-Reply", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Missing ICMPv6 Echo-Reply", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 42, + "panels": [], + "title": "Network Policy", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 43, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_drop_total{reason=\"POLICY_DENIED\"}[1m])) by (pod, reason)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Denies by Reason", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 61, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_drop_total{reason=\"POLICY_DENIED\"}[1m])) by (pod, protocol)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{protocol}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Denied Packets by Protocol", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 55, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, sum(rate(hubble_drop_total{reason=\"POLICY_DENIED\"}[1m])) by (pod, source))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top 10 Source Pods with Denied Packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 54, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, sum(rate(hubble_drop_total{reason=\"POLICY_DENIED\"}[1m])) by (pod, destination))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{destination}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top 10 Destination Pods with Denied Packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 47, + "panels": [], + "title": "HTTP", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 45, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_http_requests_total[1m])) by (pod, method)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "reqps", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 49, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_http_responses_total[1m])) by (pod, status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP responses", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 51, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, rate(hubble_http_request_duration_seconds_bucket[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP Request/Response Latency (p50)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 58, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, rate(hubble_http_request_duration_seconds_bucket[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP Request/Response Latency (p99)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 53, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": true, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_http_requests_total[5m])) by (pod, protocol)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{protocol}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP Protocol Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 6, + "panels": [], + "title": "DNS", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 70 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_dns_queries_total[1m])) by (pod, qtypes)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{qtypes}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DNS Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 70 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_dns_responses_total{rcode=\"No Error\"}[1m])) by (pod, qtypes)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{qtypes}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DNS responses", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 0.5 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "DNS Request/Response Symmetry alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 70 + }, + "id": 66, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_dns_queries_total[1m])) by (pod, qtypes) - sum(rate(hubble_dns_responses_total[1m])) by (pod, qtypes)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{qtypes}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.5 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Missing DNS Responses", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 40, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_dns_response_types_total[1m])) by (pod, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DNS Response Record Type", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 57, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_dns_responses_total{rcode=\"No Error\"}[1m])) by (pod,ips_returned)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ips_returned}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DNS Response IPs Returned", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_dns_responses_total{rcode!=\"No Error\"}[1m])) by (pod, qtypes, rcode)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{rcode}} ({{qtypes}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DNS Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 4, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 56, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10,sum(rate(hubble_dns_responses_total{rcode!=\"No Error\"}[1m])) by (pod, destination))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{destination}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pods with DNS errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 4, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 85 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, sum(rate(hubble_dns_queries_total[10m])*60) by (query, qtypes))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{query}} ({{qtypes}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top 10 DNS Queries per minute", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Hubble", + "uid": "5HftnJAWz", + "version": 24 +} diff --git a/charts/cilium/files/hubble/dashboards/hubble-l7-http-metrics-by-workload.json b/charts/cilium/files/hubble/dashboards/hubble-l7-http-metrics-by-workload.json index b8ae55e9..428c58f4 100644 --- a/charts/cilium/files/hubble/dashboards/hubble-l7-http-metrics-by-workload.json +++ b/charts/cilium/files/hubble/dashboards/hubble-l7-http-metrics-by-workload.json @@ -6,7 +6,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "9.0.5" + "version": "9.4.7" }, { "type": "datasource", @@ -118,7 +118,7 @@ }, "textMode": "auto" }, - "pluginVersion": "9.0.5", + "pluginVersion": "9.4.7", "targets": [ { "datasource": { @@ -182,7 +182,7 @@ }, "textMode": "auto" }, - "pluginVersion": "9.0.5", + "pluginVersion": "9.4.7", "targets": [ { "datasource": { @@ -212,6 +212,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -271,7 +273,8 @@ "lastNotNull" ], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -349,6 +352,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -411,7 +416,8 @@ "lastNotNull" ], "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -447,6 +453,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -509,7 +517,8 @@ "lastNotNull" ], "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -545,6 +554,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -607,7 +618,8 @@ "lastNotNull" ], "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -660,6 +672,103 @@ "title": "HTTP Request Duration by Source", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"${cluster}\", namespace=~\"${destination_namespace}\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"${cluster}\", namespace=~\"${destination_namespace}\", workload=~\"${source_workload}\"}\n) by (namespace, workload)", + "interval": "", + "legendFormat": "{{ cluster }} {{ namespace }}/{{ workload }}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage by Source", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { @@ -684,6 +793,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -746,7 +857,8 @@ "lastNotNull" ], "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -782,6 +894,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -844,7 +958,8 @@ "lastNotNull" ], "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -880,6 +995,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -941,7 +1058,8 @@ "lastNotNull" ], "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -993,10 +1111,107 @@ ], "title": "HTTP Request Duration by Destination", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"${cluster}\", namespace=~\"${destination_namespace}\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"${cluster}\", namespace=~\"${destination_namespace}\", workload=\"${destination_workload}\"}\n) by (namespace, workload)", + "interval": "", + "legendFormat": "{{ cluster }} {{ namespace }}/{{ workload }}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage by Destination", + "type": "timeseries" } ], "refresh": "30s", - "schemaVersion": 36, + "revision": 1, + "schemaVersion": 38, "style": "dark", "tags": [], "templating": { @@ -1165,6 +1380,6 @@ "timezone": "", "title": "Hubble L7 HTTP Metrics by Workload", "uid": "3g264CZVz", - "version": 2, + "version": 3, "weekStart": "" } diff --git a/charts/cilium/files/nodeinit/prestop.bash b/charts/cilium/files/nodeinit/prestop.bash index caf4ba61..e78f178a 100644 --- a/charts/cilium/files/nodeinit/prestop.bash +++ b/charts/cilium/files/nodeinit/prestop.bash @@ -4,6 +4,8 @@ set -o errexit set -o pipefail set -o nounset +{{ .Values.nodeinit.prestop.preScript }} + if stat /tmp/node-deinit.cilium.io > /dev/null 2>&1; then exit 0 fi @@ -53,4 +55,6 @@ if iptables -w -t nat -L IP-MASQ > /dev/null; then fi {{- end }} +{{ .Values.nodeinit.prestop.postScript }} + echo "Node de-initialization complete" diff --git a/charts/cilium/files/nodeinit/startup.bash b/charts/cilium/files/nodeinit/startup.bash index f0c4ce3a..975b71fa 100644 --- a/charts/cilium/files/nodeinit/startup.bash +++ b/charts/cilium/files/nodeinit/startup.bash @@ -14,6 +14,8 @@ echo "Addressing:" ip -4 a ip -6 a +{{ .Values.nodeinit.startup.preScript }} + {{- if .Values.nodeinit.removeCbrBridge }} if ip link show cbr0; then echo "Detected cbr0 bridge. Deleting interface..." @@ -97,8 +99,7 @@ then # Starting from GKE node version 1.24, containerd version used is 1.6. # Since that version containerd no longer allows missing configuration for the CNI, # not even for pods with hostNetwork set to true. Thus, we add a temporary one. - # This will be replaced with the real config by cni-install.sh script from the - # agent pod. + # This will be replaced with the real config by the agent pod. echo -e "{\n\t"cniVersion": "0.3.1",\n\t"name": "cilium",\n\t"type": "cilium-cni"\n}" > /etc/cni/net.d/05-cilium.conf fi @@ -199,4 +200,7 @@ fi {{- if .Values.nodeinit.revertReconfigureKubelet }} rm -f /tmp/node-deinit.cilium.io {{- end }} + +{{ .Values.nodeinit.startup.postScript }} + echo "Node initialization complete" diff --git a/charts/cilium/files/spire/init.bash b/charts/cilium/files/spire/init.bash new file mode 100644 index 00000000..4b6f57bd --- /dev/null +++ b/charts/cilium/files/spire/init.bash @@ -0,0 +1,48 @@ +# shellcheck disable=SC2086 +# shellcheck disable=SC2139 +set -e + +echo "Waiting for spire process to start" +while ! pgrep spire-server > /dev/null; do sleep 5; done + +SPIRE_SERVER_ROOT_PATH="/proc/$(pgrep spire-server)/root" + +alias spire_server="${SPIRE_SERVER_ROOT_PATH}/opt/spire/bin/spire-server" +SOCKET_PATH="${SPIRE_SERVER_ROOT_PATH}/tmp/spire-server/private/api.sock" +SOCKET_FLAG="-socketPath ${SOCKET_PATH}" + +echo "Checking spire-server status" +while ! spire_server entry show ${SOCKET_FLAG} &> /dev/null; do + echo "Waiting for spire-server to start..." + sleep 5 +done + +echo "Spire Server is up, initializing cilium spire entries..." + +AGENT_SPIFFE_ID="spiffe://{{ .Values.authentication.mutual.spire.trustDomain }}/ns/{{ .Values.authentication.mutual.spire.install.namespace }}/sa/spire-agent" +AGENT_SELECTORS="-selector k8s_psat:agent_ns:{{ .Values.authentication.mutual.spire.install.namespace }} -selector k8s_psat:agent_sa:spire-agent" +CILIUM_AGENT_SPIFFE_ID="spiffe://{{ .Values.authentication.mutual.spire.trustDomain }}/cilium-agent" +CILIUM_AGENT_SELECTORS="-selector k8s:ns:{{ .Release.Namespace }} -selector k8s:sa:{{ .Values.serviceAccounts.cilium.name }}" +CILIUM_OPERATOR_SPIFFE_ID="spiffe://{{ .Values.authentication.mutual.spire.trustDomain }}/cilium-operator" +CILIUM_OPERATOR_SELECTORS="-selector k8s:ns:{{ .Release.Namespace }} -selector k8s:sa:{{ .Values.serviceAccounts.operator.name }}" + +while pgrep spire-server > /dev/null; +do + echo "Ensuring agent entry" + if spire_server entry show ${SOCKET_FLAG} -spiffeID $AGENT_SPIFFE_ID $AGENT_SELECTORS | grep -q "Found 0 entries" &> /dev/null; then + spire_server entry create ${SOCKET_FLAG} -spiffeID $AGENT_SPIFFE_ID $AGENT_SELECTORS -node + fi + + echo "Ensuring cilium-agent entry (required for the delegated identity to work)" + if spire_server entry show ${SOCKET_FLAG} -spiffeID $CILIUM_AGENT_SPIFFE_ID $CILIUM_AGENT_SELECTORS | grep -q "Found 0 entries" &> /dev/null; then + spire_server entry create ${SOCKET_FLAG} -spiffeID $CILIUM_AGENT_SPIFFE_ID -parentID $AGENT_SPIFFE_ID $CILIUM_AGENT_SELECTORS + fi + + echo "Ensuring cilium-operator entry (required for creating SPIFFE identities)" + if spire_server entry show ${SOCKET_FLAG} -spiffeID $CILIUM_OPERATOR_SPIFFE_ID $CILIUM_OPERATOR_SELECTORS | grep -q "Found 0 entries" &> /dev/null; then + spire_server entry create ${SOCKET_FLAG} -spiffeID $CILIUM_OPERATOR_SPIFFE_ID -parentID $AGENT_SPIFFE_ID $CILIUM_OPERATOR_SELECTORS + fi + + echo "Cilium Spire entries are initialized successfully or already in-sync" + sleep 30; +done diff --git a/charts/cilium/files/spire/wait-for-spire.bash b/charts/cilium/files/spire/wait-for-spire.bash new file mode 100644 index 00000000..3507af1c --- /dev/null +++ b/charts/cilium/files/spire/wait-for-spire.bash @@ -0,0 +1,35 @@ +set -e + +echo "Waiting for spire server to be reachable to start" + +{{- if .Values.authentication.mutual.spire.serverAddress }} +ADDR="{{ .Values.authentication.mutual.spire.serverAddress }}" +{{- else }} +ADDR="spire-server.{{ .Values.authentication.mutual.spire.install.namespace}}.svc:8081" +{{- end }} +CONN_TIMEOUT="3" +TIMEOUT="60" + +call_tcp_endpoint_with_timeout() { + local addr="$1" + local timeout="$2" + + nc -z "$addr" -w "$timeout" &> /dev/null +} + +# wait for SPIRE server to be reachable till $TIMEOUT is reached +start_time=$(date +%s) +while true; do + if call_tcp_endpoint_with_timeout "$ADDR" "$CONN_TIMEOUT"; then + echo "SPIRE server is reachable" + break + fi + + if [ $(( $(date +%s) - start_time )) -gt "$TIMEOUT" ]; then + echo "Timed out waiting for spire server to be reachable" + exit 1 + fi + + echo "Waiting for spire server to be reachable" + sleep 1 +done diff --git a/charts/cilium/templates/cilium-agent/clusterrole.yaml b/charts/cilium/templates/cilium-agent/clusterrole.yaml index ca332baf..89e604a7 100644 --- a/charts/cilium/templates/cilium-agent/clusterrole.yaml +++ b/charts/cilium/templates/cilium-agent/clusterrole.yaml @@ -46,6 +46,18 @@ rules: # To annotate the k8s node with Cilium's metadata - patch {{- end }} +{{- if .Values.l2announcements.enabled }} +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update + - list + - delete +{{- end }} - apiGroups: - apiextensions.k8s.io resources: @@ -81,6 +93,9 @@ rules: - ciliumnetworkpolicies - ciliumnodes - ciliumnodeconfigs + - ciliumcidrgroups + - ciliuml2announcementpolicies + - ciliumpodippools verbs: - list - watch @@ -121,6 +136,7 @@ rules: - ciliumclusterwidenetworkpolicies/status - ciliumendpoints/status - ciliumendpoints + - ciliuml2announcementpolicies/status verbs: - patch {{- end }} diff --git a/charts/cilium/templates/cilium-agent/daemonset.yaml b/charts/cilium/templates/cilium-agent/daemonset.yaml index b8b35e1f..a608b69f 100644 --- a/charts/cilium/templates/cilium-agent/daemonset.yaml +++ b/charts/cilium/templates/cilium-agent/daemonset.yaml @@ -8,7 +8,7 @@ {{- $defaultKeepDeprecatedProbes = false -}} {{- end -}} -{{- $kubeProxyReplacement := (coalesce .Values.kubeProxyReplacement "disabled") -}} +{{- $kubeProxyReplacement := (coalesce .Values.kubeProxyReplacement "false") -}} --- apiVersion: apps/v1 @@ -192,18 +192,6 @@ spec: fieldPath: metadata.namespace - name: CILIUM_CLUSTERMESH_CONFIG value: /var/lib/cilium/clustermesh/ - - name: CILIUM_CNI_CHAINING_MODE - valueFrom: - configMapKeyRef: - name: cilium-config - key: cni-chaining-mode - optional: true - - name: CILIUM_CUSTOM_CNI_CONF - valueFrom: - configMapKeyRef: - name: cilium-config - key: custom-cni-conf - optional: true {{- if .Values.k8sServiceHost }} - name: KUBERNETES_SERVICE_HOST value: {{ .Values.k8sServiceHost | quote }} @@ -217,16 +205,15 @@ spec: {{- end }} {{- if .Values.cni.install }} lifecycle: + {{- if .Values.eni.enabled }} postStart: exec: command: - "bash" - "-c" - | - /cni-install.sh --enable-debug={{ .Values.debug.enabled }} --cni-exclusive={{ .Values.cni.exclusive }} --log-file={{ .Values.cni.logFile }} - {{- if .Values.eni.enabled }} - {{- tpl (.Files.Get "files/agent/poststart-eni.bash") . | nindent 16 }} - {{- end }} + {{- tpl (.Files.Get "files/agent/poststart-eni.bash") . | nindent 20 }} + {{- end }} preStop: exec: command: @@ -238,21 +225,19 @@ spec: {{- end }} {{- if or .Values.prometheus.enabled .Values.hubble.metrics.enabled }} ports: - {{- if .Values.hubble.peerService.enabled }} - name: peer-service containerPort: {{ .Values.hubble.peerService.targetPort }} hostPort: {{ .Values.hubble.peerService.targetPort }} protocol: TCP - {{- end }} {{- if .Values.prometheus.enabled }} - name: prometheus containerPort: {{ .Values.prometheus.port }} hostPort: {{ .Values.prometheus.port }} protocol: TCP - {{- if .Values.proxy.prometheus.enabled }} + {{- if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled (not .Values.envoy.enabled) }} - name: envoy-metrics - containerPort: {{ .Values.proxy.prometheus.port }} - hostPort: {{ .Values.proxy.prometheus.port }} + containerPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} + hostPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} protocol: TCP {{- end }} {{- end }} @@ -281,6 +266,16 @@ spec: {{- end }} terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + {{- if .Values.authentication.mutual.spire.enabled }} + - name: spire-agent-socket + mountPath: {{ dir .Values.authentication.mutual.spire.adminSocketPath }} + readOnly: false + {{- end }} + {{- if .Values.envoy.enabled }} + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + {{- end }} {{- if not .Values.securityContext.privileged }} # Unprivileged containers need to mount /proc/sys/net from the host # to have write access @@ -292,7 +287,7 @@ spec: name: host-proc-sys-kernel {{- end}} {{- /* CRI-O already mounts the BPF filesystem */ -}} - {{- if not (eq .Values.containerRuntime.integration "crio") }} + {{- if and .Values.bpf.autoMount.enabled (not (eq .Values.containerRuntime.integration "crio")) }} - name: bpf-maps mountPath: /sys/fs/bpf {{- if .Values.securityContext.privileged }} @@ -439,6 +434,9 @@ spec: - name: KUBERNETES_SERVICE_PORT value: {{ .Values.k8sServicePort | quote }} {{- end }} + {{- with .Values.extraEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} volumeMounts: - name: tmp mountPath: /tmp @@ -533,7 +531,7 @@ spec: - ALL {{- end}} {{- end }} - {{- if not .Values.securityContext.privileged }} + {{- if and .Values.bpf.autoMount.enabled (not .Values.securityContext.privileged) }} # Mount the bpf fs if it is not mounted. We will perform this task # from a privileged container because the mount propagation bidirectional # only works from privileged containers. @@ -550,7 +548,7 @@ spec: securityContext: privileged: true {{- /* CRI-O already mounts the BPF filesystem */ -}} - {{- if not (eq .Values.containerRuntime.integration "crio") }} + {{- if and .Values.bpf.autoMount.enabled (not (eq .Values.containerRuntime.integration "crio")) }} volumeMounts: - name: bpf-maps mountPath: /sys/fs/bpf @@ -622,7 +620,7 @@ spec: {{- end}} volumeMounts: {{- /* CRI-O already mounts the BPF filesystem */ -}} - {{- if not (eq .Values.containerRuntime.integration "crio") }} + {{- if and .Values.bpf.autoMount.enabled (not (eq .Values.containerRuntime.integration "crio")) }} - name: bpf-maps mountPath: /sys/fs/bpf {{- end }} @@ -669,6 +667,7 @@ spec: done terminationMessagePolicy: FallbackToLogsOnError {{- end }} # wait-for-kube-proxy + {{- if .Values.cni.install }} # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent - name: install-cni-binaries image: {{ include "cilium.image" .Values.image | quote }} @@ -695,6 +694,7 @@ spec: volumeMounts: - name: cni-path mountPath: /host/opt/cni/bin + {{- end }} # .Values.cni.install restartPolicy: Always priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.priorityClassName "system-node-critical") }} serviceAccount: {{ .Values.serviceAccounts.cilium.name | quote }} @@ -721,7 +721,7 @@ spec: tolerations: {{- toYaml . | trim | nindent 8 }} {{- end }} - {{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.config.enabled }} + {{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.config.enabled (not .Values.clustermesh.apiserver.kvstoremesh.enabled) }} hostAliases: {{- range $cluster := .Values.clustermesh.config.clusters }} {{- range $ip := $cluster.ips }} @@ -740,7 +740,7 @@ spec: path: {{ .Values.daemon.runPath }} type: DirectoryOrCreate {{- /* CRI-O already mounts the BPF filesystem */ -}} - {{- if not (eq .Values.containerRuntime.integration "crio") }} + {{- if and .Values.bpf.autoMount.enabled (not (eq .Values.containerRuntime.integration "crio")) }} # To keep state between restarts / upgrades for bpf maps - name: bpf-maps hostPath: @@ -778,6 +778,19 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + {{- if .Values.authentication.mutual.spire.enabled }} + - name: spire-agent-socket + hostPath: + path: {{ dir .Values.authentication.mutual.spire.adminSocketPath }} + type: DirectoryOrCreate + {{- end }} + {{- if .Values.envoy.enabled }} + # Sharing socket with Cilium Envoy on the same node by using a host path + - name: envoy-sockets + hostPath: + path: "{{ .Values.daemon.runPath }}/envoy/sockets" + type: DirectoryOrCreate + {{- end }} {{- if .Values.kubeConfigPath }} - name: kube-config hostPath: @@ -812,11 +825,36 @@ spec: {{- end }} # To read the clustermesh configuration - name: clustermesh-secrets - secret: - secretName: cilium-clustermesh + projected: # note: the leading zero means this number is in octal representation: do not remove it defaultMode: 0400 - optional: true + sources: + - secret: + name: cilium-clustermesh + optional: true + # note: items are not explicitly listed here, since the entries of this secret + # depend on the peers configured, and that would cause a restart of all agents + # at every addition/removal. Leaving the field empty makes each secret entry + # to be automatically projected into the volume as a file whose name is the key. + - secret: + name: clustermesh-apiserver-remote-cert + optional: true + items: + - key: tls.key + path: common-etcd-client.key + - key: tls.crt + path: common-etcd-client.crt + {{- if not .Values.tls.caBundle.enabled }} + - key: ca.crt + path: common-etcd-client-ca.crt + {{- else }} + - {{ .Values.tls.caBundle.useSecret | ternary "secret" "configMap" }}: + name: {{ .Values.tls.caBundle.name }} + optional: true + items: + - key: {{ .Values.tls.caBundle.key }} + path: common-etcd-client-ca.crt + {{- end }} {{- if and .Values.ipMasqAgent .Values.ipMasqAgent.enabled }} - name: ip-masq-agent configMap: @@ -861,12 +899,21 @@ spec: name: hubble-server-certs optional: true items: - - key: ca.crt - path: client-ca.crt - key: tls.crt path: server.crt - key: tls.key path: server.key + {{- if not .Values.tls.caBundle.enabled }} + - key: ca.crt + path: client-ca.crt + {{- else }} + - {{ .Values.tls.caBundle.useSecret | ternary "secret" "configMap" }}: + name: {{ .Values.tls.caBundle.name }} + optional: true + items: + - key: {{ .Values.tls.caBundle.key }} + path: client-ca.crt + {{- end }} {{- end }} {{- range .Values.extraHostPathMounts }} - name: {{ .name }} diff --git a/charts/cilium/templates/cilium-agent/dashboards-configmap.yaml b/charts/cilium/templates/cilium-agent/dashboards-configmap.yaml new file mode 100644 index 00000000..7f1ff7c5 --- /dev/null +++ b/charts/cilium/templates/cilium-agent/dashboards-configmap.yaml @@ -0,0 +1,25 @@ +{{- if .Values.dashboards.enabled }} +{{- $files := .Files.Glob "files/cilium-agent/dashboards/*.json" }} +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ $.Values.dashboards.namespace | default $.Release.Namespace }} + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + {{- if $.Values.dashboards.label }} + {{ $.Values.dashboards.label }}: {{ ternary $.Values.dashboards.labelValue "1" (not (empty $.Values.dashboards.labelValue)) | quote }} + {{- end }} + {{- with $.Values.dashboards.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} diff --git a/charts/cilium/templates/cilium-agent/role.yaml b/charts/cilium/templates/cilium-agent/role.yaml index 467cec59..abbf37d8 100644 --- a/charts/cilium/templates/cilium-agent/role.yaml +++ b/charts/cilium/templates/cilium-agent/role.yaml @@ -58,3 +58,23 @@ rules: - list - watch {{- end}} + +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create .Values.envoyConfig.enabled .Values.envoyConfig.secretsNamespace.name }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-envoy-config-secrets + namespace: {{ .Values.envoyConfig.secretsNamespace.name | quote }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +{{- end}} diff --git a/charts/cilium/templates/cilium-agent/rolebinding.yaml b/charts/cilium/templates/cilium-agent/rolebinding.yaml index 5e62aa61..ec8c5d80 100644 --- a/charts/cilium/templates/cilium-agent/rolebinding.yaml +++ b/charts/cilium/templates/cilium-agent/rolebinding.yaml @@ -55,3 +55,21 @@ subjects: namespace: {{ .Release.Namespace }} {{- end}} +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create .Values.envoyConfig.enabled .Values.envoyConfig.secretsNamespace.name}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-envoy-config-secrets + namespace: {{ .Values.envoyConfig.secretsNamespace.name | quote }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-envoy-config-secrets +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.cilium.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end}} diff --git a/charts/cilium/templates/cilium-agent/service.yaml b/charts/cilium/templates/cilium-agent/service.yaml index 18e0c7de..e0a55803 100644 --- a/charts/cilium/templates/cilium-agent/service.yaml +++ b/charts/cilium/templates/cilium-agent/service.yaml @@ -19,11 +19,13 @@ spec: port: {{ .Values.prometheus.port }} protocol: TCP targetPort: prometheus + {{- if not .Values.envoy.enabled }} - name: envoy-metrics - port: {{ .Values.proxy.prometheus.port }} + port: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} protocol: TCP targetPort: envoy-metrics -{{- else if .Values.proxy.prometheus.enabled }} + {{- end }} +{{- else if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled (not .Values.envoy.enabled) }} apiVersion: v1 kind: Service metadata: @@ -31,7 +33,7 @@ metadata: namespace: {{ .Release.Namespace }} annotations: prometheus.io/scrape: "true" - prometheus.io/port: {{ .Values.proxy.prometheus.port | quote }} + prometheus.io/port: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port | quote }} labels: k8s-app: cilium app.kubernetes.io/name: cilium-agent @@ -43,7 +45,7 @@ spec: k8s-app: cilium ports: - name: envoy-metrics - port: {{ .Values.proxy.prometheus.port }} + port: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} protocol: TCP targetPort: envoy-metrics {{- end }} diff --git a/charts/cilium/templates/cilium-ca-bundle-configmap.yaml b/charts/cilium/templates/cilium-ca-bundle-configmap.yaml new file mode 100644 index 00000000..38522458 --- /dev/null +++ b/charts/cilium/templates/cilium-ca-bundle-configmap.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.tls.caBundle.enabled .Values.tls.caBundle.content -}} +--- +apiVersion: v1 +kind: {{ .Values.tls.caBundle.useSecret | ternary "Secret" "ConfigMap" }} +metadata: + name: {{ .Values.tls.caBundle.name }} + namespace: {{ .Release.Namespace }} +{{ .Values.tls.caBundle.useSecret | ternary "stringData" "data" }}: + {{ .Values.tls.caBundle.key }}: | + {{- .Values.tls.caBundle.content | nindent 4 }} +{{- end }} diff --git a/charts/cilium/templates/cilium-ca-secret.yaml b/charts/cilium/templates/cilium-ca-secret.yaml index f512b185..5a74f787 100644 --- a/charts/cilium/templates/cilium-ca-secret.yaml +++ b/charts/cilium/templates/cilium-ca-secret.yaml @@ -1,6 +1,6 @@ {{- if or (and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm") (not .Values.clustermesh.apiserver.tls.ca.cert)) - (and (or .Values.agent .Values.hubble.relay.enabled .Values.hubble.ui.enabled) .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "helm") (not .Values.hubble.tls.ca.cert)) + (and (or .Values.agent .Values.hubble.relay.enabled .Values.hubble.ui.enabled) .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "helm")) (and .Values.tls.ca.key .Values.tls.ca.cert) -}} diff --git a/charts/cilium/templates/cilium-configmap.yaml b/charts/cilium/templates/cilium-configmap.yaml index ea251e1d..afda31ac 100644 --- a/charts/cilium/templates/cilium-configmap.yaml +++ b/charts/cilium/templates/cilium-configmap.yaml @@ -11,7 +11,7 @@ {{- $defaultBpfCtAnyMax := 262144 -}} {{- $enableIdentityMark := "true" -}} {{- $fragmentTracking := "true" -}} -{{- $defaultKubeProxyReplacement := "disabled" -}} +{{- $defaultKubeProxyReplacement := "false" -}} {{- $azureUsePrimaryAddress := "true" -}} {{- /* Default values when 1.8 was initially deployed */ -}} @@ -50,12 +50,32 @@ {{- $defaultKubeProxyReplacement = "disabled" -}} {{- end -}} +{{- /* Default values when 1.14 was initially deployed */ -}} +{{- if semverCompare ">=1.14" (default "1.14" .Values.upgradeCompatibility) -}} + {{- /* KPR default for 1.14 needed to override earlier version defaults set above when upgradeCompatibility is not specified */ -}} + {{- $defaultKubeProxyReplacement = "false" -}} +{{- end -}} + {{- $ipam := (coalesce .Values.ipam.mode $defaultIPAM) -}} {{- $bpfCtTcpMax := (coalesce .Values.bpf.ctTcpMax $defaultBpfCtTcpMax) -}} {{- $bpfCtAnyMax := (coalesce .Values.bpf.ctAnyMax $defaultBpfCtAnyMax) -}} -{{- $kubeProxyReplacement := (coalesce .Values.kubeProxyReplacement $defaultKubeProxyReplacement) -}} +{{- $stringValueKPR := (toString .Values.kubeProxyReplacement) -}} +{{- if (eq $stringValueKPR "") }} + {{- $stringValueKPR = "" -}} +{{- end}} +{{- $kubeProxyReplacement := (coalesce $stringValueKPR $defaultKubeProxyReplacement) -}} +{{- if and (ne $kubeProxyReplacement "disabled") (ne $kubeProxyReplacement "partial") (ne $kubeProxyReplacement "strict") (ne $kubeProxyReplacement "true") (ne $kubeProxyReplacement "false") }} + {{ fail "kubeProxyReplacement must be explicitly set to a valid value (true, false, disabled (deprecated), partial (deprecated), or strict (deprecated)) to continue." }} +{{- end }} {{- $azureUsePrimaryAddress = (coalesce .Values.azure.usePrimaryAddress $azureUsePrimaryAddress) -}} {{- $socketLB := (coalesce .Values.socketLB .Values.hostServices) -}} + +{{- $cniChainingMode := "none" -}} +{{- if (not (kindIs "invalid" .Values.cni.chainingMode)) -}} + {{- $cniChainingMode = .Values.cni.chainingMode -}} +{{- else if (not (kindIs "invalid" .Values.cni.chainingTarget)) -}} + {{- $cniChainingMode = "generic-veth" -}} +{{- end -}} --- apiVersion: v1 kind: ConfigMap @@ -97,6 +117,10 @@ data: conntrack-gc-interval: {{ include "validateDuration" .Values.conntrackGCInterval | quote }} {{- end }} +{{- if .Values.conntrackGCMaxInterval }} + conntrack-gc-max-interval: {{ include "validateDuration" .Values.conntrackGCMaxInterval | quote }} +{{- end }} + {{- if hasKey .Values "disableEnvoyVersionCheck" }} disable-envoy-version-check: {{ .Values.disableEnvoyVersionCheck | quote }} {{- end }} @@ -121,9 +145,9 @@ data: skip-cnp-status-startup-clean: "{{ .Values.operator.skipCNPStatusStartupClean }}" {{- end }} -{{- if hasKey .Values "disableEndpointCRD" }} +{{- if eq .Values.disableEndpointCRD true }} # Disable the usage of CiliumEndpoint CRD - disable-endpoint-crd: "{{ .Values.disableEndpointCRD }}" + disable-endpoint-crd: "true" {{- end }} {{- if .Values.identityChangeGracePeriod }} @@ -173,11 +197,6 @@ data: # NOTE that this will open the port on ALL nodes where Cilium pods are # scheduled. prometheus-serve-addr: ":{{ .Values.prometheus.port }}" - # Port to expose Envoy metrics (e.g. "9964"). Envoy metrics listener will be disabled if this - # field is not set. - {{- if .Values.proxy.prometheus.enabled }} - proxy-prometheus-port: "{{ .Values.proxy.prometheus.port }}" - {{- end }} {{- if .Values.prometheus.metrics }} # Metrics that should be enabled or disabled from the default metric # list. (+metric_foo to enable metric_foo , -metric_bar to disable @@ -188,6 +207,14 @@ data: {{- end }} {{- end }} +{{- if not .Values.envoy.enabled }} + # Port to expose Envoy metrics (e.g. "9964"). Envoy metrics listener will be disabled if this + # field is not set. + {{- if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled }} + proxy-prometheus-port: "{{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }}" + {{- end }} +{{- end }} + {{- if .Values.operator.prometheus.enabled }} # If you want metrics enabled in cilium-operator, set the port for # which the Cilium Operator will have their metrics exposed. @@ -201,8 +228,11 @@ data: skip-crd-creation: "true" {{- end }} -{{- if or .Values.ingressController.enabled .Values.gatewayAPI.enabled (and (hasKey .Values "loadBalancer") (eq .Values.loadBalancer.l7.backend "envoy")) }} +{{- if or .Values.envoyConfig.enabled .Values.ingressController.enabled .Values.gatewayAPI.enabled (and (hasKey .Values "loadBalancer") (eq .Values.loadBalancer.l7.backend "envoy")) }} enable-envoy-config: "true" + {{- if .Values.envoyConfig.enabled }} + envoy-secrets-namespace: {{ .Values.envoyConfig.secretsNamespace.name | quote }} + {{- end }} {{- end }} {{- if .Values.ingressController.enabled }} @@ -213,6 +243,10 @@ data: ingress-lb-annotation-prefixes: {{ .Values.ingressController.ingressLBAnnotationPrefixes | join " " | quote }} ingress-default-lb-mode: {{ .Values.ingressController.loadbalancerMode }} ingress-shared-lb-service-name: {{ .Values.ingressController.service.name }} + {{- if and .Values.ingressController.defaultSecretNamespace .Values.ingressController.defaultSecretName }} + ingress-default-secret-namespace: {{ .Values.ingressController.defaultSecretNamespace | quote }} + ingress-default-secret-name: {{ .Values.ingressController.defaultSecretName | quote }} + {{- end }} {{- end }} {{- if .Values.gatewayAPI.enabled }} @@ -306,12 +340,16 @@ data: {{- if (not (kindIs "invalid" .Values.bpf.hostLegacyRouting)) }} enable-host-legacy-routing: {{ .Values.bpf.hostLegacyRouting | quote }} -{{- else if ne .Values.cni.chainingMode "none" }} +{{- else if ne $cniChainingMode "none" }} # In cni chaining mode, the other chained plugin is responsible for underlying connectivity, # so cilium eBPF host routing shoud not work, and let it fall back to the legacy routing mode enable-host-legacy-routing: "true" {{- end }} +{{- if .Values.bpf.authMapMax }} + # bpf-auth-map-max specifies the maximum number of entries in the auth map + bpf-auth-map-max: {{ .Values.bpf.authMapMax | quote }} +{{- end }} {{- if or $bpfCtTcpMax $bpfCtAnyMax }} # bpf-ct-global-*-max specifies the maximum number of connections # supported across all endpoints, split by protocol: tcp or other. One pair @@ -392,21 +430,40 @@ data: # - vxlan (default) # - geneve {{- if .Values.gke.enabled }} - tunnel: "disabled" + routing-mode: "native" enable-endpoint-routes: "true" enable-local-node-route: "false" {{- else if .Values.aksbyocni.enabled }} - tunnel: "vxlan" + routing-mode: "tunnel" + tunnel-protocol: "vxlan" +{{- else if .Values.routingMode }} + routing-mode: {{ .Values.routingMode | quote }} {{- else }} - tunnel: {{ .Values.tunnel | quote }} + {{- if eq .Values.tunnel "disabled" }} + routing-mode: "native" + {{- else if eq .Values.tunnel "vxlan" }} + routing-mode: "tunnel" + tunnel-protocol: "vxlan" + {{- else if eq .Values.tunnel "geneve" }} + routing-mode: "tunnel" + tunnel-protocol: "geneve" + {{- else }} + # Default case + routing-mode: "tunnel" + tunnel-protocol: "vxlan" + {{- end }} +{{- end }} + +{{- if .Values.tunnelProtocol }} + tunnel-protocol: {{ .Values.tunnelProtocol | quote }} {{- end }} {{- if .Values.tunnelPort }} - tunnel-port: "{{ .Values.tunnelPort }}" + tunnel-port: {{ .Values.tunnelPort | quote }} {{- end }} {{- if .Values.MTU }} - mtu: "{{ .Values.MTU }}" + mtu: {{ .Values.MTU | quote }} {{- end }} {{- if .Values.eni.enabled }} @@ -460,7 +517,7 @@ data: enable-l7-proxy: {{ .Values.l7Proxy | quote }} {{- end }} -{{- if ne .Values.cni.chainingMode "none" }} +{{- if ne $cniChainingMode "none" }} # Enable chaining with another CNI plugin # # Supported modes: @@ -469,14 +526,14 @@ data: # - flannel # - generic-veth # - portmap (Enables HostPort support for Cilium) - cni-chaining-mode: {{ .Values.cni.chainingMode }} + cni-chaining-mode: {{ $cniChainingMode }} {{- if hasKey .Values "enableIdentityMark" }} enable-identity-mark: {{ .Values.enableIdentityMark | quote }} {{- else if (ne $enableIdentityMark "true") }} enable-identity-mark: "false" {{- end }} -{{- if ne .Values.cni.chainingMode "portmap" }} +{{- if ne $cniChainingMode "portmap" }} # Disable the PodCIDR route to the cilium_host interface as it is not # required. While chaining, it is the responsibility of the underlying plugin # to enable routing. @@ -485,6 +542,7 @@ data: {{- end }} enable-ipv4-masquerade: {{ .Values.enableIPv4Masquerade | quote }} + enable-ipv4-big-tcp: {{ .Values.enableIPv4BIGTCP | quote }} enable-ipv6-big-tcp: {{ .Values.enableIPv6BIGTCP | quote }} enable-ipv6-masquerade: {{ .Values.enableIPv6Masquerade | quote }} @@ -514,9 +572,11 @@ data: {{- else if .Values.encryption.interface }} encrypt-interface: {{ .Values.encryption.interface }} {{- end }} - - {{- if .Values.encryption.nodeEncryption }} - encrypt-node: {{ .Values.encryption.nodeEncryption | quote }} + {{- if hasKey .Values.encryption.ipsec "keyWatcher" }} + enable-ipsec-key-watcher: {{ .Values.encryption.ipsec.keyWatcher | quote }} + {{- end }} + {{- if .Values.encryption.ipsec.keyRotationDuration }} + ipsec-key-rotation-duration: {{ include "validateDuration" .Values.encryption.ipsec.keyRotationDuration | quote }} {{- end }} {{- else if eq .Values.encryption.type "wireguard" }} enable-wireguard: {{ .Values.encryption.enabled | quote }} @@ -524,10 +584,13 @@ data: enable-wireguard-userspace-fallback: {{ .Values.encryption.wireguard.userspaceFallback | quote }} {{- end }} {{- end }} + {{- if .Values.encryption.nodeEncryption }} + encrypt-node: {{ .Values.encryption.nodeEncryption | quote }} + {{- end }} {{- end }} enable-xt-socket-fallback: {{ .Values.enableXTSocketFallback | quote }} -{{- if or (.Values.azure.enabled) (.Values.eni.enabled) (.Values.gke.enabled) (ne .Values.cni.chainingMode "none") }} +{{- if or (.Values.azure.enabled) (.Values.eni.enabled) (.Values.gke.enabled) (ne $cniChainingMode "none") }} install-no-conntrack-iptables-rules: "false" {{- else }} install-no-conntrack-iptables-rules: {{ .Values.installNoConntrackIptablesRules | quote }} @@ -550,6 +613,10 @@ data: {{- end }} {{- end }} +{{- if .Values.highScaleIPcache.enabled }} + enable-high-scale-ipcache: {{ .Values.highScaleIPcache.enabled | quote }} +{{- end }} + {{- if hasKey .Values "localRedirectPolicy" }} enable-local-redirect-policy: {{ .Values.localRedirectPolicy | quote }} {{- end }} @@ -602,21 +669,21 @@ data: {{- end }} {{- if hasKey .Values "hostPort" }} -{{- if eq $kubeProxyReplacement "partial" }} +{{- if or (eq $kubeProxyReplacement "partial") (eq $kubeProxyReplacement "false") }} enable-host-port: {{ .Values.hostPort.enabled | quote }} {{- end }} {{- end }} {{- if hasKey .Values "externalIPs" }} -{{- if eq $kubeProxyReplacement "partial" }} +{{- if or (eq $kubeProxyReplacement "partial") (eq $kubeProxyReplacement "false") }} enable-external-ips: {{ .Values.externalIPs.enabled | quote }} {{- end }} {{- end }} {{- if hasKey .Values "nodePort" }} -{{- if eq $kubeProxyReplacement "partial" }} +{{- if or (eq $kubeProxyReplacement "partial") (eq $kubeProxyReplacement "false") }} enable-node-port: {{ .Values.nodePort.enabled | quote }} {{- end }} {{- if hasKey .Values.nodePort "range" }} - node-port-range: {{ .Values.nodePort.range | quote }} + node-port-range: {{ get .Values.nodePort "range" | quote }} {{- end }} {{- if hasKey .Values.nodePort "directRoutingDevice" }} direct-routing-device: {{ .Values.nodePort.directRoutingDevice | quote }} @@ -686,9 +753,6 @@ data: {{- if .Values.logOptions }} log-opt: {{ .Values.logOptions | toJson | quote }} {{- end }} -{{- if and .Values.sockops .Values.sockops.enabled }} - sockops-enable: {{ .Values.sockops.enabled | quote }} -{{- end }} {{- if hasKey .Values.k8s "requireIPv4PodCIDR" }} k8s-require-ipv4-pod-cidr: {{ .Values.k8s.requireIPv4PodCIDR | quote }} {{- end }} @@ -701,19 +765,34 @@ data: {{- if and .Values.endpointRoutes .Values.endpointRoutes.enabled }} enable-endpoint-routes: {{ .Values.endpointRoutes.enabled | quote }} {{- end }} +{{- if and .Values.k8sNetworkPolicy .Values.k8sNetworkPolicy.enabled }} + enable-k8s-networkpolicy: {{ .Values.k8sNetworkPolicy.enabled | quote }} +{{- end }} {{- if .Values.cni.configMap }} read-cni-conf: {{ .Values.cni.confFileMountPath }}/{{ .Values.cni.configMapKey }} +{{- if .Values.cni.customConf }} + # legacy: v1.13 and before needed cni.customConf: true with cni.configMap write-cni-conf-when-ready: {{ .Values.cni.hostConfDirMountPath }}/05-cilium.conflist +{{- end }} {{- else if .Values.cni.readCniConf }} read-cni-conf: {{ .Values.cni.readCniConf }} {{- end }} +{{- if and (not .Values.cni.customConf) .Values.cni.install }} + # Tell the agent to generate and write a CNI configuration file + write-cni-conf-when-ready: {{ .Values.cni.hostConfDirMountPath }}/05-cilium.conflist + cni-exclusive: {{ .Values.cni.exclusive | quote }} + cni-log-file: {{ .Values.cni.logFile | quote }} +{{- end }} {{- if .Values.cni.uninstall }} cni-uninstall: {{ .Values.cni.uninstall | quote }} {{- end }} +{{- if (not (kindIs "invalid" .Values.cni.chainingTarget)) }} + cni-chaining-target: {{ .Values.cni.chainingTarget | quote }} +{{- end}} {{- if .Values.kubeConfigPath }} k8s-kubeconfig-path: {{ .Values.kubeConfigPath | quote }} {{- end }} -{{- if and ( .Values.endpointHealthChecking.enabled ) (or (eq .Values.cni.chainingMode "portmap") (eq .Values.cni.chainingMode "none")) }} +{{- if and ( .Values.endpointHealthChecking.enabled ) (or (eq $cniChainingMode "portmap") (eq $cniChainingMode "none")) }} enable-endpoint-health-checking: "true" {{- else}} # Disable health checking, when chaining mode is not set to portmap or none @@ -796,24 +875,44 @@ data: ipam: {{ $ipam | quote }} {{- end }} +{{- if .Values.ipam.ciliumNodeUpdateRate }} + ipam-cilium-node-update-rate: {{ include "validateDuration" .Values.ipam.ciliumNodeUpdateRate | quote }} +{{- end }} + {{- if or (eq $ipam "cluster-pool") (eq $ipam "cluster-pool-v2beta") }} {{- if .Values.ipv4.enabled }} - {{- if .Values.ipam.operator.clusterPoolIPv4PodCIDRList }} - cluster-pool-ipv4-cidr: {{ .Values.ipam.operator.clusterPoolIPv4PodCIDRList | join " " | quote }} - {{- else }} - cluster-pool-ipv4-cidr: {{ .Values.ipam.operator.clusterPoolIPv4PodCIDR | quote }} + {{- if hasKey .Values.ipam.operator "clusterPoolIPv4PodCIDR" }} + {{- /* ipam.operator.clusterPoolIPv4PodCIDR removed in v1.14, remove this failsafe around v1.17 */ -}} + {{- fail "Value ipam.operator.clusterPoolIPv4PodCIDR removed, use ipam.operator.clusterPoolIPv4PodCIDRList instead" }} {{- end }} + cluster-pool-ipv4-cidr: {{ .Values.ipam.operator.clusterPoolIPv4PodCIDRList | join " " | quote }} cluster-pool-ipv4-mask-size: {{ .Values.ipam.operator.clusterPoolIPv4MaskSize | quote }} {{- end }} {{- if .Values.ipv6.enabled }} - {{- if .Values.ipam.operator.clusterPoolIPv6PodCIDRList }} - cluster-pool-ipv6-cidr: {{ .Values.ipam.operator.clusterPoolIPv6PodCIDRList | join " " | quote }} - {{- else }} - cluster-pool-ipv6-cidr: {{ .Values.ipam.operator.clusterPoolIPv6PodCIDR | quote }} + {{- if hasKey .Values.ipam.operator "clusterPoolIPv6PodCIDR" }} + {{- /* ipam.operator.clusterPoolIPv6PodCIDR removed in v1.14, remove this failsafe around v1.17 */ -}} + {{- fail "Value ipam.operator.clusterPoolIPv6PodCIDR removed, use ipam.operator.clusterPoolIPv6PodCIDRList instead" }} {{- end }} + cluster-pool-ipv6-cidr: {{ .Values.ipam.operator.clusterPoolIPv6PodCIDRList | join " " | quote }} cluster-pool-ipv6-mask-size: {{ .Values.ipam.operator.clusterPoolIPv6MaskSize | quote }} {{- end }} {{- end }} +{{- if (eq $ipam "multi-pool") }} + {{- $pools := list }} + {{- range $pool, $spec := .Values.ipam.operator.autoCreateCiliumPodIPPools }} + {{- $attrs := list }} + {{- if hasKey $spec "ipv4" }} + {{- $attrs = append $attrs (printf "ipv4-cidrs:%s" (join "," $spec.ipv4.cidrs)) }} + {{- $attrs = append $attrs (printf "ipv4-mask-size:%s" (toString $spec.ipv4.maskSize)) }} + {{- end }} + {{- if hasKey $spec "ipv6" }} + {{- $attrs = append $attrs (printf "ipv6-cidrs:%s" (join "," $spec.ipv6.cidrs)) }} + {{- $attrs = append $attrs (printf "ipv6-mask-size:%s" (toString $spec.ipv6.maskSize)) }} + {{- end }} + {{- $pools = append $pools (printf "%s=%s" $pool (join ";" $attrs)) }} + {{- end }} + auto-create-cilium-pod-ip-pools: {{ join "," $pools | quote }} +{{- end }} {{- if .Values.ipam.operator.externalAPILimitBurstSize }} limit-ipam-api-burst: {{ .Values.ipam.operator.externalAPILimitBurstSize | quote }} @@ -823,9 +922,10 @@ data: {{- end }} {{- if .Values.enableCnpStatusUpdates }} - disable-cnp-status-updates: {{ (not .Values.enableCnpStatusUpdates) | quote }} + disable-cnp-status-updates: "false" {{- else if (eq $defaultEnableCnpStatusUpdates "false") }} disable-cnp-status-updates: "true" + cnp-node-status-gc-interval: "0s" {{- end }} {{- if .Values.egressGateway.enabled }} @@ -834,6 +934,12 @@ data: {{- if .Values.egressGateway.installRoutes }} install-egress-gateway-routes: "true" {{- end }} +{{- if hasKey .Values.egressGateway "reconciliationTriggerInterval" }} + egress-gateway-reconciliation-trigger-interval: {{ .Values.egressGateway.reconciliationTriggerInterval | quote }} +{{- end }} +{{- if .Values.egressGateway.maxPolicyEntries }} + egress-gateway-policy-map-max: {{ .Values.egressGateway.maxPolicyEntries }} +{{- end }} {{- if hasKey .Values "vtep" }} enable-vtep: {{ .Values.vtep.enabled | quote }} @@ -873,6 +979,25 @@ data: enable-custom-calls: {{ .Values.customCalls.enabled | quote }} {{- end }} +{{- if .Values.l2announcements.enabled }} + # Enable L2 announcements + enable-l2-announcements: {{ .Values.l2announcements.enabled | quote }} + {{- if .Values.l2announcements.leaseDuration }} + l2-announcements-lease-duration: {{ .Values.l2announcements.leaseDuration | quote }} + {{- end}} + {{- if .Values.l2announcements.leaseRenewDeadline }} + l2-announcements-renew-deadline: {{ .Values.l2announcements.leaseRenewDeadline | quote }} + {{- end}} + {{- if .Values.l2announcements.leaseRetryPeriod }} + l2-announcements-retry-period: {{ .Values.l2announcements.leaseRetryPeriod | quote }} + {{- end}} +{{- end}} + +{{- if .Values.l2podAnnouncements.enabled }} + enable-l2-pod-announcements: {{ .Values.l2podAnnouncements.enabled | quote }} + l2-pod-announcements-interface: {{ .Values.l2podAnnouncements.interface | quote }} +{{- end }} + {{- if and .Values.bgp.enabled (and (not .Values.bgp.announce.loadbalancerIP) (not .Values.bgp.announce.podCIDR)) }} {{ fail "BGP was enabled, but no announcements were enabled. Please enable one or more announcements." }} {{- end }} @@ -933,9 +1058,23 @@ data: annotate-k8s-node: "true" {{- end }} +{{- if hasKey .Values "k8sClientRateLimit" }} + k8s-client-qps: {{ .Values.k8sClientRateLimit.qps | quote }} + k8s-client-burst: {{ .Values.k8sClientRateLimit.burst | quote }} +{{- end }} + +{{- if and .Values.operator.setNodeTaints (not .Values.operator.removeNodeTaints) -}} + {{ fail "Cannot have operator.setNodeTaintsMaxNodes and not operator.removeNodeTaints = false" }} +{{- end -}} {{- if .Values.operator.removeNodeTaints }} remove-cilium-node-taints: "true" {{- end }} +{{- /* set node taints if setNodeTaints is explicitly enabled or removeNodeTaints is set */ -}} +{{- if or .Values.operator.setNodeTaints + ( and (kindIs "invalid" .Values.operator.setNodeTaints) + .Values.operator.removeNodeTaints ) }} + set-cilium-node-taints: "true" +{{- end }} {{- if .Values.operator.setNodeNetworkStatus }} set-cilium-is-up-condition: "true" {{- end }} @@ -984,8 +1123,36 @@ data: agent-not-ready-taint-key: {{ .Values.agentNotReadyTaintKey | quote }} {{- end }} + mesh-auth-enabled: {{ .Values.authentication.enabled | quote }} + mesh-auth-queue-size: {{ .Values.authentication.queueSize | quote }} + mesh-auth-rotated-identities-queue-size: {{ .Values.authentication.rotatedIdentitiesQueueSize | quote }} + mesh-auth-gc-interval: {{ include "validateDuration" .Values.authentication.gcInterval | quote }} + +{{- if .Values.authentication.mutual.spire.enabled }} + mesh-auth-mutual-enabled: "true" + mesh-auth-mutual-listener-port: {{ .Values.authentication.mutual.port | quote }} + mesh-auth-spire-agent-socket: {{ .Values.authentication.mutual.spire.agentSocketPath | quote }} + {{- if .Values.authentication.mutual.spire.serverAddress }} + mesh-auth-spire-server-address: {{ .Values.authentication.mutual.spire.serverAddress | quote }} + {{- else }} + mesh-auth-spire-server-address: "spire-server.{{ .Values.authentication.mutual.spire.install.namespace}}.svc:8081" + {{- end }} + mesh-auth-spire-server-connection-timeout: {{ .Values.authentication.mutual.spire.connectionTimeout }} + mesh-auth-spire-admin-socket: {{ .Values.authentication.mutual.spire.adminSocketPath | quote }} + mesh-auth-spiffe-trust-domain: {{ .Values.authentication.mutual.spire.trustDomain | quote }} +{{- end }} + + proxy-connect-timeout: {{ .Values.envoy.connectTimeoutSeconds | quote }} + proxy-max-requests-per-connection: {{ .Values.envoy.maxRequestsPerConnection | quote }} + proxy-max-connection-duration-seconds: {{ .Values.envoy.maxConnectionDurationSeconds | quote }} + + external-envoy-proxy: {{ .Values.envoy.enabled | quote }} + +{{- if .Values.envoy.log.path }} + envoy-log: {{ .Values.envoy.log.path | quote }} {{- end }} +{{- end }} --- {{- if and .Values.ipMasqAgent.enabled .Values.ipMasqAgent.config }} apiVersion: v1 diff --git a/charts/cilium/templates/cilium-envoy/configmap.yaml b/charts/cilium/templates/cilium-envoy/configmap.yaml new file mode 100644 index 00000000..7aa0f046 --- /dev/null +++ b/charts/cilium/templates/cilium-envoy/configmap.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.envoy.enabled (not .Values.preflight.enabled) }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-envoy-config + namespace: {{ .Release.Namespace }} +data: +{{- (tpl (.Files.Glob "files/cilium-envoy/configmap/bootstrap-config.json").AsConfig .) | nindent 2 }} + +{{- end }} diff --git a/charts/cilium/templates/cilium-envoy/daemonset.yaml b/charts/cilium/templates/cilium-envoy/daemonset.yaml new file mode 100644 index 00000000..2ff9692e --- /dev/null +++ b/charts/cilium/templates/cilium-envoy/daemonset.yaml @@ -0,0 +1,254 @@ +{{- if and .Values.envoy.enabled (not .Values.preflight.enabled) }} + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium-envoy + namespace: {{ .Release.Namespace }} + labels: + k8s-app: cilium-envoy + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-envoy + name: cilium-envoy +spec: + selector: + matchLabels: + k8s-app: cilium-envoy + {{- with .Values.envoy.updateStrategy }} + updateStrategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + annotations: + {{- if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled (not .Values.envoy.prometheus.serviceMonitor.enabled) }} + prometheus.io/port: "{{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }}" + prometheus.io/scrape: "true" + {{- end }} + {{- if .Values.envoy.rollOutPods }} + # ensure pods roll when configmap updates + cilium.io/cilium-envoy-configmap-checksum: {{ include (print $.Template.BasePath "/cilium-envoy/configmap.yaml") . | sha256sum | quote }} + {{- end }} + {{- if not .Values.envoy.securityContext.privileged }} + # Set app AppArmor's profile to "unconfined". The value of this annotation + # can be modified as long users know which profiles they have available + # in AppArmor. + container.apparmor.security.beta.kubernetes.io/cilium-envoy: "unconfined" + {{- end }} + {{- with .Values.envoy.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + k8s-app: cilium-envoy + name: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + {{- with .Values.envoy.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.envoy.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: cilium-envoy + image: {{ include "cilium.image" .Values.envoy.image | quote }} + imagePullPolicy: {{ .Values.envoy.image.pullPolicy }} + command: + - /usr/bin/cilium-envoy + args: + - '-c /var/run/cilium/envoy/bootstrap-config.json' + - '--base-id 0' + {{- if and (hasKey .Values.debug "verbose") (.Values.debug.verbose) (has "envoy" ( splitList " " .Values.debug.verbose )) }} + - '--log-level trace' + {{- else if and (.Values.debug.enabled) (hasKey .Values.debug "verbose") (.Values.debug.verbose) (has "flow" ( splitList " " .Values.debug.verbose )) }} + - '--log-level debug' + {{- else }} + - '--log-level info' + {{- end }} + - '--log-format {{ .Values.envoy.log.format }}' + {{- if .Values.envoy.log.path }} + - '--log-path {{ .Values.envoy.log.path }}' + {{- end }} + {{- with .Values.envoy.extraArgs }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- if semverCompare ">=1.20-0" .Capabilities.KubeVersion.Version }} + startupProbe: + httpGet: + host: "localhost" + path: /healthz + port: {{ .Values.envoy.healthPort }} + scheme: HTTP + failureThreshold: {{ .Values.envoy.startupProbe.failureThreshold }} + periodSeconds: {{ .Values.envoy.startupProbe.periodSeconds }} + successThreshold: 1 + {{- end }} + livenessProbe: + httpGet: + host: "localhost" + path: /healthz + port: {{ .Values.envoy.healthPort }} + scheme: HTTP + {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} + # The initial delay for the liveness probe is intentionally large to + # avoid an endless kill & restart cycle if in the event that the initial + # bootstrapping takes longer than expected. + # Starting from Kubernetes 1.20, we are using startupProbe instead + # of this field. + initialDelaySeconds: 120 + {{- end }} + periodSeconds: {{ .Values.envoy.livenessProbe.periodSeconds }} + successThreshold: 1 + failureThreshold: {{ .Values.envoy.livenessProbe.failureThreshold }} + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "localhost" + path: /healthz + port: {{ .Values.envoy.healthPort }} + scheme: HTTP + {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} + initialDelaySeconds: 5 + {{- end }} + periodSeconds: {{ .Values.envoy.readinessProbe.periodSeconds }} + successThreshold: 1 + failureThreshold: {{ .Values.envoy.readinessProbe.failureThreshold }} + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + {{- if .Values.k8sServiceHost }} + - name: KUBERNETES_SERVICE_HOST + value: {{ .Values.k8sServiceHost | quote }} + {{- end }} + {{- if .Values.k8sServicePort }} + - name: KUBERNETES_SERVICE_PORT + value: {{ .Values.k8sServicePort | quote }} + {{- end }} + {{- with .Values.envoy.extraEnv }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- with .Values.envoy.resources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + {{- if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled }} + ports: + - name: envoy-metrics + containerPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} + hostPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} + protocol: TCP + {{- end }} + securityContext: + {{- if .Values.envoy.securityContext.privileged }} + privileged: true + {{- else }} + seLinuxOptions: + {{- with .Values.envoy.securityContext.seLinuxOptions }} + {{- toYaml . | nindent 12 }} + {{- end }} + capabilities: + add: + {{- with .Values.envoy.securityContext.capabilities.envoy }} + {{- toYaml . | nindent 14 }} + {{- end }} + drop: + - ALL + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + - name: envoy-config + mountPath: /var/run/cilium/envoy/ + readOnly: true + {{- if .Values.bpf.autoMount.enabled }} + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + {{- end }} + {{- range .Values.envoy.extraHostPathMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- if .mountPropagation }} + mountPropagation: {{ .mountPropagation }} + {{- end }} + {{- end }} + {{- with .Values.envoy.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.envoy.extraContainers }} + {{- toYaml .Values.envoy.extraContainers | nindent 6 }} + {{- end }} + restartPolicy: Always + priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.envoy.priorityClassName "system-node-critical") }} + serviceAccount: {{ .Values.serviceAccounts.envoy.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.envoy.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.envoy.automount }} + terminationGracePeriodSeconds: {{ .Values.envoy.terminationGracePeriodSeconds }} + hostNetwork: true + {{- if .Values.envoy.dnsPolicy }} + dnsPolicy: {{ .Values.envoy.dnsPolicy }} + {{- end }} + {{- with .Values.envoy.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.envoy.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.envoy.tolerations }} + tolerations: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + volumes: + - name: envoy-sockets + hostPath: + path: "{{ .Values.daemon.runPath }}/envoy/sockets" + type: DirectoryOrCreate + - name: envoy-config + configMap: + name: cilium-envoy-config + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + items: + - key: bootstrap-config.json + path: bootstrap-config.json + # To keep state between restarts / upgrades + {{- if and .Values.bpf.autoMount.enabled }} + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + {{- end }} + {{- range .Values.envoy.extraHostPathMounts }} + - name: {{ .name }} + hostPath: + path: {{ .hostPath }} + {{- if .hostPathType }} + type: {{ .hostPathType }} + {{- end }} + {{- end }} + {{- with .Values.envoy.extraVolumes }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/cilium/templates/cilium-envoy/service.yaml b/charts/cilium/templates/cilium-envoy/service.yaml new file mode 100644 index 00000000..f5c2a09b --- /dev/null +++ b/charts/cilium/templates/cilium-envoy/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.envoy.enabled (not .Values.preflight.enabled) .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: cilium-envoy + namespace: {{ .Release.Namespace }} + {{- if not .Values.envoy.prometheus.serviceMonitor.enabled }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port | quote }} + {{- end }} + labels: + k8s-app: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + io.cilium/app: proxy +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium-envoy + ports: + - name: envoy-metrics + port: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} + protocol: TCP + targetPort: envoy-metrics +{{- end }} diff --git a/charts/cilium/templates/cilium-envoy/serviceaccount.yaml b/charts/cilium/templates/cilium-envoy/serviceaccount.yaml new file mode 100644 index 00000000..6094ba20 --- /dev/null +++ b/charts/cilium/templates/cilium-envoy/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.envoy.enabled (not .Values.preflight.enabled) .Values.serviceAccounts.envoy.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.envoy.name | quote }} + namespace: {{ .Release.Namespace }} + {{- if .Values.serviceAccounts.envoy.annotations }} + annotations: + {{- toYaml .Values.serviceAccounts.envoy.annotations | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/cilium/templates/cilium-envoy/servicemonitor.yaml b/charts/cilium/templates/cilium-envoy/servicemonitor.yaml new file mode 100644 index 00000000..363b3776 --- /dev/null +++ b/charts/cilium/templates/cilium-envoy/servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.envoy.enabled (not .Values.preflight.enabled) .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled .Values.envoy.prometheus.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: cilium-envoy + namespace: {{ .Values.envoy.prometheus.serviceMonitor.namespace | default .Release.Namespace }} + labels: + app.kubernetes.io/part-of: cilium + {{- with .Values.envoy.prometheus.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.envoy.prometheus.serviceMonitor.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + k8s-app: cilium-envoy + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + endpoints: + - port: metrics + interval: {{ .Values.envoy.prometheus.serviceMonitor.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.envoy.prometheus.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.envoy.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + targetLabels: + - k8s-app +{{- end }} diff --git a/charts/cilium/templates/cilium-ingress-class.yaml b/charts/cilium/templates/cilium-ingress-class.yaml index 5e4da3f2..9243b377 100644 --- a/charts/cilium/templates/cilium-ingress-class.yaml +++ b/charts/cilium/templates/cilium-ingress-class.yaml @@ -3,6 +3,10 @@ apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: name: cilium + {{- if .Values.ingressController.default }} + annotations: + ingressclass.kubernetes.io/is-default-class: "true" + {{- end}} spec: controller: cilium.io/ingress-controller {{- end}} diff --git a/charts/cilium/templates/cilium-ingress-service.yaml b/charts/cilium/templates/cilium-ingress-service.yaml index 8e6ee41e..ff6269d2 100644 --- a/charts/cilium/templates/cilium-ingress-service.yaml +++ b/charts/cilium/templates/cilium-ingress-service.yaml @@ -24,6 +24,17 @@ spec: protocol: TCP nodePort: {{ .Values.ingressController.service.secureNodePort }} type: {{ .Values.ingressController.service.type }} + {{- if semverCompare ">=1.24-0" .Capabilities.KubeVersion.Version -}} + {{- if .Values.ingressController.service.loadBalancerClass }} + loadBalancerClass: {{ .Values.ingressController.service.loadBalancerClass }} + {{- end }} + {{- if (not (kindIs "invalid" .Values.ingressController.service.allocateLoadBalancerNodePorts)) }} + allocateLoadBalancerNodePorts: {{ .Values.ingressController.service.allocateLoadBalancerNodePorts }} + {{- end }} + {{- end -}} + {{- if .Values.ingressController.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.ingressController.service.loadBalancerIP }} + {{- end }} --- apiVersion: v1 kind: Endpoints diff --git a/charts/cilium/templates/cilium-nodeinit/daemonset.yaml b/charts/cilium/templates/cilium-nodeinit/daemonset.yaml index 099e1201..63de5715 100644 --- a/charts/cilium/templates/cilium-nodeinit/daemonset.yaml +++ b/charts/cilium/templates/cilium-nodeinit/daemonset.yaml @@ -79,6 +79,10 @@ spec: {{- toYaml . | trim | nindent 12 }} {{- end }} terminationMessagePolicy: FallbackToLogsOnError + {{- with .Values.nodeinit.extraVolumeMounts }} + volumeMounts: + {{- toYaml . | nindent 10 }} + {{- end }} {{- with .Values.nodeinit.affinity }} affinity: {{- toYaml . | nindent 8 }} @@ -94,4 +98,13 @@ spec: hostPID: true hostNetwork: true priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.nodeinit.priorityClassName "system-node-critical") }} + {{- if .Values.serviceAccounts.nodeinit.enabled }} + serviceAccount: {{ .Values.serviceAccounts.nodeinit.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.nodeinit.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.nodeinit.automount }} + {{- end }} + {{- with .Values.nodeinit.extraVolumes }} + volumes: + {{- toYaml . | nindent 6 }} + {{- end }} {{- end }} diff --git a/charts/cilium/templates/cilium-nodeinit/serviceaccount.yaml b/charts/cilium/templates/cilium-nodeinit/serviceaccount.yaml new file mode 100644 index 00000000..806e3706 --- /dev/null +++ b/charts/cilium/templates/cilium-nodeinit/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.nodeinit.enabled .Values.serviceAccounts.nodeinit.enabled .Values.serviceAccounts.nodeinit.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.nodeinit.name | quote }} + namespace: {{ .Release.Namespace }} + {{- if .Values.serviceAccounts.nodeinit.annotations }} + annotations: + {{- toYaml .Values.serviceAccounts.nodeinit.annotations | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/cilium/templates/cilium-operator/clusterrole.yaml b/charts/cilium/templates/cilium-operator/clusterrole.yaml index b614a956..862fac10 100644 --- a/charts/cilium/templates/cilium-operator/clusterrole.yaml +++ b/charts/cilium/templates/cilium-operator/clusterrole.yaml @@ -15,7 +15,7 @@ rules: - list - watch {{- if hasKey .Values "disableEndpointCRD" }} -{{- if (eq (.Values.disableEndpointCRD | quote ) ( "false" | quote )) }} +{{- if not .Values.disableEndpointCRD }} {{- if (and .Values.operator.unmanagedPodWatcher.restart (ne (.Values.operator.unmanagedPodWatcher.intervalSeconds | int64) 0 ) ) }} # to automatically delete [core|kube]dns pods so that are starting to being # managed by Cilium @@ -195,14 +195,24 @@ rules: - ciliumnetworkpolicies.cilium.io - ciliumnodes.cilium.io - ciliumnodeconfigs.cilium.io + - ciliumcidrgroups.cilium.io + - ciliuml2announcementpolicies.cilium.io + - ciliumpodippools.cilium.io - apiGroups: - cilium.io resources: - ciliumloadbalancerippools + - ciliumpodippools verbs: - get - list - watch +- apiGroups: + - cilium.io + resources: + - ciliumpodippools + verbs: + - create - apiGroups: - cilium.io resources: @@ -228,6 +238,7 @@ rules: - networking.k8s.io resources: - ingresses + - ingressclasses verbs: - get - list @@ -245,6 +256,7 @@ rules: resources: - gatewayclasses - gateways + - tlsroutes - httproutes - referencegrants - referencepolicies @@ -258,6 +270,7 @@ rules: - gatewayclasses/status - gateways/status - httproutes/status + - tlsroutes/status verbs: - update - patch diff --git a/charts/cilium/templates/cilium-operator/dashboards-configmap.yaml b/charts/cilium/templates/cilium-operator/dashboards-configmap.yaml new file mode 100644 index 00000000..37402bed --- /dev/null +++ b/charts/cilium/templates/cilium-operator/dashboards-configmap.yaml @@ -0,0 +1,25 @@ +{{- if .Values.operator.dashboards.enabled }} +{{- $files := .Files.Glob "files/cilium-operator/dashboards/*.json" }} +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ $.Values.operator.dashboards.namespace | default $.Release.Namespace }} + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-operator + app.kubernetes.io/part-of: cilium + {{- if $.Values.operator.dashboards.label }} + {{ $.Values.operator.dashboards.label }}: {{ ternary $.Values.operator.dashboards.labelValue "1" (not (empty $.Values.operator.dashboards.labelValue)) | quote }} + {{- end }} + {{- with $.Values.operator.dashboards.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} diff --git a/charts/cilium/templates/cilium-operator/deployment.yaml b/charts/cilium/templates/cilium-operator/deployment.yaml index 6c527d1b..44b1da2a 100644 --- a/charts/cilium/templates/cilium-operator/deployment.yaml +++ b/charts/cilium/templates/cilium-operator/deployment.yaml @@ -18,10 +18,22 @@ spec: matchLabels: io.cilium/app: operator name: cilium-operator + # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case + # of one replica and no user configured Recreate strategy. + # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the + # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. + {{- if and (eq (.Values.operator.replicas | toString) "1") (eq .Values.operator.updateStrategy.type "RollingUpdate") }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.operator.updateStrategy.rollingUpdate.maxSurge }} + maxUnavailable: 100% + type: RollingUpdate + {{- else }} {{- with .Values.operator.updateStrategy }} strategy: {{- toYaml . | trim | nindent 4 }} {{- end }} + {{- end }} template: metadata: annotations: @@ -167,6 +179,16 @@ spec: initialDelaySeconds: 60 periodSeconds: 10 timeoutSeconds: 3 + readinessProbe: + httpGet: + host: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 5 volumeMounts: - name: cilium-config-path mountPath: /tmp/cilium/config-map @@ -186,6 +208,11 @@ spec: mountPath: {{ .Values.kubeConfigPath }} readOnly: true {{- end }} + {{- if .Values.authentication.mutual.spire.enabled }} + - name: spire-agent-socket + mountPath: {{ dir .Values.authentication.mutual.spire.agentSocketPath }} + readOnly: true + {{- end }} {{- range .Values.operator.extraHostPathMounts }} - name: {{ .name }} mountPath: {{ .mountPath }} @@ -294,6 +321,12 @@ spec: configMap: name: bgp-config {{- end }} + {{- if .Values.authentication.mutual.spire.enabled }} + - name: spire-agent-socket + hostPath: + path: {{ dir .Values.authentication.mutual.spire.agentSocketPath }} + type: DirectoryOrCreate + {{- end }} {{- with .Values.operator.extraVolumes }} {{- toYaml . | nindent 6 }} {{- end }} diff --git a/charts/cilium/templates/cilium-preflight/clusterrole.yaml b/charts/cilium/templates/cilium-preflight/clusterrole.yaml index b6cafa5b..379814c8 100644 --- a/charts/cilium/templates/cilium-preflight/clusterrole.yaml +++ b/charts/cilium/templates/cilium-preflight/clusterrole.yaml @@ -46,6 +46,18 @@ rules: # To annotate the k8s node with Cilium's metadata - patch {{- end }} +{{- if .Values.l2announcements.enabled }} +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update + - list + - delete +{{- end }} - apiGroups: - apiextensions.k8s.io resources: @@ -81,6 +93,9 @@ rules: - ciliumnetworkpolicies - ciliumnodes - ciliumnodeconfigs + - ciliumcidrgroups + - ciliuml2announcementpolicies + - ciliumpodippools verbs: - list - watch @@ -121,6 +136,7 @@ rules: - ciliumclusterwidenetworkpolicies/status - ciliumendpoints/status - ciliumendpoints + - ciliuml2announcementpolicies/status verbs: - patch {{- end }} diff --git a/charts/cilium/templates/cilium-preflight/daemonset.yaml b/charts/cilium/templates/cilium-preflight/daemonset.yaml index 596ee33c..efca1ba5 100644 --- a/charts/cilium/templates/cilium-preflight/daemonset.yaml +++ b/charts/cilium/templates/cilium-preflight/daemonset.yaml @@ -24,9 +24,13 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: + {{- with .Values.preflight.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.imagePullSecrets }} imagePullSecrets: - {{- toYaml . | nindent 6 }} + {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.preflight.podSecurityContext }} securityContext: diff --git a/charts/cilium/templates/cilium-secrets-namespace.yaml b/charts/cilium/templates/cilium-secrets-namespace.yaml index 1c089e48..6e1aa43d 100644 --- a/charts/cilium/templates/cilium-secrets-namespace.yaml +++ b/charts/cilium/templates/cilium-secrets-namespace.yaml @@ -16,3 +16,17 @@ kind: Namespace metadata: name: {{ .Values.gatewayAPI.secretsNamespace.name | quote }} {{- end}} + +# Only create the namespace if it's different from Ingress and Gateway API secret namespaces (if enabled). +{{- if and .Values.envoyConfig.enabled .Values.envoyConfig.secretsNamespace.create .Values.envoyConfig.secretsNamespace.name + (and + (or (not (and .Values.ingressController.enabled .Values.ingressController.secretsNamespace.create .Values.ingressController.secretsNamespace.name)) + (ne .Values.envoyConfig.secretsNamespace.name .Values.ingressController.secretsNamespace.name)) + (or (not (and .Values.gatewayAPI.enabled .Values.gatewayAPI.secretsNamespace.create .Values.gatewayAPI.secretsNamespace.name)) + (ne .Values.envoyConfig.secretsNamespace.name .Values.gatewayAPI.secretsNamespace.name))) }} +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.envoyConfig.secretsNamespace.name | quote }} +{{- end}} diff --git a/charts/cilium/templates/clustermesh-apiserver/_helpers.tpl b/charts/cilium/templates/clustermesh-apiserver/_helpers.tpl new file mode 100644 index 00000000..19cce1f7 --- /dev/null +++ b/charts/cilium/templates/clustermesh-apiserver/_helpers.tpl @@ -0,0 +1,11 @@ +{{- define "clustermesh-apiserver-generate-certs.admin-common-name" -}} +admin-{{ .Values.cluster.name }} +{{- end -}} + +{{- define "clustermesh-apiserver-generate-certs.remote-common-name" -}} +{{- if eq .Values.clustermesh.apiserver.tls.authMode "cluster" -}} +remote-{{ .Values.cluster.name }} +{{- else -}} +remote +{{- end -}} +{{- end -}} diff --git a/charts/cilium/templates/clustermesh-apiserver/deployment.yaml b/charts/cilium/templates/clustermesh-apiserver/deployment.yaml index 95578bcc..7c0ce675 100644 --- a/charts/cilium/templates/clustermesh-apiserver/deployment.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/deployment.yaml @@ -1,4 +1,7 @@ {{- if (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) }} +{{- if not (list "legacy" "migration" "cluster" | has .Values.clustermesh.apiserver.tls.authMode) -}} +{{- fail ".Values.clustermesh.apiserver.tls.authMode must be one of legacy, migration, cluster" -}} +{{- end -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -48,18 +51,22 @@ spec: - | rm -rf /var/run/etcd/*; /usr/local/bin/etcd --data-dir=/var/run/etcd --name=clustermesh-apiserver --listen-client-urls=http://127.0.0.1:2379 --advertise-client-urls=http://127.0.0.1:2379 --initial-cluster-token=clustermesh-apiserver --initial-cluster-state=new --auto-compaction-retention=1 & - export rootpw=`head /dev/urandom | tr -dc A-Za-z0-9 | head -c 16`; - echo $rootpw | etcdctl --interactive=false user add root; + + # The following key needs to be created before that the cilium agents + # have the possibility of connecting to etcd. + etcdctl put cilium/.has-cluster-config true + + etcdctl user add root --no-password; etcdctl user grant-role root root; - export vmpw=`head /dev/urandom | tr -dc A-Za-z0-9 | head -c 16`; - echo $vmpw | etcdctl --interactive=false user add externalworkload; + etcdctl user add admin-{{ .Values.cluster.name }} --no-password; + etcdctl user grant-role admin-{{ .Values.cluster.name }} root; + etcdctl user add externalworkload --no-password; etcdctl role add externalworkload; etcdctl role grant-permission externalworkload --from-key read ''; etcdctl role grant-permission externalworkload readwrite --prefix cilium/state/noderegister/v1/; etcdctl role grant-permission externalworkload readwrite --prefix cilium/.initlock/; etcdctl user grant-role externalworkload externalworkload; - export remotepw=`head /dev/urandom | tr -dc A-Za-z0-9 | head -c 16`; - echo $remotepw | etcdctl --interactive=false user add remote; + etcdctl user add remote --no-password; etcdctl role add remote; etcdctl role grant-permission remote --from-key read ''; etcdctl user grant-role remote remote; @@ -99,6 +106,10 @@ spec: - --advertise-client-urls=https://[$(HOSTNAME_IP)]:2379 - --initial-cluster-token=clustermesh-apiserver - --auto-compaction-retention=1 + {{- if .Values.clustermesh.apiserver.metrics.etcd.enabled }} + - --listen-metrics-urls=http://[$(HOSTNAME_IP)]:{{ .Values.clustermesh.apiserver.metrics.etcd.port }} + - --metrics={{ .Values.clustermesh.apiserver.metrics.etcd.mode }} + {{- end }} env: - name: ETCDCTL_API value: "3" @@ -106,6 +117,15 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + ports: + - name: etcd + containerPort: 2379 + protocol: TCP + {{- if .Values.clustermesh.apiserver.metrics.etcd.enabled }} + - name: etcd-metrics + containerPort: {{ .Values.clustermesh.apiserver.metrics.etcd.port }} + protocol: TCP + {{- end }} volumeMounts: - name: etcd-server-secrets mountPath: /var/lib/etcd-secrets @@ -134,6 +154,17 @@ spec: - --cluster-id=$(CLUSTER_ID) - --kvstore-opt - etcd.config=/var/lib/cilium/etcd-config.yaml + {{- if ne .Values.clustermesh.apiserver.tls.authMode "legacy" }} + - --cluster-users-enabled + - --cluster-users-config-path=/var/lib/cilium/etcd-config/users.yaml + {{- end }} + - --enable-external-workloads={{ .Values.externalWorkloads.enabled }} + {{- if .Values.clustermesh.apiserver.metrics.enabled }} + - --prometheus-serve-addr=:{{ .Values.clustermesh.apiserver.metrics.port }} + {{- end }} + {{- with .Values.clustermesh.apiserver.extraArgs }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} env: - name: CLUSTER_NAME valueFrom: @@ -160,6 +191,12 @@ spec: {{- with .Values.clustermesh.apiserver.extraEnv }} {{- toYaml . | trim | nindent 8 }} {{- end }} + {{- if .Values.clustermesh.apiserver.metrics.enabled }} + ports: + - name: apiserv-metrics + containerPort: {{ .Values.clustermesh.apiserver.metrics.port }} + protocol: TCP + {{- end }} {{- with .Values.clustermesh.apiserver.resources }} resources: {{- toYaml . | nindent 10 }} @@ -168,6 +205,11 @@ spec: - name: etcd-admin-client mountPath: /var/lib/cilium/etcd-secrets readOnly: true + {{- if ne .Values.clustermesh.apiserver.tls.authMode "legacy" }} + - name: etcd-users-config + mountPath: /var/lib/cilium/etcd-config + readOnly: true + {{- end }} {{- with .Values.clustermesh.apiserver.extraVolumeMounts }} {{- toYaml . | nindent 8 }} {{- end }} @@ -176,19 +218,146 @@ spec: securityContext: {{- toYaml . | nindent 10 }} {{- end }} + {{- if .Values.clustermesh.apiserver.kvstoremesh.enabled }} + - name: kvstoremesh + image: {{ include "cilium.image" .Values.clustermesh.apiserver.kvstoremesh.image | quote }} + imagePullPolicy: {{ .Values.clustermesh.apiserver.kvstoremesh.image.pullPolicy }} + command: + - /usr/bin/kvstoremesh + args: + {{- if .Values.debug.enabled }} + - --debug + {{- end }} + - --cluster-name=$(CLUSTER_NAME) + - --cluster-id=$(CLUSTER_ID) + - --kvstore-opt=etcd.config=/var/lib/cilium/etcd-config.yaml + - --kvstore-opt=etcd.qps=100 + - --kvstore-opt=etcd.maxInflight=10 + - --clustermesh-config=/var/lib/cilium/clustermesh + {{- if .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled }} + - --prometheus-serve-addr=:{{ .Values.clustermesh.apiserver.metrics.kvstoremesh.port }} + {{- end }} + {{- with .Values.clustermesh.apiserver.kvstoremesh.extraArgs }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + env: + - name: CLUSTER_NAME + valueFrom: + configMapKeyRef: + name: cilium-config + key: cluster-name + - name: CLUSTER_ID + valueFrom: + configMapKeyRef: + name: cilium-config + key: cluster-id + {{- with .Values.clustermesh.apiserver.kvstoremesh.extraEnv }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- if .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled }} + ports: + - name: kvmesh-metrics + containerPort: {{ .Values.clustermesh.apiserver.metrics.kvstoremesh.port }} + protocol: TCP + {{- end }} + {{- with .Values.clustermesh.apiserver.kvstoremesh.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + - name: etcd-admin-client + mountPath: /var/lib/cilium/etcd-secrets + readOnly: true + - name: kvstoremesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + {{- with .Values.clustermesh.apiserver.kvstoremesh.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + {{- with .Values.clustermesh.apiserver.kvstoremesh.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} volumes: - name: etcd-server-secrets - secret: - secretName: clustermesh-apiserver-server-cert + projected: # note: the leading zero means this number is in octal representation: do not remove it defaultMode: 0400 + sources: + - secret: + name: clustermesh-apiserver-server-cert + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + {{- if not .Values.tls.caBundle.enabled }} + - key: ca.crt + path: ca.crt + {{- else }} + - {{ .Values.tls.caBundle.useSecret | ternary "secret" "configMap" }}: + name: {{ .Values.tls.caBundle.name }} + items: + - key: {{ .Values.tls.caBundle.key }} + path: ca.crt + {{- end }} - name: etcd-admin-client - secret: - secretName: clustermesh-apiserver-admin-cert + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: clustermesh-apiserver-admin-cert + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + {{- if not .Values.tls.caBundle.enabled }} + - key: ca.crt + path: ca.crt + {{- else }} + - {{ .Values.tls.caBundle.useSecret | ternary "secret" "configMap" }}: + name: {{ .Values.tls.caBundle.name }} + items: + - key: {{ .Values.tls.caBundle.key }} + path: ca.crt + {{- end }} + {{- if ne .Values.clustermesh.apiserver.tls.authMode "legacy" }} + - name: etcd-users-config + configMap: + name: clustermesh-remote-users # note: the leading zero means this number is in octal representation: do not remove it defaultMode: 0400 + {{- end }} - name: etcd-data-dir emptyDir: {} + {{- if .Values.clustermesh.apiserver.kvstoremesh.enabled }} + - name: kvstoremesh-secrets + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: cilium-kvstoremesh + optional: true + # note: items are not explicitly listed here, since the entries of this secret + # depend on the peers configured, and that would cause a restart of this pod + # at every addition/removal. Leaving the field empty makes each secret entry + # to be automatically projected into the volume as a file whose name is the key. + - secret: + name: clustermesh-apiserver-remote-cert + optional: true + items: + - key: tls.key + path: common-etcd-client.key + - key: tls.crt + path: common-etcd-client.crt + - key: ca.crt + path: common-etcd-client-ca.crt + {{- end }} {{- with .Values.clustermesh.apiserver.extraVolumes }} {{- toYaml . | nindent 6 }} {{- end }} @@ -220,4 +389,13 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- if and .Values.clustermesh.config.enabled .Values.clustermesh.apiserver.kvstoremesh.enabled }} + hostAliases: + {{- range $cluster := .Values.clustermesh.config.clusters }} + {{- range $ip := $cluster.ips }} + - ip: {{ $ip }} + hostnames: [ "{{ $cluster.name }}.{{ $.Values.clustermesh.config.domain }}" ] + {{- end }} + {{- end }} + {{- end }} {{- end }} diff --git a/charts/cilium/templates/clustermesh-apiserver/metrics-service.yaml b/charts/cilium/templates/clustermesh-apiserver/metrics-service.yaml new file mode 100644 index 00000000..3240a571 --- /dev/null +++ b/charts/cilium/templates/clustermesh-apiserver/metrics-service.yaml @@ -0,0 +1,38 @@ +{{- if and + (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) + (or .Values.clustermesh.apiserver.metrics.enabled .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled .Values.clustermesh.apiserver.metrics.etcd.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: clustermesh-apiserver-metrics + namespace: {{ .Release.Namespace }} + labels: + k8s-app: clustermesh-apiserver + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: clustermesh-apiserver + app.kubernetes.io/component: metrics +spec: + clusterIP: None + type: ClusterIP + ports: + {{- if .Values.clustermesh.apiserver.metrics.enabled }} + - name: apiserv-metrics + port: {{ .Values.clustermesh.apiserver.metrics.port }} + protocol: TCP + targetPort: apiserv-metrics + {{- end }} + {{- if .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled }} + - name: kvmesh-metrics + port: {{ .Values.clustermesh.apiserver.metrics.kvstoremesh.port }} + protocol: TCP + targetPort: kvmesh-metrics + {{- end }} + {{- if .Values.clustermesh.apiserver.metrics.etcd.enabled }} + - name: etcd-metrics + port: {{ .Values.clustermesh.apiserver.metrics.etcd.port }} + protocol: TCP + targetPort: etcd-metrics + {{- end }} + selector: + k8s-app: clustermesh-apiserver +{{- end }} diff --git a/charts/cilium/templates/clustermesh-apiserver/service.yaml b/charts/cilium/templates/clustermesh-apiserver/service.yaml index ff1a0ceb..466841d9 100644 --- a/charts/cilium/templates/clustermesh-apiserver/service.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/service.yaml @@ -24,4 +24,10 @@ spec: {{- if and (eq "LoadBalancer" .Values.clustermesh.apiserver.service.type) .Values.clustermesh.apiserver.service.loadBalancerIP }} loadBalancerIP: {{ .Values.clustermesh.apiserver.service.loadBalancerIP }} {{- end }} + {{- if .Values.clustermesh.apiserver.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.clustermesh.apiserver.service.externalTrafficPolicy }} + {{- end }} + {{- if .Values.clustermesh.apiserver.service.internalTrafficPolicy }} + internalTrafficPolicy: {{ .Values.clustermesh.apiserver.service.internalTrafficPolicy }} + {{- end }} {{- end }} diff --git a/charts/cilium/templates/clustermesh-apiserver/servicemonitor.yaml b/charts/cilium/templates/clustermesh-apiserver/servicemonitor.yaml new file mode 100644 index 00000000..402701a0 --- /dev/null +++ b/charts/cilium/templates/clustermesh-apiserver/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if and + (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) + (or .Values.clustermesh.apiserver.metrics.enabled .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled .Values.clustermesh.apiserver.metrics.etcd.enabled) + .Values.clustermesh.apiserver.metrics.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: clustermesh-apiserver + namespace: {{ .Values.clustermesh.apiserver.metrics.serviceMonitor.namespace | default .Release.Namespace }} + labels: + app.kubernetes.io/part-of: cilium + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: clustermesh-apiserver + app.kubernetes.io/component: metrics + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + endpoints: + {{- if .Values.clustermesh.apiserver.metrics.enabled }} + - port: apiserv-metrics + interval: {{ .Values.clustermesh.apiserver.metrics.serviceMonitor.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + {{- if .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled }} + - port: kvmesh-metrics + interval: {{ .Values.clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + {{- if .Values.clustermesh.apiserver.metrics.etcd.enabled }} + - port: etcd-metrics + interval: {{ .Values.clustermesh.apiserver.metrics.serviceMonitor.etcd.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.etcd.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.etcd.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/admin-secret.yaml b/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/admin-secret.yaml index 82962f2a..3bc84ae1 100644 --- a/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/admin-secret.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/admin-secret.yaml @@ -9,7 +9,7 @@ spec: issuerRef: {{- toYaml .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef | nindent 4 }} secretName: clustermesh-apiserver-admin-cert - commonName: root + commonName: {{ include "clustermesh-apiserver-generate-certs.admin-common-name" . }} dnsNames: - localhost duration: {{ printf "%dh0m0s" (mul .Values.clustermesh.apiserver.tls.auto.certValidityDuration 24) }} diff --git a/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/remote-secret.yaml b/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/remote-secret.yaml index 353b58b4..0a5e42e2 100644 --- a/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/remote-secret.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/remote-secret.yaml @@ -9,6 +9,6 @@ spec: issuerRef: {{- toYaml .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef | nindent 4 }} secretName: clustermesh-apiserver-remote-cert - commonName: remote + commonName: {{ include "clustermesh-apiserver-generate-certs.remote-common-name" . }} duration: {{ printf "%dh0m0s" (mul .Values.clustermesh.apiserver.tls.auto.certValidityDuration 24) }} {{- end }} diff --git a/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/server-secret.yaml b/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/server-secret.yaml index 90994774..86fe56f5 100644 --- a/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/server-secret.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/server-secret.yaml @@ -13,6 +13,7 @@ spec: dnsNames: - clustermesh-apiserver.cilium.io - "*.mesh.cilium.io" + - "clustermesh-apiserver.{{ .Release.Namespace }}.svc" {{- range $dns := .Values.clustermesh.apiserver.tls.server.extraDnsNames }} - {{ $dns | quote }} {{- end }} diff --git a/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/_job-spec.tpl b/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/_job-spec.tpl index 6c765caa..67503e33 100644 --- a/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/_job-spec.tpl +++ b/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/_job-spec.tpl @@ -1,6 +1,6 @@ {{- define "clustermesh-apiserver-generate-certs.job.spec" }} {{- $certValiditySecondsStr := printf "%ds" (mul .Values.clustermesh.apiserver.tls.auto.certValidityDuration 24 60 60) -}} -{{- $clustermeshServerSANs := concat (list "*.mesh.cilium.io") +{{- $clustermeshServerSANs := concat (list "*.mesh.cilium.io" (printf "clustermesh-apiserver.%s.svc" .Release.Namespace)) .Values.clustermesh.apiserver.tls.server.extraDnsNames .Values.clustermesh.apiserver.tls.server.extraIpAddresses -}} @@ -38,6 +38,7 @@ spec: - "--clustermesh-apiserver-server-cert-sans={{ join "," $clustermeshServerSANs }}" - "--clustermesh-apiserver-admin-cert-generate" - "--clustermesh-apiserver-admin-cert-validity-duration={{ $certValiditySecondsStr }}" + - "--clustermesh-apiserver-admin-cert-common-name={{ include "clustermesh-apiserver-generate-certs.admin-common-name" . }}" {{- if .Values.externalWorkloads.enabled }} - "--clustermesh-apiserver-client-cert-generate" - "--clustermesh-apiserver-client-cert-validity-duration={{ $certValiditySecondsStr }}" @@ -45,12 +46,17 @@ spec: {{- if .Values.clustermesh.useAPIServer }} - "--clustermesh-apiserver-remote-cert-generate" - "--clustermesh-apiserver-remote-cert-validity-duration={{ $certValiditySecondsStr }}" + - "--clustermesh-apiserver-remote-cert-common-name={{ include "clustermesh-apiserver-generate-certs.remote-common-name" . }}" {{- end }} {{- with .Values.certgen.extraVolumeMounts }} volumeMounts: {{- toYaml . | nindent 10 }} {{- end }} hostNetwork: true + {{- with .Values.certgen.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} serviceAccount: {{ .Values.serviceAccounts.clustermeshcertgen.name | quote }} serviceAccountName: {{ .Values.serviceAccounts.clustermeshcertgen.name | quote }} automountServiceAccountToken: {{ .Values.serviceAccounts.clustermeshcertgen.automount }} diff --git a/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/job.yaml b/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/job.yaml index 36fcac7a..06235f29 100644 --- a/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/job.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/job.yaml @@ -1,21 +1,17 @@ {{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "cronJob") }} -{{/* -Because Kubernetes job specs are immutable, Helm will fail patch this job if -the spec changes between releases. To avoid breaking the upgrade path, we -generate a name for the job here which is based on the checksum of the spec. -This will cause the name of the job to change if its content changes, -and in turn cause Helm to do delete the old job and replace it with a new one. -*/}} -{{- $jobSpec := include "clustermesh-apiserver-generate-certs.job.spec" . -}} -{{- $checkSum := $jobSpec | sha256sum | trunc 10 -}} --- apiVersion: batch/v1 kind: Job metadata: - name: clustermesh-apiserver-generate-certs-{{$checkSum}} + name: clustermesh-apiserver-generate-certs namespace: {{ .Release.Namespace }} labels: k8s-app: clustermesh-apiserver-generate-certs app.kubernetes.io/part-of: cilium -{{ $jobSpec }} + annotations: + "helm.sh/hook": post-install,post-upgrade + {{- with .Values.certgen.annotations.job }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "clustermesh-apiserver-generate-certs.job.spec" . }} {{- end }} diff --git a/charts/cilium/templates/clustermesh-apiserver/tls-helm/admin-secret.yaml b/charts/cilium/templates/clustermesh-apiserver/tls-helm/admin-secret.yaml index 43ded27e..dfa1d987 100644 --- a/charts/cilium/templates/clustermesh-apiserver/tls-helm/admin-secret.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/tls-helm/admin-secret.yaml @@ -1,6 +1,6 @@ {{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm") }} {{- $_ := include "clustermesh-apiserver-generate-certs.helm.setup-ca" . -}} -{{- $cn := "root" }} +{{- $cn := include "clustermesh-apiserver-generate-certs.admin-common-name" . -}} {{- $dns := list "localhost" }} {{- $cert := genSignedCert $cn nil $dns (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .cmca -}} --- diff --git a/charts/cilium/templates/clustermesh-apiserver/tls-helm/remote-secret.yaml b/charts/cilium/templates/clustermesh-apiserver/tls-helm/remote-secret.yaml index 2c3bf96b..4a68c747 100644 --- a/charts/cilium/templates/clustermesh-apiserver/tls-helm/remote-secret.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/tls-helm/remote-secret.yaml @@ -1,6 +1,6 @@ {{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm") }} {{- $_ := include "clustermesh-apiserver-generate-certs.helm.setup-ca" . -}} -{{- $cn := "remote" }} +{{- $cn := include "clustermesh-apiserver-generate-certs.remote-common-name" . -}} {{- $cert := genSignedCert $cn nil nil (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .cmca -}} --- apiVersion: v1 diff --git a/charts/cilium/templates/clustermesh-apiserver/tls-helm/server-secret.yaml b/charts/cilium/templates/clustermesh-apiserver/tls-helm/server-secret.yaml index 75da3d53..beacb063 100644 --- a/charts/cilium/templates/clustermesh-apiserver/tls-helm/server-secret.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/tls-helm/server-secret.yaml @@ -2,7 +2,7 @@ {{- $_ := include "clustermesh-apiserver-generate-certs.helm.setup-ca" . -}} {{- $cn := "clustermesh-apiserver.cilium.io" }} {{- $ip := concat (list "127.0.0.1" "::1") .Values.clustermesh.apiserver.tls.server.extraIpAddresses }} -{{- $dns := concat (list $cn "*.mesh.cilium.io") .Values.clustermesh.apiserver.tls.server.extraDnsNames }} +{{- $dns := concat (list $cn "*.mesh.cilium.io" (printf "clustermesh-apiserver.%s.svc" .Release.Namespace)) .Values.clustermesh.apiserver.tls.server.extraDnsNames }} {{- $cert := genSignedCert $cn $ip $dns (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .cmca -}} --- apiVersion: v1 diff --git a/charts/cilium/templates/clustermesh-apiserver/tls-provided/admin-secret.yaml b/charts/cilium/templates/clustermesh-apiserver/tls-provided/admin-secret.yaml index ae30d895..4ff56ab2 100644 --- a/charts/cilium/templates/clustermesh-apiserver/tls-provided/admin-secret.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/tls-provided/admin-secret.yaml @@ -6,7 +6,7 @@ metadata: namespace: {{ .Release.Namespace }} type: kubernetes.io/tls data: - ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert }} + ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert | default .Values.tls.ca.cert }} tls.crt: {{ .Values.clustermesh.apiserver.tls.admin.cert | required "missing clustermesh.apiserver.tls.admin.cert" }} tls.key: {{ .Values.clustermesh.apiserver.tls.admin.key | required "missing clustermesh.apiserver.tls.admin.key" }} {{- end }} diff --git a/charts/cilium/templates/clustermesh-apiserver/tls-provided/ca-secret.yaml b/charts/cilium/templates/clustermesh-apiserver/tls-provided/ca-secret.yaml index 3fb695ec..9b29a541 100644 --- a/charts/cilium/templates/clustermesh-apiserver/tls-provided/ca-secret.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/tls-provided/ca-secret.yaml @@ -5,8 +5,8 @@ metadata: name: clustermesh-apiserver-ca-cert namespace: {{ .Release.Namespace }} data: - ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert }} - {{- if .Values.clustermesh.apiserver.tls.ca.key }} - ca.key: {{ .Values.clustermesh.apiserver.tls.ca.key }} + ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert | default .Values.tls.ca.cert }} + {{- if .Values.clustermesh.apiserver.tls.ca.key | default .Values.tls.ca.key }} + ca.key: {{ .Values.clustermesh.apiserver.tls.ca.key | default .Values.tls.ca.key }} {{- end }} {{- end }} diff --git a/charts/cilium/templates/clustermesh-apiserver/tls-provided/client-secret.yaml b/charts/cilium/templates/clustermesh-apiserver/tls-provided/client-secret.yaml index b56035a7..02a180ff 100644 --- a/charts/cilium/templates/clustermesh-apiserver/tls-provided/client-secret.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/tls-provided/client-secret.yaml @@ -6,7 +6,7 @@ metadata: namespace: {{ .Release.Namespace }} type: kubernetes.io/tls data: - ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert }} + ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert | default .Values.tls.ca.cert }} tls.crt: {{ .Values.clustermesh.apiserver.tls.client.cert | required "missing clustermesh.apiserver.tls.client.cert" }} tls.key: {{ .Values.clustermesh.apiserver.tls.client.key | required "missing clustermesh.apiserver.tls.client.key" }} {{- end }} diff --git a/charts/cilium/templates/clustermesh-apiserver/tls-provided/remote-secret.yaml b/charts/cilium/templates/clustermesh-apiserver/tls-provided/remote-secret.yaml index 45873740..7850b332 100644 --- a/charts/cilium/templates/clustermesh-apiserver/tls-provided/remote-secret.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/tls-provided/remote-secret.yaml @@ -6,7 +6,7 @@ metadata: namespace: {{ .Release.Namespace }} type: kubernetes.io/tls data: - ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert }} + ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert | default .Values.tls.ca.cert }} tls.crt: {{ .Values.clustermesh.apiserver.tls.remote.cert | required "missing clustermesh.apiserver.tls.remote.cert" }} tls.key: {{ .Values.clustermesh.apiserver.tls.remote.key | required "missing clustermesh.apiserver.tls.remote.key" }} {{- end }} diff --git a/charts/cilium/templates/clustermesh-apiserver/tls-provided/server-secret.yaml b/charts/cilium/templates/clustermesh-apiserver/tls-provided/server-secret.yaml index 018e4cfe..bb160d62 100644 --- a/charts/cilium/templates/clustermesh-apiserver/tls-provided/server-secret.yaml +++ b/charts/cilium/templates/clustermesh-apiserver/tls-provided/server-secret.yaml @@ -6,7 +6,7 @@ metadata: namespace: {{ .Release.Namespace }} type: kubernetes.io/tls data: - ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert }} + ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert | default .Values.tls.ca.cert }} tls.crt: {{ .Values.clustermesh.apiserver.tls.server.cert | required "missing clustermesh.apiserver.tls.server.cert" }} tls.key: {{ .Values.clustermesh.apiserver.tls.server.key | required "missing clustermesh.apiserver.tls.server.key" }} {{- end }} diff --git a/charts/cilium/templates/clustermesh-apiserver/users-configmap.yaml b/charts/cilium/templates/clustermesh-apiserver/users-configmap.yaml new file mode 100644 index 00000000..7c278978 --- /dev/null +++ b/charts/cilium/templates/clustermesh-apiserver/users-configmap.yaml @@ -0,0 +1,21 @@ +{{- if ne .Values.clustermesh.apiserver.tls.authMode "legacy" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: clustermesh-remote-users + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/part-of: cilium +data: + users.yaml: | + users: + {{- if .Values.clustermesh.apiserver.kvstoremesh.enabled }} + - name: remote-{{ .Values.cluster.name }} + role: remote + {{- end }} + {{- range .Values.clustermesh.config.clusters }} + - name: remote-{{ .name }} + role: remote + {{- end }} +{{- end }} diff --git a/charts/cilium/templates/clustermesh-config/_helpers.tpl b/charts/cilium/templates/clustermesh-config/_helpers.tpl index e2e66dc1..f492275f 100644 --- a/charts/cilium/templates/clustermesh-config/_helpers.tpl +++ b/charts/cilium/templates/clustermesh-config/_helpers.tpl @@ -1,14 +1,25 @@ {{- define "clustermesh-config-generate-etcd-cfg" }} {{- $cluster := index . 0 -}} {{- $domain := index . 1 -}} +{{- $hasCustomCACert := index . 2 -}} +{{- $override := index . 3 -}} +{{- /* The parenthesis around $cluster.tls are required, since it can be null: https://stackoverflow.com/a/68807258 */}} +{{- $prefix := ternary "common-" (printf "%s." $cluster.name) (or (ne $override "") (empty ($cluster.tls).cert) (empty ($cluster.tls).key)) -}} endpoints: -{{- if $cluster.ips }} +{{- if ne $override "" }} +- {{ $override }} +{{- else if $cluster.ips }} - https://{{ $cluster.name }}.{{ $domain }}:{{ $cluster.port }} -{{ else }} +{{- else }} - https://{{ $cluster.address | required "missing clustermesh.apiserver.config.clusters.address" }}:{{ $cluster.port }} {{- end }} -trusted-ca-file: /var/lib/cilium/clustermesh/{{ $cluster.name }}.etcd-client-ca.crt -key-file: /var/lib/cilium/clustermesh/{{ $cluster.name }}.etcd-client.key -cert-file: /var/lib/cilium/clustermesh/{{ $cluster.name }}.etcd-client.crt +{{- if $hasCustomCACert }} +{{- /* The custom CA configuration takes effect only if a custom certificate and key are also set */}} +trusted-ca-file: /var/lib/cilium/clustermesh/{{ $prefix }}etcd-client-ca.crt +{{- else }} +trusted-ca-file: /var/lib/cilium/clustermesh/common-etcd-client-ca.crt +{{- end }} +key-file: /var/lib/cilium/clustermesh/{{ $prefix }}etcd-client.key +cert-file: /var/lib/cilium/clustermesh/{{ $prefix }}etcd-client.crt {{- end }} diff --git a/charts/cilium/templates/clustermesh-config/clustermesh-secret.yaml b/charts/cilium/templates/clustermesh-config/clustermesh-secret.yaml index 1e34def0..36e23b4e 100644 --- a/charts/cilium/templates/clustermesh-config/clustermesh-secret.yaml +++ b/charts/cilium/templates/clustermesh-config/clustermesh-secret.yaml @@ -6,10 +6,17 @@ metadata: name: cilium-clustermesh namespace: {{ .Release.Namespace }} data: + {{- $override := ternary (printf "https://clustermesh-apiserver.%s.svc:2379" .Release.Namespace) "" $.Values.clustermesh.apiserver.kvstoremesh.enabled }} {{- range .Values.clustermesh.config.clusters }} - {{ .name }}: {{ include "clustermesh-config-generate-etcd-cfg" (list . $.Values.clustermesh.config.domain) | b64enc }} - {{ .name }}.etcd-client-ca.crt: {{ $.Values.clustermesh.apiserver.tls.ca.cert }} + {{- $hasCustomCACert := or (.tls).caCert $.Values.clustermesh.apiserver.tls.ca.cert }} + {{ .name }}: {{ include "clustermesh-config-generate-etcd-cfg" (list . $.Values.clustermesh.config.domain $hasCustomCACert $override) | b64enc }} + {{- /* The parenthesis around .tls are required, since it can be null: https://stackoverflow.com/a/68807258 */}} + {{- if and (eq $override "") (.tls).cert (.tls).key }} + {{- if $hasCustomCACert }} + {{ .name }}.etcd-client-ca.crt: {{ .tls.caCert | default $.Values.clustermesh.apiserver.tls.ca.cert }} + {{- end }} {{ .name }}.etcd-client.key: {{ .tls.key }} {{ .name }}.etcd-client.crt: {{ .tls.cert }} {{- end }} + {{- end }} {{- end }} diff --git a/charts/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml b/charts/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml new file mode 100644 index 00000000..f79c4ade --- /dev/null +++ b/charts/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.config.enabled .Values.clustermesh.apiserver.kvstoremesh.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: cilium-kvstoremesh + namespace: {{ .Release.Namespace }} +data: + {{- range .Values.clustermesh.config.clusters }} + {{- $hasCustomCACert := or (.tls).caCert $.Values.clustermesh.apiserver.tls.ca.cert }} + {{ .name }}: {{ include "clustermesh-config-generate-etcd-cfg" (list . $.Values.clustermesh.config.domain $hasCustomCACert "") | b64enc }} + {{- /* The parenthesis around .tls are required, since it can be null: https://stackoverflow.com/a/68807258 */}} + {{- if and (.tls).cert (.tls).key }} + {{- if $hasCustomCACert }} + {{ .name }}.etcd-client-ca.crt: {{ .tls.caCert | default $.Values.clustermesh.apiserver.tls.ca.cert }} + {{- end }} + {{ .name }}.etcd-client.key: {{ .tls.key }} + {{ .name }}.etcd-client.crt: {{ .tls.cert }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/cilium/templates/hubble-relay/configmap.yaml b/charts/cilium/templates/hubble-relay/configmap.yaml index e78b31df..0f5a037d 100644 --- a/charts/cilium/templates/hubble-relay/configmap.yaml +++ b/charts/cilium/templates/hubble-relay/configmap.yaml @@ -12,12 +12,10 @@ metadata: data: config.yaml: | cluster-name: {{ .Values.cluster.name }} - {{- if and .Values.hubble.enabled .Values.hubble.peerService.enabled }} peer-service: "hubble-peer.{{ .Release.Namespace }}.svc.{{ .Values.hubble.peerService.clusterDomain }}:{{ $peerSvcPort }}" - {{- else }} - peer-service: unix://{{ .Values.hubble.socketPath }} - {{- end }} listen-address: {{ .Values.hubble.relay.listenHost }}:{{ .Values.hubble.relay.listenPort }} + gops: {{ .Values.hubble.relay.gops.enabled }} + gops-port: {{ .Values.hubble.relay.gops.port | quote }} {{- if .Values.hubble.relay.pprof.enabled }} pprof: {{ .Values.hubble.relay.pprof.enabled | quote }} pprof-address: {{ .Values.hubble.relay.pprof.address | quote }} @@ -31,15 +29,18 @@ data: sort-buffer-len-max: {{ .Values.hubble.relay.sortBufferLenMax }} sort-buffer-drain-timeout: {{ .Values.hubble.relay.sortBufferDrainTimeout }} {{- if .Values.hubble.tls.enabled }} - tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt - tls-client-key-file: /var/lib/hubble-relay/tls/client.key + tls-hubble-client-cert-file: /var/lib/hubble-relay/tls/client.crt + tls-hubble-client-key-file: /var/lib/hubble-relay/tls/client.key tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt {{- else }} disable-client-tls: true {{- end }} {{- if and .Values.hubble.tls.enabled .Values.hubble.relay.tls.server.enabled }} - tls-server-cert-file: /var/lib/hubble-relay/tls/server.crt - tls-server-key-file: /var/lib/hubble-relay/tls/server.key + tls-relay-server-cert-file: /var/lib/hubble-relay/tls/server.crt + tls-relay-server-key-file: /var/lib/hubble-relay/tls/server.key + {{- if .Values.hubble.relay.tls.server.mtls }} + tls-relay-client-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt + {{- end }} {{- else }} disable-server-tls: true {{- end }} diff --git a/charts/cilium/templates/hubble-relay/deployment.yaml b/charts/cilium/templates/hubble-relay/deployment.yaml index 8de6ef98..5a4148e9 100644 --- a/charts/cilium/templates/hubble-relay/deployment.yaml +++ b/charts/cilium/templates/hubble-relay/deployment.yaml @@ -1,5 +1,4 @@ {{- if and .Values.hubble.enabled .Values.hubble.relay.enabled }} -{{- $mountSocket := not .Values.hubble.peerService.enabled -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -36,7 +35,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - {{- with .Values.hubble.relay.securityContext }} + {{- with .Values.hubble.relay.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} {{- end }} @@ -46,6 +45,10 @@ spec: {{- end }} containers: - name: hubble-relay + {{- with .Values.hubble.relay.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} image: {{ include "cilium.image" .Values.hubble.relay.image | quote }} imagePullPolicy: {{ .Values.hubble.relay.image.pullPolicy }} command: @@ -78,11 +81,6 @@ spec: {{- toYaml . | trim | nindent 12 }} {{- end }} volumeMounts: - {{- if $mountSocket }} - - name: hubble-sock-dir - mountPath: {{ dir .Values.hubble.socketPath }} - readOnly: true - {{- end }} - name: config mountPath: /etc/hubble-relay readOnly: true @@ -128,12 +126,6 @@ spec: items: - key: config.yaml path: config.yaml - {{- if $mountSocket }} - - name: hubble-sock-dir - hostPath: - path: {{ dir .Values.hubble.socketPath }} - type: Directory - {{- end }} {{- if .Values.hubble.tls.enabled }} - name: tls projected: @@ -143,12 +135,20 @@ spec: - secret: name: hubble-relay-client-certs items: - - key: ca.crt - path: hubble-server-ca.crt - key: tls.crt path: client.crt - key: tls.key path: client.key + {{- if not .Values.tls.caBundle.enabled }} + - key: ca.crt + path: hubble-server-ca.crt + {{- else }} + - {{ .Values.tls.caBundle.useSecret | ternary "secret" "configMap" }}: + name: {{ .Values.tls.caBundle.name }} + items: + - key: {{ .Values.tls.caBundle.key }} + path: hubble-server-ca.crt + {{- end }} {{- if .Values.hubble.relay.tls.server.enabled }} - secret: name: hubble-relay-server-certs diff --git a/charts/cilium/templates/hubble-ui/deployment.yaml b/charts/cilium/templates/hubble-ui/deployment.yaml index f2efb164..9dfbcdb5 100644 --- a/charts/cilium/templates/hubble-ui/deployment.yaml +++ b/charts/cilium/templates/hubble-ui/deployment.yaml @@ -102,7 +102,7 @@ spec: value: "hubble-relay:80" {{- end }} {{- with .Values.hubble.ui.backend.extraEnv }} - {{- toYaml . | trim | nindent 10 }} + {{- toYaml . | trim | nindent 8 }} {{- end }} ports: - name: grpc @@ -167,12 +167,20 @@ spec: - secret: name: hubble-ui-client-certs items: - - key: ca.crt - path: hubble-relay-ca.crt - key: tls.crt path: client.crt - key: tls.key path: client.key + {{- if not .Values.tls.caBundle.enabled }} + - key: ca.crt + path: hubble-relay-ca.crt + {{- else }} + - {{ .Values.tls.caBundle.useSecret | ternary "secret" "configMap" }}: + name: {{ .Values.tls.caBundle.name }} + items: + - key: {{ .Values.tls.caBundle.key }} + path: hubble-relay-ca.crt + {{- end }} {{- end }} {{- end }} {{- with .Values.hubble.ui.frontend.extraVolumes }} diff --git a/charts/cilium/templates/hubble-ui/ingress.yaml b/charts/cilium/templates/hubble-ui/ingress.yaml index 963742bc..b48e2cb4 100644 --- a/charts/cilium/templates/hubble-ui/ingress.yaml +++ b/charts/cilium/templates/hubble-ui/ingress.yaml @@ -9,6 +9,9 @@ metadata: k8s-app: hubble-ui app.kubernetes.io/name: hubble-ui app.kubernetes.io/part-of: cilium + {{- with .Values.hubble.ui.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .Values.hubble.ui.ingress.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/charts/cilium/templates/hubble/peer-service.yaml b/charts/cilium/templates/hubble/peer-service.yaml index 574c8da7..f54fa6c6 100644 --- a/charts/cilium/templates/hubble/peer-service.yaml +++ b/charts/cilium/templates/hubble/peer-service.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.agent .Values.hubble.enabled .Values.hubble.peerService.enabled }} +{{- if and .Values.agent .Values.hubble.enabled }} apiVersion: v1 kind: Service metadata: diff --git a/charts/cilium/templates/hubble/tls-cronjob/_job-spec.tpl b/charts/cilium/templates/hubble/tls-cronjob/_job-spec.tpl index 991bd641..f3d595b7 100644 --- a/charts/cilium/templates/hubble/tls-cronjob/_job-spec.tpl +++ b/charts/cilium/templates/hubble/tls-cronjob/_job-spec.tpl @@ -25,12 +25,8 @@ spec: {{- end }} - "--ca-generate" - "--ca-reuse-secret" - {{- if .Values.hubble.tls.ca.cert }} - - "--ca-secret-name=hubble-ca-secret" - {{- else -}} - {{- if and .Values.tls.ca.cert .Values.tls.ca.key }} + {{- if and .Values.tls.ca.cert .Values.tls.ca.key }} - "--ca-secret-name=cilium-ca" - {{- end }} {{- end }} - "--hubble-server-cert-generate" - "--hubble-server-cert-common-name={{ list "*" (.Values.cluster.name | replace "." "-") "hubble-grpc.cilium.io" | join "." }}" diff --git a/charts/cilium/templates/hubble/tls-cronjob/ca-secret.yaml b/charts/cilium/templates/hubble/tls-cronjob/ca-secret.yaml deleted file mode 100644 index 1c7bab1f..00000000 --- a/charts/cilium/templates/hubble/tls-cronjob/ca-secret.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if and (or .Values.agent .Values.hubble.relay.enabled .Values.hubble.ui.enabled) .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "cronJob") }} -{{- $crt := .Values.hubble.tls.ca.cert | default .Values.tls.ca.cert -}} -{{- $key := .Values.hubble.tls.ca.key | default .Values.tls.ca.key -}} -{{- if and $crt $key }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: hubble-ca-secret - namespace: {{ .Release.Namespace }} -data: - ca.crt: {{ $crt }} - ca.key: {{ $key }} -{{- end }} -{{- end }} diff --git a/charts/cilium/templates/hubble/tls-cronjob/clusterrole.yaml b/charts/cilium/templates/hubble/tls-cronjob/clusterrole.yaml index 4e1aa326..dc075670 100644 --- a/charts/cilium/templates/hubble/tls-cronjob/clusterrole.yaml +++ b/charts/cilium/templates/hubble/tls-cronjob/clusterrole.yaml @@ -28,7 +28,6 @@ rules: - secrets resourceNames: - cilium-ca - - hubble-ca-secret verbs: - get - update diff --git a/charts/cilium/templates/hubble/tls-cronjob/cronjob.yaml b/charts/cilium/templates/hubble/tls-cronjob/cronjob.yaml index c52ab4df..5a5992cf 100644 --- a/charts/cilium/templates/hubble/tls-cronjob/cronjob.yaml +++ b/charts/cilium/templates/hubble/tls-cronjob/cronjob.yaml @@ -8,6 +8,10 @@ metadata: k8s-app: hubble-generate-certs app.kubernetes.io/name: hubble-generate-certs app.kubernetes.io/part-of: cilium + annotations: + {{- with .Values.certgen.annotations.cronJob }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: schedule: {{ .Values.hubble.tls.auto.schedule | quote }} concurrencyPolicy: Forbid diff --git a/charts/cilium/templates/hubble/tls-cronjob/job.yaml b/charts/cilium/templates/hubble/tls-cronjob/job.yaml index 1d8a1449..e748165a 100644 --- a/charts/cilium/templates/hubble/tls-cronjob/job.yaml +++ b/charts/cilium/templates/hubble/tls-cronjob/job.yaml @@ -1,22 +1,18 @@ {{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "cronJob") }} -{{/* -Because Kubernetes job specs are immutable, Helm will fail patch this job if -the spec changes between releases. To avoid breaking the upgrade path, we -generate a name for the job here which is based on the checksum of the spec. -This will cause the name of the job to change if its content changes, -and in turn cause Helm to do delete the old job and replace it with a new one. -*/}} -{{- $jobSpec := include "hubble-generate-certs.job.spec" . -}} -{{- $checkSum := $jobSpec | sha256sum | trunc 10 -}} --- apiVersion: batch/v1 kind: Job metadata: - name: hubble-generate-certs-{{$checkSum}} + name: hubble-generate-certs namespace: {{ .Release.Namespace }} labels: k8s-app: hubble-generate-certs app.kubernetes.io/name: hubble-generate-certs app.kubernetes.io/part-of: cilium -{{ $jobSpec }} + annotations: + "helm.sh/hook": post-install,post-upgrade + {{- with .Values.certgen.annotations.job }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "hubble-generate-certs.job.spec" . }} {{- end }} diff --git a/charts/cilium/templates/hubble/tls-helm/_helpers.tpl b/charts/cilium/templates/hubble/tls-helm/_helpers.tpl index e7337304..79babf94 100644 --- a/charts/cilium/templates/hubble/tls-helm/_helpers.tpl +++ b/charts/cilium/templates/hubble/tls-helm/_helpers.tpl @@ -12,24 +12,18 @@ certificate would be signed by a different CA. {{- define "hubble-generate-certs.helm.setup-ca" }} {{- if not .ca }} {{- $ca := "" -}} - {{- $crt := .Values.hubble.tls.ca.cert | default .Values.tls.ca.cert -}} - {{- $key := .Values.hubble.tls.ca.key | default .Values.tls.ca.key -}} + {{- $crt := .Values.tls.ca.cert -}} + {{- $key := .Values.tls.ca.key -}} {{- if and $crt $key }} {{- $ca = buildCustomCert $crt $key -}} {{- else }} - {{- with lookup "v1" "Secret" .Release.Namespace "hubble-ca-secret" }} + {{- $_ := include "cilium.ca.setup" . -}} + {{- with lookup "v1" "Secret" .Release.Namespace .commonCASecretName }} {{- $crt := index .data "ca.crt" }} {{- $key := index .data "ca.key" }} {{- $ca = buildCustomCert $crt $key -}} {{- else }} - {{- $_ := include "cilium.ca.setup" . -}} - {{- with lookup "v1" "Secret" .Release.Namespace .commonCASecretName }} - {{- $crt := index .data "ca.crt" }} - {{- $key := index .data "ca.key" }} - {{- $ca = buildCustomCert $crt $key -}} - {{- else }} - {{- $ca = .commonCA -}} - {{- end }} + {{- $ca = .commonCA -}} {{- end }} {{- end }} {{- $_ := set . "ca" $ca -}} diff --git a/charts/cilium/templates/hubble/tls-helm/ca-secret.yaml b/charts/cilium/templates/hubble/tls-helm/ca-secret.yaml deleted file mode 100644 index f1e8e76f..00000000 --- a/charts/cilium/templates/hubble/tls-helm/ca-secret.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if and (or .Values.agent .Values.hubble.relay.enabled .Values.hubble.ui.enabled) .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "helm") }} -{{- $_ := include "hubble-generate-certs.helm.setup-ca" . -}} ---- -apiVersion: v1 -kind: Secret -metadata: - name: hubble-ca-secret - namespace: {{ .Release.Namespace }} -data: - ca.crt: {{ .ca.Cert | b64enc }} - ca.key: {{ .ca.Key | b64enc }} -{{- end }} diff --git a/charts/cilium/templates/hubble/tls-provided/ca-secret.yaml b/charts/cilium/templates/hubble/tls-provided/ca-secret.yaml deleted file mode 100644 index 7a9ca55a..00000000 --- a/charts/cilium/templates/hubble/tls-provided/ca-secret.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if and (or .Values.agent .Values.hubble.relay.enabled .Values.hubble.ui.enabled) .Values.hubble.enabled .Values.hubble.tls.enabled (not .Values.hubble.tls.auto.enabled) }} -apiVersion: v1 -kind: Secret -metadata: - name: hubble-ca-secret - namespace: {{ .Release.Namespace }} -data: - ca.crt: {{ .Values.hubble.tls.ca.cert }} - {{- if .Values.hubble.tls.ca.key }} - ca.key: {{ .Values.hubble.tls.ca.key }} - {{- end }} -{{- end }} diff --git a/charts/cilium/templates/hubble/tls-provided/relay-client-secret.yaml b/charts/cilium/templates/hubble/tls-provided/relay-client-secret.yaml index 22cde037..4a15f646 100644 --- a/charts/cilium/templates/hubble/tls-provided/relay-client-secret.yaml +++ b/charts/cilium/templates/hubble/tls-provided/relay-client-secret.yaml @@ -6,7 +6,7 @@ metadata: namespace: {{ .Release.Namespace }} type: kubernetes.io/tls data: - ca.crt: {{ .Values.hubble.tls.ca.cert }} + ca.crt: {{ .Values.tls.ca.cert }} tls.crt: {{ .Values.hubble.relay.tls.client.cert | required "missing hubble.relay.tls.client.cert" }} tls.key: {{ .Values.hubble.relay.tls.client.key | required "missing hubble.relay.tls.client.key" }} {{- end }} diff --git a/charts/cilium/templates/hubble/tls-provided/relay-server-secret.yaml b/charts/cilium/templates/hubble/tls-provided/relay-server-secret.yaml index ccdfc249..2038f785 100644 --- a/charts/cilium/templates/hubble/tls-provided/relay-server-secret.yaml +++ b/charts/cilium/templates/hubble/tls-provided/relay-server-secret.yaml @@ -6,7 +6,7 @@ metadata: namespace: {{ .Release.Namespace }} type: kubernetes.io/tls data: - ca.crt: {{ .Values.hubble.tls.ca.cert }} + ca.crt: {{ .Values.tls.ca.cert }} tls.crt: {{ .Values.hubble.relay.tls.server.cert | required "missing hubble.relay.tls.server.cert" }} tls.key: {{ .Values.hubble.relay.tls.server.key | required "missing hubble.relay.tls.server.key" }} {{- end }} diff --git a/charts/cilium/templates/hubble/tls-provided/server-secret.yaml b/charts/cilium/templates/hubble/tls-provided/server-secret.yaml index f251b1b4..ecc0bf07 100644 --- a/charts/cilium/templates/hubble/tls-provided/server-secret.yaml +++ b/charts/cilium/templates/hubble/tls-provided/server-secret.yaml @@ -6,7 +6,7 @@ metadata: namespace: {{ .Release.Namespace }} type: kubernetes.io/tls data: - ca.crt: {{ .Values.hubble.tls.ca.cert }} + ca.crt: {{ .Values.tls.ca.cert }} tls.crt: {{ .Values.hubble.tls.server.cert | required "missing hubble.tls.server.cert" }} tls.key: {{ .Values.hubble.tls.server.key | required "missing hubble.tls.server.key" }} {{- end }} diff --git a/charts/cilium/templates/hubble/tls-provided/ui-client-certs.yaml b/charts/cilium/templates/hubble/tls-provided/ui-client-certs.yaml index 0d2d948a..7ed9472d 100644 --- a/charts/cilium/templates/hubble/tls-provided/ui-client-certs.yaml +++ b/charts/cilium/templates/hubble/tls-provided/ui-client-certs.yaml @@ -6,7 +6,7 @@ metadata: namespace: {{ .Release.Namespace }} type: kubernetes.io/tls data: - ca.crt: {{ .Values.hubble.tls.ca.cert }} + ca.crt: {{ .Values.tls.ca.cert }} tls.crt: {{ .Values.hubble.ui.tls.client.cert | required "missing hubble.ui.tls.client.cert" }} tls.key: {{ .Values.hubble.ui.tls.client.key | required "missing hubble.ui.tls.client.key" }} {{- end }} diff --git a/charts/cilium/templates/spire/agent/clusterrole.yaml b/charts/cilium/templates/spire/agent/clusterrole.yaml new file mode 100644 index 00000000..57d7ae0d --- /dev/null +++ b/charts/cilium/templates/spire/agent/clusterrole.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.agent.serviceAccount.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} +rules: +# Required cluster role to allow spire-agent to query k8s API server +- apiGroups: [ "" ] + resources: [ "pods","nodes","nodes/proxy" ] + verbs: [ "get" ] +{{- end }} diff --git a/charts/cilium/templates/spire/agent/clusterrolebinding.yaml b/charts/cilium/templates/spire/agent/clusterrolebinding.yaml new file mode 100644 index 00000000..f7290825 --- /dev/null +++ b/charts/cilium/templates/spire/agent/clusterrolebinding.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.agent.serviceAccount.create -}} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} +roleRef: + kind: ClusterRole + name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +{{- end }} diff --git a/charts/cilium/templates/spire/agent/configmap.yaml b/charts/cilium/templates/spire/agent/configmap.yaml new file mode 100644 index 00000000..21e1a70a --- /dev/null +++ b/charts/cilium/templates/spire/agent/configmap.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: spire-agent + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +data: + agent.conf: | + agent { + data_dir = "/run/spire" + log_level = "ERROR" + server_address = "spire-server" + server_port = "8081" + socket_path = {{ .Values.authentication.mutual.spire.agentSocketPath | quote }} + admin_socket_path = {{ .Values.authentication.mutual.spire.adminSocketPath | quote }} + trust_bundle_path = "/run/spire/bundle/bundle.crt" + trust_domain = {{ .Values.authentication.mutual.spire.trustDomain | quote }} + authorized_delegates = [ + "spiffe://{{ .Values.authentication.mutual.spire.trustDomain }}/cilium-agent", + ] + } + + plugins { + NodeAttestor "k8s_psat" { + plugin_data { + cluster = {{ .Values.cluster.name | quote }} + } + } + + KeyManager "memory" { + plugin_data { + } + } + + WorkloadAttestor "k8s" { + plugin_data { + skip_kubelet_verification = {{ .Values.authentication.mutual.spire.install.agent.skipKubeletVerification }} + } + } + } + + health_checks { + listener_enabled = true + bind_address = "0.0.0.0" + bind_port = "4251" + live_path = "/live" + ready_path = "/ready" + } +{{- end }} diff --git a/charts/cilium/templates/spire/agent/daemonset.yaml b/charts/cilium/templates/spire/agent/daemonset.yaml new file mode 100644 index 00000000..4bcf2c04 --- /dev/null +++ b/charts/cilium/templates/spire/agent/daemonset.yaml @@ -0,0 +1,90 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: spire-agent + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.install.server.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: spire-agent + {{- with .Values.authentication.mutual.spire.install.server.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} +spec: + selector: + matchLabels: + app: spire-agent + template: + metadata: + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + labels: + app: spire-agent + spec: + hostPID: true + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} + initContainers: + - name: init + image: docker.io/library/busybox:1.35.0@sha256:223ae047b1065bd069aac01ae3ac8088b3ca4a527827e283b85112f29385fb1b + command: + - /bin/sh + - -c + - | + {{- tpl (.Files.Get "files/spire/wait-for-spire.bash") . | nindent 14 }} + containers: + - name: spire-agent + image: {{ .Values.authentication.mutual.spire.install.agent.image }} + args: ["-config", "/run/spire/config/agent.conf"] + volumeMounts: + - name: spire-config + mountPath: /run/spire/config + readOnly: true + - name: spire-bundle + mountPath: /run/spire/bundle + - name: spire-agent-socket + mountPath: /run/spire/sockets + readOnly: false + - name: spire-agent + mountPath: /var/run/secrets/tokens + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: status.hostIP + livenessProbe: + httpGet: + path: /live + port: 4251 + failureThreshold: 2 + initialDelaySeconds: 15 + periodSeconds: 60 + timeoutSeconds: 3 + readinessProbe: + httpGet: + path: /ready + port: 4251 + initialDelaySeconds: 5 + periodSeconds: 5 + volumes: + - name: spire-config + configMap: + name: spire-agent + - name: spire-bundle + configMap: + name: spire-bundle + - name: spire-agent-socket + hostPath: + path: /run/spire/sockets + type: DirectoryOrCreate + - name: spire-agent + projected: + sources: + - serviceAccountToken: + path: spire-agent + expirationSeconds: 600 + audience: spire-server +{{- end }} diff --git a/charts/cilium/templates/spire/agent/serviceaccount.yaml b/charts/cilium/templates/spire/agent/serviceaccount.yaml new file mode 100644 index 00000000..f155b5e6 --- /dev/null +++ b/charts/cilium/templates/spire/agent/serviceaccount.yaml @@ -0,0 +1,7 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.agent.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +{{- end }} diff --git a/charts/cilium/templates/spire/bundle-configmap.yaml b/charts/cilium/templates/spire/bundle-configmap.yaml new file mode 100644 index 00000000..986e6604 --- /dev/null +++ b/charts/cilium/templates/spire/bundle-configmap.yaml @@ -0,0 +1,7 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: spire-bundle + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +{{- end }} diff --git a/charts/cilium/templates/spire/namespace.yaml b/charts/cilium/templates/spire/namespace.yaml new file mode 100644 index 00000000..a0f3e23d --- /dev/null +++ b/charts/cilium/templates/spire/namespace.yaml @@ -0,0 +1,6 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.authentication.mutual.spire.install.namespace }} +{{- end }} diff --git a/charts/cilium/templates/spire/server/clusterrole.yaml b/charts/cilium/templates/spire/server/clusterrole.yaml new file mode 100644 index 00000000..95cc911a --- /dev/null +++ b/charts/cilium/templates/spire/server/clusterrole.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.server.serviceAccount.create -}} + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} +rules: +# ClusterRole to allow spire-server node attestor to query Token Review API +- apiGroups: [ "authentication.k8s.io" ] + resources: [ "tokenreviews" ] + verbs: [ "create" ] +# Required cluster role to allow spire-server to query k8s API server +# for pods for psat attestation +- apiGroups: [ "" ] + resources: [ "pods" ] + verbs: [ "get" ] +# Required cluster role to allow spire-server to query k8s API server +# for nodes for psat attestation +- apiGroups: [ "" ] + resources: [ "nodes","nodes/proxy" ] + verbs: [ "get" ] +{{- end }} diff --git a/charts/cilium/templates/spire/server/clusterrolebinding.yaml b/charts/cilium/templates/spire/server/clusterrolebinding.yaml new file mode 100644 index 00000000..aaefe4ff --- /dev/null +++ b/charts/cilium/templates/spire/server/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.server.serviceAccount.create -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} +roleRef: + kind: ClusterRole + name: spire-server + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +{{- end }} diff --git a/charts/cilium/templates/spire/server/configmap.yaml b/charts/cilium/templates/spire/server/configmap.yaml new file mode 100644 index 00000000..9785c11e --- /dev/null +++ b/charts/cilium/templates/spire/server/configmap.yaml @@ -0,0 +1,68 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: spire-server + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +data: + server.conf: | + server { + bind_address = "0.0.0.0" + bind_port = "8081" + socket_path = "/tmp/spire-server/private/api.sock" + trust_domain = {{ .Values.authentication.mutual.spire.trustDomain | quote }} + data_dir = "/run/spire/data" + log_level = "INFO" + ca_key_type = {{ .Values.authentication.mutual.spire.install.server.ca.keyType | quote }} + + ca_subject = { + country = [{{ .Values.authentication.mutual.spire.install.server.ca.subject.country | quote }}], + organization = [{{ .Values.authentication.mutual.spire.install.server.ca.subject.organization | quote }}], + common_name = {{ .Values.authentication.mutual.spire.install.server.ca.subject.commonName | quote }}, + } + + admin_ids = [ + "spiffe://{{ .Values.authentication.mutual.spire.trustDomain }}/cilium-operator", + ] + } + + plugins { + DataStore "sql" { + plugin_data { + database_type = "sqlite3" + connection_string = "/run/spire/data/datastore.sqlite3" + } + } + + NodeAttestor "k8s_psat" { + plugin_data { + clusters = { + {{ .Values.cluster.name | quote }} = { + use_token_review_api_validation = true + service_account_allow_list = ["{{ .Values.authentication.mutual.spire.install.namespace}}:{{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }}"] + } + } + } + } + + KeyManager "disk" { + plugin_data { + keys_path = "/run/spire/data/keys.json" + } + } + + Notifier "k8sbundle" { + plugin_data { + namespace = {{ .Values.authentication.mutual.spire.install.namespace | quote }} + } + } + } + + health_checks { + listener_enabled = true + bind_address = "0.0.0.0" + bind_port = "8080" + live_path = "/live" + ready_path = "/ready" + } +{{- end }} diff --git a/charts/cilium/templates/spire/server/role.yaml b/charts/cilium/templates/spire/server/role.yaml new file mode 100644 index 00000000..38e0b113 --- /dev/null +++ b/charts/cilium/templates/spire/server/role.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.server.serviceAccount.create -}} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +rules: +# Role (namespace scoped) to be able to push certificate bundles to a configmap +- apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "patch", "get", "list" ] +{{- end }} diff --git a/charts/cilium/templates/spire/server/rolebinding.yaml b/charts/cilium/templates/spire/server/rolebinding.yaml new file mode 100644 index 00000000..a3e18731 --- /dev/null +++ b/charts/cilium/templates/spire/server/rolebinding.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: spire-server +subjects: +- kind: ServiceAccount + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }}-pod + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }}-pod +subjects: +- kind: ServiceAccount + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +{{- end }} diff --git a/charts/cilium/templates/spire/server/service.yaml b/charts/cilium/templates/spire/server/service.yaml new file mode 100644 index 00000000..31fee016 --- /dev/null +++ b/charts/cilium/templates/spire/server/service.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: spire-server + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.install.server.service.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.authentication.mutual.spire.install.server.service.labels }} + labels: + {{- toYaml . | nindent 8 }} + {{- end }} +spec: + type: {{ .Values.authentication.mutual.spire.install.server.service.type }} + ports: + - name: grpc + port: 8081 + targetPort: grpc + protocol: TCP + selector: + app: spire-server +{{- end }} diff --git a/charts/cilium/templates/spire/server/serviceaccount.yaml b/charts/cilium/templates/spire/server/serviceaccount.yaml new file mode 100644 index 00000000..2d4aa4a3 --- /dev/null +++ b/charts/cilium/templates/spire/server/serviceaccount.yaml @@ -0,0 +1,7 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.server.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +{{- end }} diff --git a/charts/cilium/templates/spire/server/statefulset.yaml b/charts/cilium/templates/spire/server/statefulset.yaml new file mode 100644 index 00000000..0ea60a62 --- /dev/null +++ b/charts/cilium/templates/spire/server/statefulset.yaml @@ -0,0 +1,95 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: spire-server + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.install.server.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: spire-server + {{- with .Values.authentication.mutual.spire.install.server.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: + app: spire-server + serviceName: spire-server + template: + metadata: + labels: + app: spire-server + spec: + serviceAccountName: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + shareProcessNamespace: true + {{- if gt (len .Values.authentication.mutual.spire.install.server.initContainers) 0 }} + initContainers: + {{- toYaml .Values.authentication.mutual.spire.install.server.initContainers | nindent 8 }} + {{- end }} + containers: + - name: cilium-init + image: docker.io/library/busybox:1.35.0@sha256:223ae047b1065bd069aac01ae3ac8088b3ca4a527827e283b85112f29385fb1b + command: + - /bin/sh + - -c + - | + {{- tpl (.Files.Get "files/spire/init.bash") . | nindent 12 }} + - name: spire-server + image: {{ .Values.authentication.mutual.spire.install.server.image }} + args: + - -config + - /run/spire/config/server.conf + ports: + - name: grpc + containerPort: 8081 + volumeMounts: + - name: spire-config + mountPath: /run/spire/config + readOnly: true + {{- if .Values.authentication.mutual.spire.install.server.dataStorage.enabled }} + - name: spire-data + mountPath: /run/spire/data + readOnly: false + {{- end }} + - name: spire-server-socket + mountPath: /tmp/spire-server/private + readOnly: false + livenessProbe: + httpGet: + path: /live + port: 8080 + failureThreshold: 2 + initialDelaySeconds: 15 + periodSeconds: 60 + timeoutSeconds: 3 + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + volumes: + - name: spire-config + configMap: + name: spire-server + - name: spire-server-socket + hostPath: + path: /var/run/spire-server/sockets + type: DirectoryOrCreate + {{- if .Values.authentication.mutual.spire.install.server.dataStorage.enabled }} + volumeClaimTemplates: + - metadata: + name: spire-data + spec: + accessModes: + - {{ .Values.authentication.mutual.spire.install.server.dataStorage.accessMode | default "ReadWriteOnce" }} + resources: + requests: + storage: {{ .Values.authentication.mutual.spire.install.server.dataStorage.size }} + storageClassName: {{ .Values.authentication.mutual.spire.install.server.dataStorage.storageClass }} + {{- end }} +{{- end }} diff --git a/charts/cilium/templates/validate.yaml b/charts/cilium/templates/validate.yaml index d3260ccc..dfb5cb30 100644 --- a/charts/cilium/templates/validate.yaml +++ b/charts/cilium/templates/validate.yaml @@ -16,9 +16,11 @@ {{- end }} {{/* validate service monitoring CRDs */}} -{{- if and .Values.prometheus.enabled (or .Values.prometheus.serviceMonitor.enabled .Values.operator.prometheus.serviceMonitor.enabled) }} +{{- if or (and .Values.prometheus.enabled .Values.prometheus.serviceMonitor.enabled) (and .Values.operator.prometheus.enabled .Values.operator.prometheus.serviceMonitor.enabled) (and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled .Values.envoy.prometheus.serviceMonitor.enabled) (and .Values.proxy.prometheus.enabled .Values.hubble.relay.prometheus.enabled .Values.hubble.relay.prometheus.serviceMonitor.enabled) }} {{- if not (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") }} - {{ fail "Service Monitor requires monitoring.coreos.com/v1 CRDs. Please refer to https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml" }} + {{- if not .Values.prometheus.serviceMonitor.trustCRDsExist }} + {{ fail "Service Monitor requires monitoring.coreos.com/v1 CRDs. Please refer to https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml or set .Values.prometheus.serviceMonitor.trustCRDsExist=true" }} + {{- end }} {{- end }} {{- end }} @@ -46,12 +48,29 @@ {{- end }} {{- end }} -{{- if or .Values.ingressController.enabled .Values.gatewayAPI.enabled }} - {{- if hasKey .Values "kubeProxyReplacement" }} - {{- if and (ne .Values.kubeProxyReplacement "partial") (ne .Values.kubeProxyReplacement "strict") }} - {{ fail "Ingress/Gateway API controller requires .Values.kubeProxyReplacement to be set to either 'partial' or 'strict'" }} +{{- if or .Values.ingressController.enabled .Values.gatewayAPI.enabled (eq .Values.loadBalancer.l7.backend "envoy") }} + {{- if hasKey .Values "l7Proxy" }} + {{- if not .Values.l7Proxy }} + {{ fail "Ingress or Gateway API controller or Envoy L7 Load Balancer requires .Values.l7Proxy to be set to 'true'" }} {{- end }} - {{- else }} - {{ fail "Ingress/Gateway API controller requires .Values.kubeProxyReplacement to be set to either 'partial' or 'strict'" }} + {{- end }} +{{- end }} + +{{- if or .Values.envoyConfig.enabled .Values.ingressController.enabled .Values.gatewayAPI.enabled }} + {{- if or (eq (toString .Values.kubeProxyReplacement) "disabled") (and (not (hasKey .Values "kubeProxyReplacement")) (not (semverCompare ">=1.14" (default "1.14" .Values.upgradeCompatibility)))) }} + {{ fail "Ingress/Gateway API controller and EnvoyConfig require .Values.kubeProxyReplacement to be explicitly set to 'false' or 'true'" }} + {{- end }} +{{- end }} + +{{- if .Values.authentication.mutual.spire.enabled }} + {{- if not .Values.authentication.enabled }} + {{ fail "SPIRE integration requires .Values.authentication.enabled=true and .Values.authentication.mutual.spire.enabled=true" }} + {{- end }} +{{- end }} + +{{/* validate Cilium operator */}} +{{- if eq .Values.enableCiliumEndpointSlice true }} + {{- if eq .Values.disableEndpointCRD true }} + {{ fail "if Cilium Endpoint Slice is enabled (.Values.enableCiliumEndpointSlice=true), it requires .Values.disableEndpointCRD=false" }} {{- end }} {{- end }} diff --git a/charts/cilium/values.yaml b/charts/cilium/values.yaml index 5c396f1e..bbfed5b0 100644 --- a/charts/cilium/values.yaml +++ b/charts/cilium/values.yaml @@ -1,6 +1,7 @@ # File generated by install/kubernetes/Makefile; DO NOT EDIT. # This file is based on install/kubernetes/cilium/values.yaml.tmpl. + # upgradeCompatibility helps users upgrading to ensure that the configMap for # Cilium will not change critical values to ensure continued operation # This is flag is not required for new installations. @@ -39,8 +40,20 @@ k8sServiceHost: "" # -- (string) Kubernetes service port k8sServicePort: "" +# -- Configure the client side rate limit for the agent and operator +# +# If the amount of requests to the Kubernetes API server exceeds the configured +# rate limit, the agent and operator will start to throttle requests by delaying +# them until there is budget or the request times out. +k8sClientRateLimit: + # -- The sustained request rate in requests per second. + qps: 5 + # -- The burst request rate in requests per second. + # The rate limiter will allow short bursts with a higher rate. + burst: 10 + cluster: - # -- Name of the cluster. Only required for Cluster Mesh. + # -- Name of the cluster. Only required for Cluster Mesh and mutual authentication with SPIRE. name: default # -- (int) Unique ID of the cluster. Must be unique across all connected # clusters and in the range of 1 to 255. Only required for Cluster Mesh, @@ -55,6 +68,22 @@ serviceAccounts: name: cilium automount: true annotations: {} + nodeinit: + create: true + # -- Enabled is temporary until https://github.com/cilium/cilium-cli/issues/1396 is implemented. + # Cilium CLI doesn't create the SAs for node-init, thus the workaround. Helm is not affected by + # this issue. Name and automount can be configured, if enabled is set to true. + # Otherwise, they are ignored. Enabled can be removed once the issue is fixed. + # Cilium-nodeinit DS must also be fixed. + enabled: false + name: cilium-nodeinit + automount: true + annotations: {} + envoy: + create: true + name: cilium-envoy + automount: true + annotations: {} etcd: create: true name: cilium-etcd-operator @@ -114,10 +143,10 @@ rollOutCiliumPods: false image: override: ~ repository: "quay.io/cilium/cilium" - tag: "v1.13.3" + tag: "v1.14.3" pullPolicy: "IfNotPresent" # cilium-digest - digest: "sha256:77176464a1e11ea7e89e984ac7db365e7af39851507e94f137dcf56c87746314" + digest: "sha256:e5ca22526e01469f8d10c14e2339a82a13ad70d9a359b879024715540eef4ace" useDigest: true # -- Affinity for cilium-agent. @@ -335,6 +364,31 @@ nat46x64Gateway: # -- Enable RFC8215-prefixed translation enabled: false +# -- EnableHighScaleIPcache enables the special ipcache mode for high scale +# clusters. The ipcache content will be reduced to the strict minimum and +# traffic will be encapsulated to carry security identities. +highScaleIPcache: + # -- Enable the high scale mode for the ipcache. + enabled: false + +# -- Configure L2 announcements +l2announcements: + # -- Enable L2 announcements + enabled: false + # -- If a lease is not renewed for X duration, the current leader is considered dead, a new leader is picked + # leaseDuration: 15s + # -- The interval at which the leader will renew the lease + # leaseRenewDeadline: 5s + # -- The timeout between retries if renewal fails + # leaseRetryPeriod: 2s + +# -- Configure L2 pod announcements +l2podAnnouncements: + # -- Enable L2 pod announcements + enabled: false + # -- Interface used for sending Gratuitous ARP pod announcements + interface: "eth0" + # -- Configure BGP bgp: # -- Enable BGP support inside Cilium; embeds a new ConfigMap for BGP inside @@ -358,16 +412,25 @@ pmtuDiscovery: enabled: false bpf: + autoMount: + # -- Enable automatic mount of BPF filesystem + # When `autoMount` is enabled, the BPF filesystem is mounted at + # `bpf.root` path on the underlying host and inside the cilium agent pod. + # If users disable `autoMount`, it's expected that users have mounted + # bpffs filesystem at the specified `bpf.root` volume, and then the + # volume will be mounted inside the cilium agent pod at the same path. + enabled: true # -- Configure the mount point for the BPF filesystem root: /sys/fs/bpf - # -- Enable BPF clock source probing for more efficient tick retrieval. - clockProbe: false - # -- Enables pre-allocation of eBPF map values. This increases # memory usage but can reduce latency. preallocateMaps: false + # -- (int) Configure the maximum number of entries in auth map. + # @default -- `524288` + authMapMax: ~ + # -- (int) Configure the maximum number of entries in the TCP connection tracking # table. # @default -- `524288` @@ -434,6 +497,9 @@ bpf: # @default -- `[]` vlanBypass: ~ +# -- Enable BPF clock source probing for more efficient tick retrieval. +bpfClockProbe: false + # -- Clean all eBPF datapath state from the initContainer of the cilium-agent # DaemonSet. # @@ -460,7 +526,7 @@ cni: # if you're removing Cilium from the cluster. Disable this to prevent the CNI # configuration file from being removed during agent upgrade, which can cause # nodes to go unmanageable. - uninstall: true + uninstall: false # -- Configure chaining on top of other CNI plugins. Possible values: # - none @@ -468,7 +534,14 @@ cni: # - flannel # - generic-veth # - portmap - chainingMode: none + chainingMode: ~ + + # -- A CNI network name in to which the Cilium plugin should be added as a chained plugin. + # This will cause the agent to watch for a CNI network with this network name. When it is + # found, this will be used as the basis for Cilium's CNI configuration file. If this is + # set, it assumes a chaining mode of generic-veth. As a special case, a chaining mode + # of aws-cni implies a chainingTarget of aws-cni. + chainingTarget: ~ # -- Make Cilium take ownership over the `/etc/cni/net.d` directory on the # node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. @@ -493,8 +566,9 @@ cni: # -- Specify the path to a CNI config to read from on agent start. # This can be useful if you want to manage your CNI # configuration outside of a Kubernetes environment. This parameter is - # mutually exclusive with the 'cni.configMap' parameter. - # readCniConf: /host/etc/cni/net.d/05-cilium.conf + # mutually exclusive with the 'cni.configMap' parameter. The agent will + # write this to 05-cilium.conflist on startup. + # readCniConf: /host/etc/cni/net.d/05-sample.conflist.input # -- When defined, configMap will mount the provided value as ConfigMap and # interpret the cniConf variable as CNI configuration file and write it @@ -517,18 +591,20 @@ cni: # @default -- `"0s"` conntrackGCInterval: "" +# -- (string) Configure the maximum frequency for the garbage collection of the +# connection tracking table. Only affects the automatic computation for the frequency +# and has no effect when 'conntrackGCInterval' is set. This can be set to more frequently +# clean up unused identities created from ToFQDN policies. +conntrackGCMaxInterval: "" + # -- Configure container runtime specific integration. +# Deprecated in favor of bpf.autoMount.enabled. To be removed in 1.15. containerRuntime: # -- Enables specific integrations for container runtimes. # Supported values: - # - containerd # - crio - # - docker # - none - # - auto (automatically detect the container runtime) integration: none - # -- Configure the path to the container runtime control socket. - # socketPath: /path/to/runtime.sock # -- (string) Configure timeout in which Cilium will exit if CRDs are not available # @default -- `"5m"` @@ -583,7 +659,7 @@ enableRuntimeDeviceDetection: false # -- Chains to ignore when installing feeder rules. # disableIptablesFeederRules: "" -# -- Limit egress masquerading to interface selector. +# -- Limit iptables-based egress masquerading to interface selector. # egressMasqueradeInterfaces: "" # -- Whether to enable CNP status updates. @@ -602,11 +678,28 @@ enableK8sEventHandover: false # -- Enable CiliumEndpointSlice feature. enableCiliumEndpointSlice: false +envoyConfig: + # -- Enable CiliumEnvoyConfig CRD + # CiliumEnvoyConfig CRD can also be implicitly enabled by other options. + enabled: false + + # -- SecretsNamespace is the namespace in which envoy SDS will retrieve secrets from. + secretsNamespace: + # -- Create secrets namespace for CiliumEnvoyConfig CRDs. + create: true + + # -- The name of the secret namespace to which Cilium agents are given read access. + name: cilium-secrets + ingressController: # -- Enable cilium ingress controller # This will automatically set enable-envoy-config as well. enabled: false + # -- Set cilium ingress controller to be the default ingress controller + # This will let cilium ingress controller route entries without ingress class set + default: false + # -- Default ingress load balancer mode # Supported values: shared, dedicated # For granular control, use the following annotations on the ingress resource @@ -621,6 +714,12 @@ ingressController: # from Ingress to the Load Balancer service ingressLBAnnotationPrefixes: ['service.beta.kubernetes.io', 'service.kubernetes.io', 'cloud.google.com'] + # -- Default secret namespace for ingresses without .spec.tls[].secretName set. + defaultSecretNamespace: + + # -- Default secret name for ingresses without .spec.tls[].secretName set. + defaultSecretName: + # -- SecretsNamespace is the namespace in which envoy SDS will retrieve TLS secrets from. secretsNamespace: # -- Create secrets namespace for Ingress. @@ -648,6 +747,13 @@ ingressController: insecureNodePort: ~ # -- Configure a specific nodePort for secure HTTPS traffic on the shared LB service secureNodePort : ~ + # -- Configure a specific loadBalancerClass on the shared LB service (requires Kubernetes 1.24+) + loadBalancerClass: ~ + # -- Configure a specific loadBalancerIP on the shared LB service + loadBalancerIP : ~ + # -- Configure if node port allocation is required for LB service + # ref: https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-nodeport-allocation + allocateLoadBalancerNodePorts: ~ gatewayAPI: # -- Enable support for Gateway API in cilium @@ -680,7 +786,7 @@ encryption: type: ipsec # -- Enable encryption for pure node to node traffic. - # This option is only effective when encryption.type is set to ipsec. + # This option is only effective when encryption.type is set to "wireguard". nodeEncryption: false ipsec: @@ -696,26 +802,34 @@ encryption: # -- The interface to use for encrypted traffic. interface: "" + # -- Enable the key watcher. If disabled, a restart of the agent will be + # necessary on key rotations. + keyWatcher: true + + # -- Maximum duration of the IPsec key rotation. The previous key will be + # removed after that delay. + keyRotationDuration: "5m" + wireguard: # -- Enables the fallback to the user-space implementation. userspaceFallback: false - # -- Deprecated in favor of encryption.ipsec.keyFile. + # -- Deprecated in favor of encryption.ipsec.keyFile. To be removed in 1.15. # Name of the key file inside the Kubernetes secret configured via secretName. # This option is only effective when encryption.type is set to ipsec. keyFile: keys - # -- Deprecated in favor of encryption.ipsec.mountPath. + # -- Deprecated in favor of encryption.ipsec.mountPath. To be removed in 1.15. # Path to mount the secret inside the Cilium pod. # This option is only effective when encryption.type is set to ipsec. mountPath: /etc/ipsec - # -- Deprecated in favor of encryption.ipsec.secretName. + # -- Deprecated in favor of encryption.ipsec.secretName. To be removed in 1.15. # Name of the Kubernetes secret containing the encryption keys. # This option is only effective when encryption.type is set to ipsec. secretName: cilium-ipsec-keys - # -- Deprecated in favor of encryption.ipsec.interface. + # -- Deprecated in favor of encryption.ipsec.interface. To be removed in 1.15. # The interface to use for encrypted traffic. # This option is only effective when encryption.type is set to ipsec. interface: "" @@ -735,11 +849,15 @@ endpointRoutes: # the cilium_host interface. enabled: false +k8sNetworkPolicy: + # -- Enable support for K8s NetworkPolicy + enabled: true + eni: # -- Enable Elastic Network Interface (ENI) integration. enabled: false # -- Update ENI Adapter limits from the EC2 API - updateEC2AdapterLimitViaAPI: false + updateEC2AdapterLimitViaAPI: true # -- Release IPs not used from the ENI awsReleaseExcessIPs: false # -- Enable ENI prefix delegation @@ -816,12 +934,18 @@ certgen: image: override: ~ repository: "quay.io/cilium/certgen" - tag: "v0.1.8@sha256:4a456552a5f192992a6edcec2febb1c54870d665173a33dc7d876129b199ddbd" + tag: "v0.1.9" + digest: "sha256:89a0847753686444daabde9474b48340993bd19c7bea66a46e45b2974b82041f" + useDigest: true pullPolicy: "IfNotPresent" # -- Seconds after which the completed job pod will be deleted ttlSecondsAfterFinished: 1800 # -- Labels to be added to hubble-certgen pods podLabels: {} + # -- Annotations to be added to the hubble-certgen initial Job and CronJob + annotations: + job: {} + cronJob: {} # -- Node tolerations for pod assignment on nodes with taints # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ tolerations: [] @@ -892,6 +1016,9 @@ hubble: replacement: ${1} # -- Metrics relabeling configs for the ServiceMonitor hubble metricRelabelings: ~ + # -- Grafana dashboards for hubble + # grafana can import dashboards based on the label and value + # ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards dashboards: enabled: false label: grafana_dashboard @@ -913,10 +1040,6 @@ hubble: skipUnknownCGroupIDs: ~ peerService: - # -- Enable a K8s Service for the Peer service, so that it can be accessed - # by a non-local client. This configuration option is deprecated, the peer - # service will be non-optional starting Cilium v1.14. - enabled: true # -- Service Port for the Peer service. # If not set, it is dynamically assigned to port 443 if TLS is enabled and to # port 80 if not. @@ -967,16 +1090,6 @@ hubble: # -- certmanager issuer used when hubble.tls.auto.method=certmanager. certManagerIssuerRef: {} - # -- Deprecated in favor of tls.ca. To be removed in 1.13. - # base64 encoded PEM values for the Hubble CA certificate and private key. - ca: - # -- Deprecated in favor of tls.ca.cert. To be removed in 1.13. - cert: "" - # -- Deprecated in favor of tls.ca.key. To be removed in 1.13. - # The CA private key (optional). If it is provided, then it will be - # used by hubble.tls.auto.method=cronJob to generate all other certificates. - # Otherwise, a ephemeral CA is generated if hubble.tls.auto.enabled=true. - key: "" # -- base64 encoded PEM values for the Hubble server certificate and private key server: cert: "" @@ -997,9 +1110,9 @@ hubble: image: override: ~ repository: "quay.io/cilium/hubble-relay" - tag: "v1.13.3" + tag: "v1.14.3" # hubble-relay-digest - digest: "sha256:19e4aae5ff72cd9fbcb7d2d16a1570533320a478acc015fc91a4d41a177cadf6" + digest: "sha256:3f5c425faca4f8a38e29d4f86db2aadd8f33c9f05debd78a2bb2a24dd9e565d8" useDigest: true pullPolicy: "IfNotPresent" @@ -1065,8 +1178,19 @@ hubble: rollingUpdate: maxUnavailable: 1 - # -- hubble-relay security context - securityContext: {} + # -- hubble-relay pod security context + podSecurityContext: + fsGroup: 65532 + + # -- hubble-relay container security context + securityContext: + # readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: + - ALL # -- hubble-relay service configuration. service: @@ -1095,6 +1219,10 @@ hubble: # When set to true, enable TLS on for Hubble Relay server # (ie: for clients connecting to the Hubble Relay API). enabled: false + # When set to true enforces mutual TLS between Hubble Relay server and its clients. + # False allow non-mutual TLS connections. + # This option has no effect when TLS is disabled. + mtls: false # These values need to be set manually if hubble.tls.auto.enabled is false. cert: "" key: "" @@ -1145,6 +1273,12 @@ hubble: # -- Metrics relabeling configs for the ServiceMonitor hubble-relay metricRelabelings: ~ + gops: + # -- Enable gops for hubble-relay + enabled: true + # -- Configure gops listen port for hubble-relay + port: 9893 + pprof: # -- Enable pprof for hubble-relay enabled: false @@ -1198,7 +1332,9 @@ hubble: image: override: ~ repository: "quay.io/cilium/hubble-ui-backend" - tag: "v0.11.0@sha256:14c04d11f78da5c363f88592abae8d2ecee3cbe009f443ef11df6ac5f692d839" + tag: "v0.12.1" + digest: "sha256:1f86f3400827a0451e6332262467f894eeb7caf0eb8779bd951e2caa9d027cbe" + useDigest: true pullPolicy: "IfNotPresent" # -- Hubble-ui backend security context. @@ -1227,7 +1363,9 @@ hubble: image: override: ~ repository: "quay.io/cilium/hubble-ui" - tag: "v0.11.0@sha256:bcb369c47cada2d4257d63d3749f7f87c91dde32e010b223597306de95d1ecc8" + tag: "v0.12.1" + digest: "sha256:9e5f81ee747866480ea1ac4630eb6975ff9227f9782b7c93919c081c33f38267" + useDigest: true pullPolicy: "IfNotPresent" # -- Hubble-ui frontend security context. @@ -1304,9 +1442,6 @@ hubble: # -- Security context to be added to Hubble UI pods securityContext: - # -- Deprecated in favor of hubble.ui.securityContext. - # Whether to set the security context on the Hubble UI pods. - enabled: true runAsUser: 1001 runAsGroup: 1001 fsGroup: 1001 @@ -1334,12 +1469,12 @@ hubble: className: "" hosts: - chart-example.local + labels: {} tls: [] # - secretName: chart-example-tls # hosts: # - chart-example.local - # -- Method to use for identity allocation (`crd` or `kvstore`). identityAllocationMode: "crd" @@ -1357,21 +1492,29 @@ ipam: # -- Configure IP Address Management mode. # ref: https://docs.cilium.io/en/stable/network/concepts/ipam/ mode: "cluster-pool" + # -- Maximum rate at which the CiliumNode custom resource is updated. + ciliumNodeUpdateRate: "15s" operator: - # -- Deprecated in favor of ipam.operator.clusterPoolIPv4PodCIDRList. - # IPv4 CIDR range to delegate to individual nodes for IPAM. - clusterPoolIPv4PodCIDR: "10.0.0.0/8" # -- IPv4 CIDR list range to delegate to individual nodes for IPAM. - clusterPoolIPv4PodCIDRList: [] + clusterPoolIPv4PodCIDRList: ["10.0.0.0/8"] # -- IPv4 CIDR mask size to delegate to individual nodes for IPAM. clusterPoolIPv4MaskSize: 24 - # -- Deprecated in favor of ipam.operator.clusterPoolIPv6PodCIDRList. - # IPv6 CIDR range to delegate to individual nodes for IPAM. - clusterPoolIPv6PodCIDR: "fd00::/104" - # -- IPv6 CIDR list range to delegate to individual nodes for IPAM. - clusterPoolIPv6PodCIDRList: [] + # -- IPv6 CIDR list range to delegate to individual nodes for IPAM. + clusterPoolIPv6PodCIDRList: ["fd00::/104"] # -- IPv6 CIDR mask size to delegate to individual nodes for IPAM. clusterPoolIPv6MaskSize: 120 + # -- IP pools to auto-create in multi-pool IPAM mode. + autoCreateCiliumPodIPPools: {} + # default: + # ipv4: + # cidrs: + # - 10.10.0.0/8 + # maskSize: 24 + # other: + # ipv6: + # cidrs: + # - fd00:100::/80 + # maskSize: 96 # -- The maximum burst size when rate limiting access to external APIs. # Also known as the token bucket capacity. # @default -- `20` @@ -1389,6 +1532,7 @@ ipMasqAgent: # config: # nonMasqueradeCIDRs: [] # masqLinkLocal: false + # masqLinkLocalIPv6: false # iptablesLockTimeout defines the iptables "--wait" option when invoked from Cilium. # iptablesLockTimeout: "5s" @@ -1435,9 +1579,9 @@ readinessProbe: periodSeconds: 30 # -- Configure the kube-proxy replacement in Cilium BPF datapath -# Valid options are "disabled", "partial", "strict". +# Valid options are "true", "false", "disabled" (deprecated), "partial" (deprecated), "strict" (deprecated). # ref: https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/ -#kubeProxyReplacement: "disabled" +#kubeProxyReplacement: "false" # -- healthz server bind address for the kube-proxy replacement. # To enable set the value to '0.0.0.0:10256' for all ipv4 @@ -1467,7 +1611,6 @@ localRedirectPolicy: false # -- Enables periodic logging of system load logSystemLoad: false - # -- Configure maglev consistent hashing maglev: {} # -- tableSize is the size (parameter M) for the backend table of one @@ -1480,9 +1623,12 @@ maglev: {} # -- Enables masquerading of IPv4 traffic leaving the node from endpoints. enableIPv4Masquerade: true -# -- Enables IPv6 BIG TCP support which increases maximum GSO/GRO limits for nodes and pods +# -- Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods enableIPv6BIGTCP: false +# -- Enables IPv4 BIG TCP support which increases maximum IPv4 GSO/GRO limits for nodes and pods +enableIPv4BIGTCP: false + # -- Enables masquerading of IPv6 traffic leaving the node from endpoints. enableIPv6Masquerade: true @@ -1493,6 +1639,10 @@ egressGateway: # -- Install egress gateway IP rules and routes in order to properly steer # egress gateway traffic to the correct ENI interface installRoutes: false + # -- Time between triggers of egress gateway state reconciliations + reconciliationTriggerInterval: 1s + # -- Maximum number of entries in egress gateway policy map + # maxPolicyEntries: 16384 vtep: # -- Enables VXLAN Tunnel Endpoint (VTEP) Integration (beta) to allow @@ -1572,7 +1722,7 @@ loadBalancer: # # Applicable values: # - envoy: Enable L7 load balancing via envoy proxy. This will automatically set enable-envoy-config as well. - # - disabled: Disable L7 load balancing. + # - disabled: Disable L7 load balancing by way of service annotation. backend: disabled # -- List of ports from service to be automatically redirected to above backend. # Any service exposing one of these ports will be automatically redirected. @@ -1642,21 +1792,225 @@ prometheus: replacement: ${1} # -- Metrics relabeling configs for the ServiceMonitor cilium-agent metricRelabelings: ~ - # -- Metrics that should be enabled or disabled from the default metric - # list. (+metric_foo to enable metric_foo , -metric_bar to disable - # metric_bar). + # -- Set to `true` and helm will not check for monitoring.coreos.com/v1 CRDs before deploying + trustCRDsExist: false + + # -- Metrics that should be enabled or disabled from the default metric list. + # The list is expected to be separated by a space. (+metric_foo to enable + # metric_foo , -metric_bar to disable metric_bar). # ref: https://docs.cilium.io/en/stable/observability/metrics/ metrics: ~ +# -- Grafana dashboards for cilium-agent +# grafana can import dashboards based on the label and value +# ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards +dashboards: + enabled: false + label: grafana_dashboard + namespace: ~ + labelValue: "1" + annotations: {} + # -- Configure Istio proxy options. proxy: + prometheus: + # -- Deprecated in favor of envoy.prometheus.enabled enabled: true - port: "9964" + # -- Deprecated in favor of envoy.prometheus.port + port: ~ # -- Regular expression matching compatible Istio sidecar istio-proxy # container image names sidecarImageRegex: "cilium/istio_proxy" +# Configure Cilium Envoy options. +envoy: + # -- Enable Envoy Proxy in standalone DaemonSet. + enabled: false + + log: + # -- The format string to use for laying out the log message metadata of Envoy. + format: "[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v" + # -- Path to a separate Envoy log file, if any. Defaults to /dev/stdout. + path: "" + + # -- Time in seconds after which a TCP connection attempt times out + connectTimeoutSeconds: 2 + # -- ProxyMaxRequestsPerConnection specifies the max_requests_per_connection setting for Envoy + maxRequestsPerConnection: 0 + # -- Set Envoy HTTP option max_connection_duration seconds. Default 0 (disable) + maxConnectionDurationSeconds: 0 + # -- Set Envoy upstream HTTP idle connection timeout seconds. + # Does not apply to connections with pending requests. Default 60s + idleTimeoutDurationSeconds: 60 + + # -- Envoy container image. + image: + override: ~ + repository: "quay.io/cilium/cilium-envoy" + tag: "v1.25.10-f71a313bd0daee41470af31ce6ea20c750fe35dd" + pullPolicy: "IfNotPresent" + digest: "sha256:bfa1e919ed02afc66e9ff36c1fd9148237fc8b8560a0b44d89acf144b0ffb08c" + useDigest: true + + # -- Additional containers added to the cilium Envoy DaemonSet. + extraContainers: [] + + # -- Additional envoy container arguments. + extraArgs: [] + + # -- Additional envoy container environment variables. + extraEnv: [] + + # -- Additional envoy hostPath mounts. + extraHostPathMounts: [] + # - name: host-mnt-data + # mountPath: /host/mnt/data + # hostPath: /mnt/data + # hostPathType: Directory + # readOnly: true + # mountPropagation: HostToContainer + + # -- Additional envoy volumes. + extraVolumes: [] + + # -- Additional envoy volumeMounts. + extraVolumeMounts: [] + + # -- Configure termination grace period for cilium-envoy DaemonSet. + terminationGracePeriodSeconds: 1 + + # -- TCP port for the health API. + healthPort: 9878 + + # -- cilium-envoy update strategy + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/#updating-a-daemonset + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 2 + # -- Roll out cilium envoy pods automatically when configmap is updated. + rollOutPods: false + + # -- Security Context for cilium-envoy pods. + podSecurityContext: {} + + # -- Annotations to be added to envoy pods + podAnnotations: {} + + # -- Labels to be added to envoy pods + podLabels: {} + + # -- Envoy resource limits & requests + # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: {} + # limits: + # cpu: 4000m + # memory: 4Gi + # requests: + # cpu: 100m + # memory: 512Mi + + startupProbe: + # -- failure threshold of startup probe. + # 105 x 2s translates to the old behaviour of the readiness probe (120s delay + 30 x 3s) + failureThreshold: 105 + # -- interval between checks of the startup probe + periodSeconds: 2 + livenessProbe: + # -- failure threshold of liveness probe + failureThreshold: 10 + # -- interval between checks of the liveness probe + periodSeconds: 30 + readinessProbe: + # -- failure threshold of readiness probe + failureThreshold: 3 + # -- interval between checks of the readiness probe + periodSeconds: 30 + + securityContext: + # -- User to run the pod with + # runAsUser: 0 + # -- Run the pod with elevated privileges + privileged: false + # -- SELinux options for the `cilium-envoy` container + seLinuxOptions: + level: 's0' + # Running with spc_t since we have removed the privileged mode. + # Users can change it to a different type as long as they have the + # type available on the system. + type: 'spc_t' + capabilities: + # -- Capabilities for the `cilium-envoy` container + envoy: + # Used since cilium proxy uses setting IPPROTO_IP/IP_TRANSPARENT + - NET_ADMIN + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + + # -- Affinity for cilium-envoy. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium-envoy + + # -- Node selector for cilium-envoy. + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for envoy scheduling to nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - operator: Exists + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- The priority class to use for cilium-envoy. + priorityClassName: ~ + + # -- DNS policy for Cilium envoy pods. + # Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy + dnsPolicy: ~ + + prometheus: + # -- Enable prometheus metrics for cilium-envoy + enabled: true + serviceMonitor: + # -- Enable service monitors. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor cilium-envoy + labels: {} + # -- Annotations to add to ServiceMonitor cilium-envoy + annotations: {} + # -- Interval for scrape metrics. + interval: "10s" + # -- Specify the Kubernetes namespace where Prometheus expects to find + # service monitors configured. + # namespace: "" + # -- Relabeling configs for the ServiceMonitor cilium-envoy + relabelings: + - sourceLabels: + - __meta_kubernetes_pod_node_name + targetLabel: node + replacement: ${1} + # -- Metrics relabeling configs for the ServiceMonitor cilium-envoy + metricRelabelings: ~ + # -- Serve prometheus metrics for cilium-envoy on the configured port + port: "9964" + # -- Enable use of the remote node identity. # ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity remoteNodeIdentity: true @@ -1682,11 +2036,6 @@ resourceQuotas: # in the node. sleepAfterInit: false -# -- Configure BPF socket operations configuration -sockops: - # enabled enables installation of socket options acceleration. - enabled: false - # -- Enable check of service source ranges (currently, only for LoadBalancer). svcSourceRangeCheck: true @@ -1703,7 +2052,8 @@ tls: secretsBackend: local # -- Base64 encoded PEM values for the CA certificate and private key. - # This can be used as common CA to generate certificates used by hubble and clustermesh components + # This can be used as common CA to generate certificates used by hubble and clustermesh components. + # It is neither required nor used when cert-manager is used to generate the certificates. ca: # -- Optional CA cert. If it is provided, it will be used by cilium to # generate all other certificates. Otherwise, an ephemeral CA is generated. @@ -1716,12 +2066,57 @@ tls: # -- Generated certificates validity duration in days. This will be used for auto generated CA. certValidityDuration: 1095 + # -- Configure the CA trust bundle used for the validation of the certificates + # leveraged by hubble and clustermesh. When enabled, it overrides the content of the + # 'ca.crt' field of the respective certificates, allowing for CA rotation with no down-time. + caBundle: + # -- Enable the use of the CA trust bundle. + enabled: false + + # -- Name of the ConfigMap containing the CA trust bundle. + name: cilium-root-ca.crt + + # -- Entry of the ConfigMap containing the CA trust bundle. + key: ca.crt + + # -- Use a Secret instead of a ConfigMap. + useSecret: false + + # If uncommented, creates the ConfigMap and fills it with the specified content. + # Otherwise, the ConfigMap is assumed to be already present in .Release.Namespace. + # + # content: | + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # -- Configure the encapsulation configuration for communication between nodes. +# Deprecated in favor of tunnelProtocol and routingMode. To be removed in 1.15. # Possible values: # - disabled -# - vxlan (default) +# - vxlan +# - geneve +# @default -- `"vxlan"` +tunnel: "" + +# -- Tunneling protocol to use in tunneling mode and for ad-hoc tunnels. +# Possible values: +# - "" +# - vxlan # - geneve -tunnel: "vxlan" +# @default -- `"vxlan"` +tunnelProtocol: "" + +# -- Enable native-routing mode or tunneling mode. +# Possible values: +# - "" +# - native +# - tunnel +# @default -- `"tunnel"` +routingMode: "" # -- Configure VXLAN and Geneve tunnel port. # @default -- Port 8472 for VXLAN, Port 6081 for Geneve @@ -1731,7 +2126,7 @@ tunnelPort: 0 MTU: 0 # -- Disable the usage of CiliumEndpoint CRD. -disableEndpointCRD: "false" +disableEndpointCRD: false wellKnownIdentities: # -- Enable the use of well-known identities. @@ -1745,7 +2140,9 @@ etcd: image: override: ~ repository: "quay.io/cilium/cilium-etcd-operator" - tag: "v2.0.7@sha256:04b8327f7f992693c2cb483b999041ed8f92efc8e14f2a5f3ab95574a65ea2dc" + tag: "v2.0.7" + digest: "sha256:04b8327f7f992693c2cb483b999041ed8f92efc8e14f2a5f3ab95574a65ea2dc" + useDigest: true pullPolicy: "IfNotPresent" # -- The priority class to use for cilium-etcd-operator @@ -1848,15 +2245,15 @@ operator: image: override: ~ repository: "quay.io/cilium/operator" - tag: "v1.13.3" + tag: "v1.14.3" # operator-generic-digest - genericDigest: "sha256:fa7003cbfdf8358cb71786afebc711b26e5e44a2ed99bd4944930bba915b8910" + genericDigest: "sha256:c9613277b72103ed36e9c0d16b9a17cafd507461d59340e432e3e9c23468b5e2" # operator-azure-digest - azureDigest: "sha256:7749b732d510954d9fb74f7e675b31b49100fd773e588c6fbbf42529acfb1be8" + azureDigest: "sha256:b5f6b37d7c90fec6d066686d59924fb03eb39288234492526e50b46f8d1e7c28" # operator-aws-digest - awsDigest: "sha256:394c40d156235d3c2004f77bb73402457092351cc6debdbc5727ba36fbd863ae" + awsDigest: "sha256:120a437f75bc9b06e969e5183ac36e346e5ad02e7eb144d5d14c2918f0455c63" # operator-alibabacloud-digest - alibabacloudDigest: "sha256:8dba4795cb38200746a2236623f5b84742ee2c56a8afda724c85f5027ea854eb" + alibabacloudDigest: "sha256:64fceb06c305f524a96138e7b165bdd868a4454af55210ddc5fc13de2b36f979" useDigest: true pullPolicy: "IfNotPresent" suffix: "" @@ -1875,8 +2272,8 @@ operator: updateStrategy: type: RollingUpdate rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 + maxSurge: 25% + maxUnavailable: 50% # -- Affinity for cilium-operator affinity: @@ -2005,6 +2402,16 @@ operator: # -- Metrics relabeling configs for the ServiceMonitor cilium-operator metricRelabelings: ~ + # -- Grafana dashboards for cilium-operator + # grafana can import dashboards based on the label and value + # ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards + dashboards: + enabled: false + label: grafana_dashboard + namespace: ~ + labelValue: "1" + annotations: {} + # -- Skip CRDs creation for cilium-operator skipCRDCreation: false @@ -2012,6 +2419,11 @@ operator: # pod running. removeNodeTaints: true + # -- Taint nodes where Cilium is scheduled but not running. This prevents pods + # from being scheduled to nodes where Cilium is not the default CNI provider. + # @default -- same as removeNodeTaints + setNodeTaints: ~ + # -- Set Node condition NetworkUnavailable to 'false' with the reason # 'CiliumIsUp' for nodes that have a healthy Cilium pod. setNodeNetworkStatus: true @@ -2031,7 +2443,7 @@ nodeinit: image: override: ~ repository: "quay.io/cilium/startup-script" - tag: "d69851597ea019af980891a4628fb36b7880ec26" + tag: "62093c5c233ea914bfa26a10ba41f8780d9b737f" pullPolicy: "IfNotPresent" # -- The priority class to use for the nodeinit pod. @@ -2044,6 +2456,12 @@ nodeinit: # -- Additional nodeinit environment variables. extraEnv: [] + # -- Additional nodeinit volumes. + extraVolumes: [] + + # -- Additional nodeinit volumeMounts. + extraVolumeMounts: [] + # -- Affinity for cilium-nodeinit affinity: {} @@ -2097,6 +2515,15 @@ nodeinit: # written by the node-init DaemonSet bootstrapFile: "/tmp/cilium-bootstrap.d/cilium-bootstrap-time" + # -- startup offers way to customize startup nodeinit script (pre and post position) + startup: + preScript: "" + postScript: "" + # -- prestop offers way to customize prestop nodeinit script (pre and post position) + prestop: + preScript: "" + postScript: "" + preflight: # -- Enable Cilium pre-flight resources (required for upgrade) enabled: false @@ -2105,9 +2532,9 @@ preflight: image: override: ~ repository: "quay.io/cilium/cilium" - tag: "v1.13.3" + tag: "v1.14.3" # cilium-digest - digest: "sha256:77176464a1e11ea7e89e984ac7db365e7af39851507e94f137dcf56c87746314" + digest: "sha256:e5ca22526e01469f8d10c14e2339a82a13ad70d9a359b879024715540eef4ace" useDigest: true pullPolicy: "IfNotPresent" @@ -2243,18 +2670,21 @@ clustermesh: # ips: # - 172.18.255.201 # # -- base64 encoded PEM values for the cluster client certificate, private key and certificate authority. + # # These fields can (and should) be omitted in case the CA is shared across clusters. In that case, the + # # "remote" private key and certificate available in the local cluster are automatically used instead. # tls: # cert: "" # key: "" + # caCert: "" apiserver: # -- Clustermesh API server image. image: override: ~ repository: "quay.io/cilium/clustermesh-apiserver" - tag: "v1.13.3" + tag: "v1.14.3" # clustermesh-apiserver-digest - digest: "sha256:5ad8e9dc17f5677d1d75b53a4e80ec2e5c4fcf4973ced8b30f8ad53933c6969a" + digest: "sha256:a148485acda60f5aa9fc217ed2376ed93b7de9dd90706addeb361c6df928c192" useDigest: true pullPolicy: "IfNotPresent" @@ -2263,7 +2693,9 @@ clustermesh: image: override: ~ repository: "quay.io/coreos/etcd" - tag: "v3.5.4@sha256:795d8660c48c439a7c3764c2330ed9222ab5db5bb524d8d0607cac76f7ba82a3" + tag: "v3.5.4" + digest: "sha256:795d8660c48c439a7c3764c2330ed9222ab5db5bb524d8d0607cac76f7ba82a3" + useDigest: true pullPolicy: "IfNotPresent" # -- Specifies the resources for etcd container in the apiserver @@ -2288,6 +2720,46 @@ clustermesh: # cpu: 100m # memory: 100Mi + kvstoremesh: + # -- Enable KVStoreMesh. KVStoreMesh caches the information retrieved + # from the remote clusters in the local etcd instance. + enabled: false + + # -- KVStoreMesh image. + image: + override: ~ + repository: "quay.io/cilium/kvstoremesh" + tag: "v1.14.3" + # kvstoremesh-digest + digest: "sha256:36d0b45c3f65ff3f391d5446b6434c5650d33576ba215a6e3159f0f8dc2ec693" + useDigest: true + pullPolicy: "IfNotPresent" + + # -- Additional KVStoreMesh arguments. + extraArgs: [] + + # -- Additional KVStoreMesh environment variables. + extraEnv: [] + + # -- Resource requests and limits for the KVStoreMesh container + resources: {} + # requests: + # cpu: 100m + # memory: 64Mi + # limits: + # cpu: 1000m + # memory: 1024M + + # -- Additional KVStoreMesh volumeMounts. + extraVolumeMounts: [] + + # -- KVStoreMesh Security context + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + service: # -- The type of service used for apiserver access. type: NodePort @@ -2309,9 +2781,18 @@ clustermesh: # For EKS LoadBalancer, use annotation service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 annotations: {} + # -- The externalTrafficPolicy of service used for apiserver access. + externalTrafficPolicy: + + # -- The internalTrafficPolicy of service used for apiserver access. + internalTrafficPolicy: + # -- Number of replicas run for the clustermesh-apiserver deployment. replicas: 1 + # -- Additional clustermesh-apiserver arguments. + extraArgs: [] + # -- Additional clustermesh-apiserver environment variables. extraEnv: [] @@ -2395,6 +2876,23 @@ clustermesh: priorityClassName: "" tls: + # -- Configure the clustermesh authentication mode. + # Supported values: + # - legacy: All clusters access remote clustermesh instances with the same + # username (i.e., remote). The "remote" certificate must be + # generated with CN=remote if provided manually. + # - migration: Intermediate mode required to upgrade from legacy to cluster + # (and vice versa) with no disruption. Specifically, it enables + # the creation of the per-cluster usernames, while still using + # the common one for authentication. The "remote" certificate must + # be generated with CN=remote if provided manually (same as legacy). + # - cluster: Each cluster accesses remote etcd instances with a username + # depending on the local cluster name (i.e., remote-). + # The "remote" certificate must be generated with CN=remote- + # if provided manually. Cluster mode is meaningful only when the same + # CA is shared across all clusters part of the mesh. + authMode: legacy + # -- Configure automatic TLS certificates generation. # A Kubernetes CronJob is used the generate any # certificates not provided by the user at installation @@ -2434,12 +2932,15 @@ clustermesh: # name: ca-issuer # -- certmanager issuer used when clustermesh.apiserver.tls.auto.method=certmanager. certManagerIssuerRef: {} - # -- base64 encoded PEM values for the ExternalWorkload CA certificate and private key. + # -- Deprecated in favor of tls.ca. To be removed in 1.15. + # base64 encoded PEM values for the ExternalWorkload CA certificate and private key. ca: - # -- Optional CA cert. If it is provided, it will be used by the 'cronJob' method to + # -- Deprecated in favor of tls.ca.cert. To be removed in 1.15. + # Optional CA cert. If it is provided, it will be used by the 'cronJob' method to # generate all other certificates. Otherwise, an ephemeral CA is generated. cert: "" - # -- Optional CA private key. If it is provided, it will be used by the 'cronJob' method to + # -- Deprecated in favor of tls.ca.key. To be removed in 1.15. + # Optional CA private key. If it is provided, it will be used by the 'cronJob' method to # generate all other certificates. Otherwise, an ephemeral CA is generated. key: "" # -- base64 encoded PEM values for the clustermesh-apiserver server certificate and private key. @@ -2467,6 +2968,62 @@ clustermesh: cert: "" key: "" + # clustermesh-apiserver Prometheus metrics configuration + metrics: + # -- Enables exporting apiserver metrics in OpenMetrics format. + enabled: true + # -- Configure the port the apiserver metric server listens on. + port: 9962 + + kvstoremesh: + # -- Enables exporting KVStoreMesh metrics in OpenMetrics format. + enabled: true + # -- Configure the port the KVStoreMesh metric server listens on. + port: 9964 + + etcd: + # -- Enables exporting etcd metrics in OpenMetrics format. + enabled: false + # -- Set level of detail for etcd metrics; specify 'extensive' to include server side gRPC histogram metrics. + mode: basic + # -- Configure the port the etcd metric server listens on. + port: 9963 + + serviceMonitor: + # -- Enable service monitor. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor clustermesh-apiserver + labels: {} + # -- Annotations to add to ServiceMonitor clustermesh-apiserver + annotations: {} + # -- Specify the Kubernetes namespace where Prometheus expects to find + # service monitors configured. + # namespace: "" + + # -- Interval for scrape metrics (apiserver metrics) + interval: "10s" + # -- Relabeling configs for the ServiceMonitor clustermesh-apiserver (apiserver metrics) + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (apiserver metrics) + metricRelabelings: ~ + + kvstoremesh: + # -- Interval for scrape metrics (KVStoreMesh metrics) + interval: "10s" + # -- Relabeling configs for the ServiceMonitor clustermesh-apiserver (KVStoreMesh metrics) + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (KVStoreMesh metrics) + metricRelabelings: ~ + + etcd: + # -- Interval for scrape metrics (etcd metrics) + interval: "10s" + # -- Relabeling configs for the ServiceMonitor clustermesh-apiserver (etcd metrics) + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (etcd metrics) + metricRelabelings: ~ + # -- Configure external workloads support externalWorkloads: # -- Enable support for external workloads, such as VMs (false by default). @@ -2515,8 +3072,11 @@ dnsProxy: idleConnectionGracePeriod: 0s # -- Maximum number of IPs to retain for expired DNS lookups with still-active connections. maxDeferredConnectionDeletes: 10000 - # -- The minimum time, in seconds, to use DNS data for toFQDNs policies. - minTtl: 3600 + # -- The minimum time, in seconds, to use DNS data for toFQDNs policies. If + # the upstream DNS server returns a DNS record with a shorter TTL, Cilium + # overwrites the TTL with this value. Setting this value to zero means that + # Cilium will honor the TTLs returned by the upstream DNS server. + minTtl: 0 # -- DNS cache data at this path is preloaded on agent startup. preCache: "" # -- Global port on which the in-agent DNS proxy should listen. Default 0 is a OS-assigned port. @@ -2528,3 +3088,104 @@ dnsProxy: sctp: # -- Enable SCTP support. NOTE: Currently, SCTP support does not support rewriting ports or multihoming. enabled: false + +# Configuration for types of authentication for Cilium (beta) +authentication: + # -- Enable authentication processing and garbage collection. + # Note that if disabled, policy enforcement will still block requests that require authentication. + # But the resulting authentication requests for these requests will not be processed, therefore the requests not be allowed. + enabled: true + # -- Buffer size of the channel Cilium uses to receive authentication events from the signal map. + queueSize: 1024 + # -- Buffer size of the channel Cilium uses to receive certificate expiration events from auth handlers. + rotatedIdentitiesQueueSize: 1024 + # -- Interval for garbage collection of auth map entries. + gcInterval: "5m0s" + # Configuration for Cilium's service-to-service mutual authentication using TLS handshakes. + # Note that this is not full mTLS support without also enabling encryption of some form. + # Current encryption options are Wireguard or IPSec, configured in encryption block above. + mutual: + # -- Port on the agent where mutual authentication handshakes between agents will be performed + port: 4250 + # Settings for SPIRE + spire: + # -- Enable SPIRE integration (beta) + enabled: false + # Settings to control the SPIRE installation and configuration + install: + # -- Enable SPIRE installation. + # This will only take effect only if authentication.mutual.spire.enabled is true + enabled: true + # -- SPIRE namespace to install into + namespace: cilium-spire + # SPIRE agent configuration + agent: + # -- SPIRE agent image + image: ghcr.io/spiffe/spire-agent:1.6.3@sha256:8eef9857bf223181ecef10d9bbcd2f7838f3689e9bd2445bede35066a732e823 + # -- SPIRE agent service account + serviceAccount: + create: true + name: spire-agent + # -- SPIRE agent annotations + annotations: { } + # -- SPIRE agent labels + labels: { } + # -- SPIRE Workload Attestor kubelet verification. + skipKubeletVerification: true + server: + # -- SPIRE server image + image: ghcr.io/spiffe/spire-server:1.6.3@sha256:f4bc49fb0bd1d817a6c46204cc7ce943c73fb0a5496a78e0e4dc20c9a816ad7f + # -- SPIRE server service account + serviceAccount: + create: true + name: spire-server + # -- SPIRE server init containers + initContainers: [] + # -- SPIRE server annotations + annotations: {} + # -- SPIRE server labels + labels: {} + # SPIRE server service configuration + service: + # -- Service type for the SPIRE server service + type: ClusterIP + # -- Annotations to be added to the SPIRE server service + annotations: {} + # -- Labels to be added to the SPIRE server service + labels: {} + # SPIRE server datastorage configuration + dataStorage: + # -- Enable SPIRE server data storage + enabled: true + # -- Size of the SPIRE server data storage + size: 1Gi + # -- Access mode of the SPIRE server data storage + accessMode: ReadWriteOnce + # -- StorageClass of the SPIRE server data storage + storageClass: null + # SPIRE CA configuration + ca: + # -- SPIRE CA key type + # AWS requires the use of RSA. EC cryptography is not supported + keyType: "rsa-4096" + # -- SPIRE CA Subject + subject: + country: "US" + organization: "SPIRE" + commonName: "Cilium SPIRE CA" + # -- SPIRE server address used by Cilium Operator + # + # If k8s Service DNS along with port number is used (e.g. ..svc(.*): format), + # Cilium Operator will resolve its address by looking up the clusterIP from Service resource. + # + # Example values: 10.0.0.1:8081, spire-server.cilium-spire.svc:8081 + serverAddress: ~ + # -- SPIFFE trust domain to use for fetching certificates + trustDomain: spiffe.cilium + # -- SPIRE socket path where the SPIRE delegated api agent is listening + adminSocketPath: /run/spire/sockets/admin.sock + # -- SPIRE socket path where the SPIRE workload agent is listening. + # Applies to both the Cilium Agent and Operator + agentSocketPath: /run/spire/sockets/agent/agent.sock + # -- SPIRE connection timeout + connectionTimeout: 30s diff --git a/charts/cilium/values.yaml.tmpl b/charts/cilium/values.yaml.tmpl index b0374a55..91be377a 100644 --- a/charts/cilium/values.yaml.tmpl +++ b/charts/cilium/values.yaml.tmpl @@ -1,3 +1,4 @@ + # upgradeCompatibility helps users upgrading to ensure that the configMap for # Cilium will not change critical values to ensure continued operation # This is flag is not required for new installations. @@ -36,8 +37,20 @@ k8sServiceHost: "" # -- (string) Kubernetes service port k8sServicePort: "" +# -- Configure the client side rate limit for the agent and operator +# +# If the amount of requests to the Kubernetes API server exceeds the configured +# rate limit, the agent and operator will start to throttle requests by delaying +# them until there is budget or the request times out. +k8sClientRateLimit: + # -- The sustained request rate in requests per second. + qps: 5 + # -- The burst request rate in requests per second. + # The rate limiter will allow short bursts with a higher rate. + burst: 10 + cluster: - # -- Name of the cluster. Only required for Cluster Mesh. + # -- Name of the cluster. Only required for Cluster Mesh and mutual authentication with SPIRE. name: default # -- (int) Unique ID of the cluster. Must be unique across all connected # clusters and in the range of 1 to 255. Only required for Cluster Mesh, @@ -52,6 +65,22 @@ serviceAccounts: name: cilium automount: true annotations: {} + nodeinit: + create: true + # -- Enabled is temporary until https://github.com/cilium/cilium-cli/issues/1396 is implemented. + # Cilium CLI doesn't create the SAs for node-init, thus the workaround. Helm is not affected by + # this issue. Name and automount can be configured, if enabled is set to true. + # Otherwise, they are ignored. Enabled can be removed once the issue is fixed. + # Cilium-nodeinit DS must also be fixed. + enabled: false + name: cilium-nodeinit + automount: true + annotations: {} + envoy: + create: true + name: cilium-envoy + automount: true + annotations: {} etcd: create: true name: cilium-etcd-operator @@ -332,6 +361,31 @@ nat46x64Gateway: # -- Enable RFC8215-prefixed translation enabled: false +# -- EnableHighScaleIPcache enables the special ipcache mode for high scale +# clusters. The ipcache content will be reduced to the strict minimum and +# traffic will be encapsulated to carry security identities. +highScaleIPcache: + # -- Enable the high scale mode for the ipcache. + enabled: false + +# -- Configure L2 announcements +l2announcements: + # -- Enable L2 announcements + enabled: false + # -- If a lease is not renewed for X duration, the current leader is considered dead, a new leader is picked + # leaseDuration: 15s + # -- The interval at which the leader will renew the lease + # leaseRenewDeadline: 5s + # -- The timeout between retries if renewal fails + # leaseRetryPeriod: 2s + +# -- Configure L2 pod announcements +l2podAnnouncements: + # -- Enable L2 pod announcements + enabled: false + # -- Interface used for sending Gratuitous ARP pod announcements + interface: "eth0" + # -- Configure BGP bgp: # -- Enable BGP support inside Cilium; embeds a new ConfigMap for BGP inside @@ -355,16 +409,25 @@ pmtuDiscovery: enabled: false bpf: + autoMount: + # -- Enable automatic mount of BPF filesystem + # When `autoMount` is enabled, the BPF filesystem is mounted at + # `bpf.root` path on the underlying host and inside the cilium agent pod. + # If users disable `autoMount`, it's expected that users have mounted + # bpffs filesystem at the specified `bpf.root` volume, and then the + # volume will be mounted inside the cilium agent pod at the same path. + enabled: true # -- Configure the mount point for the BPF filesystem root: /sys/fs/bpf - # -- Enable BPF clock source probing for more efficient tick retrieval. - clockProbe: false - # -- Enables pre-allocation of eBPF map values. This increases # memory usage but can reduce latency. preallocateMaps: false + # -- (int) Configure the maximum number of entries in auth map. + # @default -- `524288` + authMapMax: ~ + # -- (int) Configure the maximum number of entries in the TCP connection tracking # table. # @default -- `524288` @@ -431,6 +494,9 @@ bpf: # @default -- `[]` vlanBypass: ~ +# -- Enable BPF clock source probing for more efficient tick retrieval. +bpfClockProbe: false + # -- Clean all eBPF datapath state from the initContainer of the cilium-agent # DaemonSet. # @@ -457,7 +523,7 @@ cni: # if you're removing Cilium from the cluster. Disable this to prevent the CNI # configuration file from being removed during agent upgrade, which can cause # nodes to go unmanageable. - uninstall: true + uninstall: false # -- Configure chaining on top of other CNI plugins. Possible values: # - none @@ -465,7 +531,14 @@ cni: # - flannel # - generic-veth # - portmap - chainingMode: none + chainingMode: ~ + + # -- A CNI network name in to which the Cilium plugin should be added as a chained plugin. + # This will cause the agent to watch for a CNI network with this network name. When it is + # found, this will be used as the basis for Cilium's CNI configuration file. If this is + # set, it assumes a chaining mode of generic-veth. As a special case, a chaining mode + # of aws-cni implies a chainingTarget of aws-cni. + chainingTarget: ~ # -- Make Cilium take ownership over the `/etc/cni/net.d` directory on the # node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. @@ -490,8 +563,9 @@ cni: # -- Specify the path to a CNI config to read from on agent start. # This can be useful if you want to manage your CNI # configuration outside of a Kubernetes environment. This parameter is - # mutually exclusive with the 'cni.configMap' parameter. - # readCniConf: /host/etc/cni/net.d/05-cilium.conf + # mutually exclusive with the 'cni.configMap' parameter. The agent will + # write this to 05-cilium.conflist on startup. + # readCniConf: /host/etc/cni/net.d/05-sample.conflist.input # -- When defined, configMap will mount the provided value as ConfigMap and # interpret the cniConf variable as CNI configuration file and write it @@ -514,18 +588,20 @@ cni: # @default -- `"0s"` conntrackGCInterval: "" +# -- (string) Configure the maximum frequency for the garbage collection of the +# connection tracking table. Only affects the automatic computation for the frequency +# and has no effect when 'conntrackGCInterval' is set. This can be set to more frequently +# clean up unused identities created from ToFQDN policies. +conntrackGCMaxInterval: "" + # -- Configure container runtime specific integration. +# Deprecated in favor of bpf.autoMount.enabled. To be removed in 1.15. containerRuntime: # -- Enables specific integrations for container runtimes. # Supported values: - # - containerd # - crio - # - docker # - none - # - auto (automatically detect the container runtime) integration: none - # -- Configure the path to the container runtime control socket. - # socketPath: /path/to/runtime.sock # -- (string) Configure timeout in which Cilium will exit if CRDs are not available # @default -- `"5m"` @@ -580,7 +656,7 @@ enableRuntimeDeviceDetection: false # -- Chains to ignore when installing feeder rules. # disableIptablesFeederRules: "" -# -- Limit egress masquerading to interface selector. +# -- Limit iptables-based egress masquerading to interface selector. # egressMasqueradeInterfaces: "" # -- Whether to enable CNP status updates. @@ -599,11 +675,28 @@ enableK8sEventHandover: false # -- Enable CiliumEndpointSlice feature. enableCiliumEndpointSlice: false +envoyConfig: + # -- Enable CiliumEnvoyConfig CRD + # CiliumEnvoyConfig CRD can also be implicitly enabled by other options. + enabled: false + + # -- SecretsNamespace is the namespace in which envoy SDS will retrieve secrets from. + secretsNamespace: + # -- Create secrets namespace for CiliumEnvoyConfig CRDs. + create: true + + # -- The name of the secret namespace to which Cilium agents are given read access. + name: cilium-secrets + ingressController: # -- Enable cilium ingress controller # This will automatically set enable-envoy-config as well. enabled: false + # -- Set cilium ingress controller to be the default ingress controller + # This will let cilium ingress controller route entries without ingress class set + default: false + # -- Default ingress load balancer mode # Supported values: shared, dedicated # For granular control, use the following annotations on the ingress resource @@ -618,6 +711,12 @@ ingressController: # from Ingress to the Load Balancer service ingressLBAnnotationPrefixes: ['service.beta.kubernetes.io', 'service.kubernetes.io', 'cloud.google.com'] + # -- Default secret namespace for ingresses without .spec.tls[].secretName set. + defaultSecretNamespace: + + # -- Default secret name for ingresses without .spec.tls[].secretName set. + defaultSecretName: + # -- SecretsNamespace is the namespace in which envoy SDS will retrieve TLS secrets from. secretsNamespace: # -- Create secrets namespace for Ingress. @@ -645,6 +744,13 @@ ingressController: insecureNodePort: ~ # -- Configure a specific nodePort for secure HTTPS traffic on the shared LB service secureNodePort : ~ + # -- Configure a specific loadBalancerClass on the shared LB service (requires Kubernetes 1.24+) + loadBalancerClass: ~ + # -- Configure a specific loadBalancerIP on the shared LB service + loadBalancerIP : ~ + # -- Configure if node port allocation is required for LB service + # ref: https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-nodeport-allocation + allocateLoadBalancerNodePorts: ~ gatewayAPI: # -- Enable support for Gateway API in cilium @@ -677,7 +783,7 @@ encryption: type: ipsec # -- Enable encryption for pure node to node traffic. - # This option is only effective when encryption.type is set to ipsec. + # This option is only effective when encryption.type is set to "wireguard". nodeEncryption: false ipsec: @@ -693,26 +799,34 @@ encryption: # -- The interface to use for encrypted traffic. interface: "" + # -- Enable the key watcher. If disabled, a restart of the agent will be + # necessary on key rotations. + keyWatcher: true + + # -- Maximum duration of the IPsec key rotation. The previous key will be + # removed after that delay. + keyRotationDuration: "5m" + wireguard: # -- Enables the fallback to the user-space implementation. userspaceFallback: false - # -- Deprecated in favor of encryption.ipsec.keyFile. + # -- Deprecated in favor of encryption.ipsec.keyFile. To be removed in 1.15. # Name of the key file inside the Kubernetes secret configured via secretName. # This option is only effective when encryption.type is set to ipsec. keyFile: keys - # -- Deprecated in favor of encryption.ipsec.mountPath. + # -- Deprecated in favor of encryption.ipsec.mountPath. To be removed in 1.15. # Path to mount the secret inside the Cilium pod. # This option is only effective when encryption.type is set to ipsec. mountPath: /etc/ipsec - # -- Deprecated in favor of encryption.ipsec.secretName. + # -- Deprecated in favor of encryption.ipsec.secretName. To be removed in 1.15. # Name of the Kubernetes secret containing the encryption keys. # This option is only effective when encryption.type is set to ipsec. secretName: cilium-ipsec-keys - # -- Deprecated in favor of encryption.ipsec.interface. + # -- Deprecated in favor of encryption.ipsec.interface. To be removed in 1.15. # The interface to use for encrypted traffic. # This option is only effective when encryption.type is set to ipsec. interface: "" @@ -732,11 +846,15 @@ endpointRoutes: # the cilium_host interface. enabled: false +k8sNetworkPolicy: + # -- Enable support for K8s NetworkPolicy + enabled: true + eni: # -- Enable Elastic Network Interface (ENI) integration. enabled: false # -- Update ENI Adapter limits from the EC2 API - updateEC2AdapterLimitViaAPI: false + updateEC2AdapterLimitViaAPI: true # -- Release IPs not used from the ENI awsReleaseExcessIPs: false # -- Enable ENI prefix delegation @@ -814,11 +932,17 @@ certgen: override: ~ repository: "${CERTGEN_REPO}" tag: "${CERTGEN_VERSION}" + digest: "${CERTGEN_DIGEST}" + useDigest: true pullPolicy: "${PULL_POLICY}" # -- Seconds after which the completed job pod will be deleted ttlSecondsAfterFinished: 1800 # -- Labels to be added to hubble-certgen pods podLabels: {} + # -- Annotations to be added to the hubble-certgen initial Job and CronJob + annotations: + job: {} + cronJob: {} # -- Node tolerations for pod assignment on nodes with taints # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ tolerations: [] @@ -889,6 +1013,9 @@ hubble: replacement: ${1} # -- Metrics relabeling configs for the ServiceMonitor hubble metricRelabelings: ~ + # -- Grafana dashboards for hubble + # grafana can import dashboards based on the label and value + # ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards dashboards: enabled: false label: grafana_dashboard @@ -910,10 +1037,6 @@ hubble: skipUnknownCGroupIDs: ~ peerService: - # -- Enable a K8s Service for the Peer service, so that it can be accessed - # by a non-local client. This configuration option is deprecated, the peer - # service will be non-optional starting Cilium v1.14. - enabled: true # -- Service Port for the Peer service. # If not set, it is dynamically assigned to port 443 if TLS is enabled and to # port 80 if not. @@ -964,16 +1087,6 @@ hubble: # -- certmanager issuer used when hubble.tls.auto.method=certmanager. certManagerIssuerRef: {} - # -- Deprecated in favor of tls.ca. To be removed in 1.13. - # base64 encoded PEM values for the Hubble CA certificate and private key. - ca: - # -- Deprecated in favor of tls.ca.cert. To be removed in 1.13. - cert: "" - # -- Deprecated in favor of tls.ca.key. To be removed in 1.13. - # The CA private key (optional). If it is provided, then it will be - # used by hubble.tls.auto.method=cronJob to generate all other certificates. - # Otherwise, a ephemeral CA is generated if hubble.tls.auto.enabled=true. - key: "" # -- base64 encoded PEM values for the Hubble server certificate and private key server: cert: "" @@ -1062,8 +1175,19 @@ hubble: rollingUpdate: maxUnavailable: 1 - # -- hubble-relay security context - securityContext: {} + # -- hubble-relay pod security context + podSecurityContext: + fsGroup: 65532 + + # -- hubble-relay container security context + securityContext: + # readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: + - ALL # -- hubble-relay service configuration. service: @@ -1092,6 +1216,10 @@ hubble: # When set to true, enable TLS on for Hubble Relay server # (ie: for clients connecting to the Hubble Relay API). enabled: false + # When set to true enforces mutual TLS between Hubble Relay server and its clients. + # False allow non-mutual TLS connections. + # This option has no effect when TLS is disabled. + mtls: false # These values need to be set manually if hubble.tls.auto.enabled is false. cert: "" key: "" @@ -1142,6 +1270,12 @@ hubble: # -- Metrics relabeling configs for the ServiceMonitor hubble-relay metricRelabelings: ~ + gops: + # -- Enable gops for hubble-relay + enabled: true + # -- Configure gops listen port for hubble-relay + port: 9893 + pprof: # -- Enable pprof for hubble-relay enabled: false @@ -1196,6 +1330,8 @@ hubble: override: ~ repository: "${HUBBLE_UI_BACKEND_REPO}" tag: "${HUBBLE_UI_BACKEND_VERSION}" + digest: "${HUBBLE_UI_BACKEND_DIGEST}" + useDigest: true pullPolicy: "${PULL_POLICY}" # -- Hubble-ui backend security context. @@ -1225,6 +1361,8 @@ hubble: override: ~ repository: "${HUBBLE_UI_FRONTEND_REPO}" tag: "${HUBBLE_UI_FRONTEND_VERSION}" + digest: "${HUBBLE_UI_FRONTEND_DIGEST}" + useDigest: true pullPolicy: "${PULL_POLICY}" # -- Hubble-ui frontend security context. @@ -1301,9 +1439,6 @@ hubble: # -- Security context to be added to Hubble UI pods securityContext: - # -- Deprecated in favor of hubble.ui.securityContext. - # Whether to set the security context on the Hubble UI pods. - enabled: true runAsUser: 1001 runAsGroup: 1001 fsGroup: 1001 @@ -1331,12 +1466,12 @@ hubble: className: "" hosts: - chart-example.local + labels: {} tls: [] # - secretName: chart-example-tls # hosts: # - chart-example.local - # -- Method to use for identity allocation (`crd` or `kvstore`). identityAllocationMode: "crd" @@ -1354,21 +1489,29 @@ ipam: # -- Configure IP Address Management mode. # ref: https://docs.cilium.io/en/stable/network/concepts/ipam/ mode: "cluster-pool" + # -- Maximum rate at which the CiliumNode custom resource is updated. + ciliumNodeUpdateRate: "15s" operator: - # -- Deprecated in favor of ipam.operator.clusterPoolIPv4PodCIDRList. - # IPv4 CIDR range to delegate to individual nodes for IPAM. - clusterPoolIPv4PodCIDR: "10.0.0.0/8" # -- IPv4 CIDR list range to delegate to individual nodes for IPAM. - clusterPoolIPv4PodCIDRList: [] + clusterPoolIPv4PodCIDRList: ["10.0.0.0/8"] # -- IPv4 CIDR mask size to delegate to individual nodes for IPAM. clusterPoolIPv4MaskSize: 24 - # -- Deprecated in favor of ipam.operator.clusterPoolIPv6PodCIDRList. - # IPv6 CIDR range to delegate to individual nodes for IPAM. - clusterPoolIPv6PodCIDR: "fd00::/104" - # -- IPv6 CIDR list range to delegate to individual nodes for IPAM. - clusterPoolIPv6PodCIDRList: [] + # -- IPv6 CIDR list range to delegate to individual nodes for IPAM. + clusterPoolIPv6PodCIDRList: ["fd00::/104"] # -- IPv6 CIDR mask size to delegate to individual nodes for IPAM. clusterPoolIPv6MaskSize: 120 + # -- IP pools to auto-create in multi-pool IPAM mode. + autoCreateCiliumPodIPPools: {} + # default: + # ipv4: + # cidrs: + # - 10.10.0.0/8 + # maskSize: 24 + # other: + # ipv6: + # cidrs: + # - fd00:100::/80 + # maskSize: 96 # -- The maximum burst size when rate limiting access to external APIs. # Also known as the token bucket capacity. # @default -- `20` @@ -1386,6 +1529,7 @@ ipMasqAgent: # config: # nonMasqueradeCIDRs: [] # masqLinkLocal: false + # masqLinkLocalIPv6: false # iptablesLockTimeout defines the iptables "--wait" option when invoked from Cilium. # iptablesLockTimeout: "5s" @@ -1432,9 +1576,9 @@ readinessProbe: periodSeconds: 30 # -- Configure the kube-proxy replacement in Cilium BPF datapath -# Valid options are "disabled", "partial", "strict". +# Valid options are "true", "false", "disabled" (deprecated), "partial" (deprecated), "strict" (deprecated). # ref: https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/ -#kubeProxyReplacement: "disabled" +#kubeProxyReplacement: "false" # -- healthz server bind address for the kube-proxy replacement. # To enable set the value to '0.0.0.0:10256' for all ipv4 @@ -1464,7 +1608,6 @@ localRedirectPolicy: false # -- Enables periodic logging of system load logSystemLoad: false - # -- Configure maglev consistent hashing maglev: {} # -- tableSize is the size (parameter M) for the backend table of one @@ -1477,9 +1620,12 @@ maglev: {} # -- Enables masquerading of IPv4 traffic leaving the node from endpoints. enableIPv4Masquerade: true -# -- Enables IPv6 BIG TCP support which increases maximum GSO/GRO limits for nodes and pods +# -- Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods enableIPv6BIGTCP: false +# -- Enables IPv4 BIG TCP support which increases maximum IPv4 GSO/GRO limits for nodes and pods +enableIPv4BIGTCP: false + # -- Enables masquerading of IPv6 traffic leaving the node from endpoints. enableIPv6Masquerade: true @@ -1490,6 +1636,10 @@ egressGateway: # -- Install egress gateway IP rules and routes in order to properly steer # egress gateway traffic to the correct ENI interface installRoutes: false + # -- Time between triggers of egress gateway state reconciliations + reconciliationTriggerInterval: 1s + # -- Maximum number of entries in egress gateway policy map + # maxPolicyEntries: 16384 vtep: # -- Enables VXLAN Tunnel Endpoint (VTEP) Integration (beta) to allow @@ -1569,7 +1719,7 @@ loadBalancer: # # Applicable values: # - envoy: Enable L7 load balancing via envoy proxy. This will automatically set enable-envoy-config as well. - # - disabled: Disable L7 load balancing. + # - disabled: Disable L7 load balancing by way of service annotation. backend: disabled # -- List of ports from service to be automatically redirected to above backend. # Any service exposing one of these ports will be automatically redirected. @@ -1639,21 +1789,225 @@ prometheus: replacement: ${1} # -- Metrics relabeling configs for the ServiceMonitor cilium-agent metricRelabelings: ~ - # -- Metrics that should be enabled or disabled from the default metric - # list. (+metric_foo to enable metric_foo , -metric_bar to disable - # metric_bar). + # -- Set to `true` and helm will not check for monitoring.coreos.com/v1 CRDs before deploying + trustCRDsExist: false + + # -- Metrics that should be enabled or disabled from the default metric list. + # The list is expected to be separated by a space. (+metric_foo to enable + # metric_foo , -metric_bar to disable metric_bar). # ref: https://docs.cilium.io/en/stable/observability/metrics/ metrics: ~ +# -- Grafana dashboards for cilium-agent +# grafana can import dashboards based on the label and value +# ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards +dashboards: + enabled: false + label: grafana_dashboard + namespace: ~ + labelValue: "1" + annotations: {} + # -- Configure Istio proxy options. proxy: + prometheus: + # -- Deprecated in favor of envoy.prometheus.enabled enabled: true - port: "9964" + # -- Deprecated in favor of envoy.prometheus.port + port: ~ # -- Regular expression matching compatible Istio sidecar istio-proxy # container image names sidecarImageRegex: "cilium/istio_proxy" +# Configure Cilium Envoy options. +envoy: + # -- Enable Envoy Proxy in standalone DaemonSet. + enabled: false + + log: + # -- The format string to use for laying out the log message metadata of Envoy. + format: "[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v" + # -- Path to a separate Envoy log file, if any. Defaults to /dev/stdout. + path: "" + + # -- Time in seconds after which a TCP connection attempt times out + connectTimeoutSeconds: 2 + # -- ProxyMaxRequestsPerConnection specifies the max_requests_per_connection setting for Envoy + maxRequestsPerConnection: 0 + # -- Set Envoy HTTP option max_connection_duration seconds. Default 0 (disable) + maxConnectionDurationSeconds: 0 + # -- Set Envoy upstream HTTP idle connection timeout seconds. + # Does not apply to connections with pending requests. Default 60s + idleTimeoutDurationSeconds: 60 + + # -- Envoy container image. + image: + override: ~ + repository: "quay.io/cilium/cilium-envoy" + tag: "v1.25.10-f71a313bd0daee41470af31ce6ea20c750fe35dd" + pullPolicy: "${PULL_POLICY}" + digest: "sha256:bfa1e919ed02afc66e9ff36c1fd9148237fc8b8560a0b44d89acf144b0ffb08c" + useDigest: true + + # -- Additional containers added to the cilium Envoy DaemonSet. + extraContainers: [] + + # -- Additional envoy container arguments. + extraArgs: [] + + # -- Additional envoy container environment variables. + extraEnv: [] + + # -- Additional envoy hostPath mounts. + extraHostPathMounts: [] + # - name: host-mnt-data + # mountPath: /host/mnt/data + # hostPath: /mnt/data + # hostPathType: Directory + # readOnly: true + # mountPropagation: HostToContainer + + # -- Additional envoy volumes. + extraVolumes: [] + + # -- Additional envoy volumeMounts. + extraVolumeMounts: [] + + # -- Configure termination grace period for cilium-envoy DaemonSet. + terminationGracePeriodSeconds: 1 + + # -- TCP port for the health API. + healthPort: 9878 + + # -- cilium-envoy update strategy + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/#updating-a-daemonset + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 2 + # -- Roll out cilium envoy pods automatically when configmap is updated. + rollOutPods: false + + # -- Security Context for cilium-envoy pods. + podSecurityContext: {} + + # -- Annotations to be added to envoy pods + podAnnotations: {} + + # -- Labels to be added to envoy pods + podLabels: {} + + # -- Envoy resource limits & requests + # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: {} + # limits: + # cpu: 4000m + # memory: 4Gi + # requests: + # cpu: 100m + # memory: 512Mi + + startupProbe: + # -- failure threshold of startup probe. + # 105 x 2s translates to the old behaviour of the readiness probe (120s delay + 30 x 3s) + failureThreshold: 105 + # -- interval between checks of the startup probe + periodSeconds: 2 + livenessProbe: + # -- failure threshold of liveness probe + failureThreshold: 10 + # -- interval between checks of the liveness probe + periodSeconds: 30 + readinessProbe: + # -- failure threshold of readiness probe + failureThreshold: 3 + # -- interval between checks of the readiness probe + periodSeconds: 30 + + securityContext: + # -- User to run the pod with + # runAsUser: 0 + # -- Run the pod with elevated privileges + privileged: false + # -- SELinux options for the `cilium-envoy` container + seLinuxOptions: + level: 's0' + # Running with spc_t since we have removed the privileged mode. + # Users can change it to a different type as long as they have the + # type available on the system. + type: 'spc_t' + capabilities: + # -- Capabilities for the `cilium-envoy` container + envoy: + # Used since cilium proxy uses setting IPPROTO_IP/IP_TRANSPARENT + - NET_ADMIN + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + + # -- Affinity for cilium-envoy. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium-envoy + + # -- Node selector for cilium-envoy. + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for envoy scheduling to nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - operator: Exists + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- The priority class to use for cilium-envoy. + priorityClassName: ~ + + # -- DNS policy for Cilium envoy pods. + # Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy + dnsPolicy: ~ + + prometheus: + # -- Enable prometheus metrics for cilium-envoy + enabled: true + serviceMonitor: + # -- Enable service monitors. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor cilium-envoy + labels: {} + # -- Annotations to add to ServiceMonitor cilium-envoy + annotations: {} + # -- Interval for scrape metrics. + interval: "10s" + # -- Specify the Kubernetes namespace where Prometheus expects to find + # service monitors configured. + # namespace: "" + # -- Relabeling configs for the ServiceMonitor cilium-envoy + relabelings: + - sourceLabels: + - __meta_kubernetes_pod_node_name + targetLabel: node + replacement: ${1} + # -- Metrics relabeling configs for the ServiceMonitor cilium-envoy + metricRelabelings: ~ + # -- Serve prometheus metrics for cilium-envoy on the configured port + port: "9964" + # -- Enable use of the remote node identity. # ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity remoteNodeIdentity: true @@ -1679,11 +2033,6 @@ resourceQuotas: # in the node. sleepAfterInit: false -# -- Configure BPF socket operations configuration -sockops: - # enabled enables installation of socket options acceleration. - enabled: false - # -- Enable check of service source ranges (currently, only for LoadBalancer). svcSourceRangeCheck: true @@ -1700,7 +2049,8 @@ tls: secretsBackend: local # -- Base64 encoded PEM values for the CA certificate and private key. - # This can be used as common CA to generate certificates used by hubble and clustermesh components + # This can be used as common CA to generate certificates used by hubble and clustermesh components. + # It is neither required nor used when cert-manager is used to generate the certificates. ca: # -- Optional CA cert. If it is provided, it will be used by cilium to # generate all other certificates. Otherwise, an ephemeral CA is generated. @@ -1713,12 +2063,57 @@ tls: # -- Generated certificates validity duration in days. This will be used for auto generated CA. certValidityDuration: 1095 + # -- Configure the CA trust bundle used for the validation of the certificates + # leveraged by hubble and clustermesh. When enabled, it overrides the content of the + # 'ca.crt' field of the respective certificates, allowing for CA rotation with no down-time. + caBundle: + # -- Enable the use of the CA trust bundle. + enabled: false + + # -- Name of the ConfigMap containing the CA trust bundle. + name: cilium-root-ca.crt + + # -- Entry of the ConfigMap containing the CA trust bundle. + key: ca.crt + + # -- Use a Secret instead of a ConfigMap. + useSecret: false + + # If uncommented, creates the ConfigMap and fills it with the specified content. + # Otherwise, the ConfigMap is assumed to be already present in .Release.Namespace. + # + # content: | + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # -- Configure the encapsulation configuration for communication between nodes. +# Deprecated in favor of tunnelProtocol and routingMode. To be removed in 1.15. # Possible values: # - disabled -# - vxlan (default) +# - vxlan +# - geneve +# @default -- `"vxlan"` +tunnel: "" + +# -- Tunneling protocol to use in tunneling mode and for ad-hoc tunnels. +# Possible values: +# - "" +# - vxlan # - geneve -tunnel: "vxlan" +# @default -- `"vxlan"` +tunnelProtocol: "" + +# -- Enable native-routing mode or tunneling mode. +# Possible values: +# - "" +# - native +# - tunnel +# @default -- `"tunnel"` +routingMode: "" # -- Configure VXLAN and Geneve tunnel port. # @default -- Port 8472 for VXLAN, Port 6081 for Geneve @@ -1728,7 +2123,7 @@ tunnelPort: 0 MTU: 0 # -- Disable the usage of CiliumEndpoint CRD. -disableEndpointCRD: "false" +disableEndpointCRD: false wellKnownIdentities: # -- Enable the use of well-known identities. @@ -1743,6 +2138,8 @@ etcd: override: ~ repository: "${CILIUM_ETCD_OPERATOR_REPO}" tag: "${CILIUM_ETCD_OPERATOR_VERSION}" + digest: "${CILIUM_ETCD_OPERATOR_DIGEST}" + useDigest: true pullPolicy: "${PULL_POLICY}" # -- The priority class to use for cilium-etcd-operator @@ -1872,8 +2269,8 @@ operator: updateStrategy: type: RollingUpdate rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 + maxSurge: 25% + maxUnavailable: 50% # -- Affinity for cilium-operator affinity: @@ -2002,6 +2399,16 @@ operator: # -- Metrics relabeling configs for the ServiceMonitor cilium-operator metricRelabelings: ~ + # -- Grafana dashboards for cilium-operator + # grafana can import dashboards based on the label and value + # ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards + dashboards: + enabled: false + label: grafana_dashboard + namespace: ~ + labelValue: "1" + annotations: {} + # -- Skip CRDs creation for cilium-operator skipCRDCreation: false @@ -2009,6 +2416,11 @@ operator: # pod running. removeNodeTaints: true + # -- Taint nodes where Cilium is scheduled but not running. This prevents pods + # from being scheduled to nodes where Cilium is not the default CNI provider. + # @default -- same as removeNodeTaints + setNodeTaints: ~ + # -- Set Node condition NetworkUnavailable to 'false' with the reason # 'CiliumIsUp' for nodes that have a healthy Cilium pod. setNodeNetworkStatus: true @@ -2041,6 +2453,12 @@ nodeinit: # -- Additional nodeinit environment variables. extraEnv: [] + # -- Additional nodeinit volumes. + extraVolumes: [] + + # -- Additional nodeinit volumeMounts. + extraVolumeMounts: [] + # -- Affinity for cilium-nodeinit affinity: {} @@ -2094,6 +2512,15 @@ nodeinit: # written by the node-init DaemonSet bootstrapFile: "/tmp/cilium-bootstrap.d/cilium-bootstrap-time" + # -- startup offers way to customize startup nodeinit script (pre and post position) + startup: + preScript: "" + postScript: "" + # -- prestop offers way to customize prestop nodeinit script (pre and post position) + prestop: + preScript: "" + postScript: "" + preflight: # -- Enable Cilium pre-flight resources (required for upgrade) enabled: false @@ -2240,9 +2667,12 @@ clustermesh: # ips: # - 172.18.255.201 # # -- base64 encoded PEM values for the cluster client certificate, private key and certificate authority. + # # These fields can (and should) be omitted in case the CA is shared across clusters. In that case, the + # # "remote" private key and certificate available in the local cluster are automatically used instead. # tls: # cert: "" # key: "" + # caCert: "" apiserver: # -- Clustermesh API server image. @@ -2261,6 +2691,8 @@ clustermesh: override: ~ repository: "${ETCD_REPO}" tag: "${ETCD_VERSION}" + digest: "${ETCD_DIGEST}" + useDigest: true pullPolicy: "${PULL_POLICY}" # -- Specifies the resources for etcd container in the apiserver @@ -2285,6 +2717,46 @@ clustermesh: # cpu: 100m # memory: 100Mi + kvstoremesh: + # -- Enable KVStoreMesh. KVStoreMesh caches the information retrieved + # from the remote clusters in the local etcd instance. + enabled: false + + # -- KVStoreMesh image. + image: + override: ~ + repository: "${KVSTOREMESH_REPO}" + tag: "${CILIUM_VERSION}" + # kvstoremesh-digest + digest: ${KVSTOREMESH_DIGEST} + useDigest: ${USE_DIGESTS} + pullPolicy: "${PULL_POLICY}" + + # -- Additional KVStoreMesh arguments. + extraArgs: [] + + # -- Additional KVStoreMesh environment variables. + extraEnv: [] + + # -- Resource requests and limits for the KVStoreMesh container + resources: {} + # requests: + # cpu: 100m + # memory: 64Mi + # limits: + # cpu: 1000m + # memory: 1024M + + # -- Additional KVStoreMesh volumeMounts. + extraVolumeMounts: [] + + # -- KVStoreMesh Security context + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + service: # -- The type of service used for apiserver access. type: NodePort @@ -2306,9 +2778,18 @@ clustermesh: # For EKS LoadBalancer, use annotation service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 annotations: {} + # -- The externalTrafficPolicy of service used for apiserver access. + externalTrafficPolicy: + + # -- The internalTrafficPolicy of service used for apiserver access. + internalTrafficPolicy: + # -- Number of replicas run for the clustermesh-apiserver deployment. replicas: 1 + # -- Additional clustermesh-apiserver arguments. + extraArgs: [] + # -- Additional clustermesh-apiserver environment variables. extraEnv: [] @@ -2392,6 +2873,23 @@ clustermesh: priorityClassName: "" tls: + # -- Configure the clustermesh authentication mode. + # Supported values: + # - legacy: All clusters access remote clustermesh instances with the same + # username (i.e., remote). The "remote" certificate must be + # generated with CN=remote if provided manually. + # - migration: Intermediate mode required to upgrade from legacy to cluster + # (and vice versa) with no disruption. Specifically, it enables + # the creation of the per-cluster usernames, while still using + # the common one for authentication. The "remote" certificate must + # be generated with CN=remote if provided manually (same as legacy). + # - cluster: Each cluster accesses remote etcd instances with a username + # depending on the local cluster name (i.e., remote-). + # The "remote" certificate must be generated with CN=remote- + # if provided manually. Cluster mode is meaningful only when the same + # CA is shared across all clusters part of the mesh. + authMode: legacy + # -- Configure automatic TLS certificates generation. # A Kubernetes CronJob is used the generate any # certificates not provided by the user at installation @@ -2431,12 +2929,15 @@ clustermesh: # name: ca-issuer # -- certmanager issuer used when clustermesh.apiserver.tls.auto.method=certmanager. certManagerIssuerRef: {} - # -- base64 encoded PEM values for the ExternalWorkload CA certificate and private key. + # -- Deprecated in favor of tls.ca. To be removed in 1.15. + # base64 encoded PEM values for the ExternalWorkload CA certificate and private key. ca: - # -- Optional CA cert. If it is provided, it will be used by the 'cronJob' method to + # -- Deprecated in favor of tls.ca.cert. To be removed in 1.15. + # Optional CA cert. If it is provided, it will be used by the 'cronJob' method to # generate all other certificates. Otherwise, an ephemeral CA is generated. cert: "" - # -- Optional CA private key. If it is provided, it will be used by the 'cronJob' method to + # -- Deprecated in favor of tls.ca.key. To be removed in 1.15. + # Optional CA private key. If it is provided, it will be used by the 'cronJob' method to # generate all other certificates. Otherwise, an ephemeral CA is generated. key: "" # -- base64 encoded PEM values for the clustermesh-apiserver server certificate and private key. @@ -2464,6 +2965,62 @@ clustermesh: cert: "" key: "" + # clustermesh-apiserver Prometheus metrics configuration + metrics: + # -- Enables exporting apiserver metrics in OpenMetrics format. + enabled: true + # -- Configure the port the apiserver metric server listens on. + port: 9962 + + kvstoremesh: + # -- Enables exporting KVStoreMesh metrics in OpenMetrics format. + enabled: true + # -- Configure the port the KVStoreMesh metric server listens on. + port: 9964 + + etcd: + # -- Enables exporting etcd metrics in OpenMetrics format. + enabled: false + # -- Set level of detail for etcd metrics; specify 'extensive' to include server side gRPC histogram metrics. + mode: basic + # -- Configure the port the etcd metric server listens on. + port: 9963 + + serviceMonitor: + # -- Enable service monitor. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor clustermesh-apiserver + labels: {} + # -- Annotations to add to ServiceMonitor clustermesh-apiserver + annotations: {} + # -- Specify the Kubernetes namespace where Prometheus expects to find + # service monitors configured. + # namespace: "" + + # -- Interval for scrape metrics (apiserver metrics) + interval: "10s" + # -- Relabeling configs for the ServiceMonitor clustermesh-apiserver (apiserver metrics) + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (apiserver metrics) + metricRelabelings: ~ + + kvstoremesh: + # -- Interval for scrape metrics (KVStoreMesh metrics) + interval: "10s" + # -- Relabeling configs for the ServiceMonitor clustermesh-apiserver (KVStoreMesh metrics) + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (KVStoreMesh metrics) + metricRelabelings: ~ + + etcd: + # -- Interval for scrape metrics (etcd metrics) + interval: "10s" + # -- Relabeling configs for the ServiceMonitor clustermesh-apiserver (etcd metrics) + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (etcd metrics) + metricRelabelings: ~ + # -- Configure external workloads support externalWorkloads: # -- Enable support for external workloads, such as VMs (false by default). @@ -2512,8 +3069,11 @@ dnsProxy: idleConnectionGracePeriod: 0s # -- Maximum number of IPs to retain for expired DNS lookups with still-active connections. maxDeferredConnectionDeletes: 10000 - # -- The minimum time, in seconds, to use DNS data for toFQDNs policies. - minTtl: 3600 + # -- The minimum time, in seconds, to use DNS data for toFQDNs policies. If + # the upstream DNS server returns a DNS record with a shorter TTL, Cilium + # overwrites the TTL with this value. Setting this value to zero means that + # Cilium will honor the TTLs returned by the upstream DNS server. + minTtl: 0 # -- DNS cache data at this path is preloaded on agent startup. preCache: "" # -- Global port on which the in-agent DNS proxy should listen. Default 0 is a OS-assigned port. @@ -2525,3 +3085,104 @@ dnsProxy: sctp: # -- Enable SCTP support. NOTE: Currently, SCTP support does not support rewriting ports or multihoming. enabled: false + +# Configuration for types of authentication for Cilium (beta) +authentication: + # -- Enable authentication processing and garbage collection. + # Note that if disabled, policy enforcement will still block requests that require authentication. + # But the resulting authentication requests for these requests will not be processed, therefore the requests not be allowed. + enabled: true + # -- Buffer size of the channel Cilium uses to receive authentication events from the signal map. + queueSize: 1024 + # -- Buffer size of the channel Cilium uses to receive certificate expiration events from auth handlers. + rotatedIdentitiesQueueSize: 1024 + # -- Interval for garbage collection of auth map entries. + gcInterval: "5m0s" + # Configuration for Cilium's service-to-service mutual authentication using TLS handshakes. + # Note that this is not full mTLS support without also enabling encryption of some form. + # Current encryption options are Wireguard or IPSec, configured in encryption block above. + mutual: + # -- Port on the agent where mutual authentication handshakes between agents will be performed + port: 4250 + # Settings for SPIRE + spire: + # -- Enable SPIRE integration (beta) + enabled: false + # Settings to control the SPIRE installation and configuration + install: + # -- Enable SPIRE installation. + # This will only take effect only if authentication.mutual.spire.enabled is true + enabled: true + # -- SPIRE namespace to install into + namespace: cilium-spire + # SPIRE agent configuration + agent: + # -- SPIRE agent image + image: ghcr.io/spiffe/spire-agent:1.6.3@sha256:8eef9857bf223181ecef10d9bbcd2f7838f3689e9bd2445bede35066a732e823 + # -- SPIRE agent service account + serviceAccount: + create: true + name: spire-agent + # -- SPIRE agent annotations + annotations: { } + # -- SPIRE agent labels + labels: { } + # -- SPIRE Workload Attestor kubelet verification. + skipKubeletVerification: true + server: + # -- SPIRE server image + image: ghcr.io/spiffe/spire-server:1.6.3@sha256:f4bc49fb0bd1d817a6c46204cc7ce943c73fb0a5496a78e0e4dc20c9a816ad7f + # -- SPIRE server service account + serviceAccount: + create: true + name: spire-server + # -- SPIRE server init containers + initContainers: [] + # -- SPIRE server annotations + annotations: {} + # -- SPIRE server labels + labels: {} + # SPIRE server service configuration + service: + # -- Service type for the SPIRE server service + type: ClusterIP + # -- Annotations to be added to the SPIRE server service + annotations: {} + # -- Labels to be added to the SPIRE server service + labels: {} + # SPIRE server datastorage configuration + dataStorage: + # -- Enable SPIRE server data storage + enabled: true + # -- Size of the SPIRE server data storage + size: 1Gi + # -- Access mode of the SPIRE server data storage + accessMode: ReadWriteOnce + # -- StorageClass of the SPIRE server data storage + storageClass: null + # SPIRE CA configuration + ca: + # -- SPIRE CA key type + # AWS requires the use of RSA. EC cryptography is not supported + keyType: "rsa-4096" + # -- SPIRE CA Subject + subject: + country: "US" + organization: "SPIRE" + commonName: "Cilium SPIRE CA" + # -- SPIRE server address used by Cilium Operator + # + # If k8s Service DNS along with port number is used (e.g. ..svc(.*): format), + # Cilium Operator will resolve its address by looking up the clusterIP from Service resource. + # + # Example values: 10.0.0.1:8081, spire-server.cilium-spire.svc:8081 + serverAddress: ~ + # -- SPIFFE trust domain to use for fetching certificates + trustDomain: spiffe.cilium + # -- SPIRE socket path where the SPIRE delegated api agent is listening + adminSocketPath: /run/spire/sockets/admin.sock + # -- SPIRE socket path where the SPIRE workload agent is listening. + # Applies to both the Cilium Agent and Operator + agentSocketPath: /run/spire/sockets/agent/agent.sock + # -- SPIRE connection timeout + connectionTimeout: 30s diff --git a/hack/sync-charts.sh b/hack/sync-charts.sh index fe444e16..fad51327 100755 --- a/hack/sync-charts.sh +++ b/hack/sync-charts.sh @@ -30,6 +30,6 @@ CERT_MANAGER_VERSION=v1.13.1 curl -sL https://charts.jetstack.io/charts/cert-manager-${CERT_MANAGER_VERSION}.tgz \ | tar -xz -C ${ROOT}/charts -CILIUM_VERSION=1.13.3 +CILIUM_VERSION=1.14.3 curl -sL https://helm.cilium.io/cilium-${CILIUM_VERSION}.tgz \ | tar -xz -C ${ROOT}/charts