diff --git a/charts/server/templates/configmap-trust-bundle.yaml b/charts/server/templates/configmap-trust-bundle.yaml new file mode 100644 index 00000000..d18dd7d3 --- /dev/null +++ b/charts/server/templates/configmap-trust-bundle.yaml @@ -0,0 +1,10 @@ +{{- if .Values.common.trustBundle }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "zenith.componentname" (list . "trust-bundle") }} + labels: {{ include "zenith.componentLabels" (list . "trust-bundle") | nindent 4 }} +data: + ca-certificates.crt: | + {{- nindent 4 .Values.common.trustBundle }} +{{- end }} diff --git a/charts/server/templates/registrar/deployment.yaml b/charts/server/templates/registrar/deployment.yaml index 0d8011c9..8278db05 100644 --- a/charts/server/templates/registrar/deployment.yaml +++ b/charts/server/templates/registrar/deployment.yaml @@ -37,6 +37,11 @@ spec: - name: etc-zenith mountPath: /etc/zenith readOnly: true + {{- if .Values.common.trustBundle }} + - name: trust-bundle + mountPath: /etc/ssl/certs + readOnly: true + {{- end }} {{- with .Values.registrar.nodeSelector }} nodeSelector: {{ toYaml . | nindent 8 }} {{- end }} @@ -50,4 +55,9 @@ spec: - name: etc-zenith secret: secretName: {{ include "zenith.componentname" (list . "registrar-conf") }} + {{- if .Values.common.trustBundle }} + - name: trust-bundle + configMap: + name: {{ include "zenith.componentname" (list . "trust-bundle") }} + {{- end }} {{- end }} diff --git a/charts/server/templates/sshd/deployment.yaml b/charts/server/templates/sshd/deployment.yaml index 2812cd58..d9983582 100644 --- a/charts/server/templates/sshd/deployment.yaml +++ b/charts/server/templates/sshd/deployment.yaml @@ -46,6 +46,11 @@ spec: readOnly: true - name: var-run-sshd mountPath: /var/run/sshd + {{- if .Values.common.trustBundle }} + - name: trust-bundle + mountPath: /etc/ssl/certs + readOnly: true + {{- end }} {{- with .Values.sshd.nodeSelector }} nodeSelector: {{ toYaml . | nindent 8 }} {{- end }} @@ -70,4 +75,9 @@ spec: name: {{ include "zenith.componentname" (list . "sshd-conf") }} - name: var-run-sshd emptyDir: {} + {{- if .Values.common.trustBundle }} + - name: trust-bundle + configMap: + name: {{ include "zenith.componentname" (list . "trust-bundle") }} + {{- end }} {{- end }} diff --git a/charts/server/templates/sync/configmap.yaml b/charts/server/templates/sync/configmap.yaml index 775bfef2..5be1ae07 100644 --- a/charts/server/templates/sync/configmap.yaml +++ b/charts/server/templates/sync/configmap.yaml @@ -3,6 +3,9 @@ {{- $common := deepCopy .Values.common.ingress }} {{- $ingress := mergeOverwrite $global $common }} kubernetes: + {{- if .Values.common.trustBundle }} + trustBundleConfigmapName: {{ include "zenith.componentname" (list . "trust-bundle") }} + {{- end }} targetNamespace: {{ .Values.common.kubernetes.targetNamespace }} # By default, we use the same chart version for the service chart serviceChartVersion: {{ .Chart.Version }} diff --git a/charts/server/templates/sync/deployment.yaml b/charts/server/templates/sync/deployment.yaml index 909acc94..5ee9001e 100644 --- a/charts/server/templates/sync/deployment.yaml +++ b/charts/server/templates/sync/deployment.yaml @@ -46,6 +46,11 @@ spec: - name: etc-zenith mountPath: /etc/zenith readOnly: true + {{- if .Values.common.trustBundle }} + - name: trust-bundle + mountPath: /etc/ssl/certs + readOnly: true + {{- end }} - name: tmp mountPath: /tmp {{- with .Values.sync.nodeSelector }} @@ -61,6 +66,11 @@ spec: - name: etc-zenith configMap: name: {{ include "zenith.componentname" (list . "sync-conf") }} + {{- if .Values.common.trustBundle }} + - name: trust-bundle + configMap: + name: {{ include "zenith.componentname" (list . "trust-bundle") }} + {{- end }} # Mount a writable directory at /tmp - name: tmp emptyDir: {} diff --git a/charts/server/templates/sync/role-trust-bundle-reader.yaml b/charts/server/templates/sync/role-trust-bundle-reader.yaml new file mode 100644 index 00000000..6e14b39c --- /dev/null +++ b/charts/server/templates/sync/role-trust-bundle-reader.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.sync.enabled .Values.common.trustBundle }} +# This role allows the holder to read the trust bundle configmap in the release namespace +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "zenith.componentname" (list . "sync") }}-trust-bundle-reader + labels: {{ include "zenith.componentLabels" (list . "sync") | nindent 4 }} +rules: + # We only need access to the named configmap + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - {{ include "zenith.componentname" (list . "trust-bundle") }} + verbs: + - list + - get + - watch +{{- end }} diff --git a/charts/server/templates/sync/rolebinding-trust-bundle-reader.yaml b/charts/server/templates/sync/rolebinding-trust-bundle-reader.yaml new file mode 100644 index 00000000..eeebdb3b --- /dev/null +++ b/charts/server/templates/sync/rolebinding-trust-bundle-reader.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.sync.enabled .Values.common.trustBundle }} +# This role binding allows the sync service account to access the trust bundle configmap +# in the release namespace +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "zenith.componentname" (list . "sync") }}-trust-bundle-reader + labels: {{ include "zenith.componentLabels" (list . "sync") | nindent 4 }} +subjects: + - kind: ServiceAccount + namespace: {{ .Release.Namespace }} + name: {{ include "zenith.componentname" (list . "sync") }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "zenith.componentname" (list . "sync") }}-trust-bundle-reader +{{- end }} diff --git a/charts/server/values.yaml b/charts/server/values.yaml index e94b55d6..c8073df1 100644 --- a/charts/server/values.yaml +++ b/charts/server/values.yaml @@ -25,6 +25,8 @@ global: # Common configuration common: + # A bundle of trusted CAs to use instead of the defaults + trustBundle: # Ingress configuration # This overrides global.ingress, and can be overridden by component-specific settings ingress: {} diff --git a/client/Dockerfile b/client/Dockerfile index f3e57578..be5b9e78 100644 --- a/client/Dockerfile +++ b/client/Dockerfile @@ -19,6 +19,10 @@ FROM ubuntu:jammy # Don't buffer stdout and stderr as it breaks realtime logging ENV PYTHONUNBUFFERED 1 +# Make requests use the system trust roots +# By default, this means we use the roots baked into the image +ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt + # Create the user that will be used to run the client process ENV ZENITH_UID 1001 ENV ZENITH_GID 1001 diff --git a/operator/Dockerfile b/operator/Dockerfile index e4db2421..d180efec 100644 --- a/operator/Dockerfile +++ b/operator/Dockerfile @@ -23,6 +23,10 @@ FROM ubuntu:jammy # Don't buffer stdout and stderr as it breaks realtime logging ENV PYTHONUNBUFFERED 1 +# Make httpx use the system trust roots +# By default, this means we use the CAs from the ca-certificates package +ENV SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt + # Create the user that will be used to run the app ENV ZENITH_UID 1001 ENV ZENITH_GID 1001 diff --git a/registrar/Dockerfile b/registrar/Dockerfile index 303490bd..65071ebf 100644 --- a/registrar/Dockerfile +++ b/registrar/Dockerfile @@ -19,6 +19,10 @@ FROM ubuntu:jammy # Don't buffer stdout and stderr as it breaks realtime logging ENV PYTHONUNBUFFERED 1 +# Make httpx use the system trust roots +# By default, this means we use the CAs from the ca-certificates package +ENV SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt + # Create the user that will be used to run the app ENV ZENITH_UID 1001 ENV ZENITH_GID 1001 diff --git a/sshd/Dockerfile b/sshd/Dockerfile index 1379091d..986d4c09 100644 --- a/sshd/Dockerfile +++ b/sshd/Dockerfile @@ -19,6 +19,11 @@ FROM ubuntu:jammy # Don't buffer stdout and stderr as it breaks realtime logging ENV PYTHONUNBUFFERED 1 +# Make requests and httpx use the system trust roots +# By default, this means we use the CAs from the ca-certificates package +ENV SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt +ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt + # Create an unprivileged user to accept tunnel requests # The user has a home directory, a restricted shell to allow the tunnel script # to run and an empty password to allow anonymous SSH diff --git a/sync/Dockerfile b/sync/Dockerfile index f2c570d4..9e910a2b 100644 --- a/sync/Dockerfile +++ b/sync/Dockerfile @@ -59,6 +59,10 @@ FROM ubuntu:jammy # Don't buffer stdout and stderr as it breaks realtime logging ENV PYTHONUNBUFFERED 1 +# Make httpx use the system trust roots +# By default, this means we use the CAs from the ca-certificates package +ENV SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt + # Tell Helm to use /tmp for mutable data ENV HELM_CACHE_HOME /tmp/helm/cache ENV HELM_CONFIG_HOME /tmp/helm/config diff --git a/sync/zenith/sync/config.py b/sync/zenith/sync/config.py index 572883b8..7b803de4 100644 --- a/sync/zenith/sync/config.py +++ b/sync/zenith/sync/config.py @@ -212,12 +212,16 @@ class KubernetesConfig(Section): #: Default values for releases of the service chart service_default_values: t.Dict[str, t.Any] = Field(default_factory = dict) + #: The name of a configmap containing a trust bundle + #: If not given, the default trust will be used + trust_bundle_configmap_name: t.Optional[str] = None + #: The label used to indicate a managed resource created_by_label: str = "app.kubernetes.io/created-by" #: The label used to indicate the corresponding Zenith service for a resource service_name_label: str = "zenith.stackhpc.com/service-name" - #: The annotation used to record that a secret is a mirror of another secret - tls_mirror_annotation: str = "zenith.stackhpc.com/mirrors" + #: The annotation used to record that a resource is a mirror of another + mirror_annotation: str = "zenith.stackhpc.com/mirrors" #: The maximum number of concurrent reconciliations reconciliation_max_concurrency: t.Annotated[int, Field(gt = 0)] = 20 #: The maximum delay between retries when backing off diff --git a/sync/zenith/sync/processor/helm.py b/sync/zenith/sync/processor/helm.py index a1a335de..57986a81 100644 --- a/sync/zenith/sync/processor/helm.py +++ b/sync/zenith/sync/processor/helm.py @@ -1,5 +1,6 @@ import asyncio import base64 +import copy import dataclasses import json import logging @@ -141,6 +142,34 @@ async def _reconcile_oidc_cookie_secret(self, service: model.Service) -> str: raise return base64.b64decode(secret.data["cookie-secret"]).decode() + def _get_trust_values(self) -> typing.Dict[str, typing.Any]: + """ + Returns the values for configuring a custom trust bundle for the OIDC callout. + """ + if self.config.trust_bundle_configmap_name: + # Make sure that the OIDC pods trust the bundle + return { + "oidc": { + "extraVolumes": [ + { + "name": "trust-bundle", + "configMap": { + "name": self.config.trust_bundle_configmap_name, + }, + }, + ], + "extraVolumeMounts": [ + { + "name": "trust-bundle", + "mountPath": "/etc/ssl/certs", + "readOnly": True, + }, + ], + }, + } + else: + return {} + def _get_service_values(self, service: model.Service) -> typing.Dict[str, typing.Any]: """ Returns the values for the core service configuration. @@ -287,6 +316,7 @@ async def service_updated(self, service: model.Service): version = self.config.service_chart_version ), self.config.service_default_values, + self._get_trust_values(), self._get_service_values(service), self._get_ingress_enabled(service), self._get_tls_values(service), @@ -326,91 +356,126 @@ async def metrics(self) -> typing.Iterable[metrics.Metric]: helm_status_metric.add_obj(release) return [helm_status_metric] - async def _update_tls_mirror(self, source_object): + async def _watch_events(self, ekresource, name, namespace): """ - Updates the mirror secret in the target namespace. + Yields watch events for the specified object, including a synthetic add/delete event + for the initial state. """ - self.logger.info( - "Updating mirrored TLS secret '%s' in namespace '%s'", - self.config.ingress.tls.secret_name, - self.config.target_namespace - ) - await self.ekclient.apply_object( - { - "apiVersion": "v1", - "kind": "Secret", - "metadata": { - "name": self.config.ingress.tls.secret_name, - "namespace": self.config.target_namespace, - "labels": { - self.config.created_by_label: "zenith-sync", - }, - "annotations": { - self.config.tls_mirror_annotation: "{}/{}".format( - source_object["metadata"]["namespace"], - source_object["metadata"]["name"] - ), + initial_state, events = await ekresource.watch_one(name, namespace = namespace) + if initial_state: + yield { + "type": "ADDED", + "object": initial_state + } + else: + yield { + "type": "DELETED", + "object": { + "metadata": { + "name": name, + "namespace": namespace, }, - }, - "type": source_object["type"], - "data": source_object["data"], - }, - force = True - ) + } + } + async for event in events: + yield event - async def _delete_tls_mirror(self): + async def _mirror_obj(self, ekresource, name, source_namespace, target_namespace): """ - Deletes the mirror secret in the target namespace. + Mirrors the specified object from the source namespace to the target namespace. """ self.logger.info( - "Deleting mirrored TLS secret '%s' in namespace '%s'", - self.config.ingress.tls.secret_name, - self.config.target_namespace - ) - secrets = await self.ekclient.api("v1").resource("secrets") - await secrets.delete( - self.config.ingress.tls.secret_name, - namespace = self.config.target_namespace + "Mirroring object [apiVersion: %s, kind: %s, name: %s, from: %s, to: %s]", + ekresource.api_version, + ekresource.kind, + name, + source_namespace, + target_namespace ) + async for event in self._watch_events(ekresource, name, source_namespace): + # Prepare the mirror object from the source object + mirror_obj = copy.deepcopy(event["object"]) + # Make sure that the API version and kind are present + mirror_obj.setdefault("apiVersion", ekresource.api_version) + mirror_obj.setdefault("kind", ekresource.kind) + # Replace the metadata object with one containing only what we need + mirror_obj["metadata"] = { + "name": mirror_obj["metadata"]["name"], + # Set the namespace to the target namespace + "namespace": target_namespace, + "labels": { self.config.created_by_label: "zenith-sync" }, + "annotations": { self.config.mirror_annotation: f"{source_namespace}/{name}" }, + } - async def _run_tls_mirror(self): + if event["type"] == "DELETED": + self.logger.info( + "Deleting mirrored object [apiVersion: %s, kind: %s, name: %s, ns: %s]", + ekresource.api_version, + ekresource.kind, + mirror_obj["metadata"]["name"], + mirror_obj["metadata"]["namespace"] + ) + await ekresource.delete( + mirror_obj["metadata"]["name"], + namespace = mirror_obj["metadata"]["namespace"] + ) + else: + self.logger.info( + "Updating mirrored object [apiVersion: %s, kind: %s, name: %s, ns: %s]", + ekresource.api_version, + ekresource.kind, + mirror_obj["metadata"]["name"], + mirror_obj["metadata"]["namespace"] + ) + await ekresource.server_side_apply( + mirror_obj["metadata"]["name"], + mirror_obj, + namespace = mirror_obj["metadata"]["namespace"], + force = True + ) + + async def _run_trust_bundle_mirror(self): """ - Runs the TLS mirror. + Continuously mirrors the trust bundle into the target namespace. """ # We need to mirror TLS secrets alongside handling events - if self.config.ingress.tls.enabled and self.config.ingress.tls.secret_name: - self.logger.info( - "Mirroring TLS secret [secret: %s, from: %s, to: %s]", - self.config.ingress.tls.secret_name, + if self.config.trust_bundle_configmap_name: + configmaps = await self.ekclient.api("v1").resource("configmaps") + await self._mirror_obj( + configmaps, + self.config.trust_bundle_configmap_name, self.config.self_namespace, self.config.target_namespace ) - # Watch the named secret in the release namespace for changes + else: + self.logger.info("Mirroring of trust bundle configmap is not required") + while True: + await asyncio.Event().wait() + + async def _run_tls_mirror(self): + """ + Continuously mirrors the TLS secret into the target namespace. + """ + # We need to mirror TLS secrets alongside handling events + if self.config.ingress.tls.enabled and self.config.ingress.tls.secret_name: secrets = await self.ekclient.api("v1").resource("secrets") - initial_state, events = await secrets.watch_one( + await self._mirror_obj( + secrets, self.config.ingress.tls.secret_name, - namespace = self.config.self_namespace + self.config.self_namespace, + self.config.target_namespace ) - # Mirror the changes to the target namespace - if initial_state: - await self._update_tls_mirror(initial_state) - else: - await self._delete_tls_mirror() - async for event in events: - if event["type"] != "DELETED": - await self._update_tls_mirror(event["object"]) - else: - await self._delete_tls_mirror() else: self.logger.info("Mirroring of wildcard TLS secret is not required") while True: - await asyncio.sleep(86400) + await asyncio.Event().wait() async def run(self, store: store.Store): # We need to run the TLS mirror alongside the main loop done, not_done = await asyncio.wait( [ asyncio.create_task(super().run(store)), + asyncio.create_task(self._run_trust_bundle_mirror()), asyncio.create_task(self._run_tls_mirror()), ], return_when = asyncio.FIRST_COMPLETED