From aeaf70c2e2219c5e392ec944a436dbb4e403e8b5 Mon Sep 17 00:00:00 2001 From: Luke Repko <luke.repko@rackspace.com> Date: Fri, 19 Apr 2024 19:41:10 -0500 Subject: [PATCH 1/5] Add optional kustomize letsencrypt overlay When using letsencrypt, you must annotate the named ingress with the letsencrypt cluster-issuer as well as any other desired acme.cert-manager.io annotations [1]. As most services have multiple ingresses (external vs internal), we must only annotate the ingress that will be accessed externally, else the ACME solver will not work correctly. Using this optional overlay will help ensure the expected ingress is annotated. So that the challenge url is accessible externally. 1. https://cert-manager.io/docs/usage/ingress/#supported-annotations --- .../cinder/letsencrypt/kustomization.yaml | 13 ++++++++ .../glance/letsencrypt/kustomization.yaml | 13 ++++++++ kustomize/heat/letsencrypt/kustomization.yaml | 22 +++++++++++++ .../horizon/letsencrypt/kustomization.yaml | 13 ++++++++ .../keystone/letsencrypt/kustomization.yaml | 13 ++++++++ .../neutron/letsencrypt/kustomization.yaml | 13 ++++++++ kustomize/nova/letsencrypt/kustomization.yaml | 31 ++++++++++++++++++ .../octavia/letsencrypt/kustomization.yaml | 13 ++++++++ .../placement/letsencrypt/kustomization.yaml | 13 ++++++++ .../skyline/letsencrypt/kustomization.yaml | 32 +++++++++++++++++++ 10 files changed, 176 insertions(+) create mode 100644 kustomize/cinder/letsencrypt/kustomization.yaml create mode 100644 kustomize/glance/letsencrypt/kustomization.yaml create mode 100644 kustomize/heat/letsencrypt/kustomization.yaml create mode 100644 kustomize/horizon/letsencrypt/kustomization.yaml create mode 100644 kustomize/keystone/letsencrypt/kustomization.yaml create mode 100644 kustomize/neutron/letsencrypt/kustomization.yaml create mode 100644 kustomize/nova/letsencrypt/kustomization.yaml create mode 100644 kustomize/octavia/letsencrypt/kustomization.yaml create mode 100644 kustomize/placement/letsencrypt/kustomization.yaml create mode 100644 kustomize/skyline/letsencrypt/kustomization.yaml diff --git a/kustomize/cinder/letsencrypt/kustomization.yaml b/kustomize/cinder/letsencrypt/kustomization.yaml new file mode 100644 index 00000000..600c2979 --- /dev/null +++ b/kustomize/cinder/letsencrypt/kustomization.yaml @@ -0,0 +1,13 @@ +bases: + - ../base + +patches: + - target: + kind: Ingress + name: cinder-namespace-fqdn + patch: |- + - op: add + path: /metadata/annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-edit-in-place: "true" diff --git a/kustomize/glance/letsencrypt/kustomization.yaml b/kustomize/glance/letsencrypt/kustomization.yaml new file mode 100644 index 00000000..36dd9501 --- /dev/null +++ b/kustomize/glance/letsencrypt/kustomization.yaml @@ -0,0 +1,13 @@ +bases: + - ../base + +patches: + - target: + kind: Ingress + name: glance-namespace-fqdn + patch: |- + - op: add + path: /metadata/annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-edit-in-place: "true" diff --git a/kustomize/heat/letsencrypt/kustomization.yaml b/kustomize/heat/letsencrypt/kustomization.yaml new file mode 100644 index 00000000..72dcb904 --- /dev/null +++ b/kustomize/heat/letsencrypt/kustomization.yaml @@ -0,0 +1,22 @@ +bases: + - ../base + +patches: + - target: + kind: Ingress + name: heat-namespace-fqdn + patch: |- + - op: add + path: /metadata/annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-edit-in-place: "true" + - target: + kind: Ingress + name: cloudformation-namespace-fqdn + patch: |- + - op: add + path: /metadata/annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-edit-in-place: "true" diff --git a/kustomize/horizon/letsencrypt/kustomization.yaml b/kustomize/horizon/letsencrypt/kustomization.yaml new file mode 100644 index 00000000..77ac7bd9 --- /dev/null +++ b/kustomize/horizon/letsencrypt/kustomization.yaml @@ -0,0 +1,13 @@ +bases: + - ../base + +patches: + - target: + kind: Ingress + name: horizon-namespace-fqdn + patch: |- + - op: add + path: /metadata/annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-edit-in-place: "true" diff --git a/kustomize/keystone/letsencrypt/kustomization.yaml b/kustomize/keystone/letsencrypt/kustomization.yaml new file mode 100644 index 00000000..394f3d8e --- /dev/null +++ b/kustomize/keystone/letsencrypt/kustomization.yaml @@ -0,0 +1,13 @@ +bases: + - ../base + +patches: + - target: + kind: Ingress + name: keystone-namespace-fqdn + patch: |- + - op: add + path: /metadata/annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-edit-in-place: "true" diff --git a/kustomize/neutron/letsencrypt/kustomization.yaml b/kustomize/neutron/letsencrypt/kustomization.yaml new file mode 100644 index 00000000..1e6b4270 --- /dev/null +++ b/kustomize/neutron/letsencrypt/kustomization.yaml @@ -0,0 +1,13 @@ +bases: + - ../base + +patches: + - target: + kind: Ingress + name: neutron-namespace-fqdn + patch: |- + - op: add + path: /metadata/annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-edit-in-place: "true" diff --git a/kustomize/nova/letsencrypt/kustomization.yaml b/kustomize/nova/letsencrypt/kustomization.yaml new file mode 100644 index 00000000..57a890ec --- /dev/null +++ b/kustomize/nova/letsencrypt/kustomization.yaml @@ -0,0 +1,31 @@ +bases: + - ../base + +patches: + - target: + kind: Ingress + name: nova-namespace-fqdn + patch: |- + - op: add + path: /metadata/annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-edit-in-place: "true" + - target: + kind: Ingress + name: metadata-namespace-fqdn + patch: |- + - op: add + path: /metadata/annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-edit-in-place: "true" + - target: + kind: Ingress + name: novncproxy-namespace-fqdn + patch: |- + - op: add + path: /metadata/annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-edit-in-place: "true" diff --git a/kustomize/octavia/letsencrypt/kustomization.yaml b/kustomize/octavia/letsencrypt/kustomization.yaml new file mode 100644 index 00000000..c6b2199f --- /dev/null +++ b/kustomize/octavia/letsencrypt/kustomization.yaml @@ -0,0 +1,13 @@ +bases: + - ../base + +patches: + - target: + kind: Ingress + name: octavia-namespace-fqdn + patch: |- + - op: add + path: /metadata/annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-edit-in-place: "true" diff --git a/kustomize/placement/letsencrypt/kustomization.yaml b/kustomize/placement/letsencrypt/kustomization.yaml new file mode 100644 index 00000000..0c6295f8 --- /dev/null +++ b/kustomize/placement/letsencrypt/kustomization.yaml @@ -0,0 +1,13 @@ +bases: + - ../base + +patches: + - target: + kind: Ingress + name: placement-namespace-fqdn + patch: |- + - op: add + path: /metadata/annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-edit-in-place: "true" diff --git a/kustomize/skyline/letsencrypt/kustomization.yaml b/kustomize/skyline/letsencrypt/kustomization.yaml new file mode 100644 index 00000000..6427ce88 --- /dev/null +++ b/kustomize/skyline/letsencrypt/kustomization.yaml @@ -0,0 +1,32 @@ +bases: + - ../base + +patches: + - target: + kind: Ingress + name: skyline + patch: |- + - op: add + path: /spec/rules + value: + - host: skyline.dfw-ospcv2-staging.ohthree.com + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: skyline-apiserver + port: + name: "s-apiserver" + - op: add + path: /spec/tls + value: + - hosts: + - skyline.dfw-ospcv2-staging.ohthree.com + secretName: skyline-tls-public + - op: add + path: /metadata/annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-edit-in-place: "true" From 2c0cf4a1d629114d77895d02c8791add9ccadf8d Mon Sep 17 00:00:00 2001 From: Luke Repko <luke.repko@rackspace.com> Date: Mon, 22 Apr 2024 13:25:01 -0500 Subject: [PATCH 2/5] fix: Update prod example helm overrides The original key paths were incorrect. Not all services use `api` here. --- docs/infrastructure-letsencrypt.md | 0 helm-configs/prod-example-openstack-overrides.yaml | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 docs/infrastructure-letsencrypt.md diff --git a/docs/infrastructure-letsencrypt.md b/docs/infrastructure-letsencrypt.md new file mode 100644 index 00000000..e69de29b diff --git a/helm-configs/prod-example-openstack-overrides.yaml b/helm-configs/prod-example-openstack-overrides.yaml index ed5e2b49..4acc4cc9 100644 --- a/helm-configs/prod-example-openstack-overrides.yaml +++ b/helm-configs/prod-example-openstack-overrides.yaml @@ -138,7 +138,7 @@ endpoints: kind: ClusterIssuer host: metadata.nova.dfw-ospcv2-staging.ohthree.com port: - api: + metadata: public: 443 scheme: public: https @@ -154,7 +154,7 @@ endpoints: kind: ClusterIssuer host: novnc.nova.dfw-ospcv2-staging.ohthree.com port: - api: + novnc_proxy: public: 443 scheme: public: https From 23fe8516a651ffaa1a8cdc26f87d7bc83df52805 Mon Sep 17 00:00:00 2001 From: Luke Repko <luke.repko@rackspace.com> Date: Mon, 22 Apr 2024 13:42:33 -0500 Subject: [PATCH 3/5] feat: Add letsencrypt issuer to helm overrides ex Expand the prod overrides example to include the necessary issuerRef name of letsencrypt-prod, it is a better fit. Remove the fake cert, key, and anchors to them. --- .../prod-example-openstack-overrides.yaml | 70 ++++--------------- 1 file changed, 15 insertions(+), 55 deletions(-) diff --git a/helm-configs/prod-example-openstack-overrides.yaml b/helm-configs/prod-example-openstack-overrides.yaml index 4acc4cc9..1d90681b 100644 --- a/helm-configs/prod-example-openstack-overrides.yaml +++ b/helm-configs/prod-example-openstack-overrides.yaml @@ -1,14 +1,4 @@ _region: ®ion DFW3 -_certificate: &crt | - -----BEGIN CERTIFICATE----- - -----END CERTIFICATE----- - -----BEGIN CERTIFICATE----- - -----END CERTIFICATE----- - -_certificate_key: &key | - -----BEGIN RSA PRIVATE KEY----- - -----END RSA PRIVATE KEY----- - images: tags: bootstrap: "docker.io/openstackhelm/heat:2023.1-ubuntu_jammy" @@ -114,11 +104,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: nova-tls-api issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: nova.dfw-ospcv2-staging.ohthree.com port: @@ -130,11 +118,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: metadata-tls-metadata issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: metadata.nova.dfw-ospcv2-staging.ohthree.com port: @@ -146,11 +132,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: nova-novncproxy-tls-proxy issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: novnc.nova.dfw-ospcv2-staging.ohthree.com port: @@ -162,11 +146,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: heat-tls-cfn issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: cloudformation.heat.dfw-ospcv2-staging.ohthree.com port: @@ -178,11 +160,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: heat-tls-cloudwatch issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: cloudwatch.heat.dfw-ospcv2-staging.ohthree.com port: @@ -194,11 +174,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: horizon-tls-web issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: dfw-ospcv2-staging.ohthree.com port: @@ -233,11 +211,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: keystone-tls-api issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: keystone.dfw-ospcv2-staging.ohthree.com port: @@ -253,11 +229,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: glance-tls-api issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: glance.dfw-ospcv2-staging.ohthree.com port: @@ -269,11 +243,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: octavia-tls-api issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: octavia.dfw-ospcv2-staging.ohthree.com port: @@ -285,11 +257,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: neutron-tls-server issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: neutron.dfw-ospcv2-staging.ohthree.com port: @@ -301,11 +271,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: heat-tls-api issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: heat.dfw-ospcv2-staging.ohthree.com port: @@ -317,11 +285,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: placement-tls-api issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: placement.dfw-ospcv2-staging.ohthree.com port: @@ -333,11 +299,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: cinder-tls-api issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: cinder.dfw-ospcv2-staging.ohthree.com port: @@ -349,11 +313,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: cinder-tls-api issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: cinder.dfw-ospcv2-staging.ohthree.com port: @@ -365,11 +327,9 @@ endpoints: host_fqdn_override: public: tls: - crt: *crt - key: *key secretName: cinder-tls-api issuerRef: - name: ca-issuer + name: letsencrypt-prod kind: ClusterIssuer host: cinder.dfw-ospcv2-staging.ohthree.com port: From f264782a8fc7a217f9c822871376a44a7a28073a Mon Sep 17 00:00:00 2001 From: Luke Repko <luke.repko@rackspace.com> Date: Mon, 22 Apr 2024 14:44:06 -0500 Subject: [PATCH 4/5] docs: Add letsencrypt cert deployment info --- docs/infrastructure-letsencrypt.md | 86 ++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 87 insertions(+) diff --git a/docs/infrastructure-letsencrypt.md b/docs/infrastructure-letsencrypt.md index e69de29b..29850f4d 100644 --- a/docs/infrastructure-letsencrypt.md +++ b/docs/infrastructure-letsencrypt.md @@ -0,0 +1,86 @@ +# Deploying Let's Encrypt Certificates + +Are you tired of manually renewing and deploying a countless number of +certificates across your environments? Us too! + +## Apply the Let's Encrypt Cluster Issuer + +Before we can have Cert Manager start coordinating Let's Encrypt certificate +requests for us, we need to add an ACME issuer with a valid, monitored +email (for expiration reminders and other important ACME related information). + +``` yaml +read -p "Enter a valid email address for use with ACME: " ACME_EMAIL; \ +cat <<EOF | kubectl apply -f - +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: ${ACME_EMAIL} + privateKeySecretRef: + name: letsencrypt-prod + solvers: + - http01: + ingress: + ingressClassName: nginx +EOF +``` + +## Use the proper TLS issuerRef + +!!! danger "Important for later helm installations!" + You must ensure your helm configuration is such that you set the + `endpoints.$service.host_fqdn_override.public.tls.issuerRef.name` for any + given endpoint to use our `letsencrypt-prod` ClusterIssuer. Similarly, + ensure that `endpoints.$service.host_fqdn_override.public.host` + is set to the external DNS hostname you plan to expose for a given + service endpoint. + +!!! example + You can find several examples of this in the + `helm-configs/prod-example-openstack-overrides.yaml`, one such example + for glance is below for reference. + ```yaml + endpoints: + image: + host_fqdn_override: + public: + tls: + secretName: glance-tls-api + issuerRef: + name: letsencrpyt-prod + kind: ClusterIssuer + host: glance.api.your.domain.tld + port: + api: + public: 443 + scheme: + public: https + ``` + +## Helm Kustomize Post Render Args + +In order for Cert Manager to set up the ACME challenge, it needs to know which +ingress to target. We do this via a kustomize overlay that injects the +needed annotation(s) to the ingress that will be publicly exposed. The +kustomize overlay to use for that is aptly named, `letsencrypt`. + +!!! example "Example keystone installation using the letsencrypt overlay" + ```shell + helm upgrade --install keystone ./keystone \ + --namespace=openstack \ + --wait \ + --timeout 120m \ + -f /opt/genestack/helm-configs/keystone/keystone-helm-overrides.yaml \ + -f /opt/genestack/helm-configs/prod-example-openstack-overrides.yaml \ + --set endpoints.identity.auth.admin.password="$(kubectl --namespace openstack get secret keystone-admin -o jsonpath='{.data.password}' | base64 -d)" \ + --set endpoints.oslo_db.auth.admin.password="$(kubectl --namespace openstack get secret mariadb -o jsonpath='{.data.root-password}' | base64 -d)" \ + --set endpoints.oslo_db.auth.keystone.password="$(kubectl --namespace openstack get secret keystone-db-password -o jsonpath='{.data.password}' | base64 -d)" \ + --set endpoints.oslo_messaging.auth.admin.password="$(kubectl --namespace openstack get secret rabbitmq-default-user -o jsonpath='{.data.password}' | base64 -d)" \ + --set endpoints.oslo_messaging.auth.keystone.password="$(kubectl --namespace openstack get secret keystone-rabbitmq-password -o jsonpath='{.data.password}' | base64 -d)" \ + --post-renderer /opt/genestack/kustomize/kustomize.sh \ + --post-renderer-args keystone/letsencrypt + ``` diff --git a/mkdocs.yml b/mkdocs.yml index ea8141d5..2771debf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -161,6 +161,7 @@ nav: - infrastructure-overview.md - Namespace: infrastructure-namespace.md - Ingress: infrastructure-ingress.md + - LetsEncrypt: infrastructure-letsencrypt.md - MariaDB: - infrastructure-mariadb.md - RabbitMQ: From c020939ff5f8c945670f7d89b7dd3c67f6e93b93 Mon Sep 17 00:00:00 2001 From: Luke Repko <luke.repko@rackspace.com> Date: Mon, 22 Apr 2024 15:06:50 -0500 Subject: [PATCH 5/5] fix: Use existing fqdn base to avoid duplication --- .../skyline/letsencrypt/kustomization.yaml | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/kustomize/skyline/letsencrypt/kustomization.yaml b/kustomize/skyline/letsencrypt/kustomization.yaml index 6427ce88..e6271803 100644 --- a/kustomize/skyline/letsencrypt/kustomization.yaml +++ b/kustomize/skyline/letsencrypt/kustomization.yaml @@ -1,30 +1,11 @@ bases: - - ../base + - ../fqdn patches: - target: kind: Ingress name: skyline patch: |- - - op: add - path: /spec/rules - value: - - host: skyline.dfw-ospcv2-staging.ohthree.com - http: - paths: - - path: / - pathType: ImplementationSpecific - backend: - service: - name: skyline-apiserver - port: - name: "s-apiserver" - - op: add - path: /spec/tls - value: - - hosts: - - skyline.dfw-ospcv2-staging.ohthree.com - secretName: skyline-tls-public - op: add path: /metadata/annotations value: