From aeaf70c2e2219c5e392ec944a436dbb4e403e8b5 Mon Sep 17 00:00:00 2001
From: Luke Repko <luke.repko@rackspace.com>
Date: Fri, 19 Apr 2024 19:41:10 -0500
Subject: [PATCH 1/5] Add optional kustomize letsencrypt overlay

When using letsencrypt, you must annotate the named ingress with the
letsencrypt cluster-issuer as well as any other desired
acme.cert-manager.io annotations [1]. As most services have multiple
ingresses (external vs internal), we must only annotate the ingress that
will be accessed externally, else the ACME solver will not work
correctly.

Using this optional overlay will help ensure the expected ingress is
annotated. So that the challenge url is accessible externally.

 1. https://cert-manager.io/docs/usage/ingress/#supported-annotations
---
 .../cinder/letsencrypt/kustomization.yaml     | 13 ++++++++
 .../glance/letsencrypt/kustomization.yaml     | 13 ++++++++
 kustomize/heat/letsencrypt/kustomization.yaml | 22 +++++++++++++
 .../horizon/letsencrypt/kustomization.yaml    | 13 ++++++++
 .../keystone/letsencrypt/kustomization.yaml   | 13 ++++++++
 .../neutron/letsencrypt/kustomization.yaml    | 13 ++++++++
 kustomize/nova/letsencrypt/kustomization.yaml | 31 ++++++++++++++++++
 .../octavia/letsencrypt/kustomization.yaml    | 13 ++++++++
 .../placement/letsencrypt/kustomization.yaml  | 13 ++++++++
 .../skyline/letsencrypt/kustomization.yaml    | 32 +++++++++++++++++++
 10 files changed, 176 insertions(+)
 create mode 100644 kustomize/cinder/letsencrypt/kustomization.yaml
 create mode 100644 kustomize/glance/letsencrypt/kustomization.yaml
 create mode 100644 kustomize/heat/letsencrypt/kustomization.yaml
 create mode 100644 kustomize/horizon/letsencrypt/kustomization.yaml
 create mode 100644 kustomize/keystone/letsencrypt/kustomization.yaml
 create mode 100644 kustomize/neutron/letsencrypt/kustomization.yaml
 create mode 100644 kustomize/nova/letsencrypt/kustomization.yaml
 create mode 100644 kustomize/octavia/letsencrypt/kustomization.yaml
 create mode 100644 kustomize/placement/letsencrypt/kustomization.yaml
 create mode 100644 kustomize/skyline/letsencrypt/kustomization.yaml

diff --git a/kustomize/cinder/letsencrypt/kustomization.yaml b/kustomize/cinder/letsencrypt/kustomization.yaml
new file mode 100644
index 00000000..600c2979
--- /dev/null
+++ b/kustomize/cinder/letsencrypt/kustomization.yaml
@@ -0,0 +1,13 @@
+bases:
+  - ../base
+
+patches:
+  - target:
+      kind: Ingress
+      name: cinder-namespace-fqdn
+    patch: |-
+      - op: add
+        path: /metadata/annotations
+        value:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+          acme.cert-manager.io/http01-edit-in-place: "true"
diff --git a/kustomize/glance/letsencrypt/kustomization.yaml b/kustomize/glance/letsencrypt/kustomization.yaml
new file mode 100644
index 00000000..36dd9501
--- /dev/null
+++ b/kustomize/glance/letsencrypt/kustomization.yaml
@@ -0,0 +1,13 @@
+bases:
+  - ../base
+
+patches:
+  - target:
+      kind: Ingress
+      name: glance-namespace-fqdn
+    patch: |-
+      - op: add
+        path: /metadata/annotations
+        value:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+          acme.cert-manager.io/http01-edit-in-place: "true"
diff --git a/kustomize/heat/letsencrypt/kustomization.yaml b/kustomize/heat/letsencrypt/kustomization.yaml
new file mode 100644
index 00000000..72dcb904
--- /dev/null
+++ b/kustomize/heat/letsencrypt/kustomization.yaml
@@ -0,0 +1,22 @@
+bases:
+  - ../base
+
+patches:
+  - target:
+      kind: Ingress
+      name: heat-namespace-fqdn
+    patch: |-
+      - op: add
+        path: /metadata/annotations
+        value:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+          acme.cert-manager.io/http01-edit-in-place: "true"
+  - target:
+      kind: Ingress
+      name: cloudformation-namespace-fqdn
+    patch: |-
+      - op: add
+        path: /metadata/annotations
+        value:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+          acme.cert-manager.io/http01-edit-in-place: "true"
diff --git a/kustomize/horizon/letsencrypt/kustomization.yaml b/kustomize/horizon/letsencrypt/kustomization.yaml
new file mode 100644
index 00000000..77ac7bd9
--- /dev/null
+++ b/kustomize/horizon/letsencrypt/kustomization.yaml
@@ -0,0 +1,13 @@
+bases:
+  - ../base
+
+patches:
+  - target:
+      kind: Ingress
+      name: horizon-namespace-fqdn
+    patch: |-
+      - op: add
+        path: /metadata/annotations
+        value:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+          acme.cert-manager.io/http01-edit-in-place: "true"
diff --git a/kustomize/keystone/letsencrypt/kustomization.yaml b/kustomize/keystone/letsencrypt/kustomization.yaml
new file mode 100644
index 00000000..394f3d8e
--- /dev/null
+++ b/kustomize/keystone/letsencrypt/kustomization.yaml
@@ -0,0 +1,13 @@
+bases:
+  - ../base
+
+patches:
+  - target:
+      kind: Ingress
+      name: keystone-namespace-fqdn
+    patch: |-
+      - op: add
+        path: /metadata/annotations
+        value:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+          acme.cert-manager.io/http01-edit-in-place: "true"
diff --git a/kustomize/neutron/letsencrypt/kustomization.yaml b/kustomize/neutron/letsencrypt/kustomization.yaml
new file mode 100644
index 00000000..1e6b4270
--- /dev/null
+++ b/kustomize/neutron/letsencrypt/kustomization.yaml
@@ -0,0 +1,13 @@
+bases:
+  - ../base
+
+patches:
+  - target:
+      kind: Ingress
+      name: neutron-namespace-fqdn
+    patch: |-
+      - op: add
+        path: /metadata/annotations
+        value:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+          acme.cert-manager.io/http01-edit-in-place: "true"
diff --git a/kustomize/nova/letsencrypt/kustomization.yaml b/kustomize/nova/letsencrypt/kustomization.yaml
new file mode 100644
index 00000000..57a890ec
--- /dev/null
+++ b/kustomize/nova/letsencrypt/kustomization.yaml
@@ -0,0 +1,31 @@
+bases:
+  - ../base
+
+patches:
+  - target:
+      kind: Ingress
+      name: nova-namespace-fqdn
+    patch: |-
+      - op: add
+        path: /metadata/annotations
+        value:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+          acme.cert-manager.io/http01-edit-in-place: "true"
+  - target:
+      kind: Ingress
+      name: metadata-namespace-fqdn
+    patch: |-
+      - op: add
+        path: /metadata/annotations
+        value:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+          acme.cert-manager.io/http01-edit-in-place: "true"
+  - target:
+      kind: Ingress
+      name: novncproxy-namespace-fqdn
+    patch: |-
+      - op: add
+        path: /metadata/annotations
+        value:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+          acme.cert-manager.io/http01-edit-in-place: "true"
diff --git a/kustomize/octavia/letsencrypt/kustomization.yaml b/kustomize/octavia/letsencrypt/kustomization.yaml
new file mode 100644
index 00000000..c6b2199f
--- /dev/null
+++ b/kustomize/octavia/letsencrypt/kustomization.yaml
@@ -0,0 +1,13 @@
+bases:
+  - ../base
+
+patches:
+  - target:
+      kind: Ingress
+      name: octavia-namespace-fqdn
+    patch: |-
+      - op: add
+        path: /metadata/annotations
+        value:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+          acme.cert-manager.io/http01-edit-in-place: "true"
diff --git a/kustomize/placement/letsencrypt/kustomization.yaml b/kustomize/placement/letsencrypt/kustomization.yaml
new file mode 100644
index 00000000..0c6295f8
--- /dev/null
+++ b/kustomize/placement/letsencrypt/kustomization.yaml
@@ -0,0 +1,13 @@
+bases:
+  - ../base
+
+patches:
+  - target:
+      kind: Ingress
+      name: placement-namespace-fqdn
+    patch: |-
+      - op: add
+        path: /metadata/annotations
+        value:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+          acme.cert-manager.io/http01-edit-in-place: "true"
diff --git a/kustomize/skyline/letsencrypt/kustomization.yaml b/kustomize/skyline/letsencrypt/kustomization.yaml
new file mode 100644
index 00000000..6427ce88
--- /dev/null
+++ b/kustomize/skyline/letsencrypt/kustomization.yaml
@@ -0,0 +1,32 @@
+bases:
+  - ../base
+
+patches:
+  - target:
+      kind: Ingress
+      name: skyline
+    patch: |-
+      - op: add
+        path: /spec/rules
+        value:
+          - host: skyline.dfw-ospcv2-staging.ohthree.com
+            http:
+              paths:
+                - path: /
+                  pathType: ImplementationSpecific
+                  backend:
+                    service:
+                      name: skyline-apiserver
+                      port:
+                        name: "s-apiserver"
+      - op: add
+        path: /spec/tls
+        value:
+          - hosts:
+            - skyline.dfw-ospcv2-staging.ohthree.com
+            secretName: skyline-tls-public
+      - op: add
+        path: /metadata/annotations
+        value:
+          cert-manager.io/cluster-issuer: letsencrypt-prod
+          acme.cert-manager.io/http01-edit-in-place: "true"

From 2c0cf4a1d629114d77895d02c8791add9ccadf8d Mon Sep 17 00:00:00 2001
From: Luke Repko <luke.repko@rackspace.com>
Date: Mon, 22 Apr 2024 13:25:01 -0500
Subject: [PATCH 2/5] fix: Update prod example helm overrides

The original key paths were incorrect. Not all services use `api` here.
---
 docs/infrastructure-letsencrypt.md                 | 0
 helm-configs/prod-example-openstack-overrides.yaml | 4 ++--
 2 files changed, 2 insertions(+), 2 deletions(-)
 create mode 100644 docs/infrastructure-letsencrypt.md

diff --git a/docs/infrastructure-letsencrypt.md b/docs/infrastructure-letsencrypt.md
new file mode 100644
index 00000000..e69de29b
diff --git a/helm-configs/prod-example-openstack-overrides.yaml b/helm-configs/prod-example-openstack-overrides.yaml
index ed5e2b49..4acc4cc9 100644
--- a/helm-configs/prod-example-openstack-overrides.yaml
+++ b/helm-configs/prod-example-openstack-overrides.yaml
@@ -138,7 +138,7 @@ endpoints:
             kind: ClusterIssuer
         host: metadata.nova.dfw-ospcv2-staging.ohthree.com
     port:
-      api:
+      metadata:
         public: 443
     scheme:
       public: https
@@ -154,7 +154,7 @@ endpoints:
             kind: ClusterIssuer
         host: novnc.nova.dfw-ospcv2-staging.ohthree.com
     port:
-      api:
+      novnc_proxy:
         public: 443
     scheme:
       public: https

From 23fe8516a651ffaa1a8cdc26f87d7bc83df52805 Mon Sep 17 00:00:00 2001
From: Luke Repko <luke.repko@rackspace.com>
Date: Mon, 22 Apr 2024 13:42:33 -0500
Subject: [PATCH 3/5] feat: Add letsencrypt issuer to helm overrides ex

Expand the prod overrides example to include the necessary issuerRef
name of letsencrypt-prod, it is a better fit. Remove the fake cert, key,
and anchors to them.
---
 .../prod-example-openstack-overrides.yaml     | 70 ++++---------------
 1 file changed, 15 insertions(+), 55 deletions(-)

diff --git a/helm-configs/prod-example-openstack-overrides.yaml b/helm-configs/prod-example-openstack-overrides.yaml
index 4acc4cc9..1d90681b 100644
--- a/helm-configs/prod-example-openstack-overrides.yaml
+++ b/helm-configs/prod-example-openstack-overrides.yaml
@@ -1,14 +1,4 @@
 _region: &region DFW3
-_certificate: &crt |
-  -----BEGIN CERTIFICATE-----
-  -----END CERTIFICATE-----
-  -----BEGIN CERTIFICATE-----
-  -----END CERTIFICATE-----
-
-_certificate_key: &key |
-  -----BEGIN RSA PRIVATE KEY-----
-  -----END RSA PRIVATE KEY-----
-
 images:
   tags:
     bootstrap: "docker.io/openstackhelm/heat:2023.1-ubuntu_jammy"
@@ -114,11 +104,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: nova-tls-api
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: nova.dfw-ospcv2-staging.ohthree.com
     port:
@@ -130,11 +118,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: metadata-tls-metadata
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: metadata.nova.dfw-ospcv2-staging.ohthree.com
     port:
@@ -146,11 +132,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: nova-novncproxy-tls-proxy
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: novnc.nova.dfw-ospcv2-staging.ohthree.com
     port:
@@ -162,11 +146,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: heat-tls-cfn
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: cloudformation.heat.dfw-ospcv2-staging.ohthree.com
     port:
@@ -178,11 +160,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: heat-tls-cloudwatch
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: cloudwatch.heat.dfw-ospcv2-staging.ohthree.com
     port:
@@ -194,11 +174,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: horizon-tls-web
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: dfw-ospcv2-staging.ohthree.com
     port:
@@ -233,11 +211,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: keystone-tls-api
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: keystone.dfw-ospcv2-staging.ohthree.com
     port:
@@ -253,11 +229,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: glance-tls-api
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: glance.dfw-ospcv2-staging.ohthree.com
     port:
@@ -269,11 +243,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: octavia-tls-api
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: octavia.dfw-ospcv2-staging.ohthree.com
     port:
@@ -285,11 +257,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: neutron-tls-server
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: neutron.dfw-ospcv2-staging.ohthree.com
     port:
@@ -301,11 +271,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: heat-tls-api
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: heat.dfw-ospcv2-staging.ohthree.com
     port:
@@ -317,11 +285,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: placement-tls-api
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: placement.dfw-ospcv2-staging.ohthree.com
     port:
@@ -333,11 +299,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: cinder-tls-api
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: cinder.dfw-ospcv2-staging.ohthree.com
     port:
@@ -349,11 +313,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: cinder-tls-api
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: cinder.dfw-ospcv2-staging.ohthree.com
     port:
@@ -365,11 +327,9 @@ endpoints:
     host_fqdn_override:
       public:
         tls:
-          crt: *crt
-          key: *key
           secretName: cinder-tls-api
           issuerRef:
-            name: ca-issuer
+            name: letsencrypt-prod
             kind: ClusterIssuer
         host: cinder.dfw-ospcv2-staging.ohthree.com
     port:

From f264782a8fc7a217f9c822871376a44a7a28073a Mon Sep 17 00:00:00 2001
From: Luke Repko <luke.repko@rackspace.com>
Date: Mon, 22 Apr 2024 14:44:06 -0500
Subject: [PATCH 4/5] docs: Add letsencrypt cert deployment info

---
 docs/infrastructure-letsencrypt.md | 86 ++++++++++++++++++++++++++++++
 mkdocs.yml                         |  1 +
 2 files changed, 87 insertions(+)

diff --git a/docs/infrastructure-letsencrypt.md b/docs/infrastructure-letsencrypt.md
index e69de29b..29850f4d 100644
--- a/docs/infrastructure-letsencrypt.md
+++ b/docs/infrastructure-letsencrypt.md
@@ -0,0 +1,86 @@
+# Deploying Let's Encrypt Certificates
+
+Are you tired of manually renewing and deploying a countless number of
+certificates across your environments? Us too!
+
+## Apply the Let's Encrypt Cluster Issuer
+
+Before we can have Cert Manager start coordinating Let's Encrypt certificate
+requests for us, we need to add an ACME issuer with a valid, monitored
+email (for expiration reminders and other important ACME related information).
+
+``` yaml
+read -p "Enter a valid email address for use with ACME: " ACME_EMAIL; \
+cat <<EOF | kubectl apply -f -
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-prod
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: ${ACME_EMAIL}
+    privateKeySecretRef:
+      name: letsencrypt-prod
+    solvers:
+      - http01:
+          ingress:
+            ingressClassName: nginx
+EOF
+```
+
+## Use the proper TLS issuerRef
+
+!!! danger "Important for later helm installations!"
+    You must ensure your helm configuration is such that you set the
+    `endpoints.$service.host_fqdn_override.public.tls.issuerRef.name` for any
+    given endpoint to use our `letsencrypt-prod` ClusterIssuer. Similarly,
+    ensure that `endpoints.$service.host_fqdn_override.public.host`
+    is set to the external DNS hostname you plan to expose for a given
+    service endpoint.
+
+!!! example
+    You can find several examples of this in the
+    `helm-configs/prod-example-openstack-overrides.yaml`, one such example
+    for glance is below for reference.
+    ```yaml
+    endpoints:
+      image:
+        host_fqdn_override:
+          public:
+            tls:
+              secretName: glance-tls-api
+              issuerRef:
+                name: letsencrpyt-prod
+                kind: ClusterIssuer
+            host: glance.api.your.domain.tld
+        port:
+          api:
+            public: 443
+        scheme:
+          public: https
+    ```
+
+## Helm Kustomize Post Render Args
+
+In order for Cert Manager to set up the ACME challenge, it needs to know which
+ingress to target. We do this via a kustomize overlay that injects the
+needed annotation(s) to the ingress that will be publicly exposed. The
+kustomize overlay to use for that is aptly named, `letsencrypt`.
+
+!!! example "Example keystone installation using the letsencrypt overlay"
+    ```shell
+    helm upgrade --install keystone ./keystone \
+    --namespace=openstack \
+    --wait \
+    --timeout 120m \
+    -f /opt/genestack/helm-configs/keystone/keystone-helm-overrides.yaml \
+    -f /opt/genestack/helm-configs/prod-example-openstack-overrides.yaml \
+    --set endpoints.identity.auth.admin.password="$(kubectl --namespace openstack get secret keystone-admin -o jsonpath='{.data.password}' | base64 -d)" \
+    --set endpoints.oslo_db.auth.admin.password="$(kubectl --namespace openstack get secret mariadb -o jsonpath='{.data.root-password}' | base64 -d)" \
+    --set endpoints.oslo_db.auth.keystone.password="$(kubectl --namespace openstack get secret keystone-db-password -o jsonpath='{.data.password}' | base64 -d)" \
+    --set endpoints.oslo_messaging.auth.admin.password="$(kubectl --namespace openstack get secret rabbitmq-default-user -o jsonpath='{.data.password}' | base64 -d)" \
+    --set endpoints.oslo_messaging.auth.keystone.password="$(kubectl --namespace openstack get secret keystone-rabbitmq-password -o jsonpath='{.data.password}' | base64 -d)" \
+    --post-renderer /opt/genestack/kustomize/kustomize.sh \
+    --post-renderer-args keystone/letsencrypt
+    ```
diff --git a/mkdocs.yml b/mkdocs.yml
index ea8141d5..2771debf 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -161,6 +161,7 @@ nav:
               - infrastructure-overview.md
               - Namespace: infrastructure-namespace.md
               - Ingress: infrastructure-ingress.md
+              - LetsEncrypt: infrastructure-letsencrypt.md
               - MariaDB:
                   - infrastructure-mariadb.md
               - RabbitMQ:

From c020939ff5f8c945670f7d89b7dd3c67f6e93b93 Mon Sep 17 00:00:00 2001
From: Luke Repko <luke.repko@rackspace.com>
Date: Mon, 22 Apr 2024 15:06:50 -0500
Subject: [PATCH 5/5] fix: Use existing fqdn base to avoid duplication

---
 .../skyline/letsencrypt/kustomization.yaml    | 21 +------------------
 1 file changed, 1 insertion(+), 20 deletions(-)

diff --git a/kustomize/skyline/letsencrypt/kustomization.yaml b/kustomize/skyline/letsencrypt/kustomization.yaml
index 6427ce88..e6271803 100644
--- a/kustomize/skyline/letsencrypt/kustomization.yaml
+++ b/kustomize/skyline/letsencrypt/kustomization.yaml
@@ -1,30 +1,11 @@
 bases:
-  - ../base
+  - ../fqdn
 
 patches:
   - target:
       kind: Ingress
       name: skyline
     patch: |-
-      - op: add
-        path: /spec/rules
-        value:
-          - host: skyline.dfw-ospcv2-staging.ohthree.com
-            http:
-              paths:
-                - path: /
-                  pathType: ImplementationSpecific
-                  backend:
-                    service:
-                      name: skyline-apiserver
-                      port:
-                        name: "s-apiserver"
-      - op: add
-        path: /spec/tls
-        value:
-          - hosts:
-            - skyline.dfw-ospcv2-staging.ohthree.com
-            secretName: skyline-tls-public
       - op: add
         path: /metadata/annotations
         value: