From 6918887827069702a8902729217a868e3a362aa9 Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Mon, 29 Apr 2024 12:35:37 -0500 Subject: [PATCH 1/5] fix: change scripts to bash shebang --- scripts/gitops-deploy.sh | 2 +- scripts/gitops-secrets-gen.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gitops-deploy.sh b/scripts/gitops-deploy.sh index fc435f4e5..b63f0c705 100755 --- a/scripts/gitops-deploy.sh +++ b/scripts/gitops-deploy.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash usage() { echo "$(basename "$0") " >&2 diff --git a/scripts/gitops-secrets-gen.sh b/scripts/gitops-secrets-gen.sh index be835679d..eec9bc2fe 100755 --- a/scripts/gitops-secrets-gen.sh +++ b/scripts/gitops-secrets-gen.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash function usage() { echo "$(basename "$0") " >&2 From 12f01ce078cab9567c13a7b77909c58576d5a4b1 Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Mon, 29 Apr 2024 12:37:59 -0500 Subject: [PATCH 2/5] doc: switch to using OpenStack Helm repo always Drop git cloning the OpenStack Helm repos and switch to always using the OpenStack Helm repos. Now we no longer need the script to get the dependencies updated and linked as well. --- docs/install-understack-ubuntu-k3s.md | 10 ++-------- scripts/openstack-helm-depend-sync.sh | 3 --- 2 files changed, 2 insertions(+), 11 deletions(-) delete mode 100755 scripts/openstack-helm-depend-sync.sh diff --git a/docs/install-understack-ubuntu-k3s.md b/docs/install-understack-ubuntu-k3s.md index 3fbec6ae0..19e6cb18b 100644 --- a/docs/install-understack-ubuntu-k3s.md +++ b/docs/install-understack-ubuntu-k3s.md @@ -308,13 +308,7 @@ nautobot job.batch/nautobot-backup-sg6f 1/1 2m48s ### OpenStack Pre-requisites ```bash -# clone the two repos because they reference the infra one as a relative path -# so you can't use real helm commands -git clone https://github.com/openstack/openstack-helm -git clone https://github.com/openstack/openstack-helm-infra -# update the dependencies cause we can't use real helm references -./scripts/openstack-helm-depend-sync.sh ironic -# keystone can now be used from a helm repo +# add the OpenStack Helm repo we can install from helm repo add osh https://tarballs.opendev.org/openstack/openstack-helm/ ``` @@ -379,7 +373,7 @@ Install the OpenStack Ironic helm chart using our custom aio-values.yaml overrid ```bash helm --namespace openstack template \ ironic \ - ./openstack-helm/ironic/ \ + osh/ironic/ \ -f components/ironic/aio-values.yaml \ -f secret-openstack.yaml \ | kubectl -n openstack apply -f - diff --git a/scripts/openstack-helm-depend-sync.sh b/scripts/openstack-helm-depend-sync.sh deleted file mode 100755 index 18dba086f..000000000 --- a/scripts/openstack-helm-depend-sync.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -cd $(git rev-parse --show-toplevel)/openstack-helm && helm dep up "$1" From 395579aed4965bd879152bc9292dde7ad254ef97 Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Mon, 29 Apr 2024 12:38:55 -0500 Subject: [PATCH 3/5] chore: drop unused sealed secrets script --- scripts/openstack-helm-sealed-secrets.sh | 40 ------------------------ 1 file changed, 40 deletions(-) delete mode 100755 scripts/openstack-helm-sealed-secrets.sh diff --git a/scripts/openstack-helm-sealed-secrets.sh b/scripts/openstack-helm-sealed-secrets.sh deleted file mode 100755 index f85bdbdc1..000000000 --- a/scripts/openstack-helm-sealed-secrets.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -x - -# function to process each YAML file -process_yaml() { - kind=$(echo "$1" | yq e '.kind') - if [[ "${kind}" == "Secret" ]]; then - # its a match, encrypt it - echo "$1" | \ - kubeseal \ - --scope cluster-wide \ - --allow-empty-data \ - -o yaml - else - # not a match just output it - echo "---" - echo "$1" - fi -} - -NL=$'\n' - -# read the stream from stdin and break up each YAML doc -yaml_acc="" -while IFS= read -r line; do - if [[ $line =~ ^---$ ]]; then - # process each YAML file - if [[ -n $yaml_acc ]]; then - process_yaml "$yaml_acc" - yaml_acc="" - fi - else - # accumulate the lines of the current YAML doc - yaml_acc+="${line}${NL}" - - fi -done - -# process the last one -[[ -n $yaml_acc ]] && process_yaml "$yaml_acc" -exit 0 From 3189d3aeefe3e42cc04aa1bc03a09528d1f31d4d Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Mon, 29 Apr 2024 12:43:25 -0500 Subject: [PATCH 4/5] docs: drop patching of coredns Dropped the patching of CoreDNS by recommending the usage of something like sslip.io for the services. --- components/dexidp/README.md | 49 +++++++------------------------------ scripts/patch-coredns.sh | 21 ---------------- 2 files changed, 9 insertions(+), 61 deletions(-) delete mode 100755 scripts/patch-coredns.sh diff --git a/components/dexidp/README.md b/components/dexidp/README.md index 013c5e0c2..f5877b9f3 100644 --- a/components/dexidp/README.md +++ b/components/dexidp/README.md @@ -1,4 +1,4 @@ -## Overview +# Overview The [Dex IDP](https://dexidp.io/) is deployed to act as an OAuth2/OpenID Connect identity provider for Nautobot and potentially other applications in @@ -27,49 +27,19 @@ requirements: Unfortunately, these Nautobot changes alone are not enough to have authentication working in most setups. We also need to make sure that every -step of [authorziation code grant][authzcodegrant] can be completed. Given that -we use ephemeral clusters for development and they use `.local` domain, -following needs to work: +step of [authorziation code grant][authzcodegrant] can be completed. -- Nautobot container can reach the Dex using the issuer URL (`https://dexidp.local`). -- End-user's browser must be able to reach Dex using exactly the same URL. -- Nautobot needs to be reachable using the DNS name, i.e. `https://nautobot.local` -- All of these needs to happen over HTTPS. +For ephemeral clusters in development it is recommended to use a service like +[sslip.io](https://sslip.io) to use hostnames like: -When Dex and Nautobot are hosted in the same cluster, by default they will try -to communicate over the internal networking and plain HTTP. This clearly -violates the requirements listed above, so we have to force the communication -between the pods to happen over the Ingress (which provides TLS termination and -stable hostname). +- dex.127-0-0-1.sslip.io +- nautobot-127-0-0-1.sslip.io -### Fixing DNS on development cluster +And creating HTTPS certificates for them. Then you can access the services +via DNS names and not have to patch your `/etc/hosts` or the DNS resolution in the cluster. -In development cluster, this can be done by reconfiguring [CoreDNS][coredns] -component. We have provided -[`scripts/patch-coredns.sh`](../../scripts/patch-coredns.sh) script to make the -necessary changes automatically. +## Azure authentication -```shell -$ ./scripts/patch-coredns.sh -[*] Patching coredns ConfigMap -configmap/coredns replaced -[*] Restarting CoreDNS -deployment.apps/coredns restarted -$ -``` - -### Making components accessible from your machine - -If running development cluster on your machine, you may need to create create -an entry in your `/etc/hosts` file that looks similar to this: - -```hosts -# Nautobot kind cluster -127.0.0.1 argocd.local nautobot.local keystone keystone.openstack dexidp.local workflows.local -``` - - -### Azure authentication Dex can optionally be configured to allow authentication through Azure SSO. The exact steps to configure this are available in [01-secrets/README.md](../01-secrets/README.md). @@ -77,4 +47,3 @@ exact steps to configure this are available in [socialauth]: https://python-social-auth.readthedocs.io/en/latest/backends/oidc.html [disco]: https://openid.net/specs/openid-connect-discovery-1_0.html [authzcodegrant]: https://datatracker.ietf.org/doc/html/rfc6749#section-4.1 -[coredns]: https://kubernetes.io/docs/tasks/administer-cluster/coredns/#about-coredns diff --git a/scripts/patch-coredns.sh b/scripts/patch-coredns.sh deleted file mode 100755 index 2dd3538e7..000000000 --- a/scripts/patch-coredns.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -e -EXISTING_COREFILE=$(kubectl -n kube-system get cm coredns -o jsonpath='{.data.Corefile}') - -ADD_LINE=" rewrite name dexidp.local ingress-nginx-controller.ingress-nginx.svc.cluster.local" - -if grep -q "$ADD_LINE" <(echo "$EXISTING_COREFILE"); then - echo "Configmap already patched." - exit 0 -fi -# shellcheck disable=SC2001 -PATCHED_COREFILE=$(echo "$EXISTING_COREFILE" | sed -e "s/^}$/${ADD_LINE}\n\}/") - - -echo "[*] Patching coredns ConfigMap" -kubectl -n kube-system --dry-run=client create cm coredns \ - --from-literal=Corefile="$PATCHED_COREFILE" -o yaml \ - | kubectl -n kube-system replace -f - - -echo "[*] Restarting CoreDNS" -kubectl -n kube-system rollout restart deployment coredns From 35d036a3e1b40ef7269bddaf3dca39f79628d049 Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Mon, 29 Apr 2024 15:00:40 -0500 Subject: [PATCH 5/5] docs: clean up some path references to repo and deploy Make it more clear what paths the user needs to run commands from by cleaning up the paths. --- docs/gitops-install.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/gitops-install.md b/docs/gitops-install.md index 2bed6775f..c2552766c 100644 --- a/docs/gitops-install.md +++ b/docs/gitops-install.md @@ -47,7 +47,7 @@ To avoid defining many environment variables we'll simplify by creating an `.env` file for our deployment. In this case we'll call it `my-k3s.env` and place it where we've cloned understack. A complete file would like like -```bash title="/path/to/understack/my-k3s.env" +```bash title="/path/to/uc-deploy/my-k3s.env" UC_REPO="$HOME/devel/understack" UC_DEPLOY="$HOME/devel/uc-deploy" DEPLOY_NAME="my-k3s" @@ -90,9 +90,17 @@ which needs to be set into the `UC_DEPLOY_EMAIL` variable. All Ingress DNS names will be created as subdomains of the value you put into the `DNS_ZONE` variable. +#### Saving the Environment Config + +```bash +cd /path/to/uc-deploy +git add my-k3s.env +git commit -m "my-k3s: save environment config" +``` + #### Getting Ready to Generate Secrets and Configs -You can run `source /path/to/understack/my-k3s.env` to have `$UC_DEPLOY` in +You can run `source /path/to/uc-deploy/my-k3s.env` to have `$UC_DEPLOY` in your shell. ### Populating the infrastructure @@ -111,7 +119,8 @@ using random data to sucessfully continue the installation. TODO: probably give at least one secure example ```bash -./scripts/gitops-secrets-gen.sh ./my-k3s.env +# from your understack checkout +./scripts/gitops-secrets-gen.sh ${UC_DEPLOY}/my-k3s.env pushd "${UC_DEPLOY}" git add secrets/my-k3s git commit -m "my-k3s: secrets generation" @@ -124,7 +133,8 @@ In this section we will use the [App of Apps][app-of-apps] pattern to define the deployment of all the components of UnderStack. ```bash -./scripts/gitops-deploy.sh ./my-k3s.env +# from your understack checkout +./scripts/gitops-deploy.sh ${UC_DEPLOY}/my-k3s.env pushd "${UC_DEPLOY}" git add clusters/my-k3s git add helm-configs/my-k3s