From 19f2bc85e9fb2c60f21bcb144aad9ed337fe2a37 Mon Sep 17 00:00:00 2001 From: Cory Ringdahl Date: Mon, 10 Jun 2024 12:12:23 -0500 Subject: [PATCH 1/4] incorporate sealed secrets into gen scripts - argocd Application 'sources' correctly reads 'kind: SealedSecret' and creates Secrets - will prevent committing bare secrets to git --- docs/gitops-install.md | 9 +-- scripts/easy-secrets-gen.sh | 100 ++++++++++------------------------ scripts/gitops-secrets-gen.sh | 14 ++++- 3 files changed, 45 insertions(+), 78 deletions(-) diff --git a/docs/gitops-install.md b/docs/gitops-install.md index 7ac7b0f8d..17a3b2e39 100644 --- a/docs/gitops-install.md +++ b/docs/gitops-install.md @@ -118,12 +118,9 @@ TODO: some examples and documentation on how to build out a cluster Secrets in their very nature are sensitive pieces of data. The ultimate storage and injection of these in a production environment needs to be -carefully considered. For the purposes of this document no specific -choice has been made but tools like Vault, Sealed Secrets, SOPS, etc -should be considered. This will only generate the necessary secrets -using random data to successfully continue the installation. - -TODO: probably give at least one secure example +carefully considered. For the purposes of this document, Sealed Secrets +has been chosen; other tools like Vault, SOPS, etc should be considered +for production deployments. ```bash # from your understack checkout diff --git a/scripts/easy-secrets-gen.sh b/scripts/easy-secrets-gen.sh index acdca6e40..087c8e25b 100755 --- a/scripts/easy-secrets-gen.sh +++ b/scripts/easy-secrets-gen.sh @@ -4,6 +4,21 @@ cd $(git rev-parse --show-toplevel) DEST_DIR=${1:-.} +if ! type -p kubeseal kubectl > /dev/null; then + echo "You must have kubeseal & kubectl installed to use this script" >&2 + exit 1 +fi + +function secret-seal-stdin() { + # this is meant to be piped to + # $1 is output file, -w + kubeseal \ + --scope cluster-wide \ + --allow-empty-data \ + -o yaml \ + -w $1 +} + [ ! -f "${DEST_DIR}/secret-mariadb.yaml" ] && \ kubectl --namespace openstack \ create secret generic mariadb \ @@ -12,7 +27,7 @@ kubectl --namespace openstack \ --type Opaque \ --from-literal=root-password="$(./scripts/pwgen.sh)" \ --from-literal=password="$(./scripts/pwgen.sh)" \ - > "${DEST_DIR}/secret-mariadb.yaml" + | secret-seal-stdin "${DEST_DIR}/secret-mariadb.yaml" NAUTOBOT_SECRET_KEY="$(./scripts/pwgen.sh)" if [ ! -f "${DEST_DIR}/secret-nautobot-django.yaml" ]; then @@ -22,7 +37,7 @@ if [ ! -f "${DEST_DIR}/secret-nautobot-django.yaml" ]; then -o yaml \ --type Opaque \ --from-literal="NAUTOBOT_SECRET_KEY=${NAUTOBOT_SECRET_KEY}" \ - > "${DEST_DIR}/secret-nautobot-django.yaml" + | secret-seal-stdin "${DEST_DIR}/secret-nautobot-django.yaml" fi [ ! -f "${DEST_DIR}/secret-nautobot-redis.yaml" ] && \ @@ -32,7 +47,7 @@ kubectl --namespace nautobot \ -o yaml \ --type Opaque \ --from-literal=redis-password="$(./scripts/pwgen.sh)" \ - > "${DEST_DIR}/secret-nautobot-redis.yaml" + | secret-seal-stdin "${DEST_DIR}/secret-nautobot-redis.yaml" NAUTOBOT_SSO_SECRET=$(./scripts/pwgen.sh) for ns in nautobot dex; do @@ -43,7 +58,7 @@ for ns in nautobot dex; do -o yaml \ --type Opaque \ --from-literal=client-secret="$NAUTOBOT_SSO_SECRET" \ - > "${DEST_DIR}/secret-nautobot-sso-$ns.yaml" + | secret-seal-stdin "${DEST_DIR}/secret-nautobot-sso-$ns.yaml" done unset NAUTOBOT_SSO_SECRET @@ -57,7 +72,7 @@ for ns in argo argo-events dex; do --type Opaque \ --from-literal=client-secret="$ARGO_SSO_SECRET" \ --from-literal=client-id=argo \ - > "${DEST_DIR}/secret-argo-sso-$ns.yaml" + | secret-seal-stdin "${DEST_DIR}/secret-argo-sso-$ns.yaml" done unset ARGO_SSO_SECRET @@ -73,10 +88,9 @@ for ns in argocd dex; do --from-literal=client-secret="$ARGOCD_SSO_SECRET" \ --from-literal=client-id=argocd \ | yq '.metadata.labels |= {"app.kubernetes.io/part-of": "argocd"}' \ - > "${DEST_DIR}/secret-argocd-sso-$ns.yaml" + | secret-seal-stdin "${DEST_DIR}/secret-argocd-sso-$ns.yaml" done unset ARGOCD_SSO_SECRET -rm -rf "${DEST_DIR}/secret-argo-sso-argocd.yaml" # create constant OpenStack memcache key to avoid cache invalidation on deploy export MEMCACHE_SECRET_KEY="$(./scripts/pwgen.sh 64)" @@ -100,7 +114,7 @@ kubectl --namespace openstack \ --from-literal=username="keystone" \ --from-literal=password="${KEYSTONE_RABBITMQ_PASSWORD}" \ --dry-run=client -o yaml \ - > "${DEST_DIR}/secret-keystone-rabbitmq-password.yaml" + | secret-seal-stdin "${DEST_DIR}/secret-keystone-rabbitmq-password.yaml" [ ! -f "${DEST_DIR}/secret-keystone-db-password.yaml" ] && \ kubectl --namespace openstack \ @@ -108,7 +122,7 @@ kubectl --namespace openstack \ --type Opaque \ --from-literal=password="${KEYSTONE_DB_PASSWORD}" \ --dry-run=client -o yaml \ - > "${DEST_DIR}/secret-keystone-db-password.yaml" + | secret-seal-stdin "${DEST_DIR}/secret-keystone-db-password.yaml" [ ! -f "${DEST_DIR}/secret-keystone-admin.yaml" ] && \ kubectl --namespace openstack \ @@ -116,7 +130,7 @@ kubectl --namespace openstack \ --type Opaque \ --from-literal=password="${KEYSTONE_ADMIN_PASSWORD}" \ --dry-run=client -o yaml \ - > "${DEST_DIR}/secret-keystone-admin.yaml" + | secret-seal-stdin "${DEST_DIR}/secret-keystone-admin.yaml" # ironic credentials [ ! -f "${DEST_DIR}/secret-ironic-rabbitmq-password.yaml" ] && \ @@ -125,14 +139,14 @@ kubectl --namespace openstack \ --type Opaque \ --from-literal=username="ironic" \ --from-literal=password="${IRONIC_RABBITMQ_PASSWORD}" \ - --dry-run=client -o yaml > "${DEST_DIR}/secret-ironic-rabbitmq-password.yaml" + --dry-run=client -o yaml | secret-seal-stdin "${DEST_DIR}/secret-ironic-rabbitmq-password.yaml" [ ! -f "${DEST_DIR}/secret-ironic-db-password.yaml" ] && \ kubectl --namespace openstack \ create secret generic ironic-db-password \ --type Opaque \ --from-literal=password="${IRONIC_DB_PASSWORD}" \ - --dry-run=client -o yaml > "${DEST_DIR}/secret-ironic-db-password.yaml" + --dry-run=client -o yaml | secret-seal-stdin "${DEST_DIR}/secret-ironic-db-password.yaml" [ ! -f "${DEST_DIR}/secret-ironic-keystone-password.yaml" ] && \ kubectl --namespace openstack \ @@ -140,70 +154,16 @@ kubectl --namespace openstack \ --type Opaque \ --from-literal=username="ironic" \ --from-literal=password="${IRONIC_KEYSTONE_PASSWORD}" \ - --dry-run=client -o yaml > "${DEST_DIR}/secret-ironic-keystone-password.yaml" + --dry-run=client -o yaml | secret-seal-stdin "${DEST_DIR}/secret-ironic-keystone-password.yaml" if [ "x${DO_TMPL_VALUES}" = "xy" ]; then [ ! -f "${DEST_DIR}/secret-openstack.yaml" ] && \ yq '(.. | select(tag == "!!str")) |= envsubst' \ "./components/openstack-secrets.tpl.yaml" \ - > "${DEST_DIR}/secret-openstack.yaml" + | secret-seal-stdin "${DEST_DIR}/secret-openstack.yaml" fi -if [ "x${SKIP_KUBESEAL}" = "xy" ]; then - echo "Skipping kubeseal" - exit 0 -fi - -kubeseal \ - --scope cluster-wide \ - --allow-empty-data \ - -o yaml \ - -f "${DEST_DIR}/secret-mariadb.yaml" \ - -w components/01-secrets/encrypted-mariadb.yaml - -kubeseal \ - --scope cluster-wide \ - --allow-empty-data \ - -o yaml \ - -f "${DEST_DIR}/secret-nautobot-env.yaml" \ - -w components/01-secrets/encrypted-nautobot-env.yaml - -kubeseal \ - --scope cluster-wide \ - --allow-empty-data \ - -o yaml \ - -f "${DEST_DIR}/secret-nautobot-redis.yaml" \ - -w components/01-secrets/encrypted-nautobot-redis.yaml - -for skrt in $(find "${DEST_DIR}" -maxdepth 1 -name "secret-keystone*.yaml" -o -name "secret-ironic*.yaml"); do - encskrt=$(echo "${skrt}" | sed -e 's/secret-/components\/01-secrets\/encrypted-/') - kubeseal \ - --scope cluster-wide \ - --allow-empty-data \ - -o yaml \ - -f "${skrt}" \ - -w "${encskrt}" -done - -for ns in nautobot dex; do - kubeseal \ - --scope cluster-wide \ - --allow-empty-data \ - -o yaml \ - -f "${DEST_DIR}/secret-nautobot-sso-$ns.yaml" \ - -w components/01-secrets/encrypted-nautobot-sso-$ns.yaml -done - -for ns in argo argo-events argocd dex; do - kubeseal \ - --scope cluster-wide \ - --allow-empty-data \ - -o yaml \ - -f secret-argo-sso-$ns.yaml \ - -w components/01-secrets/encrypted-argo-sso-$ns.yaml -done - -cd components/01-secrets/ +cd ${DEST_DIR} rm -f kustomization.yaml kustomize create --autodetect -cd ../.. +cd - diff --git a/scripts/gitops-secrets-gen.sh b/scripts/gitops-secrets-gen.sh index 3f0caa197..cedf2a9bb 100755 --- a/scripts/gitops-secrets-gen.sh +++ b/scripts/gitops-secrets-gen.sh @@ -7,6 +7,16 @@ function usage() { exit 1 } +function secret-seal-stdin() { + # this is meant to be piped to + # $1 is output file, -w + kubeseal \ + --scope cluster-wide \ + --allow-empty-data \ + -o yaml \ + -w $1 +} + if [ $# -ne 1 ]; then usage fi @@ -66,7 +76,6 @@ fi export DNS_ZONE export DEPLOY_NAME -export SKIP_KUBESEAL=y export DO_TMPL_VALUES=y mkdir -p "${UC_DEPLOY}/secrets/${DEPLOY_NAME}" "${SCRIPTS_DIR}/easy-secrets-gen.sh" "${UC_DEPLOY}/secrets/${DEPLOY_NAME}" @@ -74,7 +83,7 @@ mkdir -p "${UC_DEPLOY}/secrets/${DEPLOY_NAME}" if [ "x${NO_SECRET_DEPLOY}" = "x" ]; then echo "Creating ArgoCD config" mkdir -p "${UC_DEPLOY}/secrets/${DEPLOY_NAME}/argocd" - cat << EOF > "${UC_DEPLOY}/secrets/${DEPLOY_NAME}/argocd/secret-deploy-repo.yaml" + cat << EOF | secret-seal-stdin "${UC_DEPLOY}/secrets/${DEPLOY_NAME}/argocd/secret-deploy-repo.yaml" apiVersion: v1 kind: Secret metadata: @@ -106,6 +115,7 @@ spec: ingressClassName: nginx EOF +# Placeholders don't need sealing if [ ! -f "${UC_DEPLOY}/secrets/${DEPLOY_NAME}/secret-metallb.yaml" ]; then echo "Creating metallb secret placeholder" echo "---" > "${UC_DEPLOY}/secrets/${DEPLOY_NAME}/secret-metallb.yaml" From 5be17f0996e20dd294b58fd16c17734e19a68d83 Mon Sep 17 00:00:00 2001 From: Cory Ringdahl Date: Mon, 10 Jun 2024 15:41:59 -0500 Subject: [PATCH 2/4] argocd, helm-secrets, and multi-source aren't playing nice yet --- scripts/easy-secrets-gen.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/easy-secrets-gen.sh b/scripts/easy-secrets-gen.sh index 087c8e25b..dccc78922 100755 --- a/scripts/easy-secrets-gen.sh +++ b/scripts/easy-secrets-gen.sh @@ -160,7 +160,7 @@ if [ "x${DO_TMPL_VALUES}" = "xy" ]; then [ ! -f "${DEST_DIR}/secret-openstack.yaml" ] && \ yq '(.. | select(tag == "!!str")) |= envsubst' \ "./components/openstack-secrets.tpl.yaml" \ - | secret-seal-stdin "${DEST_DIR}/secret-openstack.yaml" + > "${DEST_DIR}/secret-openstack.yaml" fi cd ${DEST_DIR} From ccdfad65ed06ce36ee7c206f7e64d0e27db9c834 Mon Sep 17 00:00:00 2001 From: Cory Ringdahl Date: Tue, 18 Jun 2024 10:00:52 -0500 Subject: [PATCH 3/4] simple "is sealed secrets installed" check; both secrets gens do this validation --- scripts/easy-secrets-gen.sh | 5 +++++ scripts/gitops-secrets-gen.sh | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/scripts/easy-secrets-gen.sh b/scripts/easy-secrets-gen.sh index dccc78922..345c116a3 100755 --- a/scripts/easy-secrets-gen.sh +++ b/scripts/easy-secrets-gen.sh @@ -9,6 +9,11 @@ if ! type -p kubeseal kubectl > /dev/null; then exit 1 fi +if ! $(kubectl api-resources | grep -q sealedsecrets); then + echo "Your cluster doesn't appear to have the sealed secrets operator installed." >&2 + exit 1 +fi + function secret-seal-stdin() { # this is meant to be piped to # $1 is output file, -w diff --git a/scripts/gitops-secrets-gen.sh b/scripts/gitops-secrets-gen.sh index cedf2a9bb..9f71a8bfa 100755 --- a/scripts/gitops-secrets-gen.sh +++ b/scripts/gitops-secrets-gen.sh @@ -7,6 +7,16 @@ function usage() { exit 1 } +if ! type -p kubeseal kubectl > /dev/null; then + echo "You must have kubeseal & kubectl installed to use this script" >&2 + exit 1 +fi + +if ! $(kubectl api-resources | grep -q sealedsecrets); then + echo "Your cluster doesn't appear to have the sealed secrets operator installed." >&2 + exit 1 +fi + function secret-seal-stdin() { # this is meant to be piped to # $1 is output file, -w From 0711f1f7b621a393896627278939a4fd94f94832 Mon Sep 17 00:00:00 2001 From: Andrew Harris Date: Tue, 18 Jun 2024 16:02:54 -0400 Subject: [PATCH 4/4] fix: moving bootstrap step (argocd & sealed-secrets) to before the secrets generation --- docs/gitops-install.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/gitops-install.md b/docs/gitops-install.md index 17a3b2e39..0c6871dd5 100644 --- a/docs/gitops-install.md +++ b/docs/gitops-install.md @@ -114,6 +114,15 @@ your shell. TODO: some examples and documentation on how to build out a cluster +### Bootstrapping ArgoCD and Sealed-Secrets + +If you do not have ArgoCD deployed then you can use the following: + +```bash +cd ${UC_DEPLOY} +kubectl kustomize --enable-helm https://github.com/rackerlabs/understack/bootstrap/ | kubectl apply -f - +``` + ### Generating secrets Secrets in their very nature are sensitive pieces of data. The ultimate @@ -166,14 +175,7 @@ At this point we will use our configs to make the actual deployment. Make sure everything you've committed to your deployment repo is pushed to your git server so that ArgoCD can access it. -If you do not have ArgoCD deployed then you can use the following: - -```bash -cd ${UC_DEPLOY} -kubectl kustomize --enable-helm https://github.com/rackerlabs/understack/bootstrap/ | kubectl apply -f - -``` - -Now configure ArgoCD to be able to authenticate against Dex IdP. +Configure ArgoCD to be able to authenticate against Dex IdP. ```bash kubectl -n argocd apply -f "${UC_DEPLOY}/secrets/${DEPLOY_NAME}/secret-argocd-sso-argocd.yaml"