From dd978fc25f626650c1ddbb863c35c377cff88d8e Mon Sep 17 00:00:00 2001 From: Josh Diamond Date: Fri, 20 Sep 2024 09:47:32 -0700 Subject: [PATCH] enhancement to allow proxy packages to use staging registry --- packages/aws/rancher-proxy.yaml | 21 +++++ packages/aws/rancher-registry.yaml | 13 ++-- templates/proxy-standalone/manifest.yaml | 26 +++++++ .../overlay/opt/basic-proxy/squid/squid.conf | 77 +++++++++++++++++++ .../overlay/opt/corral/proxy/proxy-install.sh | 18 +++++ templates/rancher-proxy/manifest.yaml | 52 +++++++++++++ .../corral/rancher/install-cert-manager.sh | 12 +++ .../opt/corral/rancher/install-rancher.sh | 67 ++++++++++++++++ .../overlay/opt/corral/rancher/preflight.sh | 18 +++++ .../opt/corral/rancher/wait-for-password.sh | 47 +++++++++++ 10 files changed, 345 insertions(+), 6 deletions(-) create mode 100644 packages/aws/rancher-proxy.yaml create mode 100644 templates/proxy-standalone/manifest.yaml create mode 100644 templates/proxy-standalone/overlay/opt/basic-proxy/squid/squid.conf create mode 100755 templates/proxy-standalone/overlay/opt/corral/proxy/proxy-install.sh create mode 100644 templates/rancher-proxy/manifest.yaml create mode 100644 templates/rancher-proxy/overlay/opt/corral/rancher/install-cert-manager.sh create mode 100644 templates/rancher-proxy/overlay/opt/corral/rancher/install-rancher.sh create mode 100644 templates/rancher-proxy/overlay/opt/corral/rancher/preflight.sh create mode 100644 templates/rancher-proxy/overlay/opt/corral/rancher/wait-for-password.sh diff --git a/packages/aws/rancher-proxy.yaml b/packages/aws/rancher-proxy.yaml new file mode 100644 index 0000000..a7b4c18 --- /dev/null +++ b/packages/aws/rancher-proxy.yaml @@ -0,0 +1,21 @@ +manifest: + name: rancher-proxy + description: rancher-proxy + variables: + server_count: + default: 1 +templates: + - aws/registry_nodes + - aws/cluster_nodes + - proxy-standalone + - rke2 + - rancher-proxy +variables: + cni: + - calico + proxy_setup: + - true + docker_compose_version: + - 2.15.1 + cert_manager_version: + - 1.11.0 \ No newline at end of file diff --git a/packages/aws/rancher-registry.yaml b/packages/aws/rancher-registry.yaml index 9a2dc3d..cd8fad4 100644 --- a/packages/aws/rancher-registry.yaml +++ b/packages/aws/rancher-registry.yaml @@ -13,17 +13,18 @@ templates: variables: cni: - calico - kubernetes_version: - - v1.25.16+rke2r1 - - v1.26.14+rke2r1 + # kubernetes_version: + # - v1.25.16+rke2r1 + # - v1.26.14+rke2r1 registry_auth: - global - enabled - disabled docker_compose_version: - 2.18.1 - rancher_version: - - 2.7.10 - - 2.8.2 + # rancher_version: + # - 2.7.10 + # - 2.8.2 cert_manager_version: + - 1.15.0 - 1.11.0 diff --git a/templates/proxy-standalone/manifest.yaml b/templates/proxy-standalone/manifest.yaml new file mode 100644 index 0000000..7488d95 --- /dev/null +++ b/templates/proxy-standalone/manifest.yaml @@ -0,0 +1,26 @@ +name: proxy +description: | + A docker squid proxy +variables: + registry_ip: + type: string + readOnly: true + description: "Public IP address of the proxy bastion node" + registry_private_ip: + type: string + readOnly: true + description: "Private IP address of the proxy bastion node" + rancher_version: + type: string + description: "The rancher version to download the images for" + cert_manager_version: + type: string + description: "The cert-manager version for HA rancher install" + proxy_setup: + type: boolean + description: "Boolean to set a registry_node as a proxy bastion node" + default: true +commands: + - command: /opt/corral/proxy/proxy-install.sh + node_pools: + - registry diff --git a/templates/proxy-standalone/overlay/opt/basic-proxy/squid/squid.conf b/templates/proxy-standalone/overlay/opt/basic-proxy/squid/squid.conf new file mode 100644 index 0000000..6add618 --- /dev/null +++ b/templates/proxy-standalone/overlay/opt/basic-proxy/squid/squid.conf @@ -0,0 +1,77 @@ +# +# Recommended minimum configuration: +# + +# Example rule allowing access from your local networks. +# Adapt to list your (internal) IP networks from where browsing +# should be allowed +acl localnet src 10.0.0.0/8 # RFC1918 possible internal network +acl localnet src 172.0.0.0/8 # RFC1918 possible internal network +acl localnet src 192.168.0.0/16 # RFC1918 possible internal network +acl localnet src fc00::/7 # RFC 4193 local private network range +acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines + +acl Safe_ports port 22 # ssh +acl Safe_ports port 2376 # docker port +acl Safe_ports port 8443 # keycloak +acl SSL_ports port 22 +acl SSL_ports port 2376 + +acl SSL_ports port 443 +acl SSL_ports port 6443 +acl SSL_ports port 8443 # keycloak +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http +acl Safe_ports port 6443 # k8s +acl CONNECT method CONNECT + +# Recommended minimum Access Permission configuration: +# Deny requests to certain unsafe ports +http_access deny !Safe_ports + +# Deny CONNECT to other than secure SSL ports +http_access deny CONNECT !SSL_ports + +# Only allow cachemgr access from localhost +http_access allow localhost manager +http_access deny manager + +# We strongly recommend the following be uncommented to protect innocent +# web applications running on the proxy server who think the only +# one who can access services on "localhost" is a local user +#http_access deny to_localhost + +# Example rule allowing access from your local networks. +# Adapt localnet in the ACL section to list your (internal) IP networks +# from where browsing should be allowed +http_access allow localnet +http_access allow localhost +http_access allow all + +# Squid normally listens to port 3128 +http_port 3219 + +# Uncomment and adjust the following to add a disk cache directory. +#cache_dir ufs /var/cache/squid 100 16 256 + +# Leave coredumps in the first cache dir +coredump_dir /var/cache/squid + +# Add any of your own refresh_pattern entries above these. +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern . 0 20% 4320 + + + + + diff --git a/templates/proxy-standalone/overlay/opt/corral/proxy/proxy-install.sh b/templates/proxy-standalone/overlay/opt/corral/proxy/proxy-install.sh new file mode 100755 index 0000000..772d458 --- /dev/null +++ b/templates/proxy-standalone/overlay/opt/corral/proxy/proxy-install.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -ex + +function corral_set() { + echo "corral_set $1=$2" +} + +function corral_log() { + echo "corral_log $1" +} + +echo "$CORRAL_corral_user_public_key" >> "$HOME"/.ssh/authorized_key + +docker run -d -v /opt/basic-proxy/squid/squid.conf:/etc/squid/squid.conf -p 3219:3219 ubuntu/squid + +CORRAL_squid_container=$(docker ps --format {{.ID}}) + +echo "corral_set squid_container=$CORRAL_squid_container" diff --git a/templates/rancher-proxy/manifest.yaml b/templates/rancher-proxy/manifest.yaml new file mode 100644 index 0000000..874b2e8 --- /dev/null +++ b/templates/rancher-proxy/manifest.yaml @@ -0,0 +1,52 @@ +name: rancher-proxy +description: Install rancher on the airgapped kubernetes cluster behind a proxy. +variables: + rancher_version: + type: string + description: "Specify rancher version to install. Defaults to latest stable version." + bootstrap_password: + readOnly: true + type: string + description: "Initial password for the rancher `admin` user." + rancher_host: + readOnly: true + type: string + description: "Host of newly created rancher instance." + rancher_url: + readOnly: true + type: string + description: "Host of newly created rancher instance." + cert_manager_version: + type: string + description: "The cert-manager version for rancher install" + rancher_chart_repo: + optional: false + type: string + default: "latest" + description: "Name of Helm chart to use for Rancher install. Example: latest, alpha, stable, prime or staging" + rancher_image: + type: string + optional: true + description: "Specify rancher image for rancher image in a separate docker hub/registry" + rancher_image_tag: + type: string + optional: true + description: "Specify rancher image tag for the latest commit/version of rancher." + rancher_chart_url: + type: string + description: "the URL of the helm repo where rancher chart exists. i.e. https://releases.rancher.com/server-charts/latest" + optional: true +commands: + - command: "/opt/corral/rancher/preflight.sh" + node_pools: + - bastion + - server + - command: "/opt/corral/rancher/install-cert-manager.sh" + node_pools: + - bastion + - command: "/opt/corral/rancher/install-rancher.sh" + node_pools: + - bastion + - command: "/opt/corral/rancher/wait-for-password.sh" + node_pools: + - bastion \ No newline at end of file diff --git a/templates/rancher-proxy/overlay/opt/corral/rancher/install-cert-manager.sh b/templates/rancher-proxy/overlay/opt/corral/rancher/install-cert-manager.sh new file mode 100644 index 0000000..92f5c3e --- /dev/null +++ b/templates/rancher-proxy/overlay/opt/corral/rancher/install-cert-manager.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -ex + +helm repo add jetstack https://charts.jetstack.io +helm repo update +kubectl create namespace cert-manager +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v$CORRAL_cert_manager_version/cert-manager.crds.yaml + +helm upgrade --install cert-manager jetstack/cert-manager --namespace cert-manager --version v$CORRAL_cert_manager_version --set http_proxy=http://$CORRAL_registry_private_ip:3219 --set https_proxy=http://$CORRAL_registry_private_ip:3219 --set no_proxy=127.0.0.0/8\\,10.0.0.0/8\\,172.0.0.0/8\\,192.168.0.0/16\\,.svc\\,.cluster.local\\,cattle-system.svc\\,169.254.169.254 +# when attempting to install rancher right after the cert-manager install there is some intermitten issues +# allowing it to sleep for at least a 1m fixes the issue. +sleep 1m diff --git a/templates/rancher-proxy/overlay/opt/corral/rancher/install-rancher.sh b/templates/rancher-proxy/overlay/opt/corral/rancher/install-rancher.sh new file mode 100644 index 0000000..4af5f90 --- /dev/null +++ b/templates/rancher-proxy/overlay/opt/corral/rancher/install-rancher.sh @@ -0,0 +1,67 @@ +#!/bin/bash +set -ex +repos=("latest" "alpha" "stable" "staging" "prime") +if [[ ! ${repos[*]} =~ ${CORRAL_rancher_chart_repo} ]]; then + echo 'Error: `rancher_chart_repo` must be one of ["latest", "alpha", "stable", "staging", "prime"]' + exit 1 +fi + +CORRAL_rancher_host=${CORRAL_rancher_host:="${CORRAL_fqdn}"} +CORRAL_rancher_version=${CORRAL_rancher_version:=$(helm search repo rancher-latest/rancher -o json | jq -r .[0].version)} +minor_version=$(echo "$CORRAL_kubernetes_version" | cut -d. -f2) + +kubectl create namespace cattle-system + +community=("latest" "alpha" "stable") + +if [ "$minor_version" -gt 24 ]; then + + args=("rancher-$CORRAL_rancher_chart_repo/rancher" "--namespace cattle-system" "--set global.cattle.psp.enabled=false" "--set hostname=$CORRAL_rancher_host" "--version=$CORRAL_rancher_version" "--set proxy=http://$CORRAL_registry_private_ip:3219") + + if [[ ${community[*]} =~ ${CORRAL_rancher_chart_repo} ]]; then + if [ ! -z "$CORRAL_rancher_chart_url" ]; then + helm repo add "rancher-$CORRAL_rancher_chart_repo" "$CORRAL_rancher_chart_url" + else + helm repo add "rancher-$CORRAL_rancher_chart_repo" "https://releases.rancher.com/server-charts/$CORRAL_rancher_chart_repo" + fi + args2=("") + fi + + if [[ "$CORRAL_rancher_chart_repo" == "prime" ]]; then + helm repo add "rancher-prime" "https://charts.rancher.com/server-charts/prime" + args2=("--set rancherImage=registry.suse.com/rancher/rancher") + fi + + if [[ "$CORRAL_rancher_chart_repo" == "staging" ]]; then + helm repo add "rancher-staging" "https://charts.optimus.rancher.io/server-charts/latest" + args2=("--set rancherImage=stgregistry.suse.com/rancher/rancher") + + if [ ! -z "$CORRAL_rancher_image_tag" ]; then + args2+=("--set rancherImageTag=$CORRAL_rancher_image_tag") + fi + + helm repo update + + if [ ! -z "$CORRAL_rancher_image" ]; then + helm upgrade --install rancher ${args[*]} --set noProxy=localhost\\,127.0.0.0/8\\,10.0.0.0/8\\,172.0.0.0/8\\,192.168.0.0/16\\,.svc\\,.cluster.local\\,cattle-system.svc\\,169.254.169.254 ${args2[*]} --set 'extraEnv[0].name=CATTLE_AGENT_IMAGE' --set 'extraEnv[0].value=stgregistry.suse.com/rancher/rancher-agent:'$CORRAL_rancher_version'' + else + helm upgrade --install rancher ${args[*]} --set noProxy=localhost\\,127.0.0.0/8\\,10.0.0.0/8\\,172.0.0.0/8\\,192.168.0.0/16\\,.svc\\,.cluster.local\\,cattle-system.svc\\,169.254.169.254 ${args2[*]} + fi + echo "corral_set rancher_version=$CORRAL_rancher_version" + echo "corral_set rancher_host=$CORRAL_rancher_host" + exit 0 + fi + + helm repo update + + if [ ! -z "$CORRAL_rancher_image_tag" ]; then + args2+=("--set rancherImageTag=$CORRAL_rancher_image_tag") + fi + + helm upgrade --install rancher ${args[*]} --set noProxy=localhost\\,127.0.0.0/8\\,10.0.0.0/8\\,172.0.0.0/8\\,192.168.0.0/16\\,.svc\\,.cluster.local\\,cattle-system.svc\\,169.254.169.254 ${args2[*]} +else + helm upgrade --install rancher rancher-$CORRAL_rancher_chart_repo/rancher --namespace cattle-system --set hostname=$CORRAL_rancher_host --version=$CORRAL_rancher_version --set proxy=http://$CORRAL_registry_private_ip:3219 --set noProxy=127.0.0.0/8\\,10.0.0.0/8\\,172.0.0.0/8\\,192.168.0.0/16\\,.svc\\,.cluster.local\\,cattle-system.svc\\,169.254.169.254 +fi + +echo "corral_set rancher_version=$CORRAL_rancher_version" +echo "corral_set rancher_host=$CORRAL_rancher_host" diff --git a/templates/rancher-proxy/overlay/opt/corral/rancher/preflight.sh b/templates/rancher-proxy/overlay/opt/corral/rancher/preflight.sh new file mode 100644 index 0000000..a0bf421 --- /dev/null +++ b/templates/rancher-proxy/overlay/opt/corral/rancher/preflight.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -ex + +apt-get update || true + +apt install -y jq || true + +curl --proxy http://$CORRAL_registry_private_ip:3219 https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + +curl --proxy http://$CORRAL_registry_private_ip:3219 -LO https://storage.googleapis.com/kubernetes-release/release/$(curl --proxy http://$CORRAL_registry_private_ip:3219 -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl + +chmod +x ./kubectl +mv ./kubectl /usr/local/bin/kubectl + +mkdir ~/.kube + +echo $CORRAL_kubeconfig | base64 -d > ~/.kube/config +chmod 400 ~/.kube/config \ No newline at end of file diff --git a/templates/rancher-proxy/overlay/opt/corral/rancher/wait-for-password.sh b/templates/rancher-proxy/overlay/opt/corral/rancher/wait-for-password.sh new file mode 100644 index 0000000..45b76be --- /dev/null +++ b/templates/rancher-proxy/overlay/opt/corral/rancher/wait-for-password.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -ex + +function corral_set() { + echo "corral_set $1=$2" +} + +function corral_log() { + echo "corral_log $1" +} + +if [[ $CORRAL_rancher_version == "2.5.*" ]]; then + echo "corral_set bootstrap_password=admin" + return 0 +fi + +if [ ${CORRAL_bootstrap_password} -ne "" ]; then + echo "bootstrap_password=${CORRAL_bootstrap_password}" + exit 0 +fi + +echo "waiting for bootstrap password" +until [ "$(kubectl -n cattle-system get secret/bootstrap-secret -o json --ignore-not-found=true | jq -r '.data.bootstrapPassword | length > 0')" == "true" ]; do + sleep 0.1 + echo -n "." +done +echo + +echo "corral_set bootstrap_password=$(kubectl -n cattle-system get secret/bootstrap-secret -o json | jq -r '.data.bootstrapPassword' | base64 -d)" +bootstrap_password=$(kubectl -n cattle-system get secret/bootstrap-secret -o json | jq -r '.data.bootstrapPassword' | base64 -d) + + +corral_log "Bastion public address: ${CORRAL_registry_ip}" + +corral_log "Bastion private address: ${CORRAL_registry_private_ip}" + +corral_log "Save private key: echo \"${CORRAL_corral_private_key}\" | tr -d '\"' > id_rsa" + +corral_log "Save public key: echo \"${CORRAL_corral_public_key}\" | tr -d '\"' > id_rsa.pub" + +corral_log "Follow squid proxy logs: ssh -i id_rsa root@${CORRAL_registry_ip} \"sudo docker exec $CORRAL_squid_container tail -f /var/log/squid/access.log\" " + +corral_log "Connect to bastion node: ssh -i id_rsa root@${CORRAL_registry_ip}" + +corral_log "From bastion, connect to rancher server node with: ssh ubuntu@${CORRAL_kube_api_host}" + +corral_log "Rancher instance running at: https://$CORRAL_rancher_host/dashboard/?setup=$bootstrap_password"