enhancement to allow proxy packages to use staging registry

packages/aws/rancher-proxy.yaml

@@ -0,0 +1,21 @@
+manifest:
+  name: rancher-proxy
+  description: rancher-proxy
+  variables:
+    server_count:
+      default: 1
+templates:
+  - aws/registry_nodes
+  - aws/cluster_nodes
+  - proxy-standalone
+  - rke2
+  - rancher-proxy
+variables:
+  cni:
+    - calico
+  proxy_setup:
+    - true
+  docker_compose_version:
+    - 2.15.1
+  cert_manager_version:
+    - 1.11.0

packages/aws/rancher-registry.yaml

@@ -13,17 +13,18 @@ templates:
 variables:
   cni:
     - calico
-  kubernetes_version:
-    - v1.25.16+rke2r1
-    - v1.26.14+rke2r1
+  # kubernetes_version:
+  #   - v1.25.16+rke2r1
+  #   - v1.26.14+rke2r1
   registry_auth:
     - global
     - enabled
     - disabled
   docker_compose_version:
     - 2.18.1
-  rancher_version:
-    - 2.7.10
-    - 2.8.2
+  # rancher_version:
+  #   - 2.7.10
+  #   - 2.8.2
   cert_manager_version:
+    - 1.15.0
     - 1.11.0

templates/proxy-standalone/manifest.yaml

@@ -0,0 +1,26 @@
+name: proxy
+description: |
+  A docker squid proxy
+variables:
+  registry_ip:
+    type: string
+    readOnly: true
+    description: "Public IP address of the proxy bastion node"
+  registry_private_ip:
+    type: string
+    readOnly: true
+    description: "Private IP address of the proxy bastion node"
+  rancher_version:
+    type: string
+    description: "The rancher version to download the images for"
+  cert_manager_version:
+    type: string
+    description: "The cert-manager version for HA rancher install"
+  proxy_setup:
+    type: boolean
+    description: "Boolean to set a registry_node as a proxy bastion node"
+    default: true
+commands:
+  - command: /opt/corral/proxy/proxy-install.sh
+    node_pools:
+      - registry

templates/proxy-standalone/overlay/opt/basic-proxy/squid/squid.conf

@@ -0,0 +1,77 @@
+#
+# Recommended minimum configuration:
+#
+
+# Example rule allowing access from your local networks.
+# Adapt to list your (internal) IP networks from where browsing
+# should be allowed
+acl localnet src 10.0.0.0/8	# RFC1918 possible internal network
+acl localnet src 172.0.0.0/8	# RFC1918 possible internal network
+acl localnet src 192.168.0.0/16	# RFC1918 possible internal network
+acl localnet src fc00::/7       # RFC 4193 local private network range
+acl localnet src fe80::/10      # RFC 4291 link-local (directly plugged) machines
+
+acl Safe_ports port 22      # ssh
+acl Safe_ports port 2376    # docker port
+acl Safe_ports port 8443    # keycloak
+acl SSL_ports port 22
+acl SSL_ports port 2376
+
+acl SSL_ports port 443
+acl SSL_ports port 6443
+acl SSL_ports port 8443     # keycloak
+acl Safe_ports port 80		# http
+acl Safe_ports port 21		# ftp
+acl Safe_ports port 443		# https
+acl Safe_ports port 70		# gopher
+acl Safe_ports port 210		# wais
+acl Safe_ports port 1025-65535	# unregistered ports
+acl Safe_ports port 280		# http-mgmt
+acl Safe_ports port 488		# gss-http
+acl Safe_ports port 591		# filemaker
+acl Safe_ports port 777		# multiling http
+acl Safe_ports port 6443        # k8s
+acl CONNECT method CONNECT
+
+# Recommended minimum Access Permission configuration:
+# Deny requests to certain unsafe ports
+http_access deny !Safe_ports
+
+# Deny CONNECT to other than secure SSL ports
+http_access deny CONNECT !SSL_ports
+
+# Only allow cachemgr access from localhost
+http_access allow localhost manager
+http_access deny manager
+
+# We strongly recommend the following be uncommented to protect innocent
+# web applications running on the proxy server who think the only
+# one who can access services on "localhost" is a local user
+#http_access deny to_localhost
+
+# Example rule allowing access from your local networks.
+# Adapt localnet in the ACL section to list your (internal) IP networks
+# from where browsing should be allowed
+http_access allow localnet
+http_access allow localhost
+http_access allow all
+
+# Squid normally listens to port 3128
+http_port 3219
+
+# Uncomment and adjust the following to add a disk cache directory.
+#cache_dir ufs /var/cache/squid 100 16 256
+
+# Leave coredumps in the first cache dir
+coredump_dir /var/cache/squid
+
+# Add any of your own refresh_pattern entries above these.
+refresh_pattern ^ftp:		1440	20%	10080
+refresh_pattern ^gopher:	1440	0%	1440
+refresh_pattern -i (/cgi-bin/|\?) 0	0%	0
+refresh_pattern .		0	20%	4320
+
+
+
+
+

templates/proxy-standalone/overlay/opt/corral/proxy/proxy-install.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -ex
+
+function corral_set() {
+    echo "corral_set $1=$2"
+}
+
+function corral_log() {
+    echo "corral_log $1"
+}
+
+echo "$CORRAL_corral_user_public_key" >> "$HOME"/.ssh/authorized_key
+
+docker run -d -v /opt/basic-proxy/squid/squid.conf:/etc/squid/squid.conf -p 3219:3219 ubuntu/squid
+
+CORRAL_squid_container=$(docker ps --format {{.ID}})
+
+echo "corral_set squid_container=$CORRAL_squid_container"

templates/rancher-proxy/manifest.yaml

@@ -0,0 +1,52 @@
+name: rancher-proxy
+description: Install rancher on the airgapped kubernetes cluster behind a proxy.
+variables:
+  rancher_version:
+    type: string
+    description: "Specify rancher version to install. Defaults to latest stable version."
+  bootstrap_password:
+    readOnly: true
+    type: string
+    description: "Initial password for the rancher `admin` user."
+  rancher_host:
+    readOnly: true
+    type: string
+    description: "Host of newly created rancher instance."
+  rancher_url:
+    readOnly: true
+    type: string
+    description: "Host of newly created rancher instance."
+  cert_manager_version:
+    type: string
+    description: "The cert-manager version for rancher install"
+  rancher_chart_repo:
+    optional: false
+    type: string
+    default: "latest"
+    description: "Name of Helm chart to use for Rancher install. Example: latest, alpha, stable, prime or staging"
+  rancher_image:
+    type: string
+    optional: true
+    description: "Specify rancher image for rancher image in a separate docker hub/registry"
+  rancher_image_tag:
+    type: string
+    optional: true
+    description: "Specify rancher image tag for the latest commit/version of rancher."
+  rancher_chart_url:
+    type: string
+    description: "the URL of the helm repo where rancher chart exists. i.e. https://releases.rancher.com/server-charts/latest"
+    optional: true
+commands:
+  - command: "/opt/corral/rancher/preflight.sh"
+    node_pools:
+      - bastion
+      - server
+  - command: "/opt/corral/rancher/install-cert-manager.sh"
+    node_pools:
+      - bastion
+  - command: "/opt/corral/rancher/install-rancher.sh"
+    node_pools:
+      - bastion
+  - command: "/opt/corral/rancher/wait-for-password.sh"
+    node_pools:
+      - bastion

templates/rancher-proxy/overlay/opt/corral/rancher/install-cert-manager.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -ex
+
+helm repo add jetstack https://charts.jetstack.io
+helm repo update
+kubectl create namespace cert-manager
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v$CORRAL_cert_manager_version/cert-manager.crds.yaml
+
+helm upgrade --install cert-manager jetstack/cert-manager --namespace cert-manager --version v$CORRAL_cert_manager_version --set http_proxy=http://$CORRAL_registry_private_ip:3219 --set https_proxy=http://$CORRAL_registry_private_ip:3219 --set no_proxy=127.0.0.0/8\\,10.0.0.0/8\\,172.0.0.0/8\\,192.168.0.0/16\\,.svc\\,.cluster.local\\,cattle-system.svc\\,169.254.169.254
+# when attempting to install rancher right after the cert-manager install there is some intermitten issues
+# allowing it to sleep for at least a 1m fixes the issue.
+sleep 1m

templates/rancher-proxy/overlay/opt/corral/rancher/install-rancher.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+set -ex
+repos=("latest" "alpha" "stable" "staging" "prime")
+if [[ ! ${repos[*]} =~ ${CORRAL_rancher_chart_repo} ]]; then
+  echo 'Error: `rancher_chart_repo` must be one of ["latest", "alpha", "stable", "staging", "prime"]'
+  exit 1
+fi
+
+CORRAL_rancher_host=${CORRAL_rancher_host:="${CORRAL_fqdn}"}
+CORRAL_rancher_version=${CORRAL_rancher_version:=$(helm search repo rancher-latest/rancher -o json | jq -r .[0].version)}
+minor_version=$(echo "$CORRAL_kubernetes_version" | cut -d. -f2)
+
+kubectl create namespace cattle-system
+
+community=("latest" "alpha" "stable")
+
+if [ "$minor_version" -gt 24 ]; then
+
+    args=("rancher-$CORRAL_rancher_chart_repo/rancher" "--namespace cattle-system" "--set global.cattle.psp.enabled=false" "--set hostname=$CORRAL_rancher_host" "--version=$CORRAL_rancher_version" "--set proxy=http://$CORRAL_registry_private_ip:3219")
+
+    if [[  ${community[*]} =~ ${CORRAL_rancher_chart_repo} ]]; then
+      if [ ! -z "$CORRAL_rancher_chart_url" ]; then
+        helm repo add "rancher-$CORRAL_rancher_chart_repo" "$CORRAL_rancher_chart_url"
+      else
+        helm repo add "rancher-$CORRAL_rancher_chart_repo" "https://releases.rancher.com/server-charts/$CORRAL_rancher_chart_repo"
+      fi
+      args2=("")
+    fi
+
+    if [[ "$CORRAL_rancher_chart_repo" == "prime" ]]; then
+     helm repo add "rancher-prime" "https://charts.rancher.com/server-charts/prime"
+     args2=("--set rancherImage=registry.suse.com/rancher/rancher")
+    fi
+
+    if [[ "$CORRAL_rancher_chart_repo" == "staging" ]]; then
+     helm repo add "rancher-staging" "https://charts.optimus.rancher.io/server-charts/latest"
+     args2=("--set rancherImage=stgregistry.suse.com/rancher/rancher")
+
+      if [ ! -z "$CORRAL_rancher_image_tag" ]; then
+        args2+=("--set rancherImageTag=$CORRAL_rancher_image_tag")
+      fi
+
+      helm repo update
+
+      if [ ! -z "$CORRAL_rancher_image" ]; then
+          helm upgrade --install rancher ${args[*]} --set noProxy=localhost\\,127.0.0.0/8\\,10.0.0.0/8\\,172.0.0.0/8\\,192.168.0.0/16\\,.svc\\,.cluster.local\\,cattle-system.svc\\,169.254.169.254 ${args2[*]} --set 'extraEnv[0].name=CATTLE_AGENT_IMAGE' --set 'extraEnv[0].value=stgregistry.suse.com/rancher/rancher-agent:'$CORRAL_rancher_version''
+      else
+        helm upgrade --install rancher ${args[*]} --set noProxy=localhost\\,127.0.0.0/8\\,10.0.0.0/8\\,172.0.0.0/8\\,192.168.0.0/16\\,.svc\\,.cluster.local\\,cattle-system.svc\\,169.254.169.254 ${args2[*]}
+      fi
+      echo "corral_set rancher_version=$CORRAL_rancher_version"
+      echo "corral_set rancher_host=$CORRAL_rancher_host"
+      exit 0
+    fi
+
+    helm repo update
+
+    if [ ! -z "$CORRAL_rancher_image_tag" ]; then
+      args2+=("--set rancherImageTag=$CORRAL_rancher_image_tag")
+    fi
+
+    helm upgrade --install rancher ${args[*]} --set noProxy=localhost\\,127.0.0.0/8\\,10.0.0.0/8\\,172.0.0.0/8\\,192.168.0.0/16\\,.svc\\,.cluster.local\\,cattle-system.svc\\,169.254.169.254 ${args2[*]}
+else
+    helm upgrade --install rancher rancher-$CORRAL_rancher_chart_repo/rancher --namespace cattle-system --set hostname=$CORRAL_rancher_host --version=$CORRAL_rancher_version --set proxy=http://$CORRAL_registry_private_ip:3219 --set noProxy=127.0.0.0/8\\,10.0.0.0/8\\,172.0.0.0/8\\,192.168.0.0/16\\,.svc\\,.cluster.local\\,cattle-system.svc\\,169.254.169.254
+fi
+
+echo "corral_set rancher_version=$CORRAL_rancher_version"
+echo "corral_set rancher_host=$CORRAL_rancher_host"

templates/rancher-proxy/overlay/opt/corral/rancher/preflight.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -ex
+
+apt-get update || true
+
+apt install -y jq || true
+
+curl --proxy http://$CORRAL_registry_private_ip:3219 https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+
+curl --proxy http://$CORRAL_registry_private_ip:3219 -LO https://storage.googleapis.com/kubernetes-release/release/$(curl --proxy http://$CORRAL_registry_private_ip:3219 -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
+
+chmod +x ./kubectl
+mv ./kubectl /usr/local/bin/kubectl
+
+mkdir ~/.kube
+
+echo $CORRAL_kubeconfig | base64 -d > ~/.kube/config
+chmod 400 ~/.kube/config

templates/rancher-proxy/overlay/opt/corral/rancher/wait-for-password.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -ex
+
+function corral_set() {
+    echo "corral_set $1=$2"
+}
+
+function corral_log() {
+    echo "corral_log $1"
+}
+
+if [[ $CORRAL_rancher_version == "2.5.*" ]]; then
+  echo "corral_set bootstrap_password=admin"
+  return 0
+fi
+
+if [ ${CORRAL_bootstrap_password} -ne "" ]; then
+  echo "bootstrap_password=${CORRAL_bootstrap_password}"
+  exit 0
+fi
+
+echo "waiting for bootstrap password"
+until [ "$(kubectl -n cattle-system get secret/bootstrap-secret -o json --ignore-not-found=true | jq -r '.data.bootstrapPassword | length > 0')" == "true" ]; do
+  sleep 0.1
+  echo -n "."
+done
+echo
+
+echo "corral_set bootstrap_password=$(kubectl -n cattle-system get secret/bootstrap-secret -o json | jq -r '.data.bootstrapPassword' | base64 -d)"
+bootstrap_password=$(kubectl -n cattle-system get secret/bootstrap-secret -o json | jq -r '.data.bootstrapPassword' | base64 -d)
+
+
+corral_log "Bastion public address: ${CORRAL_registry_ip}" 
+
+corral_log "Bastion private address: ${CORRAL_registry_private_ip}"
+
+corral_log "Save private key: echo \"${CORRAL_corral_private_key}\" | tr -d '\"' > id_rsa"
+
+corral_log "Save public key: echo \"${CORRAL_corral_public_key}\" | tr -d '\"' > id_rsa.pub"
+
+corral_log "Follow squid proxy logs: ssh -i id_rsa root@${CORRAL_registry_ip} \"sudo docker exec $CORRAL_squid_container tail -f /var/log/squid/access.log\" "
+
+corral_log "Connect to bastion node: ssh -i id_rsa root@${CORRAL_registry_ip}"
+
+corral_log "From bastion, connect to rancher server node with: ssh ubuntu@${CORRAL_kube_api_host}"
+
+corral_log "Rancher instance running at: https://$CORRAL_rancher_host/dashboard/?setup=$bootstrap_password"