Improve UX around setting Proxy configurations in the spec

KU-1801
canonical · Oct 11, 2024 · a0b9e29 · a0b9e29
1 parent 1a107cc
commit a0b9e29
Show file tree

Hide file tree

Showing 12 changed files with 365 additions and 9 deletions.
diff --git a/bootstrap/api/v1beta2/ck8sconfig_types.go b/bootstrap/api/v1beta2/ck8sconfig_types.go
@@ -66,6 +66,18 @@ type CK8sConfigSpec struct {
 	// +optional
 	SnapstoreProxyID string `json:"snapstoreProxyId,omitempty"`
 
+	// HTTPSProxy is optional https proxy configuration
+	// +optional
+	HTTPSProxy string `json:"httpsProxy,omitempty"`
+
+	// HTTPProxy is optional http proxy configuration
+	// +optional
+	HTTPProxy string `json:"httpProxy,omitempty"`
+
+	// NoProxy is optional no proxy configuration
+	// +optional
+	NoProxy string `json:"noProxy,omitempty"`
+
 	// CK8sControlPlaneConfig is configuration for the control plane node.
 	// +optional
 	ControlPlaneConfig CK8sControlPlaneConfig `json:"controlPlane,omitempty"`

diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigs.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigs.yaml
@@ -165,6 +165,12 @@ spec:
                   - path
                   type: object
                 type: array
+              httpProxy:
+                description: HTTPProxy is optional http proxy configuration
+                type: string
+              httpsProxy:
+                description: HTTPSProxy is optional https proxy configuration
+                type: string
               initConfig:
                 description: CK8sInitConfig is configuration for the initializing
                   the cluster features.
@@ -192,6 +198,9 @@ spec:
                       the default CNI.
                     type: boolean
                 type: object
+              noProxy:
+                description: NoProxy is optional no proxy configuration
+                type: string
               nodeName:
                 description: |-
                   NodeName is the name to use for the kubelet of this node. It is needed for clouds

diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigtemplates.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigtemplates.yaml
@@ -174,6 +174,12 @@ spec:
                           - path
                           type: object
                         type: array
+                      httpProxy:
+                        description: HTTPProxy is optional http proxy configuration
+                        type: string
+                      httpsProxy:
+                        description: HTTPSProxy is optional https proxy configuration
+                        type: string
                       initConfig:
                         description: CK8sInitConfig is configuration for the initializing
                           the cluster features.
@@ -201,6 +207,9 @@ spec:
                               enable the default CNI.
                             type: boolean
                         type: object
+                      noProxy:
+                        description: NoProxy is optional no proxy configuration
+                        type: string
                       nodeName:
                         description: |-
                           NodeName is the name to use for the kubelet of this node. It is needed for clouds

diff --git a/bootstrap/controllers/ck8sconfig_controller.go b/bootstrap/controllers/ck8sconfig_controller.go
@@ -356,6 +356,9 @@ func (r *CK8sConfigReconciler) joinWorker(ctx context.Context, scope *Scope) err
 			ConfigFileContents:   string(joinConfig),
 			MicroclusterAddress:  scope.Config.Spec.ControlPlaneConfig.MicroclusterAddress,
 			MicroclusterPort:     microclusterPort,
+			ContainerdHTTPProxy:  scope.Config.Spec.HTTPProxy,
+			ContainerdHTTPSProxy: scope.Config.Spec.HTTPSProxy,
+			ContainerdNoProxy:    scope.Config.Spec.NoProxy,
 			AirGapped:            scope.Config.Spec.AirGapped,
 			SnapstoreProxyScheme: scope.Config.Spec.SnapstoreProxyScheme,
 			SnapstoreProxyDomain: scope.Config.Spec.SnapstoreProxyDomain,
@@ -590,6 +593,9 @@ func (r *CK8sConfigReconciler) handleClusterNotInitialized(ctx context.Context,
 			MicroclusterAddress:  scope.Config.Spec.ControlPlaneConfig.MicroclusterAddress,
 			MicroclusterPort:     microclusterPort,
 			NodeName:             scope.Config.Spec.NodeName,
+			ContainerdHTTPProxy:  scope.Config.Spec.HTTPProxy,
+			ContainerdHTTPSProxy: scope.Config.Spec.HTTPSProxy,
+			ContainerdNoProxy:    scope.Config.Spec.NoProxy,
 			AirGapped:            scope.Config.Spec.AirGapped,
 			SnapstoreProxyScheme: scope.Config.Spec.SnapstoreProxyScheme,
 			SnapstoreProxyDomain: scope.Config.Spec.SnapstoreProxyDomain,

diff --git a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanes.yaml b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanes.yaml
@@ -362,6 +362,12 @@ spec:
                       - path
                       type: object
                     type: array
+                  httpProxy:
+                    description: HTTPProxy is optional http proxy configuration
+                    type: string
+                  httpsProxy:
+                    description: HTTPSProxy is optional https proxy configuration
+                    type: string
                   initConfig:
                     description: CK8sInitConfig is configuration for the initializing
                       the cluster features.
@@ -389,6 +395,9 @@ spec:
                           the default CNI.
                         type: boolean
                     type: object
+                  noProxy:
+                    description: NoProxy is optional no proxy configuration
+                    type: string
                   nodeName:
                     description: |-
                       NodeName is the name to use for the kubelet of this node. It is needed for clouds

diff --git a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanetemplates.yaml b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanetemplates.yaml
@@ -339,6 +339,12 @@ spec:
                               - path
                               type: object
                             type: array
+                          httpProxy:
+                            description: HTTPProxy is optional http proxy configuration
+                            type: string
+                          httpsProxy:
+                            description: HTTPSProxy is optional https proxy configuration
+                            type: string
                           initConfig:
                             description: CK8sInitConfig is configuration for the initializing
                               the cluster features.
@@ -366,6 +372,9 @@ spec:
                                   to enable the default CNI.
                                 type: boolean
                             type: object
+                          noProxy:
+                            description: NoProxy is optional no proxy configuration
+                            type: string
                           nodeName:
                             description: |-
                               NodeName is the name to use for the kubelet of this node. It is needed for clouds

diff --git a/pkg/cloudinit/common.go b/pkg/cloudinit/common.go
@@ -47,6 +47,12 @@ type BaseUserData struct {
 	SnapstoreProxyDomain string
 	// The snap store proxy ID
 	SnapstoreProxyID string
+	// ContainerdHTTPProxy is http_proxy configuration for containerd.
+	ContainerdHTTPProxy string
+	// ContainerdHTTPSProxy is https_proxy configuration for containerd.
+	ContainerdHTTPSProxy string
+	// ContainerdNoProxy is no_proxy configuration for containerd.
+	ContainerdNoProxy string
 	// MicroclusterAddress is the address to use for microcluster.
 	MicroclusterAddress string
 	// MicroclusterPort is the port to use for microcluster.
@@ -91,6 +97,12 @@ func NewBaseCloudConfig(data BaseUserData) (CloudConfig, error) {
 		config.RunCommands = append(config.RunCommands, "/capi/scripts/configure-snapstore-proxy.sh")
 	}
 
+	// containerd proxy configuration
+	if containerdProxyConfigFiles := getContainerdProxyConfigFiles(data); containerdProxyConfigFiles != nil {
+		config.WriteFiles = append(config.WriteFiles, containerdProxyConfigFiles...)
+		config.RunCommands = append(config.RunCommands, "/capi/scripts/configure-containerd-proxy.sh")
+	}
+
 	// write files
 	config.WriteFiles = append(
 		config.WriteFiles,
@@ -129,6 +141,8 @@ func NewBaseCloudConfig(data BaseUserData) (CloudConfig, error) {
 		)...,
 	)
 
+
+
 	// boot commands
 	config.BootCommands = data.BootCommands
 
@@ -179,3 +193,29 @@ func getSnapstoreProxyConfigFiles(data BaseUserData) []File {
 
 	return []File{schemeFile, domainFile, storeIDFile}
 }
+
+func getContainerdProxyConfigFiles(data BaseUserData) []File {
+	if data.ContainerdHTTPSProxy == "" || data.ContainerdHTTPProxy == "" {
+		return nil
+	}
+	return []File{
+		{
+			Path:        "/capi/etc/containerd-http-proxy",
+			Content:     data.ContainerdHTTPProxy,
+			Permissions: "0400",
+			Owner:       "root:root",
+		},
+		{
+			Path:        "/capi/etc/containerd-https-proxy",
+			Content:     data.ContainerdHTTPSProxy,
+			Permissions: "0400",
+			Owner:       "root:root",
+		},
+		{
+			Path:        "/capi/etc/containerd-no-proxy",
+			Content:     data.ContainerdNoProxy,
+			Permissions: "0400",
+			Owner:       "root:root",
+		},
+	}
+}
diff --git a/pkg/cloudinit/controlplane_init_test.go b/pkg/cloudinit/controlplane_init_test.go
@@ -83,6 +83,7 @@ func TestNewInitControlPlane(t *testing.T) {
 		HaveField("Path", "/capi/scripts/wait-apiserver-ready.sh"),
 		HaveField("Path", "/capi/scripts/deploy-manifests.sh"),
 		HaveField("Path", "/capi/scripts/configure-auth-token.sh"),
+		HaveField("Path", "/capi/scripts/configure-containerd-proxy.sh"),
 		HaveField("Path", "/capi/scripts/configure-node-token.sh"),
 		HaveField("Path", "/capi/scripts/create-sentinel-bootstrap.sh"),
 		HaveField("Path", "/capi/scripts/configure-snapstore-proxy.sh"),
@@ -100,6 +101,80 @@ func TestNewInitControlPlane(t *testing.T) {
 	), "Some /capi/scripts files are missing")
 }
 
+func TestNewInitControlPlaneWithProxy(t *testing.T) {
+	g := NewWithT(t)
+
+	config, err := cloudinit.NewInitControlPlane(cloudinit.InitControlPlaneInput{
+		BaseUserData: cloudinit.BaseUserData{
+			KubernetesVersion: "v1.30.0",
+			BootCommands:      []string{"bootcmd"},
+			PreRunCommands:    []string{"prerun1", "prerun2"},
+			PostRunCommands:   []string{"postrun1", "postrun2"},
+			ExtraFiles: []cloudinit.File{{
+				Path:        "/tmp/file",
+				Content:     "test file",
+				Permissions: "0400",
+				Owner:       "root:root",
+			}},
+			ContainerdHTTPProxy:  "http://proxy.internal",
+			ContainerdHTTPSProxy: "https://proxy.internal",
+			ContainerdNoProxy:    "10.0.0.0/8,10.152.183.1,192.168.0.0/16",
+			ConfigFileContents:   "### config file ###",
+			MicroclusterAddress:  "10.0.0.0/8",
+		},
+		AuthToken:          "test-token",
+		K8sdProxyDaemonSet: "test-daemonset",
+	})
+
+	g.Expect(err).ToNot(HaveOccurred())
+
+	// Verify the boot commands.
+	g.Expect(config.BootCommands).To(Equal([]string{"bootcmd"}))
+
+	// Verify the run commands.
+	g.Expect(config.RunCommands).To(Equal([]string{
+		"set -x",
+		"/capi/scripts/configure-containerd-proxy.sh",
+		"prerun1",
+		"prerun2",
+		"/capi/scripts/install.sh",
+		"/capi/scripts/bootstrap.sh",
+		"/capi/scripts/load-images.sh",
+		"/capi/scripts/wait-apiserver-ready.sh",
+		"/capi/scripts/deploy-manifests.sh",
+		"/capi/scripts/configure-auth-token.sh",
+		"/capi/scripts/configure-node-token.sh",
+		"/capi/scripts/create-sentinel-bootstrap.sh",
+		"postrun1",
+		"postrun2",
+	}))
+
+	// NOTE (mateoflorido): Keep this test in sync with the expected paths in the controlplane_init.go file.
+	g.Expect(config.WriteFiles).To(ConsistOf(
+		HaveField("Path", "/capi/scripts/install.sh"),
+		HaveField("Path", "/capi/scripts/bootstrap.sh"),
+		HaveField("Path", "/capi/scripts/load-images.sh"),
+		HaveField("Path", "/capi/scripts/join-cluster.sh"),
+		HaveField("Path", "/capi/scripts/wait-apiserver-ready.sh"),
+		HaveField("Path", "/capi/scripts/deploy-manifests.sh"),
+		HaveField("Path", "/capi/scripts/configure-auth-token.sh"),
+		HaveField("Path", "/capi/scripts/configure-containerd-proxy.sh"),
+		HaveField("Path", "/capi/scripts/configure-node-token.sh"),
+		HaveField("Path", "/capi/scripts/create-sentinel-bootstrap.sh"),
+		HaveField("Path", "/capi/etc/config.yaml"),
+		HaveField("Path", "/capi/etc/containerd-http-proxy"),
+		HaveField("Path", "/capi/etc/containerd-https-proxy"),
+		HaveField("Path", "/capi/etc/containerd-no-proxy"),
+		HaveField("Path", "/capi/etc/microcluster-address"),
+		HaveField("Path", "/capi/etc/node-name"),
+		HaveField("Path", "/capi/etc/node-token"),
+		HaveField("Path", "/capi/etc/token"),
+		HaveField("Path", "/capi/etc/snap-channel"),
+		HaveField("Path", "/capi/manifests/00-k8sd-proxy.yaml"),
+		HaveField("Path", "/tmp/file"),
+	), "Some /capi/scripts files are missing")
+}
+
 func TestNewInitControlPlaneInvalidVersionError(t *testing.T) {
 	g := NewWithT(t)
 

diff --git a/pkg/cloudinit/controlplane_join_test.go b/pkg/cloudinit/controlplane_join_test.go
@@ -64,6 +64,7 @@ func TestNewJoinControlPlane(t *testing.T) {
 		HaveField("Path", "/capi/scripts/wait-apiserver-ready.sh"),
 		HaveField("Path", "/capi/scripts/deploy-manifests.sh"),
 		HaveField("Path", "/capi/scripts/configure-auth-token.sh"),
+		HaveField("Path", "/capi/scripts/configure-containerd-proxy.sh"),
 		HaveField("Path", "/capi/scripts/configure-node-token.sh"),
 		HaveField("Path", "/capi/scripts/create-sentinel-bootstrap.sh"),
 		HaveField("Path", "/capi/scripts/configure-snapstore-proxy.sh"),
@@ -80,6 +81,76 @@ func TestNewJoinControlPlane(t *testing.T) {
 	), "Some /capi/scripts files are missing")
 }
 
+func TestNewJoinControlPlaneWithProxy(t *testing.T) {
+	g := NewWithT(t)
+
+	config, err := cloudinit.NewJoinControlPlane(cloudinit.JoinControlPlaneInput{
+		BaseUserData: cloudinit.BaseUserData{
+			KubernetesVersion: "v1.30.0",
+			BootCommands:      []string{"bootcmd"},
+			PreRunCommands:    []string{"prerun1", "prerun2"},
+			PostRunCommands:   []string{"postrun1", "postrun2"},
+			ExtraFiles: []cloudinit.File{{
+				Path:        "/tmp/file",
+				Content:     "test file",
+				Permissions: "0400",
+				Owner:       "root:root",
+			}},
+			ContainerdHTTPProxy:  "http://proxy.internal",
+			ContainerdHTTPSProxy: "https://proxy.internal",
+			ContainerdNoProxy:    "10.0.0.0/8,10.152.183.1,192.168.0.0/16",
+			ConfigFileContents:   "### config file ###",
+			MicroclusterAddress:  "10.0.0.11",
+		},
+		JoinToken: "test-token",
+	})
+
+	g.Expect(err).NotTo(HaveOccurred())
+
+	// Verify the boot commands.
+	g.Expect(config.BootCommands).To(Equal([]string{"bootcmd"}))
+
+	// Verify the run commands.
+	g.Expect(config.RunCommands).To(Equal([]string{
+		"set -x",
+		"/capi/scripts/configure-containerd-proxy.sh",
+		"prerun1",
+		"prerun2",
+		"/capi/scripts/install.sh",
+		"/capi/scripts/load-images.sh",
+		"/capi/scripts/join-cluster.sh",
+		"/capi/scripts/wait-apiserver-ready.sh",
+		"/capi/scripts/configure-node-token.sh",
+		"/capi/scripts/create-sentinel-bootstrap.sh",
+		"postrun1",
+		"postrun2",
+	}))
+
+	// NOTE (mateoflorido): Keep this test in sync with the expected paths in the controlplane_join.go file.
+	g.Expect(config.WriteFiles).To(ConsistOf(
+		HaveField("Path", "/capi/scripts/install.sh"),
+		HaveField("Path", "/capi/scripts/bootstrap.sh"),
+		HaveField("Path", "/capi/scripts/load-images.sh"),
+		HaveField("Path", "/capi/scripts/join-cluster.sh"),
+		HaveField("Path", "/capi/scripts/wait-apiserver-ready.sh"),
+		HaveField("Path", "/capi/scripts/deploy-manifests.sh"),
+		HaveField("Path", "/capi/scripts/configure-auth-token.sh"),
+		HaveField("Path", "/capi/scripts/configure-containerd-proxy.sh"),
+		HaveField("Path", "/capi/scripts/configure-node-token.sh"),
+		HaveField("Path", "/capi/scripts/create-sentinel-bootstrap.sh"),
+		HaveField("Path", "/capi/etc/config.yaml"),
+		HaveField("Path", "/capi/etc/containerd-http-proxy"),
+		HaveField("Path", "/capi/etc/containerd-https-proxy"),
+		HaveField("Path", "/capi/etc/containerd-no-proxy"),
+		HaveField("Path", "/capi/etc/microcluster-address"),
+		HaveField("Path", "/capi/etc/node-name"),
+		HaveField("Path", "/capi/etc/node-token"),
+		HaveField("Path", "/capi/etc/join-token"),
+		HaveField("Path", "/capi/etc/snap-channel"),
+		HaveField("Path", "/tmp/file"),
+	), "Some /capi/scripts files are missing")
+}
+
 func TestNewJoinControlPlaneInvalidVersionError(t *testing.T) {
 	g := NewWithT(t)