Add support for updating vSphere credentials

Makefile

@@ -84,6 +84,10 @@ TOOLS_BIN_DIR_ABS := $(shell pwd)/$(TOOLS_BIN_DIR)
 OUTPUT_DIR := _output
 OUTPUT_BIN_DIR := ${OUTPUT_DIR}/bin
 
+MOCKGEN_BIN := mockgen
+MOCKGEN := $(TOOLS_BIN_DIR)/$(MOCKGEN_BIN) --build_flags=--mod=mod
+MOCKGEN_VERSION := v1.6.0
+
 KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
 KUSTOMIZE_VERSION := 4.2.0
 
@@ -100,12 +104,19 @@ BUILDKIT := $(BUILD_LIB)/buildkit.sh
 
 CONTROLLER_GEN_BIN := controller-gen
 CONTROLLER_GEN := $(TOOLS_BIN_DIR)/$(CONTROLLER_GEN_BIN)
+CONTROLLER_GEN_VERSION := v0.6.1
 
 GO_VULNCHECK_BIN := govulncheck
 GO_VULNCHECK := $(TOOLS_BIN_DIR)/$(GO_VULNCHECK_BIN)
+GO_VULNCHECK_VERSION := latest
 
 SETUP_ENVTEST_BIN := setup-envtest
 SETUP_ENVTEST := $(TOOLS_BIN_DIR)/$(SETUP_ENVTEST_BIN)
+SETUP_ENVTEST_VERSION := v0.0.0-20240215124517-56159419231e
+
+GCI_BIN := gci
+GCI := $(TOOLS_BIN_DIR)/$(GCI_BIN)
+GCI_VERSION := v0.8.0
 
 BINARY_NAME=eks-anywhere-cluster-controller
 ifdef CODEBUILD_SRC_DIR
@@ -276,6 +287,9 @@ $(CONTROLLER_MANIFEST_OUTPUT_DIR):
 $(TOOLS_BIN_DIR):
 	mkdir -p $(TOOLS_BIN_DIR)
 
+$(MOCKGEN): $(TOOLS_BIN_DIR)
+	GOBIN=$(TOOLS_BIN_DIR_ABS) $(GO) install github.com/golang/mock/mockgen@$(MOCKGEN_VERSION)
+
 $(KUSTOMIZE): $(TOOLS_BIN_DIR) $(KUSTOMIZE_OUTPUT_BIN_DIR)
 	-rm $(TOOLS_BIN_DIR)/kustomize
 	cd $(TOOLS_BIN_DIR) && curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s $(KUSTOMIZE_VERSION)
@@ -286,13 +300,16 @@ $(KUBEBUILDER): $(TOOLS_BIN_DIR)
 	chmod +x $(KUBEBUILDER)
 
 $(CONTROLLER_GEN): $(TOOLS_BIN_DIR)
-	GOBIN=$(TOOLS_BIN_DIR_ABS) $(GO) install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1
+	GOBIN=$(TOOLS_BIN_DIR_ABS) $(GO) install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_GEN_VERSION)
 
 $(GO_VULNCHECK): $(TOOLS_BIN_DIR)
-	GOBIN=$(TOOLS_BIN_DIR_ABS) $(GO) install golang.org/x/vuln/cmd/govulncheck@latest
+	GOBIN=$(TOOLS_BIN_DIR_ABS) $(GO) install golang.org/x/vuln/cmd/govulncheck@$(GO_VULNCHECK_VERSION)
 
 $(SETUP_ENVTEST): $(TOOLS_BIN_DIR)
-	GOBIN=$(TOOLS_BIN_DIR_ABS) $(GO) install sigs.k8s.io/controller-runtime/tools/[email protected]
+	GOBIN=$(TOOLS_BIN_DIR_ABS) $(GO) install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(SETUP_ENVTEST_VERSION)
+
+$(GCI): $(TOOLS_BIN_DIR)
+	GOBIN=$(TOOLS_BIN_DIR_ABS) $(GO) install github.com/daixiang0/gci@$(GCI_VERSION)
 
 envtest-setup: $(SETUP_ENVTEST)
 	$(eval KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path --arch $(GO_ARCH) $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION)))
@@ -316,12 +333,9 @@ vulncheck: $(GO_VULNCHECK)
 
 LS_FILES_CMD = git ls-files --exclude-standard | grep '\.go$$' | grep -v '/mocks/\|zz_generated\.'
 
-$(TOOLS_BIN_DIR)/gci:
-	GOBIN=$(TOOLS_BIN_DIR_ABS) $(GO) install github.com/daixiang0/[email protected]
-
 .PHONY: run-gci
-run-gci: $(TOOLS_BIN_DIR)/gci ## Run gci against code.
-	$(LS_FILES_CMD) | xargs $(TOOLS_BIN_DIR)/gci write --skip-generated -s standard,default -s "prefix($(shell $(GO) list -m))"
+run-gci: $(GCI) ## Run gci against code.
+	$(LS_FILES_CMD) | xargs $(GCI) write --skip-generated -s standard,default -s "prefix($(shell $(GO) list -m))"
 
 .PHONY: build-cross-platform
 build-cross-platform: eks-a-cross-platform
@@ -546,10 +560,8 @@ packages-e2e-test: build-all-test-binaries ## Run Curated Packages tests
 	./bin/e2e.test -test.v -test.run $(PACKAGES_E2E_TESTS)
 
 .PHONY: mocks
-mocks: export PATH := $(GO_VERSION):$(PATH)
-mocks: MOCKGEN := ${GOPATH}/bin/mockgen --build_flags=--mod=mod
+mocks: $(MOCKGEN)
 mocks: ## Generate mocks
-	$(GO) install github.com/golang/mock/[email protected]
 	${MOCKGEN} -destination=controllers/mocks/snow_machineconfig_controller.go -package=mocks -source "controllers/snow_machineconfig_controller.go"
 	${MOCKGEN} -destination=pkg/providers/mocks/providers.go -package=mocks "github.com/aws/eks-anywhere/pkg/providers" Provider,DatacenterConfig,MachineConfig
 	${MOCKGEN} -destination=pkg/executables/mocks/executables.go -package=mocks "github.com/aws/eks-anywhere/pkg/executables" Executable,DockerClient,DockerContainer

controllers/cluster_controller_test.go

@@ -1795,7 +1795,7 @@ func createSecret() *apiv1.Secret {
 	return &apiv1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: "eksa-system",
-			Name:      vsphere.CredentialsObjectName,
+			Name:      constants.VSphereCredentialsName,
 		},
 		Data: map[string][]byte{
 			"username": []byte("test"),

pkg/constants/constants.go

@@ -42,10 +42,11 @@ const (
 	// DefaultNutanixPrismCentralPort is the default port for Nutanix Prism Central.
 	DefaultNutanixPrismCentralPort = 9440
 
-	VSphereCredentialsName = "vsphere-credentials"
-	NutanixCredentialsName = "nutanix-credentials"
-	EksaLicenseName        = "eksa-license"
-	EksaPackagesName       = "eksa-packages"
+	VSphereCredentialsName              = "vsphere-credentials"
+	VSphereCloudProviderCredentialsName = "cloud-provider-vsphere-credentials"
+	NutanixCredentialsName              = "nutanix-credentials"
+	EksaLicenseName                     = "eksa-license"
+	EksaPackagesName                    = "eksa-packages"
 	// UpgraderConfigMapName is the name of config map that stores the upgrader images.
 	UpgraderConfigMapName = "in-place-upgrade"
 	// KubeVipConfigMapName is the name of config map that stores the kube-vip config.

pkg/providers/vsphere/reconciler/reconciler.go

@@ -14,6 +14,7 @@ import (
 	anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1"
 	c "github.com/aws/eks-anywhere/pkg/cluster"
 	"github.com/aws/eks-anywhere/pkg/config"
+	"github.com/aws/eks-anywhere/pkg/constants"
 	"github.com/aws/eks-anywhere/pkg/controller"
 	"github.com/aws/eks-anywhere/pkg/controller/clientutil"
 	"github.com/aws/eks-anywhere/pkg/controller/clusters"
@@ -63,7 +64,7 @@ func VsphereCredentials(ctx context.Context, cli client.Client) (*apiv1.Secret,
 	secret := &apiv1.Secret{}
 	secretKey := client.ObjectKey{
 		Namespace: "eksa-system",
-		Name:      vsphere.CredentialsObjectName,
+		Name:      constants.VSphereCredentialsName,
 	}
 	if err := cli.Get(ctx, secretKey, secret); err != nil {
 		return nil, err

pkg/providers/vsphere/vsphere.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	_ "embed"
+	"encoding/base64"
 	"fmt"
 	"os"
 	"reflect"
@@ -13,8 +14,10 @@ import (
 	"github.com/Masterminds/sprig"
 	etcdv1 "github.com/aws/etcdadm-controller/api/v1beta1"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+	"sigs.k8s.io/yaml"
 
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
 	"github.com/aws/eks-anywhere/pkg/bootstrapper"
@@ -34,7 +37,6 @@ import (
 )
 
 const (
-	CredentialsObjectName    = "vsphere-credentials"
 	eksaLicense              = "EKSA_LICENSE"
 	vSphereUsernameKey       = "VSPHERE_USERNAME"
 	vSpherePasswordKey       = "VSPHERE_PASSWORD"
@@ -51,6 +53,18 @@ const (
 	MemoryAvailable          = "Memory_Available"
 )
 
+const cloudProviderVsphereCredentialsDataTemplate = `
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cloud-provider-vsphere-credentials
+  namespace: kube-system
+data:
+  %[1]s.password: %s
+  %[1]s.username: %s
+type: Opaque
+`
+
 //go:embed config/template-cp.yaml
 var defaultCAPIConfigCP string
 
@@ -690,17 +704,13 @@ func (p *vsphereProvider) updatePrevClusterMemoryUsage(ctx context.Context, clus
 	return nil
 }
 
-func (p *vsphereProvider) UpdateSecrets(ctx context.Context, cluster *types.Cluster, _ *cluster.Spec) error {
+func (p *vsphereProvider) UpdateSecrets(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec) error {
 	var contents bytes.Buffer
-	err := p.createSecret(ctx, cluster, &contents)
+	err := p.updateAllVsphereSecrets(ctx, cluster, clusterSpec, &contents)
 	if err != nil {
 		return err
 	}
 
-	err = p.providerKubectlClient.ApplyKubeSpecFromBytes(ctx, cluster, contents.Bytes())
-	if err != nil {
-		return fmt.Errorf("loading secrets object: %v", err)
-	}
 	return nil
 }
 
@@ -946,16 +956,16 @@ func (p *vsphereProvider) GenerateCAPISpecForCreate(ctx context.Context, _ *type
 	return controlPlaneSpec, workersSpec, nil
 }
 
-func (p *vsphereProvider) createSecret(ctx context.Context, cluster *types.Cluster, contents *bytes.Buffer) error {
+func (p *vsphereProvider) updateAllVsphereSecrets(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec, contents *bytes.Buffer) error {
 	t, err := template.New("tmpl").Funcs(sprig.TxtFuncMap()).Parse(defaultSecretObject)
 	if err != nil {
 		return fmt.Errorf("creating secret object template: %v", err)
 	}
 	vuc := config.NewVsphereUserConfig()
 
 	values := map[string]string{
-		"vspherePassword":           os.Getenv(vSpherePasswordKey),
-		"vsphereUsername":           os.Getenv(vSphereUsernameKey),
+		"vspherePassword":           vuc.EksaVspherePassword,
+		"vsphereUsername":           vuc.EksaVsphereUsername,
 		"eksaCloudProviderUsername": vuc.EksaVsphereCPUsername,
 		"eksaCloudProviderPassword": vuc.EksaVsphereCPPassword,
 		"eksaLicense":               os.Getenv(eksaLicense),
@@ -967,11 +977,69 @@ func (p *vsphereProvider) createSecret(ctx context.Context, cluster *types.Clust
 	if err != nil {
 		return fmt.Errorf("substituting values for secret object template: %v", err)
 	}
+	err = p.providerKubectlClient.ApplyKubeSpecFromBytes(ctx, cluster, contents.Bytes())
+	if err != nil {
+		return fmt.Errorf("applying secret object: %v", err)
+	}
+
+	clusterVsphereCredentialsSecret := &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: corev1.SchemeGroupVersion.Version,
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: constants.EksaSystemNamespace,
+			Name:      fmt.Sprintf("%s-%s", cluster.Name, constants.VSphereCredentialsName),
+			Labels: map[string]string{
+				constants.ClusterctlMoveLabelName: "true",
+			},
+		},
+		Data: map[string][]byte{
+			"username": []byte(vuc.EksaVsphereUsername),
+			"password": []byte(vuc.EksaVspherePassword),
+		},
+	}
+
+	encodedCloudProviderUsername := base64.StdEncoding.EncodeToString([]byte(vuc.EksaVsphereCPUsername))
+	encodedCloudProviderPassword := base64.StdEncoding.EncodeToString([]byte(vuc.EksaVsphereCPPassword))
+	cloudProviderVsphereCredentialsData := fmt.Sprintf(cloudProviderVsphereCredentialsDataTemplate, clusterSpec.VSphereDatacenter.Spec.Server, encodedCloudProviderPassword, encodedCloudProviderUsername)
+	cloudProviderVsphereCredentialsCRSSecret := &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: corev1.SchemeGroupVersion.Version,
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: constants.EksaSystemNamespace,
+			Name:      fmt.Sprintf("%s-%s", cluster.Name, constants.VSphereCloudProviderCredentialsName),
+		},
+		Data: map[string][]byte{
+			"data": []byte(cloudProviderVsphereCredentialsData),
+		},
+		Type: "addons.cluster.x-k8s.io/resource-set",
+	}
+
+	secretsToBeUpdated := []*corev1.Secret{
+		clusterVsphereCredentialsSecret,
+		cloudProviderVsphereCredentialsCRSSecret,
+	}
+
+	for _, secret := range secretsToBeUpdated {
+		objBytes, err := yaml.Marshal(secret)
+		if err != nil {
+			return fmt.Errorf("marshalling secret object: %v", err)
+		}
+
+		err = p.providerKubectlClient.ApplyKubeSpecFromBytes(ctx, cluster, objBytes)
+		if err != nil {
+			return fmt.Errorf("applying secret object: %v", err)
+		}
+	}
+
 	return nil
 }
 
 func (p *vsphereProvider) PreCAPIInstallOnBootstrap(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec) error {
-	return p.UpdateSecrets(ctx, cluster, nil)
+	return p.UpdateSecrets(ctx, cluster, clusterSpec)
 }
 
 func (p *vsphereProvider) PostBootstrapSetup(ctx context.Context, clusterConfig *v1alpha1.Cluster, cluster *types.Cluster) error {
@@ -1102,14 +1170,6 @@ func (p *vsphereProvider) ValidateNewSpec(ctx context.Context, cluster *types.Cl
 		return fmt.Errorf("spec.network is immutable. Previous value %s, new value %s", oSpec.Network, nSpec.Network)
 	}
 
-	secretChanged, err := p.secretContentsChanged(ctx, cluster)
-	if err != nil {
-		return err
-	}
-
-	if secretChanged {
-		return fmt.Errorf("the VSphere credentials derived from %s and %s are immutable; please use the same credentials for the upgraded cluster", vSpherePasswordKey, vSphereUsernameKey)
-	}
 	return nil
 }
 
@@ -1158,24 +1218,6 @@ func (p *vsphereProvider) validateMachineConfigImmutability(ctx context.Context,
 	return nil
 }
 
-func (p *vsphereProvider) secretContentsChanged(ctx context.Context, workloadCluster *types.Cluster) (bool, error) {
-	nPassword := os.Getenv(vSpherePasswordKey)
-	oSecret, err := p.providerKubectlClient.GetSecretFromNamespace(ctx, workloadCluster.KubeconfigFile, CredentialsObjectName, constants.EksaSystemNamespace)
-	if err != nil {
-		return false, fmt.Errorf("obtaining VSphere secret %s from workload cluster: %v", CredentialsObjectName, err)
-	}
-
-	if string(oSecret.Data["password"]) != nPassword {
-		return true, nil
-	}
-
-	nUser := os.Getenv(vSphereUsernameKey)
-	if string(oSecret.Data["username"]) != nUser {
-		return true, nil
-	}
-	return false, nil
-}
-
 // ChangeDiff returns the component change diff for the provider.
 func (p *vsphereProvider) ChangeDiff(currentComponents, newComponents *cluster.ManagementComponents) *types.ComponentChangeDiff {
 	if currentComponents.VSphere.Version == newComponents.VSphere.Version {

pkg/providers/vsphere/vsphere_test.go

@@ -21,7 +21,6 @@ import (
 	. "github.com/onsi/gomega"
 	"github.com/stretchr/testify/assert"
 	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 
@@ -2888,21 +2887,11 @@ func TestValidateNewSpecSuccess(t *testing.T) {
 	kubectl := mocks.NewMockProviderKubectlClient(mockCtrl)
 	provider.providerKubectlClient = kubectl
 
-	clusterVsphereSecret := &v1.Secret{
-		TypeMeta:   metav1.TypeMeta{},
-		ObjectMeta: metav1.ObjectMeta{},
-		Data: map[string][]byte{
-			"username": []byte("vsphere_username"),
-			"password": []byte("vsphere_password"),
-		},
-	}
-
 	kubectl.EXPECT().GetEksaCluster(context.TODO(), gomock.Any(), gomock.Any()).Return(clusterSpec.Cluster, nil)
 	kubectl.EXPECT().GetEksaVSphereDatacenterConfig(context.TODO(), clusterSpec.Cluster.Spec.DatacenterRef.Name, gomock.Any(), clusterSpec.Cluster.Namespace).Return(clusterSpec.VSphereDatacenter, nil)
 	for _, config := range clusterSpec.VSphereMachineConfigs {
 		kubectl.EXPECT().GetEksaVSphereMachineConfig(context.TODO(), gomock.Any(), gomock.Any(), clusterSpec.Cluster.Namespace).Return(config, nil)
 	}
-	kubectl.EXPECT().GetSecretFromNamespace(gomock.Any(), gomock.Any(), CredentialsObjectName, gomock.Any()).Return(clusterVsphereSecret, nil)
 
 	err := provider.ValidateNewSpec(context.TODO(), &types.Cluster{}, newClusterSpec)
 	assert.NoError(t, err, "No error should be returned when previous spec == new spec")
@@ -2923,21 +2912,11 @@ func TestValidateNewSpecMutableFields(t *testing.T) {
 		config.Spec.Folder = "new=" + config.Spec.Folder
 	}
 
-	clusterVsphereSecret := &v1.Secret{
-		TypeMeta:   metav1.TypeMeta{},
-		ObjectMeta: metav1.ObjectMeta{},
-		Data: map[string][]byte{
-			"username": []byte("vsphere_username"),
-			"password": []byte("vsphere_password"),
-		},
-	}
-
 	kubectl.EXPECT().GetEksaCluster(context.TODO(), gomock.Any(), gomock.Any()).Return(clusterSpec.Cluster, nil)
 	kubectl.EXPECT().GetEksaVSphereDatacenterConfig(context.TODO(), clusterSpec.Cluster.Spec.DatacenterRef.Name, gomock.Any(), gomock.Any()).Return(clusterSpec.VSphereDatacenter, nil)
 	for _, config := range clusterSpec.VSphereMachineConfigs {
 		kubectl.EXPECT().GetEksaVSphereMachineConfig(context.TODO(), gomock.Any(), gomock.Any(), clusterSpec.Cluster.Namespace).Return(config, nil)
 	}
-	kubectl.EXPECT().GetSecretFromNamespace(gomock.Any(), gomock.Any(), CredentialsObjectName, gomock.Any()).Return(clusterVsphereSecret, nil)
 
 	err := provider.ValidateNewSpec(context.TODO(), &types.Cluster{}, newClusterSpec)
 	assert.NoError(t, err, "No error should be returned when modifying mutable fields")

pkg/workflows/management/upgrade_cluster.go

@@ -46,7 +46,7 @@ func (s *upgradeCluster) Run(ctx context.Context, commandContext *task.CommandCo
 }
 
 func (s *upgradeCluster) Name() string {
-	return "upgrade-workload-cluster"
+	return "upgrade-management-cluster"
 }
 
 func (s *upgradeCluster) Checkpoint() *task.CompletedTask {