feat: Enhance MirrorPeer and DRPolicy controllers for StorageClientTy…

…pe handling - Updated `MirrorPeerReconciler` to handle `StorageClientType` by adding conditions in `Reconcile` method. - Introduced `createStorageClusterPeer` function for creating `StorageClusterPeer` objects. - Added utility functions `fetchClientInfoConfigMap` and `getClientInfoFromConfigMap` for handling client info. - Modified `processManagedClusterAddon` to utilize new config structure. - Enhanced `DRPolicyReconciler` to skip certain operations based on `StorageClientType`. Signed-off-by: vbadrina <[email protected]>
red-hat-storage · Jul 17, 2024 · e43ec63 · e43ec63
1 parent 6570168
commit e43ec63
Show file tree

Hide file tree

Showing 12 changed files with 296 additions and 18 deletions.
diff --git a/addons/agent_mirrorpeer_controller.go b/addons/agent_mirrorpeer_controller.go
@@ -123,7 +123,7 @@ func (r *MirrorPeerReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, err
 	}
 
-	if mirrorPeer.Spec.Type == multiclusterv1alpha1.Async {
+	if mirrorPeer.Spec.Type == multiclusterv1alpha1.Async && !utils.IsStorageClientType(mirrorPeer.Spec.Items) {
 		clusterFSIDs := make(map[string]string)
 		logger.Info("Fetching clusterFSIDs")
 		err = r.fetchClusterFSIDs(ctx, &mirrorPeer, clusterFSIDs)

diff --git a/api/v1alpha1/mirrorpeer_types.go b/api/v1alpha1/mirrorpeer_types.go
@@ -35,7 +35,9 @@ const (
 
 // StorageClusterRef holds a reference to a StorageCluster
 type StorageClusterRef struct {
-	Name      string `json:"name"`
+	Name string `json:"name"`
+
+	// +kubebuilder:validation:Optional
 	Namespace string `json:"namespace"`
 }
 

diff --git a/bundle/manifests/multicluster.odf.openshift.io_mirrorpeers.yaml b/bundle/manifests/multicluster.odf.openshift.io_mirrorpeers.yaml
@@ -60,7 +60,6 @@ spec:
                           type: string
                       required:
                       - name
-                      - namespace
                       type: object
                   required:
                   - clusterName

diff --git a/bundle/manifests/odf-multicluster-orchestrator.clusterserviceversion.yaml b/bundle/manifests/odf-multicluster-orchestrator.clusterserviceversion.yaml
@@ -36,7 +36,7 @@ metadata:
       ]
     capabilities: Basic Install
     console.openshift.io/plugins: '["odf-multicluster-console"]'
-    createdAt: "2024-04-03T11:58:23Z"
+    createdAt: "2024-07-17T07:39:20Z"
     olm.skipRange: ""
     operators.openshift.io/infrastructure-features: '["disconnected"]'
     operators.operatorframework.io/builder: operator-sdk-v1.34.1

diff --git a/config/crd/bases/multicluster.odf.openshift.io_mirrorpeers.yaml b/config/crd/bases/multicluster.odf.openshift.io_mirrorpeers.yaml
@@ -60,7 +60,6 @@ spec:
                           type: string
                       required:
                       - name
-                      - namespace
                       type: object
                   required:
                   - clusterName

diff --git a/controllers/drpolicy_controller.go b/controllers/drpolicy_controller.go
@@ -113,6 +113,11 @@ func (r *DRPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 		return ctrl.Result{}, err
 	}
 
+	if utils.IsStorageClientType(mirrorPeer.Spec.Items) {
+		logger.Info("MirrorPeer contains StorageClient reference. Skipping creation of VolumeReplicationClasses", "MirrorPeer", mirrorPeer.Name)
+		return ctrl.Result{}, nil
+	}
+
 	if mirrorPeer.Spec.Type == multiclusterv1alpha1.Async {
 		clusterFSIDs := make(map[string]string)
 		logger.Info("Fetching cluster FSIDs")

diff --git a/controllers/mirrorpeer_controller.go b/controllers/mirrorpeer_controller.go
@@ -18,12 +18,15 @@ package controllers
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"log/slog"
 	"os"
 
 	"github.com/red-hat-storage/odf-multicluster-orchestrator/addons/setup"
 
 	ramenv1alpha1 "github.com/ramendr/ramen/api/v1alpha1"
+	ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
 	addons "github.com/red-hat-storage/odf-multicluster-orchestrator/addons"
 	multiclusterv1alpha1 "github.com/red-hat-storage/odf-multicluster-orchestrator/api/v1alpha1"
 	"github.com/red-hat-storage/odf-multicluster-orchestrator/controllers/utils"
@@ -38,6 +41,7 @@ import (
 	"k8s.io/klog/v2"
 	"open-cluster-management.io/addon-framework/pkg/agent"
 	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
+	workv1 "open-cluster-management.io/api/work/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -53,8 +57,32 @@ type MirrorPeerReconciler struct {
 	Logger *slog.Logger
 }
 
-const mirrorPeerFinalizer = "hub.multicluster.odf.openshift.io"
-const spokeClusterRoleBindingName = "spoke-clusterrole-bindings"
+const (
+	mirrorPeerFinalizer         = "hub.multicluster.odf.openshift.io"
+	spokeClusterRoleBindingName = "spoke-clusterrole-bindings"
+	ClientInfoConfigMapName     = "odf-client-info"
+	ClientConfigMapKeyTemplate  = "%s/%s"
+)
+
+// Temp
+type ProviderInfo struct {
+	Version                    string               `json:"version"`
+	DeploymentType             string               `json:"deploymentType"`
+	StorageSystemName          string               `json:"storageSystemName"`
+	ProviderManagedClusterName string               `json:"providerManagedClusterName"`
+	NamespacedName             types.NamespacedName `json:"namespacedName"`
+	StorageProviderEndpoint    string               `json:"storageProviderEndpoint"`
+	CephClusterFSID            string               `json:"cephClusterFSID"`
+}
+
+type ClientInfo struct {
+	ClusterID                string       `json:"clusterId"`
+	Name                     string       `json:"name"`
+	ProviderInfo             ProviderInfo `json:"providerInfo,omitempty"`
+	ClientManagedClusterName string       `json:"clientManagedClusterName,omitempty"`
+}
+
+// Temp
 
 //+kubebuilder:rbac:groups=multicluster.odf.openshift.io,resources=mirrorpeers,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=multicluster.odf.openshift.io,resources=mirrorpeers/status,verbs=get;update;patch
@@ -240,45 +268,217 @@ func (r *MirrorPeerReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, err
 	}
 
+	if utils.IsStorageClientType(mirrorPeer.Spec.Items) {
+		result, err := createStorageClusterPeer(ctx, r.Client, logger, mirrorPeer)
+		if err != nil {
+			logger.Error("Failed to create StorageClusterPeer", "error", err)
+			return result, err
+		}
+	}
 	return r.updateMirrorPeerStatus(ctx, mirrorPeer)
 }
 
+func getKey(clusterName, clientName string) string {
+	return fmt.Sprintf(ClientConfigMapKeyTemplate, clusterName, clientName)
+}
+
+func createStorageClusterPeer(ctx context.Context, client client.Client, logger *slog.Logger, mirrorPeer multiclusterv1alpha1.MirrorPeer) (ctrl.Result, error) {
+	logger = logger.With("MirrorPeer", mirrorPeer.Name)
+	clientInfoMap, err := fetchClientInfoConfigMap(ctx, client)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			logger.Info("Client info config map not found. Retrying request another time...")
+			return ctrl.Result{Requeue: true}, nil
+		}
+		return ctrl.Result{}, err
+	}
+	items := mirrorPeer.Spec.Items
+	clientInfo := make([]ClientInfo, 0)
+
+	for _, item := range items {
+		ci, err := getClientInfoFromConfigMap(clientInfoMap.Data, getKey(item.ClusterName, item.StorageClusterRef.Name))
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+		clientInfo = append(clientInfo, ci)
+	}
+
+	for i := range items {
+		var storageClusterPeerName string
+		var oppositeClient ClientInfo
+		currentClient := clientInfo[i]
+		// Provider A StorageClusterPeer contains info of Provider B endpoint and ticket, hence this
+		if i == 0 {
+			oppositeClient := clientInfo[1]
+			storageClusterPeerName = getStorageClusterPeerName(oppositeClient.ProviderInfo.ProviderManagedClusterName)
+		} else {
+			oppositeClient = clientInfo[0]
+			storageClusterPeerName = getStorageClusterPeerName(oppositeClient.ProviderInfo.ProviderManagedClusterName)
+		}
+
+		storageClusterPeer := ocsv1.StorageClusterPeer{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: storageClusterPeerName,
+				// This provider A namespace on which the Storage object exists
+				Namespace: currentClient.ProviderInfo.NamespacedName.Namespace,
+			},
+			Spec: ocsv1.StorageClusterPeerSpec{
+				// Endpoint for StorageClusterPeer in Provider A will be the Provider B's endpoint and vice versa
+				APIServerEndpoint: oppositeClient.ProviderInfo.StorageProviderEndpoint,
+				// Same above logic for onboarding token
+				OnboardingTicket: fetchOnboardingTicket(oppositeClient),
+			},
+		}
+		storageClusterPeerJson, err := json.Marshal(storageClusterPeer)
+		if err != nil {
+			logger.Error("Failed to marshal StorageClusterPeer to JSON", "StorageClusterPeer", storageClusterPeerName)
+			return ctrl.Result{}, err
+		}
+
+		ownerRef := metav1.OwnerReference{
+			APIVersion: mirrorPeer.APIVersion,
+			Kind:       mirrorPeer.Kind,
+			Name:       mirrorPeer.Name,
+			UID:        mirrorPeer.UID,
+		}
+
+		// ManifestWork created for Provider A will be called storageclusterpeer-{ProviderA} since that is where Manifests will be applied
+		// Provider names are unique hence only 1 ManifestWork per ProviderCluster
+		manifestWorkName := fmt.Sprintf("storageclusterpeer-%s", currentClient.ProviderInfo.ProviderManagedClusterName)
+
+		// The namespace of Provider A is where this ManifestWork will be created on the hub
+		namespace := currentClient.ProviderInfo.ProviderManagedClusterName
+
+		operationResult, err := utils.CreateOrUpdateManifestWork(ctx, client, manifestWorkName, namespace, storageClusterPeerJson, ownerRef)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+
+		logger.Info(fmt.Sprintf("ManifestWork was %s for StorageClusterPeer %s", operationResult, storageClusterPeerName))
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func fetchOnboardingTicket(clientInfo ClientInfo) string {
+	panic("unimplemented")
+}
+
+func getStorageClusterPeerName(providerClusterName string) string {
+	// Provider A will have SCP named {ProviderB}-peer
+	return fmt.Sprintf("%s-peer", providerClusterName)
+}
+
+func fetchClientInfoConfigMap(ctx context.Context, c client.Client) (*corev1.ConfigMap, error) {
+	currentNamespace := os.Getenv("POD_NAMESPACE")
+	if currentNamespace == "" {
+		return nil, fmt.Errorf("cannot detect the current namespace")
+	}
+	clientInfoMap, err := utils.FetchConfigMap(ctx, c, ClientInfoConfigMapName, currentNamespace)
+	if err != nil {
+		return nil, err
+	}
+	return clientInfoMap, nil
+}
+
+type ManagedClusterAddonConfig struct {
+	// Namespace on the managedCluster where it will be deployed
+	InstallNamespace string
+
+	// Name of the MCA
+	Name string
+
+	// Namespace on the hub where MCA will be created, it represents the Managed cluster where the addons will be deployed
+	Namespace string
+}
+
+// Helper function to extract and unmarshal ClientInfo from ConfigMap
+func getClientInfoFromConfigMap(clientInfoMap map[string]string, key string) (ClientInfo, error) {
+	clientInfoJSON, ok := clientInfoMap[key]
+	if !ok {
+		return ClientInfo{}, fmt.Errorf("client info for %s not found in ConfigMap", key)
+	}
+
+	var clientInfo ClientInfo
+	if err := json.Unmarshal([]byte(clientInfoJSON), &clientInfo); err != nil {
+		return ClientInfo{}, fmt.Errorf("failed to unmarshal client info for %s: %v", key, err)
+	}
+
+	return clientInfo, nil
+}
+
+func getConfig(ctx context.Context, c client.Client, mp multiclusterv1alpha1.MirrorPeer) ([]ManagedClusterAddonConfig, error) {
+	managedClusterAddonsConfig := make([]ManagedClusterAddonConfig, 0)
+	if utils.IsStorageClientType(mp.Spec.Items) {
+		clientInfoMap, err := fetchClientInfoConfigMap(ctx, c)
+		if err != nil {
+			return []ManagedClusterAddonConfig{}, err
+		}
+		for _, item := range mp.Spec.Items {
+			clientName := item.StorageClusterRef.Name
+			clientInfo, err := getClientInfoFromConfigMap(clientInfoMap.Data, getKey(item.ClusterName, clientName))
+			if err != nil {
+				return []ManagedClusterAddonConfig{}, err
+			}
+			config := ManagedClusterAddonConfig{
+				Name:             setup.TokenExchangeName,
+				Namespace:        clientInfo.ProviderInfo.ProviderManagedClusterName,
+				InstallNamespace: clientInfo.ProviderInfo.NamespacedName.Namespace,
+			}
+			managedClusterAddonsConfig = append(managedClusterAddonsConfig, config)
+		}
+	} else {
+		for _, item := range mp.Spec.Items {
+			managedClusterAddonsConfig = append(managedClusterAddonsConfig, ManagedClusterAddonConfig{
+				Name:             setup.TokenExchangeName,
+				Namespace:        item.ClusterName,
+				InstallNamespace: item.StorageClusterRef.Namespace,
+			})
+		}
+	}
+	return managedClusterAddonsConfig, nil
+}
+
 // processManagedClusterAddon creates an addon for the cluster management in all the peer refs,
 // the resources gets an owner ref of the mirrorpeer to let the garbage collector handle it if the mirrorpeer gets deleted
 func (r *MirrorPeerReconciler) processManagedClusterAddon(ctx context.Context, mirrorPeer multiclusterv1alpha1.MirrorPeer) error {
 	logger := r.Logger.With("MirrorPeer", mirrorPeer.Name)
 	logger.Info("Processing ManagedClusterAddons for MirrorPeer")
 
-	for _, item := range mirrorPeer.Spec.Items {
-		logger.Info("Handling ManagedClusterAddon for cluster", "ClusterName", item.ClusterName)
+	addonConfigs, err := getConfig(ctx, r.Client, mirrorPeer)
+	if err != nil {
+		return fmt.Errorf("failed to get managedclusteraddon config %w", err)
+	}
+	for _, config := range addonConfigs {
+		logger.Info("Handling ManagedClusterAddon for cluster", "ClusterName", config.Namespace)
 
 		var managedClusterAddOn addonapiv1alpha1.ManagedClusterAddOn
 		namespacedName := types.NamespacedName{
-			Name:      setup.TokenExchangeName,
-			Namespace: item.ClusterName,
+			Name:      config.Name,
+			Namespace: config.Namespace,
 		}
 
 		err := r.Client.Get(ctx, namespacedName, &managedClusterAddOn)
 		if err != nil {
 			if k8serrors.IsNotFound(err) {
-				logger.Info("ManagedClusterAddon not found, will create a new one", "ClusterName", item.ClusterName)
+				logger.Info("ManagedClusterAddon not found, will create a new one", "ClusterName", config.Namespace)
 
 				annotations := make(map[string]string)
 				annotations[utils.DRModeAnnotationKey] = string(mirrorPeer.Spec.Type)
 				managedClusterAddOn = addonapiv1alpha1.ManagedClusterAddOn{
 					ObjectMeta: metav1.ObjectMeta{
 						Name:        setup.TokenExchangeName,
-						Namespace:   item.ClusterName,
+						Namespace:   config.Namespace,
 						Annotations: annotations,
 					},
 				}
 			}
 		}
 
 		_, err = controllerutil.CreateOrUpdate(ctx, r.Client, &managedClusterAddOn, func() error {
-			managedClusterAddOn.Spec.InstallNamespace = item.StorageClusterRef.Namespace
+			managedClusterAddOn.Spec.InstallNamespace = config.InstallNamespace
 			if err := controllerutil.SetOwnerReference(&mirrorPeer, &managedClusterAddOn, r.Scheme); err != nil {
-				logger.Error("Failed to set owner reference on ManagedClusterAddon", "error", err, "ClusterName", item.ClusterName)
+				logger.Error("Failed to set owner reference on ManagedClusterAddon", "error", err, "ClusterName", config.Namespace)
 				return err
 			}
 			return nil
@@ -289,7 +489,7 @@ func (r *MirrorPeerReconciler) processManagedClusterAddon(ctx context.Context, m
 			return err
 		}
 
-		logger.Info("Successfully reconciled ManagedClusterAddOn", "ClusterName", item.ClusterName)
+		logger.Info("Successfully reconciled ManagedClusterAddOn", "ClusterName", config.Namespace)
 	}
 
 	logger.Info("Successfully processed all ManagedClusterAddons for MirrorPeer")
@@ -497,6 +697,7 @@ func (r *MirrorPeerReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&multiclusterv1alpha1.MirrorPeer{}, builder.WithPredicates(mpPredicate)).
+		Owns(&workv1.ManifestWork{}).
 		Complete(r)
 
 }

diff --git a/controllers/utils/configmap.go b/controllers/utils/configmap.go
@@ -0,0 +1,22 @@
+package utils
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// FetchConfigMap fetches a ConfigMap with a given name from a given namespace
+func FetchConfigMap(ctx context.Context, c client.Client, name, namespace string) (*corev1.ConfigMap, error) {
+	configMap := &corev1.ConfigMap{}
+	err := c.Get(ctx, client.ObjectKey{
+		Name:      name,
+		Namespace: namespace,
+	}, configMap)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch ConfigMap %s in namespace %s: %v", name, namespace, err)
+	}
+	return configMap, nil
+}