legacy: Create app CR for per cluster app-operator instance (#1356)

* Add appfinalizer resource

* Create app CR for app-operator

* Fix logging in appversionlabel resource

* Keep finalizers until all apps are deleted

* Add changelog

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Create app CR for per cluster app-operator instance.
+- Add `appfinalizer` resource to remove finalizers from workload cluster app CRs.
+
 ## [0.24.2] - 2021-02-25
 
 ### Changed

pkg/label/app.go

@@ -1,9 +1,12 @@
 package label
 
 const (
-	// App is a standard label for tenant resources.
+	// App label is deprecated and is replaced by AppKubernetesName.
 	App = "app"
 
+	// AppKubernetesName is a standard label for Kubernetes resources.
+	AppKubernetesName = "app.kubernetes.io/name"
+
 	// AppOperatorHelmMajorVersion is a label for chart-operator app CRs.
 	AppOperatorHelmMajorVersion = "app-operator.giantswarm.io/helm-major-version"
 )

service/controller/aws/resource_set.go

@@ -27,6 +27,7 @@ import (
 	"github.com/giantswarm/cluster-operator/service/controller/aws/key"
 	"github.com/giantswarm/cluster-operator/service/controller/controllercontext"
 	"github.com/giantswarm/cluster-operator/service/controller/resource/app"
+	"github.com/giantswarm/cluster-operator/service/controller/resource/appfinalizer"
 	"github.com/giantswarm/cluster-operator/service/controller/resource/appversionlabel"
 	"github.com/giantswarm/cluster-operator/service/controller/resource/certconfig"
 	"github.com/giantswarm/cluster-operator/service/controller/resource/clusterconfigmap"
@@ -126,6 +127,20 @@ func newResourceSet(config resourceSetConfig) (*controller.ResourceSet, error) {
 		}
 	}
 
+	var appFinalizerResource resource.Interface
+	{
+		c := appfinalizer.Config{
+			GetClusterConfigFunc: getClusterConfig,
+			G8sClient:            config.K8sClient.G8sClient(),
+			Logger:               config.Logger,
+		}
+
+		appFinalizerResource, err = appfinalizer.New(c)
+		if err != nil {
+			return nil, microerror.Mask(err)
+		}
+	}
+
 	var appVersionLabelResource resource.Interface
 	{
 		c := appversionlabel.Config{
@@ -296,6 +311,7 @@ func newResourceSet(config resourceSetConfig) (*controller.ResourceSet, error) {
 		clusterConfigMapResource,
 		kubeConfigResource,
 		appResource,
+		appFinalizerResource,
 		appVersionLabelResource,
 	}
 

service/controller/azure/resource_set.go

@@ -27,6 +27,7 @@ import (
 	"github.com/giantswarm/cluster-operator/service/controller/azure/key"
 	"github.com/giantswarm/cluster-operator/service/controller/controllercontext"
 	"github.com/giantswarm/cluster-operator/service/controller/resource/app"
+	"github.com/giantswarm/cluster-operator/service/controller/resource/appfinalizer"
 	"github.com/giantswarm/cluster-operator/service/controller/resource/appversionlabel"
 	"github.com/giantswarm/cluster-operator/service/controller/resource/certconfig"
 	"github.com/giantswarm/cluster-operator/service/controller/resource/clusterconfigmap"
@@ -123,6 +124,20 @@ func newResourceSet(config resourceSetConfig) (*controller.ResourceSet, error) {
 		}
 	}
 
+	var appFinalizerResource resource.Interface
+	{
+		c := appfinalizer.Config{
+			GetClusterConfigFunc: getClusterConfig,
+			G8sClient:            config.K8sClient.G8sClient(),
+			Logger:               config.Logger,
+		}
+
+		appFinalizerResource, err = appfinalizer.New(c)
+		if err != nil {
+			return nil, microerror.Mask(err)
+		}
+	}
+
 	var appVersionLabelResource resource.Interface
 	{
 		c := appversionlabel.Config{
@@ -293,6 +308,7 @@ func newResourceSet(config resourceSetConfig) (*controller.ResourceSet, error) {
 		clusterConfigMapResource,
 		kubeConfigResource,
 		appResource,
+		appFinalizerResource,
 		appVersionLabelResource,
 	}
 

service/controller/key/types.go

@@ -3,6 +3,7 @@ package key
 // AppSpec is used to define app custom resources.
 type AppSpec struct {
 	App     string
+	AppName string
 	Catalog string
 	Chart   string
 	// Whether app is installed for clusterapi clusters only.
@@ -11,6 +12,9 @@ type AppSpec struct {
 	// is used.
 	ConfigMapName string
 	// Whether app is installed for legacy clusters only.
+	// InCluster determines if the app CR should use in cluster. Otherwise the
+	// cluster kubeconfig is specified.
+	InCluster       bool
 	LegacyOnly      bool
 	Namespace       string
 	UseUpgradeForce bool

service/controller/kvm/resource_set.go

@@ -27,6 +27,7 @@ import (
 	"github.com/giantswarm/cluster-operator/service/controller/controllercontext"
 	"github.com/giantswarm/cluster-operator/service/controller/kvm/key"
 	"github.com/giantswarm/cluster-operator/service/controller/resource/app"
+	"github.com/giantswarm/cluster-operator/service/controller/resource/appfinalizer"
 	"github.com/giantswarm/cluster-operator/service/controller/resource/appversionlabel"
 	"github.com/giantswarm/cluster-operator/service/controller/resource/certconfig"
 	"github.com/giantswarm/cluster-operator/service/controller/resource/clusterconfigmap"
@@ -122,6 +123,20 @@ func newResourceSet(config resourceSetConfig) (*controller.ResourceSet, error) {
 		}
 	}
 
+	var appFinalizerResource resource.Interface
+	{
+		c := appfinalizer.Config{
+			GetClusterConfigFunc: getClusterConfig,
+			G8sClient:            config.K8sClient.G8sClient(),
+			Logger:               config.Logger,
+		}
+
+		appFinalizerResource, err = appfinalizer.New(c)
+		if err != nil {
+			return nil, microerror.Mask(err)
+		}
+	}
+
 	var appVersionLabelResource resource.Interface
 	{
 		c := appversionlabel.Config{
@@ -292,6 +307,7 @@ func newResourceSet(config resourceSetConfig) (*controller.ResourceSet, error) {
 		clusterConfigMapResource,
 		kubeConfigResource,
 		appResource,
+		appFinalizerResource,
 		appVersionLabelResource,
 	}
 

service/controller/resource/app/desired.go

@@ -13,6 +13,7 @@ import (
 	"github.com/ghodss/yaml"
 	g8sv1alpha1 "github.com/giantswarm/apiextensions/pkg/apis/application/v1alpha1"
 	"github.com/giantswarm/apiextensions/pkg/apis/core/v1alpha1"
+	releasev1alpha1 "github.com/giantswarm/apiextensions/pkg/apis/release/v1alpha1"
 	"github.com/giantswarm/backoff"
 	"github.com/giantswarm/clusterclient/service/release/searcher"
 	"github.com/giantswarm/microerror"
@@ -77,11 +78,20 @@ func (r *Resource) GetDesiredState(ctx context.Context, obj interface{}) ([]*g8s
 	}
 
 	// put `v` as a prefix of release version since all releases CRs keep this format.
-	appOperatorVersion, err := r.getComponentVersion(fmt.Sprintf("v%s", clusterConfig.ReleaseVersion), "app-operator")
+	appOperatorComponent, err := r.getReleaseComponent(fmt.Sprintf("v%s", clusterConfig.ReleaseVersion), appOperatorComponentName)
 	if err != nil {
 		return nil, microerror.Mask(err)
 	}
 
+	appOperatorVersion := appOperatorComponent.Version
+	if appOperatorVersion == "" {
+		return nil, microerror.Maskf(notFoundError, "%#q release component not found", appOperatorComponentName)
+	}
+
+	// Define app CR for app-operator in the management cluster namespace.
+	appOperatorAppSpec := newAppOperatorAppSpec(clusterConfig, appOperatorComponent)
+	apps = append(apps, r.newApp(clusterConfig, appOperatorAppSpec, g8sv1alpha1.AppSpecUserConfig{}, uniqueOperatorVersion))
+
 	for _, appSpec := range appSpecs {
 		userConfig, err := newUserConfig(clusterConfig, appSpec, configMaps, secrets)
 		if err != nil {
@@ -170,6 +180,45 @@ func (r *Resource) newApp(clusterConfig v1alpha1.ClusterGuestConfig, appSpec key
 		configMapName = appSpec.ConfigMapName
 	}
 
+	var appName string
+
+	if appSpec.AppName != "" {
+		appName = appSpec.AppName
+	} else {
+		appName = appSpec.App
+	}
+
+	var config g8sv1alpha1.AppSpecConfig
+
+	if appSpec.InCluster {
+		config = g8sv1alpha1.AppSpecConfig{}
+	} else {
+		config = g8sv1alpha1.AppSpecConfig{
+			ConfigMap: g8sv1alpha1.AppSpecConfigConfigMap{
+				Name:      configMapName,
+				Namespace: clusterConfig.ID,
+			},
+		}
+	}
+
+	var kubeConfig g8sv1alpha1.AppSpecKubeConfig
+
+	if appSpec.InCluster {
+		kubeConfig = g8sv1alpha1.AppSpecKubeConfig{
+			InCluster: true,
+		}
+	} else {
+		kubeConfig = g8sv1alpha1.AppSpecKubeConfig{
+			Context: g8sv1alpha1.AppSpecKubeConfigContext{
+				Name: key.KubeConfigSecretName(clusterConfig),
+			},
+			Secret: g8sv1alpha1.AppSpecKubeConfigSecret{
+				Name:      key.KubeConfigSecretName(clusterConfig),
+				Namespace: clusterConfig.ID,
+			},
+		}
+	}
+
 	return &g8sv1alpha1.App{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "App",
@@ -180,40 +229,23 @@ func (r *Resource) newApp(clusterConfig v1alpha1.ClusterGuestConfig, appSpec key
 				annotation.ForceHelmUpgrade: strconv.FormatBool(appSpec.UseUpgradeForce),
 			},
 			Labels: map[string]string{
-				label.App:                appSpec.App,
+				label.AppKubernetesName:  appSpec.App,
 				label.AppOperatorVersion: appOperatorVersion,
 				label.Cluster:            clusterConfig.ID,
 				label.ManagedBy:          project.Name(),
 				label.Organization:       clusterConfig.Owner,
 				label.ServiceType:        label.ServiceTypeManaged,
 			},
-			Name:      appSpec.App,
+			Name:      appName,
 			Namespace: clusterConfig.ID,
 		},
 		Spec: g8sv1alpha1.AppSpec{
-			Catalog:   appSpec.Catalog,
-			Name:      appSpec.Chart,
-			Namespace: appSpec.Namespace,
-			Version:   appSpec.Version,
-
-			Config: g8sv1alpha1.AppSpecConfig{
-				ConfigMap: g8sv1alpha1.AppSpecConfigConfigMap{
-					Name:      configMapName,
-					Namespace: clusterConfig.ID,
-				},
-			},
-
-			KubeConfig: g8sv1alpha1.AppSpecKubeConfig{
-				Context: g8sv1alpha1.AppSpecKubeConfigContext{
-					Name: key.KubeConfigSecretName(clusterConfig),
-				},
-				InCluster: false,
-				Secret: g8sv1alpha1.AppSpecKubeConfigSecret{
-					Name:      key.KubeConfigSecretName(clusterConfig),
-					Namespace: clusterConfig.ID,
-				},
-			},
-
+			Catalog:    appSpec.Catalog,
+			Name:       appSpec.Chart,
+			Namespace:  appSpec.Namespace,
+			Version:    appSpec.Version,
+			Config:     config,
+			KubeConfig: kubeConfig,
 			UserConfig: userConfig,
 		},
 	}
@@ -328,19 +360,33 @@ func (r *Resource) newAppSpecs(ctx context.Context, cr v1alpha1.ClusterGuestConf
 	return specs, nil
 }
 
-func (r *Resource) getComponentVersion(releaseVersion, component string) (string, error) {
+func newAppOperatorAppSpec(clusterConfig v1alpha1.ClusterGuestConfig, component releasev1alpha1.ReleaseSpecComponent) key.AppSpec {
+	return key.AppSpec{
+		App: appOperatorComponentName,
+		// Override app name to include the cluster ID.
+		AppName:         fmt.Sprintf("%s-%s", appOperatorComponentName, clusterConfig.ID),
+		Catalog:         controlPlaneCatalog,
+		Chart:           appOperatorComponentName,
+		InCluster:       true,
+		Namespace:       clusterConfig.ID,
+		UseUpgradeForce: false,
+		Version:         component.Version,
+	}
+}
+
+func (r *Resource) getReleaseComponent(releaseVersion, component string) (releasev1alpha1.ReleaseSpecComponent, error) {
 	release, err := r.g8sClient.ReleaseV1alpha1().Releases().Get(releaseVersion, metav1.GetOptions{})
 	if err != nil {
-		return "", microerror.Mask(err)
+		return releasev1alpha1.ReleaseSpecComponent{}, microerror.Mask(err)
 	}
 
 	for _, c := range release.Spec.Components {
 		if c.Name == component {
-			return c.Version, nil
+			return c, nil
 		}
 	}
 
-	return "", microerror.Maskf(notFoundError, fmt.Sprintf("can't find the release version %#q", releaseVersion))
+	return releasev1alpha1.ReleaseSpecComponent{}, microerror.Maskf(notFoundError, fmt.Sprintf("can't find the %#q component for %#q", component, releaseVersion))
 }
 
 func newUserConfig(clusterConfig v1alpha1.ClusterGuestConfig, appSpec key.AppSpec, configMaps map[string]corev1.ConfigMap, secrets map[string]corev1.Secret) (g8sv1alpha1.AppSpecUserConfig, error) {

service/controller/resource/app/resource.go

@@ -13,6 +13,10 @@ import (
 
 const (
 	Name = "app"
+
+	appOperatorComponentName = "app-operator"
+	controlPlaneCatalog      = "control-plane-catalog"
+	uniqueOperatorVersion    = "0.0.0"
 )
 
 // Config represents the configuration used to create a new chartconfig service.

service/controller/resource/appfinalizer/create.go

@@ -0,0 +1,9 @@
+package appfinalizer
+
+import (
+	"context"
+)
+
+func (r *Resource) EnsureCreated(ctx context.Context, obj interface{}) error {
+	return nil
+}