[Feature]: Add used quota updating logic

Makefile

@@ -119,8 +119,6 @@ e2e-test: docker-build # Run e2e tests
 e2e-test-local: docker-build # Run e2e tests for local development purposes
 	kind load docker-image $(IMG)
 	kubectl delete pod -n s3-operator-system -l control-plane=controller-manager
-	kubectl delete s3b --all -A
-	kubectl delete s3u --all -A
 	kubectl kuttl test --start-kind=false
 
 ##@ Build

api/v1alpha1/quota_handler.go

@@ -0,0 +1,110 @@
+package v1alpha1
+
+import (
+	"context"
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/snapp-incubator/s3-operator/pkg/consts"
+)
+
+func CalculateNamespaceUsedQuota(ctx context.Context, uncachedReader client.Reader,
+	suc *S3UserClaim, cleanPhase bool) (*TotalQuota, error) {
+	totalUsedQuota := TotalQuota{}
+	// List all s3UserClaims in the namespace
+	s3UserClaimList := &S3UserClaimList{}
+	if err := uncachedReader.List(ctx, s3UserClaimList, client.InNamespace(suc.Namespace)); err != nil {
+		return &totalUsedQuota, fmt.Errorf("failed to list s3 user claims, %w", err)
+	}
+
+	// Sum all resource requests
+	for _, claim := range s3UserClaimList.Items {
+		if claim.Name != suc.Name {
+			totalUsedQuota.MaxObjects.Add(claim.Spec.Quota.MaxObjects)
+			totalUsedQuota.MaxSize.Add(claim.Spec.Quota.MaxSize)
+			totalUsedQuota.MaxBuckets += int64(claim.Spec.Quota.MaxBuckets)
+		}
+	}
+	// Don't add the current user quota if the function is called by the cleaner
+	if !cleanPhase {
+		totalUsedQuota.MaxObjects.Add(suc.Spec.Quota.MaxObjects)
+		totalUsedQuota.MaxSize.Add(suc.Spec.Quota.MaxSize)
+		totalUsedQuota.MaxBuckets += int64(suc.Spec.Quota.MaxBuckets)
+	}
+	return &totalUsedQuota, nil
+}
+
+func CalculateClusterUsedQuota(ctx context.Context, runtimeClient client.Client,
+	suc *S3UserClaim, cleanPhase bool) (*TotalQuota, string, error) {
+	totalClusterUsedQuota := TotalQuota{}
+	// Find team's clusterResourceQuota
+	team, err := findTeam(ctx, runtimeClient, suc)
+	if err != nil {
+		return &totalClusterUsedQuota, "", fmt.Errorf("failed to find team, %w", err)
+	}
+
+	// Sum all resource requests in team's namespaces
+	namespaces, err := findTeamNamespaces(ctx, runtimeClient, team)
+	if err != nil {
+		return &totalClusterUsedQuota, team, fmt.Errorf("failed to find team namespaces, %w", err)
+	}
+	for _, ns := range namespaces {
+		s3UserClaimList := &S3UserClaimList{}
+		if err := uncachedReader.List(ctx, s3UserClaimList, client.InNamespace(ns)); err != nil {
+			return &totalClusterUsedQuota, team, fmt.Errorf("failed to list s3UserClaims, %w", err)
+		}
+
+		for _, claim := range s3UserClaimList.Items {
+			if claim.Name != suc.Name || claim.Namespace != suc.Namespace {
+				totalClusterUsedQuota.MaxObjects.Add(claim.Spec.Quota.MaxObjects)
+				totalClusterUsedQuota.MaxSize.Add(claim.Spec.Quota.MaxSize)
+				totalClusterUsedQuota.MaxBuckets += int64(claim.Spec.Quota.MaxBuckets)
+			}
+		}
+	}
+	// Don't add the current user quota if the function is called by the cleaner
+	if !cleanPhase {
+		totalClusterUsedQuota.MaxObjects.Add(suc.Spec.Quota.MaxObjects)
+		totalClusterUsedQuota.MaxSize.Add(suc.Spec.Quota.MaxSize)
+		totalClusterUsedQuota.MaxBuckets += int64(suc.Spec.Quota.MaxBuckets)
+	}
+	return &totalClusterUsedQuota, team, nil
+}
+
+func findTeam(ctx context.Context, runtimeClient client.Client, suc *S3UserClaim) (string, error) {
+	ns := &v1.Namespace{}
+	if err := runtimeClient.Get(ctx, types.NamespacedName{Name: suc.ObjectMeta.Namespace}, ns); err != nil {
+		return "", fmt.Errorf("failed to get namespace, %w", err)
+	}
+
+	team, ok := ns.ObjectMeta.Labels[consts.LabelTeam]
+	if !ok {
+		return "", fmt.Errorf("namespace %s doesn't have team label", ns.ObjectMeta.Name)
+	}
+
+	return team, nil
+}
+
+func findTeamNamespaces(ctx context.Context, runtimeClient client.Client, team string) ([]string, error) {
+	var namespaces []string
+
+	namespaceList := &v1.NamespaceList{}
+	if err := runtimeClient.List(ctx, namespaceList); err != nil {
+		return namespaces, fmt.Errorf("failed to list namespaces, %w", err)
+	}
+
+	for _, ns := range namespaceList.Items {
+		labels := ns.ObjectMeta.Labels
+		if labels == nil {
+			labels = map[string]string{}
+		}
+		if nsTeam := labels[consts.LabelTeam]; nsTeam == team {
+			namespaces = append(namespaces, ns.ObjectMeta.Name)
+		}
+	}
+
+	return namespaces, nil
+}

api/v1alpha1/s3userclaim_webhook.go

@@ -25,7 +25,6 @@ import (
 	openshiftquota "github.com/openshift/api/quota/v1"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/validation/field"
@@ -156,46 +155,29 @@ func validateQuota(suc *S3UserClaim, allErrs field.ErrorList) field.ErrorList {
 }
 
 func validateAgainstNamespaceQuota(ctx context.Context, suc *S3UserClaim) error {
-	// List all s3UserClaims in the namespace
-	s3UserClaimList := &S3UserClaimList{}
-	if err := uncachedReader.List(ctx, s3UserClaimList, client.InNamespace(suc.Namespace)); err != nil {
-		return fmt.Errorf("failed to list s3 user claims, %w", err)
-	}
-
-	// Sum all resource requests
-	totalMaxObjects := resource.Quantity{}
-	totalMaxSize := resource.Quantity{}
-	totalMaxBuckets := int64(0)
-	for _, claim := range s3UserClaimList.Items {
-		if claim.Name != suc.Name {
-			totalMaxObjects.Add(claim.Spec.Quota.MaxObjects)
-			totalMaxSize.Add(claim.Spec.Quota.MaxSize)
-			totalMaxBuckets += int64(claim.Spec.Quota.MaxBuckets)
-		}
+	totalUsedQuota, err := CalculateNamespaceUsedQuota(ctx, uncachedReader, suc, false)
+	if err != nil {
+		return fmt.Errorf("failed to calculate namespace used quota , %w", err)
 	}
-	totalMaxObjects.Add(suc.Spec.Quota.MaxObjects)
-	totalMaxSize.Add(suc.Spec.Quota.MaxSize)
-	totalMaxBuckets += int64(suc.Spec.Quota.MaxBuckets)
-
 	// List all quotas in the namespace and validate against them
 	resourceQuotaList := &v1.ResourceQuotaList{}
-	err := runtimeClient.List(ctx, resourceQuotaList, client.InNamespace(suc.Namespace))
+	err = runtimeClient.List(ctx, resourceQuotaList, client.InNamespace(suc.Namespace))
 	if err != nil {
 		return fmt.Errorf("failed to list resource quotas, %w", err)
 	}
 	for _, quota := range resourceQuotaList.Items {
 		if maxObjects, ok := quota.Spec.Hard[consts.ResourceNameS3MaxObjects]; ok {
-			if totalMaxObjects.Cmp(maxObjects) > 0 {
+			if totalUsedQuota.MaxObjects.Cmp(maxObjects) > 0 {
 				return consts.ErrExceededNamespaceQuota
 			}
 		}
 		if maxSize, ok := quota.Spec.Hard[consts.ResourceNameS3MaxSize]; ok {
-			if totalMaxSize.Cmp(maxSize) > 0 {
+			if totalUsedQuota.MaxSize.Cmp(maxSize) > 0 {
 				return consts.ErrExceededNamespaceQuota
 			}
 		}
 		if maxBuckets, ok := quota.Spec.Hard[consts.ResourceNameS3MaxBuckets]; ok {
-			if totalMaxBuckets > maxBuckets.Value() {
+			if totalUsedQuota.MaxBuckets > maxBuckets.Value() {
 				return consts.ErrExceededNamespaceQuota
 			}
 		}
@@ -205,62 +187,35 @@ func validateAgainstNamespaceQuota(ctx context.Context, suc *S3UserClaim) error
 }
 
 func validateAgainstClusterQuota(ctx context.Context, suc *S3UserClaim) error {
-	// Find team's clusterResourceQuota
-	team, err := findTeam(ctx, suc)
+	totalClusterUsedQuota, team, err := CalculateClusterUsedQuota(ctx, runtimeClient, suc, false)
 	if err != nil {
-		return fmt.Errorf("failed to find team, %w", err)
+		return fmt.Errorf("failed to calculate cluster resource used quota , %w", err)
 	}
+
 	clusterQuota := &openshiftquota.ClusterResourceQuota{}
 	if err := runtimeClient.Get(ctx, types.NamespacedName{Name: team}, clusterQuota); err != nil {
 		if apierrors.IsNotFound(err) {
 			return fmt.Errorf("%w, team=%s", consts.ErrClusterQuotaNotDefined, team)
 		}
 		return fmt.Errorf("failed to get clusterQuota, %w", err)
 	}
-
-	// Sum all resource requests in team's namespaces
-	totalMaxObjects := resource.Quantity{}
-	totalMaxSize := resource.Quantity{}
-	totalMaxBuckets := int64(0)
-	namespaces, err := findTeamNamespaces(ctx, team)
-	if err != nil {
-		return fmt.Errorf("failed to find team namespaces, %w", err)
-	}
-	for _, ns := range namespaces {
-		s3UserClaimList := &S3UserClaimList{}
-		if err := uncachedReader.List(ctx, s3UserClaimList, client.InNamespace(ns)); err != nil {
-			return fmt.Errorf("failed to list s3UserClaims, %w", err)
-		}
-
-		for _, claim := range s3UserClaimList.Items {
-			if claim.Name != suc.Name || claim.Namespace != suc.Namespace {
-				totalMaxObjects.Add(claim.Spec.Quota.MaxObjects)
-				totalMaxSize.Add(claim.Spec.Quota.MaxSize)
-				totalMaxBuckets += int64(claim.Spec.Quota.MaxBuckets)
-			}
-		}
-	}
-	totalMaxObjects.Add(suc.Spec.Quota.MaxObjects)
-	totalMaxSize.Add(suc.Spec.Quota.MaxSize)
-	totalMaxBuckets += int64(suc.Spec.Quota.MaxBuckets)
-
 	// Validate against clusterResourceQuota
 	if maxObjects, ok := clusterQuota.Spec.Quota.Hard[consts.ResourceNameS3MaxObjects]; ok {
-		if totalMaxObjects.Cmp(maxObjects) > 0 {
+		if totalClusterUsedQuota.MaxObjects.Cmp(maxObjects) > 0 {
 			return consts.ErrExceededClusterQuota
 		}
 	} else {
 		return fmt.Errorf("%w, team=%s", consts.ErrClusterQuotaNotDefined, team)
 	}
 	if maxSize, ok := clusterQuota.Spec.Quota.Hard[consts.ResourceNameS3MaxSize]; ok {
-		if totalMaxSize.Cmp(maxSize) > 0 {
+		if totalClusterUsedQuota.MaxSize.Cmp(maxSize) > 0 {
 			return consts.ErrExceededClusterQuota
 		}
 	} else {
 		return fmt.Errorf("%w, team=%s", consts.ErrClusterQuotaNotDefined, team)
 	}
 	if maxBuckets, ok := clusterQuota.Spec.Quota.Hard[consts.ResourceNameS3MaxBuckets]; ok {
-		if totalMaxBuckets > maxBuckets.Value() {
+		if totalClusterUsedQuota.MaxBuckets > maxBuckets.Value() {
 			return consts.ErrExceededClusterQuota
 		}
 	} else {
@@ -269,38 +224,3 @@ func validateAgainstClusterQuota(ctx context.Context, suc *S3UserClaim) error {
 
 	return nil
 }
-
-func findTeam(ctx context.Context, suc *S3UserClaim) (string, error) {
-	ns := &v1.Namespace{}
-	if err := runtimeClient.Get(ctx, types.NamespacedName{Name: suc.ObjectMeta.Namespace}, ns); err != nil {
-		return "", fmt.Errorf("failed to get namespace, %w", err)
-	}
-
-	team, ok := ns.ObjectMeta.Labels[consts.LabelTeam]
-	if !ok {
-		return "", fmt.Errorf("namespace %s doesn't have team label", ns.ObjectMeta.Name)
-	}
-
-	return team, nil
-}
-
-func findTeamNamespaces(ctx context.Context, team string) ([]string, error) {
-	var namespaces []string
-
-	namespaceList := &v1.NamespaceList{}
-	if err := runtimeClient.List(ctx, namespaceList); err != nil {
-		return namespaces, fmt.Errorf("failed to list namespaces, %w", err)
-	}
-
-	for _, ns := range namespaceList.Items {
-		labels := ns.ObjectMeta.Labels
-		if labels == nil {
-			labels = map[string]string{}
-		}
-		if nsTeam := labels[consts.LabelTeam]; nsTeam == team {
-			namespaces = append(namespaces, ns.ObjectMeta.Name)
-		}
-	}
-
-	return namespaces, nil
-}

api/v1alpha1/types.go

@@ -12,6 +12,16 @@ type UserQuota struct {
 	MaxBuckets int `json:"maxBuckets,omitempty"`
 }
 
+// TotalUserQuota specifies the total used quota for a list of users in Ceph
+type TotalQuota struct {
+	// max number of bytes the user can store
+	MaxSize resource.Quantity `json:"maxSize,omitempty"`
+	// max number of objects the user can store
+	MaxObjects resource.Quantity `json:"maxObjects,omitempty"`
+	// max number of buckets the user can create
+	MaxBuckets int64 `json:"maxBuckets,omitempty"`
+}
+
 // +kubebuilder:validation:Pattern=^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
 type Subuser string
 type SubuserBinding struct {

config/rbac/cluster_resource_quota_clusterrole.yaml

@@ -0,0 +1,23 @@
+# These access controles are needed for updating the cluster resouce quota
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: clusterresourcequota-updater
+rules:
+- apiGroups: ["quota.openshift.io"] 
+  resources: ["clusterresourcequotas/status"]
+  verbs: ["update", "patch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: clusterresourcequota-updater-binding
+subjects:
+- kind: ServiceAccount
+  name: controller-manager 
+  namespace: system 
+roleRef:
+  kind: ClusterRole
+  name: clusterresourcequota-updater
+  apiGroup: rbac.authorization.k8s.io

config/rbac/kustomization.yaml

@@ -9,6 +9,8 @@ resources:
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
+- resource_quota_clusterrole.yaml
+- cluster_resource_quota_clusterrole.yaml
 # Comment the following 4 lines if you want to disable
 # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
 # which protects your /metrics endpoint.

config/rbac/resource_quota_clusterrole.yaml

@@ -0,0 +1,24 @@
+# These access controles are needed for updating resouce quota of the namespaces
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: resourcequota-status-updater
+rules:
+- apiGroups: [""]
+  resources: ["resourcequotas/status"]
+  verbs: ["update", "patch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: resourcequota-status-updater-binding
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system
+roleRef:
+  kind: ClusterRole
+  name: resourcequota-status-updater
+  apiGroup: rbac.authorization.k8s.io
+