issue #86: adding --poll, --sync, --max-reconcile-rate container arguments

cmd/provider/main.go

@@ -19,27 +19,34 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"time"
+
+	"github.com/crossplane-contrib/provider-gitlab/apis"
+	"github.com/crossplane-contrib/provider-gitlab/pkg/controller"
+	xpcontroller "github.com/crossplane/crossplane-runtime/pkg/controller"
+	"github.com/crossplane/crossplane-runtime/pkg/feature"
+	"github.com/crossplane/crossplane-runtime/pkg/logging"
+	"github.com/crossplane/crossplane-runtime/pkg/ratelimiter"
+	"go.uber.org/zap/zapcore"
 
 	"gopkg.in/alecthomas/kingpin.v2"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-
-	"github.com/crossplane/crossplane-runtime/pkg/logging"
-
-	"github.com/crossplane-contrib/provider-gitlab/apis"
-	"github.com/crossplane-contrib/provider-gitlab/pkg/controller"
 )
 
 func main() {
 	var (
-		app            = kingpin.New(filepath.Base(os.Args[0]), "Cluster API support for Crossplane.").DefaultEnvars()
-		debug          = app.Flag("debug", "Run with debug logging.").Short('d').Bool()
-		syncPeriod     = app.Flag("sync", "Controller manager sync period such as 300ms, 1.5h, or 2h45m").Short('s').Default("1h").Duration()
-		leaderElection = app.Flag("leader-election", "Use leader election for the conroller manager.").Short('l').Default("false").OverrideDefaultFromEnvar("LEADER_ELECTION").Bool()
+		app              = kingpin.New(filepath.Base(os.Args[0]), "Cluster API support for Crossplane.").DefaultEnvars()
+		debug            = app.Flag("debug", "Run with debug logging.").Short('d').Bool()
+		syncPeriod       = app.Flag("sync", "Controller manager sync period such as 300ms, 1.5h, or 2h45m").Short('s').Default("1h").Duration()
+		pollInterval     = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("1m").Duration()
+		leaderElection   = app.Flag("leader-election", "Use leader election for the conroller manager.").Short('l').Default("false").OverrideDefaultFromEnvar("LEADER_ELECTION").Bool()
+		maxReconcileRate = app.Flag("max-reconcile-rate", "The global maximum rate per second at which resources may checked for drift from the desired state.").Default("10").Int()
 	)
 	kingpin.MustParse(app.Parse(os.Args[1:]))
 
-	zl := zap.New(zap.UseDevMode(*debug))
+	zl := zap.New(zap.UseDevMode(*debug), UseISO8601())
 	log := logging.NewLogrLogger(zl.WithName("provider-gitlab"))
 	if *debug {
 		// The controller-runtime runs with a no-op logger by default. It is
@@ -53,14 +60,46 @@ func main() {
 	cfg, err := ctrl.GetConfig()
 	kingpin.FatalIfError(err, "Cannot get API server rest config")
 
-	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
-		LeaderElection:   *leaderElection,
-		LeaderElectionID: "crossplane-leader-election-provider-gitlab",
-		SyncPeriod:       syncPeriod,
+	// mgr, err := ctrl.NewManager(cfg, ctrl.Options{
+	// 	LeaderElection:   *leaderElection,
+	// 	LeaderElectionID: "crossplane-leader-election-provider-gitlab",
+	// 	SyncPeriod:       syncPeriod,
+	// })
+	mgr, err := ctrl.NewManager(ratelimiter.LimitRESTConfig(cfg, *maxReconcileRate), ctrl.Options{
+		SyncPeriod: syncPeriod,
+
+		// controller-runtime uses both ConfigMaps and Leases for leader
+		// election by default. Leases expire after 15 seconds, with a
+		// 10 second renewal deadline. We've observed leader loss due to
+		// renewal deadlines being exceeded when under high load - i.e.
+		// hundreds of reconciles per second and ~200rps to the API
+		// server. Switching to Leases only and longer leases appears to
+		// alleviate this.
+		LeaderElection:             *leaderElection,
+		LeaderElectionID:           "crossplane-leader-election-provider-kubernetes",
+		LeaderElectionResourceLock: resourcelock.LeasesResourceLock,
+		LeaseDuration:              func() *time.Duration { d := 60 * time.Second; return &d }(),
+		RenewDeadline:              func() *time.Duration { d := 50 * time.Second; return &d }(),
 	})
-	kingpin.FatalIfError(err, "Cannot create controller manager")
 
+	kingpin.FatalIfError(err, "Cannot create controller manager")
 	kingpin.FatalIfError(apis.AddToScheme(mgr.GetScheme()), "Cannot add Gitlab APIs to scheme")
-	kingpin.FatalIfError(controller.Setup(mgr, log), "Cannot setup Gitlab controllers")
+
+	o := xpcontroller.Options{
+		Logger:                  log,
+		MaxConcurrentReconciles: *maxReconcileRate,
+		PollInterval:            *pollInterval,
+		GlobalRateLimiter:       ratelimiter.NewGlobal(*maxReconcileRate),
+		Features:                &feature.Flags{},
+	}
+
+	kingpin.FatalIfError(controller.Setup(mgr, o), "Cannot setup Gitlab controllers")
 	kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager")
 }
+
+// UseISO8601 sets the logger to use ISO8601 timestamp format
+func UseISO8601() zap.Opts {
+	return func(o *zap.Options) {
+		o.TimeEncoder = zapcore.ISO8601TimeEncoder
+	}
+}

pkg/controller/config/config.go

@@ -20,8 +20,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
+	"github.com/crossplane/crossplane-runtime/pkg/controller"
 	"github.com/crossplane/crossplane-runtime/pkg/event"
-	"github.com/crossplane/crossplane-runtime/pkg/logging"
 	"github.com/crossplane/crossplane-runtime/pkg/reconciler/providerconfig"
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
 
@@ -30,7 +30,7 @@ import (
 
 // Setup adds a controller that reconciles ProviderConfigs by accounting for
 // their current usage.
-func Setup(mgr ctrl.Manager, l logging.Logger) error {
+func Setup(mgr ctrl.Manager, o controller.Options) error {
 	name := providerconfig.ControllerName(v1beta1.ProviderConfigGroupKind)
 
 	of := resource.ProviderConfigKinds{
@@ -43,6 +43,6 @@ func Setup(mgr ctrl.Manager, l logging.Logger) error {
 		For(&v1beta1.ProviderConfig{}).
 		Watches(&source.Kind{Type: &v1beta1.ProviderConfigUsage{}}, &resource.EnqueueRequestForProviderConfig{}).
 		Complete(providerconfig.NewReconciler(mgr, of,
-			providerconfig.WithLogger(l.WithValues("controller", name)),
+			providerconfig.WithLogger(o.Logger.WithValues("controller", name)),
 			providerconfig.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name)))))
 }

pkg/controller/gitlab.go

@@ -19,7 +19,7 @@ package controller
 import (
 	ctrl "sigs.k8s.io/controller-runtime"
 
-	"github.com/crossplane/crossplane-runtime/pkg/logging"
+	"github.com/crossplane/crossplane-runtime/pkg/controller"
 
 	"github.com/crossplane-contrib/provider-gitlab/pkg/controller/config"
 	"github.com/crossplane-contrib/provider-gitlab/pkg/controller/groups"
@@ -38,8 +38,8 @@ import (
 
 // Setup creates all Gitlab API controllers with the supplied logger and adds
 // them to the supplied manager.
-func Setup(mgr ctrl.Manager, l logging.Logger) error {
-	for _, setup := range []func(ctrl.Manager, logging.Logger) error{
+func Setup(mgr ctrl.Manager, o controller.Options) error {
+	for _, setup := range []func(ctrl.Manager, controller.Options) error{
 		config.Setup,
 		groups.SetupGroup,
 		groupsMembers.SetupMember,
@@ -54,7 +54,7 @@ func Setup(mgr ctrl.Manager, l logging.Logger) error {
 		projectsDeployKeys.SetupDeployKey,
 		projectsPipelineschedules.SetupPipelineSchedule,
 	} {
-		if err := setup(mgr, l); err != nil {
+		if err := setup(mgr, o); err != nil {
 			return err
 		}
 	}

pkg/controller/groups/deploytokens/controller.go

@@ -29,8 +29,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
+	"github.com/crossplane/crossplane-runtime/pkg/controller"
 	"github.com/crossplane/crossplane-runtime/pkg/event"
-	"github.com/crossplane/crossplane-runtime/pkg/logging"
 	"github.com/crossplane/crossplane-runtime/pkg/meta"
 	"github.com/crossplane/crossplane-runtime/pkg/reconciler/managed"
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
@@ -50,7 +50,7 @@ const (
 )
 
 // SetupDeployToken adds a controller that reconciles GroupDeployTokens.
-func SetupDeployToken(mgr ctrl.Manager, l logging.Logger) error {
+func SetupDeployToken(mgr ctrl.Manager, o controller.Options) error {
 	name := managed.ControllerName(v1alpha1.DeployTokenKind)
 
 	return ctrl.NewControllerManagedBy(mgr).
@@ -60,7 +60,7 @@ func SetupDeployToken(mgr ctrl.Manager, l logging.Logger) error {
 			resource.ManagedKind(v1alpha1.DeployTokenGroupVersionKind),
 			managed.WithExternalConnecter(&connector{kube: mgr.GetClient(), newGitlabClientFn: groups.NewDeployTokenClient}),
 			managed.WithInitializers(managed.NewDefaultProviderConfig(mgr.GetClient())),
-			managed.WithLogger(l.WithValues("controller", name)),
+			managed.WithLogger(o.Logger.WithValues("controller", name)),
 			managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name)))))
 }
 

pkg/controller/groups/group.go

@@ -30,8 +30,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/crossplane/crossplane-runtime/pkg/controller"
 	"github.com/crossplane/crossplane-runtime/pkg/event"
-	"github.com/crossplane/crossplane-runtime/pkg/logging"
 	"github.com/crossplane/crossplane-runtime/pkg/meta"
 	"github.com/crossplane/crossplane-runtime/pkg/reconciler/managed"
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
@@ -56,7 +56,7 @@ const (
 )
 
 // SetupGroup adds a controller that reconciles Groups.
-func SetupGroup(mgr ctrl.Manager, l logging.Logger) error {
+func SetupGroup(mgr ctrl.Manager, o controller.Options) error {
 	name := managed.ControllerName(v1alpha1.GroupKind)
 
 	return ctrl.NewControllerManagedBy(mgr).
@@ -66,7 +66,7 @@ func SetupGroup(mgr ctrl.Manager, l logging.Logger) error {
 			resource.ManagedKind(v1alpha1.GroupKubernetesGroupVersionKind),
 			managed.WithExternalConnecter(&connector{kube: mgr.GetClient(), newGitlabClientFn: groups.NewGroupClient}),
 			managed.WithInitializers(managed.NewDefaultProviderConfig(mgr.GetClient())),
-			managed.WithLogger(l.WithValues("controller", name)),
+			managed.WithLogger(o.Logger.WithValues("controller", name)),
 			managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name)))))
 }
 

pkg/controller/groups/members/controller.go

@@ -24,8 +24,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/crossplane/crossplane-runtime/pkg/controller"
 	"github.com/crossplane/crossplane-runtime/pkg/event"
-	"github.com/crossplane/crossplane-runtime/pkg/logging"
 	"github.com/crossplane/crossplane-runtime/pkg/reconciler/managed"
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
 
@@ -48,7 +48,7 @@ const (
 )
 
 // SetupMember adds a controller that reconciles Group Members.
-func SetupMember(mgr ctrl.Manager, l logging.Logger) error {
+func SetupMember(mgr ctrl.Manager, o controller.Options) error {
 	name := managed.ControllerName(v1alpha1.MemberKind)
 
 	return ctrl.NewControllerManagedBy(mgr).
@@ -61,7 +61,7 @@ func SetupMember(mgr ctrl.Manager, l logging.Logger) error {
 				newGitlabClientFn: groups.NewMemberClient,
 				newUserClientFn:   users.NewUserClient}),
 			managed.WithInitializers(managed.NewDefaultProviderConfig(mgr.GetClient())),
-			managed.WithLogger(l.WithValues("controller", name)),
+			managed.WithLogger(o.Logger.WithValues("controller", name)),
 			managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name)))))
 }
 

pkg/controller/groups/variables/controller.go

@@ -27,9 +27,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
+	"github.com/crossplane/crossplane-runtime/pkg/controller"
 	"github.com/crossplane/crossplane-runtime/pkg/errors"
 	"github.com/crossplane/crossplane-runtime/pkg/event"
-	"github.com/crossplane/crossplane-runtime/pkg/logging"
 	"github.com/crossplane/crossplane-runtime/pkg/reconciler/managed"
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
 
@@ -50,7 +50,7 @@ const (
 )
 
 // SetupVariable adds a controller that reconciles Variables.
-func SetupVariable(mgr ctrl.Manager, l logging.Logger) error {
+func SetupVariable(mgr ctrl.Manager, o controller.Options) error {
 	name := managed.ControllerName(v1alpha1.VariableKind)
 
 	return ctrl.NewControllerManagedBy(mgr).
@@ -60,7 +60,7 @@ func SetupVariable(mgr ctrl.Manager, l logging.Logger) error {
 			resource.ManagedKind(v1alpha1.VariableGroupVersionKind),
 			managed.WithExternalConnecter(&connector{kube: mgr.GetClient(), newGitlabClientFn: groups.NewVariableClient}),
 			managed.WithInitializers(managed.NewDefaultProviderConfig(mgr.GetClient())),
-			managed.WithLogger(l.WithValues("controller", name)),
+			managed.WithLogger(o.Logger.WithValues("controller", name)),
 			managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name)))))
 }
 

pkg/controller/projects/accesstokens/controller.go

@@ -30,8 +30,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
+	"github.com/crossplane/crossplane-runtime/pkg/controller"
 	"github.com/crossplane/crossplane-runtime/pkg/event"
-	"github.com/crossplane/crossplane-runtime/pkg/logging"
 	"github.com/crossplane/crossplane-runtime/pkg/meta"
 	"github.com/crossplane/crossplane-runtime/pkg/reconciler/managed"
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
@@ -53,7 +53,7 @@ const (
 )
 
 // SetupAccessToken adds a controller that reconciles ProjectAccessTokens.
-func SetupAccessToken(mgr ctrl.Manager, l logging.Logger) error {
+func SetupAccessToken(mgr ctrl.Manager, o controller.Options) error {
 	name := managed.ControllerName(v1alpha1.AccessTokenKind)
 
 	return ctrl.NewControllerManagedBy(mgr).
@@ -63,7 +63,7 @@ func SetupAccessToken(mgr ctrl.Manager, l logging.Logger) error {
 			resource.ManagedKind(v1alpha1.AccessTokenGroupVersionKind),
 			managed.WithExternalConnecter(&connector{kube: mgr.GetClient(), newGitlabClientFn: projects.NewAccessTokenClient}),
 			managed.WithInitializers(managed.NewDefaultProviderConfig(mgr.GetClient())),
-			managed.WithLogger(l.WithValues("controller", name)),
+			managed.WithLogger(o.Logger.WithValues("controller", name)),
 			managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name)))))
 }
 

pkg/controller/projects/deploykeys/controller.go

@@ -5,8 +5,8 @@ import (
 	"strconv"
 
 	xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
+	crpc "github.com/crossplane/crossplane-runtime/pkg/controller"
 	"github.com/crossplane/crossplane-runtime/pkg/event"
-	"github.com/crossplane/crossplane-runtime/pkg/logging"
 	"github.com/crossplane/crossplane-runtime/pkg/meta"
 	"github.com/crossplane/crossplane-runtime/pkg/reconciler/managed"
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
@@ -48,7 +48,7 @@ type connector struct {
 }
 
 // SetupDeployKey adds a controller that reconciles ProjectDeployKey.
-func SetupDeployKey(manager controller.Manager, log logging.Logger) error {
+func SetupDeployKey(manager controller.Manager, o crpc.Options) error {
 	name := managed.ControllerName(v1alpha1.DeployKeyKind)
 
 	connector := &connector{kube: manager.GetClient(), newGitlabClientFn: newDeployKeyClient}
@@ -57,7 +57,7 @@ func SetupDeployKey(manager controller.Manager, log logging.Logger) error {
 		resource.ManagedKind(v1alpha1.DeployKeyGroupVersionKind),
 		managed.WithExternalConnecter(connector),
 		managed.WithInitializers(managed.NewDefaultProviderConfig(manager.GetClient())),
-		managed.WithLogger(log.WithValues("controller", name)),
+		managed.WithLogger(o.Logger.WithValues("controller", name)),
 		managed.WithRecorder(event.NewAPIRecorder(manager.GetEventRecorderFor(name))))
 
 	return controller.NewControllerManagedBy(manager).

pkg/controller/projects/deploytokens/controller.go

@@ -29,8 +29,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
+	"github.com/crossplane/crossplane-runtime/pkg/controller"
 	"github.com/crossplane/crossplane-runtime/pkg/event"
-	"github.com/crossplane/crossplane-runtime/pkg/logging"
 	"github.com/crossplane/crossplane-runtime/pkg/meta"
 	"github.com/crossplane/crossplane-runtime/pkg/reconciler/managed"
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
@@ -50,7 +50,7 @@ const (
 )
 
 // SetupDeployToken adds a controller that reconciles ProjectDeployTokens.
-func SetupDeployToken(mgr ctrl.Manager, l logging.Logger) error {
+func SetupDeployToken(mgr ctrl.Manager, o controller.Options) error {
 	name := managed.ControllerName(v1alpha1.DeployTokenKind)
 
 	return ctrl.NewControllerManagedBy(mgr).
@@ -60,7 +60,7 @@ func SetupDeployToken(mgr ctrl.Manager, l logging.Logger) error {
 			resource.ManagedKind(v1alpha1.DeployTokenGroupVersionKind),
 			managed.WithExternalConnecter(&connector{kube: mgr.GetClient(), newGitlabClientFn: projects.NewDeployTokenClient}),
 			managed.WithInitializers(managed.NewDefaultProviderConfig(mgr.GetClient())),
-			managed.WithLogger(l.WithValues("controller", name)),
+			managed.WithLogger(o.Logger.WithValues("controller", name)),
 			managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name)))))
 }
 

pkg/controller/projects/hooks/controller.go

@@ -28,8 +28,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
+	"github.com/crossplane/crossplane-runtime/pkg/controller"
 	"github.com/crossplane/crossplane-runtime/pkg/event"
-	"github.com/crossplane/crossplane-runtime/pkg/logging"
 	"github.com/crossplane/crossplane-runtime/pkg/meta"
 	"github.com/crossplane/crossplane-runtime/pkg/reconciler/managed"
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
@@ -50,7 +50,7 @@ const (
 )
 
 // SetupHook adds a controller that reconciles Hooks.
-func SetupHook(mgr ctrl.Manager, l logging.Logger) error {
+func SetupHook(mgr ctrl.Manager, o controller.Options) error {
 	name := managed.ControllerName(v1alpha1.HookKind)
 
 	return ctrl.NewControllerManagedBy(mgr).
@@ -60,7 +60,7 @@ func SetupHook(mgr ctrl.Manager, l logging.Logger) error {
 			resource.ManagedKind(v1alpha1.HookGroupVersionKind),
 			managed.WithExternalConnecter(&connector{kube: mgr.GetClient(), newGitlabClientFn: projects.NewHookClient}),
 			managed.WithInitializers(managed.NewDefaultProviderConfig(mgr.GetClient())),
-			managed.WithLogger(l.WithValues("controller", name)),
+			managed.WithLogger(o.Logger.WithValues("controller", name)),
 			managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name)))))
 }