From 872819006388d3b52fd543295bb8cb8ce17d5313 Mon Sep 17 00:00:00 2001 From: Taketoshi Fujiwara Date: Thu, 21 Nov 2024 09:58:40 +0900 Subject: [PATCH] gha: customize client-go rate limiter params (#4) --- .../templates/deployment.yaml | 6 ++++++ .../gha-runner-scale-set-controller/values.yaml | 4 ++++ .../ephemeralrunner_controller.go | 15 +++++++++++---- main.go | 11 ++++++++++- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/charts/gha-runner-scale-set-controller/templates/deployment.yaml b/charts/gha-runner-scale-set-controller/templates/deployment.yaml index bb3e7f5dd5..10428912cb 100644 --- a/charts/gha-runner-scale-set-controller/templates/deployment.yaml +++ b/charts/gha-runner-scale-set-controller/templates/deployment.yaml @@ -97,6 +97,12 @@ spec: {{- with .Values.flags.maxConcurrentReconcilesForAutoscalingListener }} - "--max-concurrent-reconciles-for-autoscaling-listener={{ . }}" {{- end }} + {{- with .Values.flags.clientGoRateLimiterQPS }} + - "--client-go-rate-limiter-qps={{ . }}" + {{- end }} + {{- with .Values.flags.clientGoRateLimiterBurst }} + - "--client-go-rate-limiter-burst={{ . }}" + {{- end }} command: - "/manager" {{- if or .Values.metrics .Values.pprof }} diff --git a/charts/gha-runner-scale-set-controller/values.yaml b/charts/gha-runner-scale-set-controller/values.yaml index 9b5fce819a..621001ea16 100644 --- a/charts/gha-runner-scale-set-controller/values.yaml +++ b/charts/gha-runner-scale-set-controller/values.yaml @@ -140,3 +140,7 @@ flags: # maxConcurrentReconcilesForEphemeralRunnerSet: 1 # maxConcurrentReconcilesForEphemeralRunner: 1 # maxConcurrentReconcilesForAutoscalingListener: 1 + + ## Defines the client-go rate limiter parameters. + # clientGoRateLimiterQPS: 20 + # clientGoRateLimiterBurst: 30 diff --git a/controllers/actions.github.com/ephemeralrunner_controller.go b/controllers/actions.github.com/ephemeralrunner_controller.go index 015461aa9b..00393adf1c 100644 --- a/controllers/actions.github.com/ephemeralrunner_controller.go +++ b/controllers/actions.github.com/ephemeralrunner_controller.go @@ -71,6 +71,7 @@ type EphemeralRunnerReconciler struct { // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.6.4/pkg/reconcile func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("ephemeralrunner", req.NamespacedName) + log.Info("Start Reconcile") ephemeralRunner := new(v1alpha1.EphemeralRunner) if err := r.Get(ctx, req.NamespacedName, ephemeralRunner); err != nil { @@ -183,6 +184,7 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ return r.updateStatusWithRunnerConfig(ctx, ephemeralRunner, log) } + now := time.Now() secret := new(corev1.Secret) if err := r.Get(ctx, req.NamespacedName, secret); err != nil { if !kerrors.IsNotFound(err) { @@ -193,6 +195,7 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ log.Info("Creating new ephemeral runner secret for jitconfig.") return r.createSecret(ctx, ephemeralRunner, log) } + log.Info("Get Secret", "duration_ms", time.Since(now).Milliseconds()) pod := new(corev1.Pod) if err := r.Get(ctx, req.NamespacedName, pod); err != nil { @@ -344,8 +347,10 @@ func (r *EphemeralRunnerReconciler) cleanupResources(ctx context.Context, epheme log.Info("Pod is deleted") log.Info("Cleaning up the runner jitconfig secret") + now := time.Now() secret := new(corev1.Secret) err = r.Get(ctx, types.NamespacedName{Namespace: ephemeralRunner.Namespace, Name: ephemeralRunner.Name}, secret) + log.Info("Get Secret for cleanup", "duration_ms", time.Since(now).Milliseconds()) switch { case err == nil: if secret.ObjectMeta.DeletionTimestamp.IsZero() { @@ -516,7 +521,7 @@ func (r *EphemeralRunnerReconciler) deletePodAsFailed(ctx context.Context, ephem func (r *EphemeralRunnerReconciler) updateStatusWithRunnerConfig(ctx context.Context, ephemeralRunner *v1alpha1.EphemeralRunner, log logr.Logger) (ctrl.Result, error) { // Runner is not registered with the service. We need to register it first log.Info("Creating ephemeral runner JIT config") - actionsClient, err := r.actionsClientFor(ctx, ephemeralRunner) + actionsClient, err := r.actionsClientFor(ctx, ephemeralRunner, log) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to get actions client for generating JIT config: %v", err) } @@ -712,11 +717,13 @@ func (r *EphemeralRunnerReconciler) updateRunStatusFromPod(ctx context.Context, return nil } -func (r *EphemeralRunnerReconciler) actionsClientFor(ctx context.Context, runner *v1alpha1.EphemeralRunner) (actions.ActionsService, error) { +func (r *EphemeralRunnerReconciler) actionsClientFor(ctx context.Context, runner *v1alpha1.EphemeralRunner, log logr.Logger) (actions.ActionsService, error) { + now := time.Now() secret := new(corev1.Secret) if err := r.Get(ctx, types.NamespacedName{Namespace: runner.Namespace, Name: runner.Spec.GitHubConfigSecret}, secret); err != nil { return nil, fmt.Errorf("failed to get secret: %w", err) } + log.Info("Get Secret for GitHub client", "duration_ms", time.Since(now).Milliseconds()) opts, err := r.actionsClientOptionsFor(ctx, runner) if err != nil { @@ -782,7 +789,7 @@ func (r *EphemeralRunnerReconciler) actionsClientOptionsFor(ctx context.Context, // runnerRegisteredWithService checks if the runner is still registered with the service // Returns found=false and err=nil if ephemeral runner does not exist in GitHub service and should be deleted func (r EphemeralRunnerReconciler) runnerRegisteredWithService(ctx context.Context, runner *v1alpha1.EphemeralRunner, log logr.Logger) (found bool, err error) { - actionsClient, err := r.actionsClientFor(ctx, runner) + actionsClient, err := r.actionsClientFor(ctx, runner, log) if err != nil { return false, fmt.Errorf("failed to get Actions client for ScaleSet: %w", err) } @@ -809,7 +816,7 @@ func (r EphemeralRunnerReconciler) runnerRegisteredWithService(ctx context.Conte } func (r *EphemeralRunnerReconciler) deleteRunnerFromService(ctx context.Context, ephemeralRunner *v1alpha1.EphemeralRunner, log logr.Logger) error { - client, err := r.actionsClientFor(ctx, ephemeralRunner) + client, err := r.actionsClientFor(ctx, ephemeralRunner, log) if err != nil { return fmt.Errorf("failed to get actions client for runner: %v", err) } diff --git a/main.go b/main.go index bf5ebe8654..3a3aa00bd5 100644 --- a/main.go +++ b/main.go @@ -109,6 +109,9 @@ func main() { maxConcurrentReconcilesForEphemeralRunnerSet int maxConcurrentReconcilesForEphemeralRunner int maxConcurrentReconcilesForAutoscalingListener int + + rateLimiterQPS int + rateLimiterBurst int ) var c github.Config err = envconfig.Process("github", &c) @@ -156,6 +159,8 @@ func main() { flag.IntVar(&maxConcurrentReconcilesForEphemeralRunnerSet, "max-concurrent-reconciles-for-ephemeral-runner-set", 1, "The maximum number of concurrent reconciles for EphemeralRunnerSet.") flag.IntVar(&maxConcurrentReconcilesForEphemeralRunner, "max-concurrent-reconciles-for-ephemeral-runner", 1, "The maximum number of concurrent reconciles for EphemeralRunner.") flag.IntVar(&maxConcurrentReconcilesForAutoscalingListener, "max-concurrent-reconciles-for-autoscaling-listener", 1, "The maximum number of concurrent reconciles for AutoscalingListener.") + flag.IntVar(&rateLimiterQPS, "client-go-rate-limiter-qps", 20, "The QPS value of client-go rate limiter.") + flag.IntVar(&rateLimiterBurst, "client-go-rate-limiter-burst", 30, "The burst value of client-go rate limiter.") flag.Parse() runnerPodDefaults.RunnerImagePullSecrets = runnerImagePullSecrets @@ -225,7 +230,11 @@ func main() { }) } - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + cfg := ctrl.GetConfigOrDie() + cfg.QPS = float32(rateLimiterQPS) + cfg.Burst = rateLimiterBurst + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{ BindAddress: metricsAddr,