From ed8dbe757c2f56dff8e5171e1a004851e9797c79 Mon Sep 17 00:00:00 2001 From: Paulin Todev Date: Mon, 2 Dec 2024 13:33:19 +0000 Subject: [PATCH] Retry running a CRD Manager if it fails --- .../prometheus/operator/common/component.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/internal/component/prometheus/operator/common/component.go b/internal/component/prometheus/operator/common/component.go index ca2ac4a98d..5c8cd286a5 100644 --- a/internal/component/prometheus/operator/common/component.go +++ b/internal/component/prometheus/operator/common/component.go @@ -104,11 +104,22 @@ func (c *Component) Run(ctx context.Context) error { innerCtx, cancel = context.WithCancel(ctx) wg.Add(1) go func() { - if err := manager.Run(innerCtx); err != nil { - level.Error(c.opts.Logger).Log("msg", "error running crd manager", "err", err) - errChan <- err + defer wg.Done() + retryInterval := 0 * time.Second + for { + select { + case <-innerCtx.Done(): + //TODO: Log that the manager is exiting? + return + case <-time.After(retryInterval): + if err := manager.Run(innerCtx); err != nil { + level.Error(c.opts.Logger).Log("msg", "error running crd manager", "err", err) + errChan <- err + //TODO: What is a good default value for this? + retryInterval = 30 * time.Second + } + } } - wg.Done() }() c.mut.Unlock() }