From d82f44b567b2c8f6af504f61b7ceae1481638622 Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Fri, 29 Nov 2024 05:41:09 -0500
Subject: [PATCH] prometheus.operator.*: allow setting informer_sync_timeout
 (#2161)

* prometheus.operator.*: allow setting informer_sync_timeout

* default to 1m

* docs
---
 CHANGELOG.md                                              | 2 ++
 .../prometheus/prometheus.operator.podmonitors.md         | 1 +
 .../components/prometheus/prometheus.operator.probes.md   | 1 +
 .../prometheus/prometheus.operator.servicemonitors.md     | 1 +
 .../component/prometheus/operator/common/crdmanager.go    | 8 +++-----
 internal/component/prometheus/operator/types.go           | 8 ++++++--
 6 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 98066c9cdd..2e0fe4427f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -124,6 +124,8 @@ v1.5.0
 
 - Add `proxy_url` to `otelcol.exporter.otlphttp`. (@wildum)
 
+- Allow setting `informer_sync_timeout` in prometheus.operator.* components. (@captncraig)
+
 ### Bugfixes
 
 - Fixed a bug in `import.git` which caused a `"non-fast-forward update"` error message. (@ptodev)
diff --git a/docs/sources/reference/components/prometheus/prometheus.operator.podmonitors.md b/docs/sources/reference/components/prometheus/prometheus.operator.podmonitors.md
index e39dcabf51..2ef2ca30b3 100644
--- a/docs/sources/reference/components/prometheus/prometheus.operator.podmonitors.md
+++ b/docs/sources/reference/components/prometheus/prometheus.operator.podmonitors.md
@@ -34,6 +34,7 @@ Name | Type | Description | Default | Required
 ---- | ---- | ----------- | ------- | --------
 `forward_to` | `list(MetricsReceiver)` | List of receivers to send scraped metrics to. | | yes
 `namespaces` | `list(string)` | List of namespaces to search for PodMonitor resources. If not specified, all namespaces will be searched. || no
+`informer_sync_timeout` | `duration` | Timeout for initial sync of PodMonitor resources. | `1m` | no
 
 ## Blocks
 
diff --git a/docs/sources/reference/components/prometheus/prometheus.operator.probes.md b/docs/sources/reference/components/prometheus/prometheus.operator.probes.md
index ab05421c5e..d1d6421173 100644
--- a/docs/sources/reference/components/prometheus/prometheus.operator.probes.md
+++ b/docs/sources/reference/components/prometheus/prometheus.operator.probes.md
@@ -37,6 +37,7 @@ Name | Type | Description | Default | Required
 ---- | ---- | ----------- | ------- | --------
 `forward_to` | `list(MetricsReceiver)` | List of receivers to send scraped metrics to. | | yes
 `namespaces` | `list(string)` | List of namespaces to search for Probe resources. If not specified, all namespaces will be searched. || no
+`informer_sync_timeout` | `duration` | Timeout for initial sync of Probe resources. | `1m` | no
 
 ## Blocks
 
diff --git a/docs/sources/reference/components/prometheus/prometheus.operator.servicemonitors.md b/docs/sources/reference/components/prometheus/prometheus.operator.servicemonitors.md
index 67b2fcc4d3..0fc51ba0fc 100644
--- a/docs/sources/reference/components/prometheus/prometheus.operator.servicemonitors.md
+++ b/docs/sources/reference/components/prometheus/prometheus.operator.servicemonitors.md
@@ -37,6 +37,7 @@ Name | Type | Description
 `forward_to` | `list(MetricsReceiver)` | List of receivers to send scraped metrics to.                                                                              |           | yes
 `namespaces` | `list(string)` | List of namespaces to search for ServiceMonitor resources. If not specified, all namespaces will be searched.              |           | no
 `kubernetes_role` | `string` | The Kubernetes role used for discovery. Supports `endpoints` or `endpointslice`. | `endpoints` | no
+`informer_sync_timeout` | `duration` | Timeout for initial sync of ServiceMonitor resources. | `1m` | no
 
 ## Blocks
 
diff --git a/internal/component/prometheus/operator/common/crdmanager.go b/internal/component/prometheus/operator/common/crdmanager.go
index e24b99159b..36b27d7596 100644
--- a/internal/component/prometheus/operator/common/crdmanager.go
+++ b/internal/component/prometheus/operator/common/crdmanager.go
@@ -4,12 +4,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	promk8s "github.com/prometheus/prometheus/discovery/kubernetes"
 	"sort"
 	"strings"
 	"sync"
 	"time"
 
+	promk8s "github.com/prometheus/prometheus/discovery/kubernetes"
+
 	"github.com/go-kit/log"
 	"github.com/grafana/ckit/shard"
 	promopv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
@@ -38,9 +39,6 @@ import (
 	"github.com/grafana/alloy/internal/util"
 )
 
-// Generous timeout period for configuring all informers
-const informerSyncTimeout = 10 * time.Second
-
 type crdManagerInterface interface {
 	Run(ctx context.Context) error
 	ClusteringUpdated()
@@ -333,7 +331,7 @@ func (c *crdManager) configureInformers(ctx context.Context, informers cache.Inf
 		return fmt.Errorf("unknown kind to configure Informers: %s", c.kind)
 	}
 
-	informerCtx, cancel := context.WithTimeout(ctx, informerSyncTimeout)
+	informerCtx, cancel := context.WithTimeout(ctx, c.args.InformerSyncTimeout)
 	defer cancel()
 
 	informer, err := informers.GetInformer(informerCtx, prototype)
diff --git a/internal/component/prometheus/operator/types.go b/internal/component/prometheus/operator/types.go
index ed5be3d6e0..6ac2e99d55 100644
--- a/internal/component/prometheus/operator/types.go
+++ b/internal/component/prometheus/operator/types.go
@@ -2,9 +2,10 @@ package operator
 
 import (
 	"fmt"
-	promk8s "github.com/prometheus/prometheus/discovery/kubernetes"
 	"time"
 
+	promk8s "github.com/prometheus/prometheus/discovery/kubernetes"
+
 	"github.com/grafana/alloy/internal/component/common/config"
 	"github.com/grafana/alloy/internal/component/common/kubernetes"
 	alloy_relabel "github.com/grafana/alloy/internal/component/common/relabel"
@@ -36,6 +37,8 @@ type Arguments struct {
 	RelabelConfigs []*alloy_relabel.Config `alloy:"rule,block,optional"`
 
 	Scrape ScrapeOptions `alloy:"scrape,block,optional"`
+
+	InformerSyncTimeout time.Duration `alloy:"informer_sync_timeout,attr,optional"`
 }
 
 // ScrapeOptions holds values that configure scraping behavior.
@@ -58,7 +61,8 @@ var DefaultArguments = Arguments{
 	Client: kubernetes.ClientArguments{
 		HTTPClientConfig: config.DefaultHTTPClientConfig,
 	},
-	KubernetesRole: string(promk8s.RoleEndpoint),
+	KubernetesRole:      string(promk8s.RoleEndpoint),
+	InformerSyncTimeout: time.Minute,
 }
 
 // SetToDefault implements syntax.Defaulter.