Upgrade to latest mimir-prometheus with Prometheus 3.0 changes

go.mod

@@ -76,16 +76,17 @@ require (
 	github.com/prometheus/procfs v0.15.1
 	github.com/shirou/gopsutil/v4 v4.24.10
 	github.com/thanos-io/objstore v0.0.0-20241111205755-d1dd89d41f97
+	github.com/tjhop/slog-gokit v0.1.2
 	github.com/twmb/franz-go v1.18.0
 	github.com/twmb/franz-go/pkg/kadm v1.14.0
 	github.com/twmb/franz-go/pkg/kfake v0.0.0-20241111190909-81ceb1a064be
 	github.com/twmb/franz-go/pkg/kmsg v1.9.0
 	github.com/twmb/franz-go/plugin/kotel v1.5.0
 	github.com/twmb/franz-go/plugin/kprom v1.1.0
 	github.com/xlab/treeprint v1.2.0
-	go.opentelemetry.io/collector/pdata v1.15.0
-	go.opentelemetry.io/otel v1.30.0
-	go.opentelemetry.io/otel/trace v1.30.0
+	go.opentelemetry.io/collector/pdata v1.16.0
+	go.opentelemetry.io/otel v1.31.0
+	go.opentelemetry.io/otel/trace v1.31.0
 	go.uber.org/multierr v1.11.0
 	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f
 	golang.org/x/term v0.26.0
@@ -113,7 +114,7 @@ require (
 	github.com/go-test/deep v1.1.0 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
-	github.com/golang/glog v1.2.1 // indirect
+	github.com/golang/glog v1.2.2 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
@@ -136,14 +137,13 @@ require (
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.55.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.30.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.30.0 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/mail.v2 v2.3.1 // indirect
 	gopkg.in/telebot.v3 v3.2.1 // indirect
-	k8s.io/apimachinery v0.31.0 // indirect
-	k8s.io/client-go v0.31.0 // indirect
+	k8s.io/apimachinery v0.31.1 // indirect
+	k8s.io/client-go v0.31.1 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 )
 
@@ -248,7 +248,7 @@ require (
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/common/sigv4 v0.1.0 // indirect
-	github.com/prometheus/exporter-toolkit v0.12.0 // indirect
+	github.com/prometheus/exporter-toolkit v0.13.0 // indirect
 	github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect
 	github.com/rs/cors v1.11.0 // indirect
 	github.com/rs/xid v1.6.0 // indirect
@@ -266,9 +266,9 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.4 // indirect
 	go.mongodb.org/mongo-driver v1.14.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/collector/semconv v0.108.1
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect
-	go.opentelemetry.io/otel/metric v1.30.0 // indirect
+	go.opentelemetry.io/collector/semconv v0.110.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect
+	go.opentelemetry.io/otel/metric v1.31.0 // indirect
 	go.uber.org/zap v1.21.0 // indirect
 	golang.org/x/mod v0.22.0 // indirect
 	golang.org/x/oauth2 v0.24.0 // indirect
@@ -284,7 +284,7 @@ require (
 )
 
 // Using a fork of Prometheus with Mimir-specific changes.
-replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20241120160701-db938c3ceac8
+replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20241121154224-81e8a18ac508
 
 // Replace memberlist with our fork which includes some fixes that haven't been
 // merged upstream yet:

integration/distributor_high_concurrency_test.go

@@ -127,7 +127,7 @@ func testDistributorHighConcurrency(t *testing.T, cachingUnmarshalDataEnabled bo
 			require.Equal(t, samples, added)
 
 			// query all samples back
-			query := fmt.Sprintf("%s[%s]", serName, model.Duration(timeRange))
+			query := fmt.Sprintf("%s[%s]", serName, model.Duration(timeRange+time.Millisecond)) // Add millisecond to ensure we get the first point (ranges are left-open).
 			result, err := client.Query(query, writeEnd)
 			require.NoError(t, err)
 			require.Equal(t, exp, result)

integration/query_frontend_cache_test.go

@@ -167,10 +167,11 @@ func generateExpectedFloats(start time.Time, end time.Time, step time.Duration,
 	val := expectedVector[0].Value
 
 	const lookbackPeriod = 5 * time.Minute
+	rangeEnd := sampleTime.Add(lookbackPeriod)
 
 	values := []model.SamplePair(nil)
 	for ts := start; !ts.After(end); ts = ts.Add(step) {
-		if ts.Before(sampleTime) || ts.After(sampleTime.Add(lookbackPeriod)) {
+		if ts.Before(sampleTime) || ts.After(rangeEnd) || ts.Equal(rangeEnd) {
 			continue
 		}
 		values = append(values, model.SamplePair{
@@ -192,10 +193,11 @@ func generateExpectedHistograms(start time.Time, end time.Time, step time.Durati
 	hist := expectedVector[0].Histogram
 
 	const lookbackPeriod = 5 * time.Minute
+	rangeEnd := sampleTime.Add(lookbackPeriod)
 
 	histograms := []model.SampleHistogramPair(nil)
 	for ts := start; !ts.After(end); ts = ts.Add(step) {
-		if ts.Before(sampleTime) || ts.After(sampleTime.Add(lookbackPeriod)) {
+		if ts.Before(sampleTime) || ts.After(rangeEnd) || ts.Equal(rangeEnd) {
 			continue
 		}
 		histograms = append(histograms, model.SampleHistogramPair{

integration/query_frontend_test.go

@@ -720,7 +720,7 @@ overrides:
 				return c.QueryRangeRaw(`sum_over_time(metric[31d:1s])`, now.Add(-time.Minute), now, time.Minute)
 			},
 			expStatusCode: http.StatusUnprocessableEntity,
-			expJSON:       fmt.Sprintf(`{"error":"expanding series: %s", "errorType":"execution", "status":"error"}`, mimirquerier.NewMaxQueryLengthError((744*time.Hour)+(6*time.Minute), 720*time.Hour)),
+			expJSON:       fmt.Sprintf(`{"error":"expanding series: %s", "errorType":"execution", "status":"error"}`, mimirquerier.NewMaxQueryLengthError((744*time.Hour)+(6*time.Minute)-time.Millisecond, 720*time.Hour)),
 		},
 		{
 			name: "query remote read time range exceeds the limit",

integration/ruler_test.go

@@ -749,10 +749,10 @@ func TestRulerMetricsForInvalidQueriesAndNoFetchedSeries(t *testing.T) {
 		zeroSeriesQueries = getZeroSeriesQueriesTotal()
 
 		const groupName4 = "good_rule_with_fetched_series_and_samples_and_non_series_selector"
-		const expression4 = `sum(metric{foo=~"1|2"}) + vector(1.2345)`
+		const expression4 = `sum(metric{foo=~"1|2"}) * vector(1.2345)`
 		addNewRuleAndWait(groupName4, expression4, false)
 
-		// Ensure that samples were not returned.
+		// Ensure that samples were returned.
 		require.Less(t, 0, getLastEvalSamples())
 
 		// Ensure that the metric for no fetched series was not incremented.

pkg/api/handlers.go

@@ -38,6 +38,7 @@ import (
 	"github.com/grafana/mimir/pkg/usagestats"
 	"github.com/grafana/mimir/pkg/util"
 	"github.com/grafana/mimir/pkg/util/chunkinfologger"
+	util_log "github.com/grafana/mimir/pkg/util/log"
 	"github.com/grafana/mimir/pkg/util/validation"
 )
 
@@ -268,13 +269,15 @@ func NewQuerierHandler(
 		nil,   // Only needed for admin APIs.
 		"",    // This is for snapshots, which is disabled when admin APIs are disabled. Hence empty.
 		false, // Disable admin APIs.
-		logger,
+		util_log.SlogFromGoKit(logger),
 		func(context.Context) v1.RulesRetriever { return &querier.DummyRulesRetriever{} },
 		0, 0, 0, // Remote read samples and concurrency limit.
 		false, // Not an agent.
 		regexp.MustCompile(".*"),
 		func() (v1.RuntimeInfo, error) { return v1.RuntimeInfo{}, errors.New("not implemented") },
 		&v1.PrometheusVersion{},
+		nil,
+		nil,
 		// This is used for the stats API which we should not support. Or find other ways to.
 		prometheus.GathererFunc(func() ([]*dto.MetricFamily, error) { return nil, nil }),
 		reg,

pkg/blockbuilder/blockbuilder_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/grafana/dskit/services"
 	"github.com/prometheus/client_golang/prometheus"
 	promtest "github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/prometheus/common/promslog"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/stretchr/testify/assert"
@@ -164,7 +165,7 @@ func TestBlockBuilder_StartWithExistingCommit(t *testing.T) {
 	expSamples := producedSamples[1+(len(producedSamples)/2):]
 
 	bucketDir := path.Join(cfg.BlocksStorage.Bucket.Filesystem.Directory, "1")
-	db, err := tsdb.Open(bucketDir, log.NewNopLogger(), nil, nil, nil)
+	db, err := tsdb.Open(bucketDir, promslog.NewNopLogger(), nil, nil, nil)
 	require.NoError(t, err)
 	t.Cleanup(func() { require.NoError(t, db.Close()) })
 	compareQuery(t,
@@ -291,7 +292,7 @@ func TestBlockBuilder_ReachHighWatermarkBeforeLastCycleSection(t *testing.T) {
 	`), "cortex_blockbuilder_consumer_lag_records"))
 
 	bucketDir := path.Join(cfg.BlocksStorage.Bucket.Filesystem.Directory, "1")
-	db, err := tsdb.Open(bucketDir, log.NewNopLogger(), nil, nil, nil)
+	db, err := tsdb.Open(bucketDir, promslog.NewNopLogger(), nil, nil, nil)
 	require.NoError(t, err)
 	t.Cleanup(func() { require.NoError(t, db.Close()) })
 	compareQuery(t,
@@ -351,7 +352,7 @@ func TestBlockBuilder_WithMultipleTenants(t *testing.T) {
 
 	for _, tenant := range tenants {
 		bucketDir := path.Join(cfg.BlocksStorage.Bucket.Filesystem.Directory, tenant)
-		db, err := tsdb.Open(bucketDir, log.NewNopLogger(), nil, nil, nil)
+		db, err := tsdb.Open(bucketDir, promslog.NewNopLogger(), nil, nil, nil)
 		require.NoError(t, err)
 		t.Cleanup(func() { require.NoError(t, db.Close()) })
 
@@ -446,7 +447,7 @@ func TestBlockBuilder_WithNonMonotonicRecordTimestamps(t *testing.T) {
 			require.NoError(t, bb.nextConsumeCycle(ctx, end))
 
 			bucketDir := path.Join(cfg.BlocksStorage.Bucket.Filesystem.Directory, tenantID)
-			db, err := tsdb.Open(bucketDir, log.NewNopLogger(), nil, nil, nil)
+			db, err := tsdb.Open(bucketDir, promslog.NewNopLogger(), nil, nil, nil)
 			require.NoError(t, err)
 			t.Cleanup(func() { require.NoError(t, db.Close()) })
 
@@ -517,7 +518,7 @@ func TestBlockBuilder_RetryOnTransientErrors(t *testing.T) {
 	require.Eventually(t, func() bool { return kafkaCommits.Load() >= 1 }, 50*time.Second, 100*time.Millisecond, "expected kafka commits")
 
 	bucketDir := path.Join(cfg.BlocksStorage.Bucket.Filesystem.Directory, "1")
-	db, err := tsdb.Open(bucketDir, log.NewNopLogger(), nil, nil, nil)
+	db, err := tsdb.Open(bucketDir, promslog.NewNopLogger(), nil, nil, nil)
 	require.NoError(t, err)
 	t.Cleanup(func() { require.NoError(t, db.Close()) })
 	compareQuery(t,

pkg/blockbuilder/tsdb.go

@@ -256,7 +256,7 @@ func (b *TSDBBuilder) newTSDB(tenant tsdbTenant) (*userTSDB, error) {
 	userID := tenant.tenantID
 	userLogger := util_log.WithUserID(userID, b.logger)
 
-	db, err := tsdb.Open(udir, userLogger, nil, &tsdb.Options{
+	db, err := tsdb.Open(udir, util_log.SlogFromGoKit(userLogger), nil, &tsdb.Options{
 		RetentionDuration:           0,
 		MinBlockDuration:            2 * time.Hour.Milliseconds(),
 		MaxBlockDuration:            2 * time.Hour.Milliseconds(),

pkg/blockbuilder/tsdb_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/grafana/dskit/flagext"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
@@ -265,7 +266,7 @@ func TestTSDBBuilder(t *testing.T) {
 			require.NoError(t, err)
 			require.Nil(t, builder.tsdbs[tenant])
 
-			newDB, err := tsdb.Open(shipperDir, log.NewNopLogger(), nil, nil, nil)
+			newDB, err := tsdb.Open(shipperDir, promslog.NewNopLogger(), nil, nil, nil)
 			require.NoError(t, err)
 
 			// One for the in-order current range. Two for the out-of-order blocks: one for the current range

pkg/compactor/bucket_compactor_e2e_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	promtest "github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/prometheus/common/promslog"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/index"
@@ -37,6 +38,7 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/grafana/mimir/pkg/storage/tsdb/block"
+	util_log "github.com/grafana/mimir/pkg/util/log"
 )
 
 func TestSyncer_GarbageCollect_e2e(t *testing.T) {
@@ -232,7 +234,7 @@ func TestGroupCompactE2E(t *testing.T) {
 		sy, err := newMetaSyncer(nil, nil, bkt, metaFetcher, duplicateBlocksFilter, blocksMarkedForDeletion)
 		require.NoError(t, err)
 
-		comp, err := tsdb.NewLeveledCompactor(ctx, reg, logger, []int64{1000, 3000}, nil, nil)
+		comp, err := tsdb.NewLeveledCompactor(ctx, reg, util_log.SlogFromGoKit(logger), []int64{1000, 3000}, nil, nil)
 		require.NoError(t, err)
 
 		planner := NewSplitAndMergePlanner([]int64{1000, 3000})
@@ -690,7 +692,7 @@ func createBlockWithOptions(
 	if err := g.Wait(); err != nil {
 		return id, err
 	}
-	c, err := tsdb.NewLeveledCompactor(ctx, nil, log.NewNopLogger(), []int64{maxt - mint}, nil, nil)
+	c, err := tsdb.NewLeveledCompactor(ctx, nil, promslog.NewNopLogger(), []int64{maxt - mint}, nil, nil)
 	if err != nil {
 		return id, errors.Wrap(err, "create compactor")
 	}

pkg/compactor/split_merge_compactor.go

@@ -8,6 +8,8 @@ import (
 	"github.com/go-kit/log"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/prometheus/tsdb"
+
+	util_log "github.com/grafana/mimir/pkg/util/log"
 )
 
 func splitAndMergeGrouperFactory(_ context.Context, cfg Config, cfgProvider ConfigProvider, userID string, logger log.Logger, _ prometheus.Registerer) Grouper {
@@ -21,7 +23,7 @@ func splitAndMergeGrouperFactory(_ context.Context, cfg Config, cfgProvider Conf
 
 func splitAndMergeCompactorFactory(ctx context.Context, cfg Config, logger log.Logger, reg prometheus.Registerer) (Compactor, Planner, error) {
 	// We don't need to customise the TSDB compactor so we're just using the Prometheus one.
-	compactor, err := tsdb.NewLeveledCompactor(ctx, reg, logger, cfg.BlockRanges.ToMilliseconds(), nil, nil)
+	compactor, err := tsdb.NewLeveledCompactor(ctx, reg, util_log.SlogFromGoKit(logger), cfg.BlockRanges.ToMilliseconds(), nil, nil)
 	if err != nil {
 		return nil, nil, err
 	}

pkg/compactor/split_merge_compactor_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/grafana/mimir/pkg/storage/sharding"
 	mimir_tsdb "github.com/grafana/mimir/pkg/storage/tsdb"
 	"github.com/grafana/mimir/pkg/storage/tsdb/block"
+	util_log "github.com/grafana/mimir/pkg/util/log"
 	util_test "github.com/grafana/mimir/pkg/util/test"
 )
 
@@ -774,7 +775,7 @@ func TestMultitenantCompactor_ShouldGuaranteeSeriesShardingConsistencyOverTheTim
 	for _, actualMeta := range actualMetas {
 		expectedSeriesIDs := expectedSeriesIDByShard[actualMeta.Thanos.Labels[mimir_tsdb.CompactorShardIDExternalLabel]]
 
-		b, err := tsdb.OpenBlock(logger, filepath.Join(storageDir, userID, actualMeta.ULID.String()), nil)
+		b, err := tsdb.OpenBlock(util_log.SlogFromGoKit(logger), filepath.Join(storageDir, userID, actualMeta.ULID.String()), nil)
 		require.NoError(t, err)
 
 		indexReader, err := b.Index()

pkg/distributor/otel.go

@@ -415,7 +415,7 @@ func otelMetricsToTimeseries(ctx context.Context, tenantID string, addSuffixes,
 		AddMetricSuffixes:                   addSuffixes,
 		EnableCreatedTimestampZeroIngestion: enableCTZeroIngestion,
 		PromoteResourceAttributes:           promoteResourceAttributes,
-	}, logger)
+	}, utillog.SlogFromGoKit(logger))
 	mimirTS := converter.TimeSeries()
 	if errs != nil {
 		dropped := len(multierr.Errors(errs))