Propagate metadata from the WAL to remote_write

Squashed commit of the following: commit 5fc3ac0 Author: Nicolás Pazos <[email protected]> Date: Thu Dec 21 15:41:01 2023 -0300 fix build and small bugfix Signed-off-by: Nicolás Pazos <[email protected]> commit 7dff2c9 Merge: 10b43bc baebe1c Author: Nicolás Pazos <[email protected]> Date: Thu Dec 21 15:30:44 2023 -0300 Merge branch 'alexnico-remote-write-1-1' into njpm/metadata-remote-write-wiring-2 Signed-off-by: Nicolás Pazos <[email protected]> commit 10b43bc Merge: bc87310 175bd21 Author: Nicolás Pazos <[email protected]> Date: Thu Dec 21 15:27:36 2023 -0300 Merge branch 'alexnico-remote-write-1-1' into njpm/metadata-remote-write-wiring-2 Signed-off-by: Nicolás Pazos <[email protected]> commit bc87310 Author: Paschalis Tsilias <[email protected]> Date: Mon Jun 19 11:51:03 2023 +0300 Fix test helper; remove unnecessary conversion Signed-off-by: Paschalis Tsilias <[email protected]> commit 04a08ef Author: Paschalis Tsilias <[email protected]> Date: Mon Jun 19 11:45:48 2023 +0300 Re-introduce the StoreMetadata method; make metadata to always be sent alongside any sample Signed-off-by: Paschalis Tsilias <[email protected]> commit 208922b Author: Paschalis Tsilias <[email protected]> Date: Fri Apr 28 14:00:17 2023 +0300 Mutate the correct field when choosing metadata mechanism Signed-off-by: Paschalis Tsilias <[email protected]> commit a5c1def Author: Paschalis Tsilias <[email protected]> Date: Fri Apr 28 13:57:56 2023 +0300 Move logic for implicitly choosing one of the two mechanism into NewQueueManager Signed-off-by: Paschalis Tsilias <[email protected]> commit 6f05973 Author: Paschalis Tsilias <[email protected]> Date: Fri Apr 28 12:05:37 2023 +0300 Rename old interface method to AppendWatcherMetadata; new to AppendMetadata Signed-off-by: Paschalis Tsilias <[email protected]> commit 42fc288 Author: Paschalis Tsilias <[email protected]> Date: Fri Apr 28 11:55:34 2023 +0300 Add mechanism for choosing between the two metadata-sending methods, defaulting to the new one Signed-off-by: Paschalis Tsilias <[email protected]> commit 7ee3ae6 Author: Paschalis Tsilias <[email protected]> Date: Fri Apr 28 10:48:22 2023 +0300 Remove repeated keyword from TimeSeries' metadata field Signed-off-by: Paschalis Tsilias <[email protected]> commit 3acfa1b Author: Paschalis Tsilias <[email protected]> Date: Fri Apr 28 10:23:21 2023 +0300 Revert "Change logic around AppendWALMetadata; introduce StoreMetadata similar to StoreSeries" This reverts commit bd41c24. commit bd41c24 Author: Paschalis Tsilias <[email protected]> Date: Thu Apr 27 18:32:53 2023 +0300 Change logic around AppendWALMetadata; introduce StoreMetadata similar to StoreSeries Signed-off-by: Paschalis Tsilias <[email protected]> commit 3f967e1 Author: Paschalis Tsilias <[email protected]> Date: Mon Apr 24 19:51:42 2023 +0300 Use labels.FromStrings helper to circumvent linter error Signed-off-by: Paschalis Tsilias <[email protected]> commit 5227af2 Author: Paschalis Tsilias <[email protected]> Date: Mon Apr 24 19:13:39 2023 +0300 Add a few more tests; remove unused helpers Signed-off-by: Paschalis Tsilias <[email protected]> commit 32cf14c Author: Paschalis Tsilias <[email protected]> Date: Mon Apr 24 18:50:08 2023 +0300 Yet another linter fix Signed-off-by: Paschalis Tsilias <[email protected]> commit 3cc7a7f Author: Paschalis Tsilias <[email protected]> Date: Mon Apr 24 18:22:39 2023 +0300 Fix sendMetadata field usage Signed-off-by: Paschalis Tsilias <[email protected]> commit 514b6a3 Author: Paschalis Tsilias <[email protected]> Date: Mon Apr 24 18:10:01 2023 +0300 Fix linter error Signed-off-by: Paschalis Tsilias <[email protected]> commit 403385f Author: Paschalis Tsilias <[email protected]> Date: Mon Apr 24 17:11:49 2023 +0300 Fix linting errors Signed-off-by: Paschalis Tsilias <[email protected]> commit 0093563 Author: Paschalis Tsilias <[email protected]> Date: Mon Apr 24 16:43:38 2023 +0300 Alternative approach: bundle metadata in TimeSeries protobuf Signed-off-by: Paschalis Tsilias <[email protected]> commit 49cccc6 Author: Paschalis Tsilias <[email protected]> Date: Mon Nov 28 17:24:06 2022 +0200 Add first test; rebase with main Signed-off-by: Paschalis Tsilias <[email protected]> commit ac83b3a Author: Paschalis Tsilias <[email protected]> Date: Mon Nov 28 15:02:52 2022 +0200 Approach bundling metadata along with samples and exemplars Signed-off-by: Paschalis Tsilias <[email protected]> Co-authored-by: Paschalis Tsilias <[email protected]> Signed-off-by: Nicolás Pazos <[email protected]>
grafana · Dec 21, 2023 · 6be4cc9 · 6be4cc9
1 parent baebe1c
commit 6be4cc9
Show file tree

Hide file tree

Showing 16 changed files with 255 additions and 51 deletions.
diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
@@ -187,6 +187,9 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
 			case "extra-scrape-metrics":
 				c.scrape.ExtraMetrics = true
 				level.Info(logger).Log("msg", "Experimental additional scrape metrics enabled")
+			case "metadata-storage":
+				c.scrape.EnableMetadataStorage = true
+				level.Info(logger).Log("msg", "Experimental in-memory metadata storage enabled")
 			case "new-service-discovery-manager":
 				c.enableNewSDManager = true
 				level.Info(logger).Log("msg", "Experimental service discovery manager")
@@ -431,7 +434,7 @@ func main() {
 	a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates.").
 		Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval)
 
-	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
+	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver, metadata-storage. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
 		Default("").StringsVar(&cfg.featureList)
 
 	a.Flag("remote-write-format", "remote write proto format to use, valid options: 0 (1.0), 1 (reduced format), 3 (min64 format)").
@@ -521,6 +524,12 @@ func main() {
 		cfg.tsdb.OutOfOrderTimeWindow = cfgFile.StorageConfig.TSDBConfig.OutOfOrderTimeWindow
 	}
 
+	for _, rwc := range cfgFile.RemoteWriteConfigs {
+		if rwc.SendWALMetadata && !cfg.scrape.EnableMetadataStorage {
+			level.Warn(logger).Log("msg", "the 'send_metadata' remote_write parameter must be set along the --enable-features=metadata-storage flag to take effect")
+		}
+	}
+
 	// Now that the validity of the config is established, set the config
 	// success metrics accordingly, although the config isn't really loaded
 	// yet. This will happen later (including setting these metrics again),

diff --git a/config/config.go b/config/config.go
@@ -1022,6 +1022,7 @@ type RemoteWriteConfig struct {
 	Name                 string            `yaml:"name,omitempty"`
 	SendExemplars        bool              `yaml:"send_exemplars,omitempty"`
 	SendNativeHistograms bool              `yaml:"send_native_histograms,omitempty"`
+	SendWALMetadata      bool              `yaml:"send_metadata,omitempty"` // TODO(@tpaschalis) Adding an extra field to enable us to remove the `metadata_config` struct in the future.
 
 	// We cannot do proper Go type embedding below as the parser will then parse
 	// values arbitrarily into the overflow maps of further-down types.

diff --git a/docs/command-line/prometheus.md b/docs/command-line/prometheus.md
@@ -52,7 +52,7 @@ The Prometheus monitoring server
 | <code class="text-nowrap">--query.timeout</code> | Maximum time a query may take before being aborted. Use with server mode only. | `2m` |
 | <code class="text-nowrap">--query.max-concurrency</code> | Maximum number of queries executed concurrently. Use with server mode only. | `20` |
 | <code class="text-nowrap">--query.max-samples</code> | Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return. Use with server mode only. | `50000000` |
-| <code class="text-nowrap">--enable-feature</code> | Comma separated feature names to enable. Valid options: agent, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. |  |
+| <code class="text-nowrap">--enable-feature</code> | Comma separated feature names to enable. Valid options: agent, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver, metadata-storage. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. |  |
 | <code class="text-nowrap">--remote-write-format</code> | remote write proto format to use, valid options: 0 (1.0), 1 (reduced format), 3 (min64 format) | `0` |
 | <code class="text-nowrap">--log.level</code> | Only log messages with the given severity or above. One of: [debug, info, warn, error] | `info` |
 | <code class="text-nowrap">--log.format</code> | Output format of log messages. One of: [logfmt, json] | `logfmt` |

diff --git a/docs/feature_flags.md b/docs/feature_flags.md
@@ -185,7 +185,7 @@ This should **only** be applied to metrics that currently produce such labels.
 `--enable-feature=otlp-write-receiver`
 
 The OTLP receiver allows Prometheus to accept [OpenTelemetry](https://opentelemetry.io/) metrics writes.
-Prometheus is best used as a Pull based system, and staleness, `up` metric, and other Pull enabled features 
+Prometheus is best used as a Pull based system, and staleness, `up` metric, and other Pull enabled features
 won't work when you push OTLP metrics.
 
 ## Experimental PromQL functions
@@ -204,3 +204,12 @@ Enables ingestion of created timestamp. Created timestamps are injected as 0 val
 Currently Prometheus supports created timestamps only on the traditional Prometheus Protobuf protocol (WIP for other protocols). As a result, when enabling this feature, the Prometheus protobuf scrape protocol will be prioritized (See `scrape_config.scrape_protocols` settings for more details).
 
 Besides enabling this feature in Prometheus, created timestamps need to be exposed by the application being scraped.
+
+## Metadata Storage
+`--enable-features=metadata-storage`
+
+When enabled, Prometheus will store metadata in-memory and keep track of
+metadata changes as WAL records. This should be used along the remote write's
+`send_metadata` parameter.
+This new way of storing and sending metadata is mutually exclusive with the
+`metadata_config.send` field and the current way of sending metadata.
diff --git a/prompb/types.proto b/prompb/types.proto
@@ -38,7 +38,6 @@ message MetricMetadata {
   string unit = 5;
 }
 
-
 message Sample {
   double value    = 1;
   // timestamp is in ms format, see model/timestamp/timestamp.go for

diff --git a/storage/remote/codec.go b/storage/remote/codec.go
@@ -33,6 +33,7 @@ import (
 	"github.com/prometheus/prometheus/model/exemplar"
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/model/metadata"
 	"github.com/prometheus/prometheus/model/textparse"
 	"github.com/prometheus/prometheus/prompb"
 	writev2 "github.com/prometheus/prometheus/prompb/write/v2"
@@ -629,6 +630,13 @@ func exemplarProtoToExemplar(ep prompb.Exemplar) exemplar.Exemplar {
 	}
 }
 
+func minMetadataProtoToMetadata(mp writev2.Metadata, symbols []string) metadata.Metadata {
+	return metadata.Metadata{
+		Type: metricTypeFromProtoEquivalent(mp.Type),
+		Unit: symbols[mp.UnitRef], // TODO: check for overflow
+		Help: symbols[mp.HelpRef], // TODO: check for overflow
+	}
+}
 func minExemplarProtoToExemplar(ep writev2.Exemplar, symbols []string) exemplar.Exemplar {
 	timestamp := ep.Timestamp
 
@@ -925,6 +933,21 @@ func metricTypeToMetricTypeProto(t textparse.MetricType) prompb.MetricMetadata_M
 	return prompb.MetricMetadata_MetricType(v)
 }
 
+func metricTypeToProtoEquivalent(t textparse.MetricType) writev2.Metadata_MetricType {
+	mt := strings.ToUpper(string(t))
+	v, ok := writev2.Metadata_MetricType_value[mt]
+	if !ok {
+		return writev2.Metadata_UNKNOWN
+	}
+
+	return writev2.Metadata_MetricType(v)
+}
+
+func metricTypeFromProtoEquivalent(t writev2.Metadata_MetricType) textparse.MetricType {
+	mt := strings.ToLower(t.String())
+	return textparse.MetricType(mt) // TODO(@tpaschalis) a better way for this?
+}
+
 // DecodeWriteRequest from an io.Reader into a prompb.WriteRequest, handling
 // snappy decompression.
 func DecodeWriteRequest(r io.Reader) (*prompb.WriteRequest, error) {

diff --git a/storage/remote/codec_test.go b/storage/remote/codec_test.go
@@ -92,6 +92,10 @@ var writeRequestMinimizedFixture = func() *writev2.WriteRequest {
 	for _, s := range []string{
 		"f", "g", // 10, 11
 		"h", "i", // 12, 13
+		"help text 1", //14
+		"unit text 1", //15
+		"help text 2", //16
+		"unit text 2", //17
 	} {
 		st.RefStr(s)
 	}
@@ -103,12 +107,14 @@ var writeRequestMinimizedFixture = func() *writev2.WriteRequest {
 				Samples:    []writev2.Sample{{Value: 1, Timestamp: 0}},
 				Exemplars:  []writev2.Exemplar{{LabelsRefs: []uint32{10, 11}, Value: 1, Timestamp: 0}},
 				Histograms: []writev2.Histogram{HistogramToMinHistogramProto(0, &testHistogram), FloatHistogramToMinHistogramProto(1, testHistogram.ToFloat(nil))},
+				Metadata:   writev2.Metadata{Type: writev2.Metadata_COUNTER, HelpRef: 14, UnitRef: 15},
 			},
 			{
 				LabelsRefs: labels,
 				Samples:    []writev2.Sample{{Value: 2, Timestamp: 1}},
 				Exemplars:  []writev2.Exemplar{{LabelsRefs: []uint32{12, 13}, Value: 2, Timestamp: 1}},
 				Histograms: []writev2.Histogram{HistogramToMinHistogramProto(2, &testHistogram), FloatHistogramToMinHistogramProto(3, testHistogram.ToFloat(nil))},
+				Metadata:   writev2.Metadata{Type: writev2.Metadata_GAUGE, HelpRef: 16, UnitRef: 17},
 			},
 		},
 		Symbols: st.LabelsStrings(),

diff --git a/storage/remote/metadata_watcher.go b/storage/remote/metadata_watcher.go
@@ -27,7 +27,7 @@ import (
 
 // MetadataAppender is an interface used by the Metadata Watcher to send metadata, It is read from the scrape manager, on to somewhere else.
 type MetadataAppender interface {
-	AppendMetadata(context.Context, []scrape.MetricMetadata)
+	AppendWatcherMetadata(context.Context, []scrape.MetricMetadata)
 }
 
 // Watchable represents from where we fetch active targets for metadata.
@@ -146,7 +146,7 @@ func (mw *MetadataWatcher) collect() {
 	}
 
 	// Blocks until the metadata is sent to the remote write endpoint or hardShutdownContext is expired.
-	mw.writer.AppendMetadata(mw.hardShutdownCtx, metadata)
+	mw.writer.AppendWatcherMetadata(mw.hardShutdownCtx, metadata)
 }
 
 func (mw *MetadataWatcher) ready() bool {

diff --git a/storage/remote/metadata_watcher_test.go b/storage/remote/metadata_watcher_test.go
@@ -58,7 +58,7 @@ type writeMetadataToMock struct {
 	metadataAppended int
 }
 
-func (mwtm *writeMetadataToMock) AppendMetadata(_ context.Context, m []scrape.MetricMetadata) {
+func (mwtm *writeMetadataToMock) AppendWatcherMetadata(_ context.Context, m []scrape.MetricMetadata) {
 	mwtm.metadataAppended += len(m)
 }