Merge branch 'main' into fix-broken-link-index

grafana · Oct 16, 2023 · b2125d1 · b2125d1
2 parents ddb4a7d + a871c34
commit b2125d1
Show file tree

Hide file tree

Showing 29 changed files with 193 additions and 105 deletions.
diff --git a/.drone/drone.yml b/.drone/drone.yml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -29,16 +29,33 @@ Main (unreleased)
 
 - Fixed an issue where `loki.process` validation for stage `metric.counter` was 
   allowing invalid combination of configuration options. (@thampiotr)
+
+### Enhancements
+
+- The `loki.write` WAL now has snappy compression enabled by default. (@thepalbi)
+
+- Allow converting labels to structured metadata with Loki's structured_metadata stage. (@gonzalesraul)
+
+v0.37.2 (2023-10-16)
+-----------------
+
+### Bugfixes
 
 - Fix the handling of the `--cluster.join-addresses` flag causing an invalid
   comparison with the mutually-exclusive `--cluster.discover-peers`. (@tpaschalis)
 
 - Fix an issue with the static to flow converter for blackbox exporter modules
   config not being included in the river output. (@erikbaranowski)
-  
+
 ### Enhancements
 
-- The `loki.write` WAL now has snappy compression enabled by default. (@thepalbi)
+- Update Prometheus dependency to v2.47.2. (@tpaschalis)
+
+- Allow Out of Order writing to the WAL for metrics. (@mattdurham)
+
+### Other changes
+
+- Use Go 1.21.3 for builds. (@tpaschalis)
 
 v0.37.1 (2023-10-10)
 -----------------

diff --git a/cmd/grafana-agent-operator/Dockerfile b/cmd/grafana-agent-operator/Dockerfile
@@ -4,7 +4,7 @@
 # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set
 # in environment variables.
 
-FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.2 as build
+FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.3 as build
 ARG BUILDPLATFORM
 ARG TARGETPLATFORM
 ARG TARGETOS

diff --git a/cmd/grafana-agent/Dockerfile b/cmd/grafana-agent/Dockerfile
@@ -4,7 +4,7 @@
 # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set
 # in environment variables.
 
-FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.2 as build
+FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.3 as build
 ARG BUILDPLATFORM
 ARG TARGETPLATFORM
 ARG TARGETOS

diff --git a/cmd/grafana-agent/Dockerfile.windows b/cmd/grafana-agent/Dockerfile.windows
@@ -1,4 +1,4 @@
-FROM grafana/agent-build-image:0.30.2-windows as builder
+FROM grafana/agent-build-image:0.30.3-windows as builder
 ARG VERSION
 ARG RELEASE_BUILD=1
 

diff --git a/cmd/grafana-agentctl/Dockerfile b/cmd/grafana-agentctl/Dockerfile
@@ -4,7 +4,7 @@
 # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set
 # in environment variables.
 
-FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.2 as build
+FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.3 as build
 ARG BUILDPLATFORM
 ARG TARGETPLATFORM
 ARG TARGETOS

diff --git a/cmd/grafana-agentctl/Dockerfile.windows b/cmd/grafana-agentctl/Dockerfile.windows
@@ -1,4 +1,4 @@
-FROM grafana/agent-build-image:0.30.2-windows as builder
+FROM grafana/agent-build-image:0.30.3-windows as builder
 ARG VERSION
 ARG RELEASE_BUILD=1
 

diff --git a/component/loki/process/stages/structured_metadata.go b/component/loki/process/stages/structured_metadata.go
@@ -32,6 +32,26 @@ func (s *structuredMetadataStage) Run(in chan Entry) chan Entry {
 		processLabelsConfigs(s.logger, e.Extracted, s.cfgs, func(labelName model.LabelName, labelValue model.LabelValue) {
 			e.StructuredMetadata = append(e.StructuredMetadata, logproto.LabelAdapter{Name: string(labelName), Value: string(labelValue)})
 		})
-		return e
+		return s.extractFromLabels(e)
 	})
 }
+
+func (s *structuredMetadataStage) extractFromLabels(e Entry) Entry {
+	labels := e.Labels
+	foundLabels := []model.LabelName{}
+
+	for lName, lSrc := range s.cfgs.Values {
+		labelKey := model.LabelName(*lSrc)
+		if lValue, ok := labels[labelKey]; ok {
+			e.StructuredMetadata = append(e.StructuredMetadata, logproto.LabelAdapter{Name: lName, Value: string(lValue)})
+			foundLabels = append(foundLabels, labelKey)
+		}
+	}
+
+	// Remove found labels, do this after append to structure metadata
+	for _, fl := range foundLabels {
+		delete(labels, fl)
+	}
+	e.Labels = labels
+	return e
+}
diff --git a/component/loki/process/stages/structured_metadata_test.go b/component/loki/process/stages/structured_metadata_test.go
@@ -71,6 +71,26 @@ stage.labels {
 }
 `
 
+var pipelineStagesStructuredMetadataFromStaticLabels = `
+stage.static_labels {
+	values = {"component" = "querier", "pod" = "loki-querier-664f97db8d-qhnwg"}
+}
+
+stage.structured_metadata {
+	values = {"pod" = ""}
+}
+`
+
+var pipelineStagesStructuredMetadataFromStaticLabelsDifferentKey = `
+stage.static_labels {
+	values = {"component" = "querier", "pod" = "loki-querier-664f97db8d-qhnwg"}
+}
+
+stage.structured_metadata {
+	values = {"pod_name" = "pod"}
+}
+`
+
 func Test_StructuredMetadataStage(t *testing.T) {
 	tests := map[string]struct {
 		pipelineStagesYaml         string
@@ -104,6 +124,18 @@ func Test_StructuredMetadataStage(t *testing.T) {
 			expectedStructuredMetadata: push.LabelsAdapter{push.LabelAdapter{Name: "app", Value: "loki"}},
 			expectedLabels:             model.LabelSet{model.LabelName("component"): model.LabelValue("ingester")},
 		},
+		"expected structured metadata and regular labels to be extracted with static labels stage and to be added to entry": {
+			pipelineStagesYaml:         pipelineStagesStructuredMetadataFromStaticLabels,
+			logLine:                    `sample log line`,
+			expectedStructuredMetadata: push.LabelsAdapter{push.LabelAdapter{Name: "pod", Value: "loki-querier-664f97db8d-qhnwg"}},
+			expectedLabels:             model.LabelSet{model.LabelName("component"): model.LabelValue("querier")},
+		},
+		"expected structured metadata and regular labels to be extracted with static labels stage using different structured key": {
+			pipelineStagesYaml:         pipelineStagesStructuredMetadataFromStaticLabelsDifferentKey,
+			logLine:                    `sample log line`,
+			expectedStructuredMetadata: push.LabelsAdapter{push.LabelAdapter{Name: "pod_name", Value: "loki-querier-664f97db8d-qhnwg"}},
+			expectedLabels:             model.LabelSet{model.LabelName("component"): model.LabelValue("querier")},
+		},
 	}
 	for name, test := range tests {
 		t.Run(name, func(t *testing.T) {

diff --git a/docs/sources/_index.md b/docs/sources/_index.md
@@ -8,7 +8,7 @@ title: Grafana Agent
 description: Grafana Agent is a flexible, performant, vendor-neutral, telemetry collector
 weight: 350
 cascade:
-  AGENT_RELEASE: v0.37.1
+  AGENT_RELEASE: v0.37.2
 ---
 
 # Grafana Agent

diff --git a/go.mod b/go.mod
@@ -675,8 +675,8 @@ replace (
 // * There is a release of Prometheus which contains
 // prometheus/prometheus#12677 and prometheus/prometheus#12729.
 // We use the last v1-related tag as the replace statement does not work for v2
-// tags without the v2 suffix to the module root
-replace github.com/prometheus/prometheus => github.com/grafana/prometheus v1.8.2-0.20231003113207-17e15326a784 // grafana:prometheus:v0.46.0-retry-improvements
+// tags without the v2 suffix to the module root.
+replace github.com/prometheus/prometheus => github.com/grafana/prometheus v1.8.2-0.20231016083943-46550094220d // grafana:prometheus:v0.47.2-retry-improvements
 
 replace gopkg.in/yaml.v2 => github.com/rfratto/go-yaml v0.0.0-20211119180816-77389c3526dc
 

diff --git a/go.sum b/go.sum
@@ -1071,8 +1071,8 @@ github.com/grafana/perflib_exporter v0.1.1-0.20230511173423-6166026bd090 h1:Ko80
 github.com/grafana/perflib_exporter v0.1.1-0.20230511173423-6166026bd090/go.mod h1:MinSWm88jguXFFrGsP56PtleUb4Qtm4tNRH/wXNXRTI=
 github.com/grafana/postgres_exporter v0.8.1-0.20210722175051-db35d7c2f520 h1:HnFWqxhoSF3WC7sKAdMZ+SRXvHLVZlZ3sbQjuUlTqkw=
 github.com/grafana/postgres_exporter v0.8.1-0.20210722175051-db35d7c2f520/go.mod h1:+HPXgiOV0InDHcZ2jNijL1SOKvo0eEPege5fQA0+ICI=
-github.com/grafana/prometheus v1.8.2-0.20231003113207-17e15326a784 h1:B8AAFKq7WQPUYdoGwWjgxFARn5XodYlKJNHCYUopah8=
-github.com/grafana/prometheus v1.8.2-0.20231003113207-17e15326a784/go.mod h1:10L5IJE5CEsjee1FnOcVswYXlPIscDWWt3IJ2UDYrz4=
+github.com/grafana/prometheus v1.8.2-0.20231016083943-46550094220d h1:hr0QEXSfpdakWdHw2sZeT/5GnGwIkHnNO0YBkfRj5zk=
+github.com/grafana/prometheus v1.8.2-0.20231016083943-46550094220d/go.mod h1:J/bmOSjgH7lFxz2gZhrWEZs2i64vMS+HIuZfmYNhJ/M=
 github.com/grafana/pyroscope-go/godeltaprof v0.1.3 h1:eunWpv1B3Z7ZK9o4499EmQGlY+CsDmSZ4FbxjRx37uk=
 github.com/grafana/pyroscope-go/godeltaprof v0.1.3/go.mod h1:1HSPtjU8vLG0jE9JrTdzjgFqdJ/VgN7fvxBNq3luJko=
 github.com/grafana/pyroscope/api v0.2.0 h1:TzOxL0s6SiaLEy944ZAKgHcx/JDRJXu4O8ObwkqR6p4=

diff --git a/pkg/metrics/wal/wal.go b/pkg/metrics/wal/wal.go
@@ -699,11 +699,6 @@ func (a *appender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v flo
 	series.Lock()
 	defer series.Unlock()
 
-	if t < series.lastTs {
-		a.w.metrics.totalOutOfOrderSamples.Inc()
-		return 0, storage.ErrOutOfOrderSample
-	}
-
 	// NOTE(rfratto): always modify pendingSamples and sampleSeries together.
 	a.pendingSamples = append(a.pendingSamples, record.RefSample{
 		Ref: series.ref,
@@ -824,11 +819,6 @@ func (a *appender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int
 	series.Lock()
 	defer series.Unlock()
 
-	if t < series.lastTs {
-		a.w.metrics.totalOutOfOrderSamples.Inc()
-		return 0, storage.ErrOutOfOrderSample
-	}
-
 	switch {
 	case h != nil:
 		// NOTE(rfratto): always modify pendingHistograms and histogramSeries

diff --git a/pkg/metrics/wal/wal_test.go b/pkg/metrics/wal/wal_test.go
@@ -434,6 +434,35 @@ func TestGlobalReferenceID_Normal(t *testing.T) {
 	require.True(t, ref3 == 2)
 }
 
+func TestDBAllowOOOSamples(t *testing.T) {
+	walDir := t.TempDir()
+
+	s, err := NewStorage(log.NewNopLogger(), nil, walDir)
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, s.Close())
+	}()
+
+	app := s.Appender(context.Background())
+
+	// Write some samples
+	payload := buildSeries([]string{"foo", "bar", "baz"})
+	for _, metric := range payload {
+		metric.Write(t, app)
+	}
+
+	require.NoError(t, app.Commit())
+	for _, metric := range payload {
+		for _, sa := range metric.samples {
+			// We want to set the timestamp to before. This should no longer trigger an out of order.
+			sa.ts = sa.ts - 10_000
+		}
+	}
+	for _, metric := range payload {
+		metric.Write(t, app)
+	}
+}
+
 func BenchmarkAppendExemplar(b *testing.B) {
 	walDir := b.TempDir()
 

diff --git a/pkg/operator/defaults.go b/pkg/operator/defaults.go
@@ -2,7 +2,7 @@ package operator
 
 // Supported versions of the Grafana Agent.
 var (
-	DefaultAgentVersion   = "v0.37.1"
+	DefaultAgentVersion   = "v0.37.2"
 	DefaultAgentBaseImage = "grafana/agent"
 	DefaultAgentImage     = DefaultAgentBaseImage + ":" + DefaultAgentVersion
 )

diff --git a/production/kubernetes/agent-bare.yaml b/production/kubernetes/agent-bare.yaml
@@ -83,7 +83,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        image: grafana/agent:v0.37.1
+        image: grafana/agent:v0.37.2
         imagePullPolicy: IfNotPresent
         name: grafana-agent
         ports:

diff --git a/production/kubernetes/agent-loki.yaml b/production/kubernetes/agent-loki.yaml
@@ -65,7 +65,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        image: grafana/agent:v0.37.1
+        image: grafana/agent:v0.37.2
         imagePullPolicy: IfNotPresent
         name: grafana-agent-logs
         ports:

diff --git a/production/kubernetes/agent-traces.yaml b/production/kubernetes/agent-traces.yaml
@@ -114,7 +114,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        image: grafana/agent:v0.37.1
+        image: grafana/agent:v0.37.2
         imagePullPolicy: IfNotPresent
         name: grafana-agent-traces
         ports:

diff --git a/production/kubernetes/build/lib/version.libsonnet b/production/kubernetes/build/lib/version.libsonnet
@@ -1 +1 @@
-'grafana/agent:v0.37.1'
+'grafana/agent:v0.37.2'
diff --git a/production/kubernetes/build/templates/operator/main.jsonnet b/production/kubernetes/build/templates/operator/main.jsonnet
@@ -23,8 +23,8 @@ local ksm = import 'kube-state-metrics/kube-state-metrics.libsonnet';
   local this = self,
 
   _images:: {
-    agent: 'grafana/agent:v0.37.1',
-    agent_operator: 'grafana/agent-operator:v0.37.1',
+    agent: 'grafana/agent:v0.37.2',
+    agent_operator: 'grafana/agent-operator:v0.37.2',
     ksm: 'registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.5.0',
   },
 

diff --git a/production/kubernetes/install-bare.sh b/production/kubernetes/install-bare.sh
@@ -25,7 +25,7 @@ check_installed() {
 check_installed curl
 check_installed envsubst
 
-MANIFEST_BRANCH=v0.37.1
+MANIFEST_BRANCH=v0.37.2
 MANIFEST_URL=${MANIFEST_URL:-https://raw.githubusercontent.com/grafana/agent/${MANIFEST_BRANCH}/production/kubernetes/agent-bare.yaml}
 NAMESPACE=${NAMESPACE:-default}
 

diff --git a/production/operator/templates/agent-operator.yaml b/production/operator/templates/agent-operator.yaml
@@ -372,7 +372,7 @@ spec:
       containers:
       - args:
         - --kubelet-service=default/kubelet
-        image: grafana/agent-operator:v0.37.1
+        image: grafana/agent-operator:v0.37.2
         imagePullPolicy: IfNotPresent
         name: grafana-agent-operator
       serviceAccount: grafana-agent-operator
@@ -436,7 +436,7 @@ metadata:
   name: grafana-agent
   namespace: ${NAMESPACE}
 spec:
-  image: grafana/agent:v0.37.1
+  image: grafana/agent:v0.37.2
   integrations:
     selector:
       matchLabels:

diff --git a/production/tanka/grafana-agent/v1/main.libsonnet b/production/tanka/grafana-agent/v1/main.libsonnet
@@ -15,8 +15,8 @@ local service = k.core.v1.service;
 (import './lib/traces.libsonnet') +
 {
   _images:: {
-    agent: 'grafana/agent:v0.37.1',
-    agentctl: 'grafana/agentctl:v0.37.1',
+    agent: 'grafana/agent:v0.37.2',
+    agentctl: 'grafana/agentctl:v0.37.2',
   },
 
   // new creates a new DaemonSet deployment of the grafana-agent. By default,

diff --git a/production/tanka/grafana-agent/v2/internal/base.libsonnet b/production/tanka/grafana-agent/v2/internal/base.libsonnet
@@ -11,8 +11,8 @@ function(name='grafana-agent', namespace='') {
   local this = self,
 
   _images:: {
-    agent: 'grafana/agent:v0.37.1',
-    agentctl: 'grafana/agentctl:v0.37.1',
+    agent: 'grafana/agent:v0.37.2',
+    agentctl: 'grafana/agentctl:v0.37.2',
   },
   _config:: {
     name: name,

diff --git a/production/tanka/grafana-agent/v2/internal/syncer.libsonnet b/production/tanka/grafana-agent/v2/internal/syncer.libsonnet
@@ -14,7 +14,7 @@ function(
 ) {
   local _config = {
     api: error 'api must be set',
-    image: 'grafana/agentctl:v0.37.1',
+    image: 'grafana/agentctl:v0.37.2',
     schedule: '*/5 * * * *',
     configs: [],
   } + config,

diff --git a/tools/crow/Dockerfile b/tools/crow/Dockerfile
@@ -4,7 +4,7 @@
 # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set
 # in environment variables.
 
-FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.2 as build
+FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.3 as build
 ARG BUILDPLATFORM
 ARG TARGETPLATFORM
 ARG TARGETOS

diff --git a/tools/gen-versioned-files/agent-version.txt b/tools/gen-versioned-files/agent-version.txt
@@ -1 +1 @@
-v0.37.1
+v0.37.2
diff --git a/tools/make/build-container.mk b/tools/make/build-container.mk
@@ -34,7 +34,7 @@
 # variable names should be passed through to the container.
 
 USE_CONTAINER       ?= 0
-BUILD_IMAGE_VERSION ?= 0.30.2
+BUILD_IMAGE_VERSION ?= 0.30.3
 BUILD_IMAGE         ?= grafana/agent-build-image:$(BUILD_IMAGE_VERSION)
 DOCKER_OPTS         ?= -it
 

diff --git a/tools/smoke/Dockerfile b/tools/smoke/Dockerfile
@@ -4,7 +4,7 @@
 # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set
 # in environment variables.
 
-FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.2 as build
+FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.3 as build
 ARG BUILDPLATFORM
 ARG TARGETPLATFORM
 ARG TARGETOS