From 566527d64498fe04cba1d4ab31c029db40f732d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 14:28:58 +0000 Subject: [PATCH 01/12] build(deps): bump golang.org/x/crypto from 0.29.0 to 0.31.0 (#2273) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.29.0 to 0.31.0. - [Commits](https://github.com/golang/crypto/compare/v0.29.0...v0.31.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index df23ad2b3..4f1e607e2 100644 --- a/go.mod +++ b/go.mod @@ -248,13 +248,13 @@ require ( go.uber.org/goleak v1.3.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.29.0 + golang.org/x/crypto v0.31.0 golang.org/x/crypto/x509roots/fallback v0.0.0-20240208163226-62c9f1799c91 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 golang.org/x/net v0.31.0 golang.org/x/oauth2 v0.23.0 - golang.org/x/sys v0.27.0 - golang.org/x/text v0.20.0 + golang.org/x/sys v0.28.0 + golang.org/x/text v0.21.0 golang.org/x/time v0.6.0 golang.org/x/tools v0.25.0 google.golang.org/api v0.188.0 @@ -803,8 +803,8 @@ require ( go4.org/netipx v0.0.0-20230125063823-8449b0a6169f // indirect golang.org/x/arch v0.7.0 // indirect golang.org/x/mod v0.21.0 // indirect - golang.org/x/sync v0.9.0 - golang.org/x/term v0.26.0 // indirect + golang.org/x/sync v0.10.0 + golang.org/x/term v0.27.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect gonum.org/v1/gonum v0.15.1 // indirect diff --git a/go.sum b/go.sum index 9a1d317f2..f5ea529ff 100644 --- a/go.sum +++ b/go.sum @@ -3546,8 +3546,8 @@ golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto/x509roots/fallback v0.0.0-20240208163226-62c9f1799c91 h1:Lyizcy9jX02jYR0ceBkL6S+jRys8Uepf7wt1vrz6Ras= golang.org/x/crypto/x509roots/fallback v0.0.0-20240208163226-62c9f1799c91/go.mod h1:kNa9WdvYnzFwC79zRpLRMJbdEFlhyM5RPFBBZp/wWH8= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -3759,8 +3759,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= -golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -3901,8 +3901,8 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -3921,8 +3921,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= -golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -3945,8 +3945,8 @@ golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From c9d0c11d2852d4b05f8ac5727c0ba5d0246d2b00 Mon Sep 17 00:00:00 2001 From: Sam DeHaan Date: Thu, 12 Dec 2024 13:49:47 -0500 Subject: [PATCH 02/12] fix: Prevent the constant logging of cgroup errors on non-linux machines (#2264) * Prevent the constant logging of cgroup errors on non-linux machines * Update changelog * Document AUTOMEMLIMIT_EXPERIMENT * Add test --- CHANGELOG.md | 2 ++ .../reference/cli/environment-variables.md | 2 ++ go.mod | 2 +- go.sum | 4 +-- .../automemlimit_cgroups_unsupported.go | 25 ++++++++++++++++ internal/alloycli/automemlimit_linux.go | 12 ++++++++ .../alloycli/automemlimit_nonlinux_test.go | 30 +++++++++++++++++++ internal/alloycli/cmd_run.go | 6 ++-- 8 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 internal/alloycli/automemlimit_cgroups_unsupported.go create mode 100644 internal/alloycli/automemlimit_linux.go create mode 100644 internal/alloycli/automemlimit_nonlinux_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 733bc72dc..9da7c5266 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,8 @@ Main (unreleased) ### Bugfixes +- Fixed issue with automemlimit logging bad messages and trying to access cgroup on non-linux builds (@dehaansa) + - Fixed issue with reloading configuration and prometheus metrics duplication in `prometheus.write.queue`. (@mattdurham) - Updated `prometheus.write.queue` to fix issue with TTL comparing different scales of time. (@mattdurham) diff --git a/docs/sources/reference/cli/environment-variables.md b/docs/sources/reference/cli/environment-variables.md index a0633188b..c5eefcdc0 100644 --- a/docs/sources/reference/cli/environment-variables.md +++ b/docs/sources/reference/cli/environment-variables.md @@ -20,6 +20,7 @@ The following environment variables are supported: * `PPROF_BLOCK_PROFILING_RATE` * `GOMEMLIMIT` * `AUTOMEMLIMIT` +* `AUTOMEMLIMIT_EXPERIMENT` * `GOGC` * `GOMAXPROCS` * `GOTRACEBACK` @@ -80,6 +81,7 @@ For example, if you want to keep memory usage below `10GiB`, use `GOMEMLIMIT=9Gi The `GOMEMLIMIT` environment variable is either automatically set to 90% of an available `cgroup` value using the [`automemlimit`][automemlimit] module, or you can explicitly set the `GOMEMLIMIT` environment variable before you run {{< param "PRODUCT_NAME" >}}. You can also change the 90% ratio by setting the `AUTOMEMLIMIT` environment variable to a float value between `0` and `1.0`. No changes occur if the limit can't be determined and you didn't explicitly define a `GOMEMLIMIT` value. +The `AUTOMEMLIMIT_EXPERIMENT` variable can be set to `system` to use the [`automemlimit`][automemlimit] module's System provider, which sets `GOMEMLIMIT` based on the same ratio applied to the total system memory. As `cgroup` is a Linux specific concept, this is the only way to use the `automemlimit` module to automatically set `GOMEMLIMIT` on non-Linux OSes. ## GOGC diff --git a/go.mod b/go.mod index 4f1e607e2..0c662da96 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/Azure/go-autorest/autorest v0.11.29 github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/IBM/sarama v1.43.3 - github.com/KimMachineGun/automemlimit v0.6.0 + github.com/KimMachineGun/automemlimit v0.6.1 github.com/Lusitaniae/apache_exporter v0.11.1-0.20220518131644-f9522724dab4 github.com/Masterminds/sprig/v3 v3.2.3 github.com/PuerkitoBio/rehttp v1.4.0 diff --git a/go.sum b/go.sum index f5ea529ff..77ad0b47c 100644 --- a/go.sum +++ b/go.sum @@ -917,8 +917,8 @@ github.com/IBM/sarama v1.43.3/go.mod h1:FVIRaLrhK3Cla/9FfRF5X9Zua2KpS3SYIXxhac1H github.com/Jeffail/gabs v1.1.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= -github.com/KimMachineGun/automemlimit v0.6.0 h1:p/BXkH+K40Hax+PuWWPQ478hPjsp9h1CPDhLlA3Z37E= -github.com/KimMachineGun/automemlimit v0.6.0/go.mod h1:T7xYht7B8r6AG/AqFcUdc7fzd2bIdBKmepfP2S1svPY= +github.com/KimMachineGun/automemlimit v0.6.1 h1:ILa9j1onAAMadBsyyUJv5cack8Y1WT26yLj/V+ulKp8= +github.com/KimMachineGun/automemlimit v0.6.1/go.mod h1:T7xYht7B8r6AG/AqFcUdc7fzd2bIdBKmepfP2S1svPY= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Lusitaniae/apache_exporter v0.11.1-0.20220518131644-f9522724dab4 h1:UEm6tMvLH2t2honcWG0q+h0qr/60++9cuh/fy7hLyMc= github.com/Lusitaniae/apache_exporter v0.11.1-0.20220518131644-f9522724dab4/go.mod h1:IfUbHkoXypdKnVGHWQ3DLRRRZzIU/JIExQAd9xgxRfw= diff --git a/internal/alloycli/automemlimit_cgroups_unsupported.go b/internal/alloycli/automemlimit_cgroups_unsupported.go new file mode 100644 index 000000000..e61cfddbd --- /dev/null +++ b/internal/alloycli/automemlimit_cgroups_unsupported.go @@ -0,0 +1,25 @@ +//go:build !linux +// +build !linux + +package alloycli + +import ( + "log/slog" + "os" + "slices" + "strings" + + "github.com/KimMachineGun/automemlimit/memlimit" + "github.com/grafana/alloy/internal/runtime/logging" +) + +func applyAutoMemLimit(l *logging.Logger) { + // For non-linux builds without cgroups, memlimit will always report an error. + // However, if the system experiment is requested, we can use the system memory limit provider. + // This logic is similar to https://github.com/KimMachineGun/automemlimit/blob/main/memlimit/experiment.go + if v, ok := os.LookupEnv("AUTOMEMLIMIT_EXPERIMENT"); ok { + if slices.Contains(strings.Split(v, ","), "system") { + memlimit.SetGoMemLimitWithOpts(memlimit.WithProvider(memlimit.FromSystem), memlimit.WithLogger(slog.New(l.Handler()))) + } + } +} diff --git a/internal/alloycli/automemlimit_linux.go b/internal/alloycli/automemlimit_linux.go new file mode 100644 index 000000000..148c160a4 --- /dev/null +++ b/internal/alloycli/automemlimit_linux.go @@ -0,0 +1,12 @@ +package alloycli + +import ( + "log/slog" + + "github.com/KimMachineGun/automemlimit/memlimit" + "github.com/grafana/alloy/internal/runtime/logging" +) + +func applyAutoMemLimit(l *logging.Logger) { + memlimit.SetGoMemLimitWithOpts(memlimit.WithLogger(slog.New(l.Handler()))) +} diff --git a/internal/alloycli/automemlimit_nonlinux_test.go b/internal/alloycli/automemlimit_nonlinux_test.go new file mode 100644 index 000000000..6b6fefaaf --- /dev/null +++ b/internal/alloycli/automemlimit_nonlinux_test.go @@ -0,0 +1,30 @@ +//go:build !linux +// +build !linux + +package alloycli + +import ( + "bytes" + "log/slog" + "testing" + + "github.com/KimMachineGun/automemlimit/memlimit" + "github.com/grafana/alloy/internal/runtime/logging" + "github.com/stretchr/testify/require" +) + +func TestNoMemlimitErrorLogs(t *testing.T) { + buffer := bytes.NewBuffer(nil) + + l, err := logging.New(buffer, logging.DefaultOptions) + require.NoError(t, err) + + applyAutoMemLimit(l) + + require.Equal(t, "", buffer.String()) + + // Linux behavior, to confirm error is logged + memlimit.SetGoMemLimitWithOpts(memlimit.WithLogger(slog.New(l.Handler()))) + + require.Contains(t, buffer.String(), "cgroups is not supported on this system") +} diff --git a/internal/alloycli/cmd_run.go b/internal/alloycli/cmd_run.go index 43f676825..7ef521ed1 100644 --- a/internal/alloycli/cmd_run.go +++ b/internal/alloycli/cmd_run.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io/fs" - "log/slog" "os" "os/signal" "path/filepath" @@ -16,7 +15,6 @@ import ( "syscall" "time" - "github.com/KimMachineGun/automemlimit/memlimit" "github.com/fatih/color" "github.com/go-kit/log" "github.com/grafana/ckit/advertise" @@ -221,8 +219,8 @@ func (fr *alloyRun) Run(cmd *cobra.Command, configPath string) error { level.Info(l).Log("boringcrypto enabled", boringcrypto.Enabled) // Set the memory limit, this will honor GOMEMLIMIT if set - // If there is a cgroup will follow that - memlimit.SetGoMemLimitWithOpts(memlimit.WithLogger(slog.New(l.Handler()))) + // If there is a cgroup on linux it will use that + applyAutoMemLimit(l) // Enable the profiling. setMutexBlockProfiling(l) From 905a1648c6129b7372a8d1fb48cf79b356c909d3 Mon Sep 17 00:00:00 2001 From: Sam DeHaan Date: Thu, 12 Dec 2024 13:51:13 -0500 Subject: [PATCH 03/12] Update uses of prometheus registration to be less likely to panic (#2178) --- internal/component/faro/receiver/exporters.go | 6 +++++- internal/component/faro/receiver/handler.go | 3 ++- internal/component/faro/receiver/server.go | 7 ++++++- internal/component/faro/receiver/sourcemaps.go | 5 ++++- internal/component/loki/relabel/metrics.go | 13 ++++++------- .../component/loki/rules/kubernetes/rules.go | 13 ++++++------- .../source/aws_firehose/internal/metrics.go | 15 ++++++--------- .../internal/cloudflaretarget/metrics.go | 11 ++++++----- .../docker/internal/dockertarget/metrics.go | 11 ++++++----- internal/component/loki/source/file/metrics.go | 17 +++++++++-------- .../gcplog/internal/gcplogtarget/metrics.go | 17 +++++++++-------- .../loki/source/gelf/internal/target/metrics.go | 11 ++++++----- .../heroku/internal/herokutarget/metrics.go | 8 ++++++-- .../syslog/internal/syslogtarget/metrics.go | 13 +++++++------ .../exporter/loki/internal/convert/metrics.go | 9 ++++----- internal/component/pyroscope/ebpf/metrics.go | 15 +++++++-------- internal/component/pyroscope/write/metrics.go | 17 +++++++++-------- internal/component/remote/vault/metrics.go | 15 ++++++++------- internal/static/metrics/wal/wal.go | 17 ++++++++--------- 19 files changed, 120 insertions(+), 103 deletions(-) diff --git a/internal/component/faro/receiver/exporters.go b/internal/component/faro/receiver/exporters.go index 5ce602ed4..1543b126c 100644 --- a/internal/component/faro/receiver/exporters.go +++ b/internal/component/faro/receiver/exporters.go @@ -17,6 +17,7 @@ import ( "github.com/grafana/alloy/internal/component/faro/receiver/internal/payload" "github.com/grafana/alloy/internal/component/otelcol" "github.com/grafana/alloy/internal/runtime/logging/level" + "github.com/grafana/alloy/internal/util" ) type exporter interface { @@ -57,7 +58,10 @@ func newMetricsExporter(reg prometheus.Registerer) *metricsExporter { }), } - reg.MustRegister(exp.totalLogs, exp.totalExceptions, exp.totalMeasurements, exp.totalEvents) + exp.totalLogs = util.MustRegisterOrGet(reg, exp.totalLogs).(prometheus.Counter) + exp.totalMeasurements = util.MustRegisterOrGet(reg, exp.totalMeasurements).(prometheus.Counter) + exp.totalExceptions = util.MustRegisterOrGet(reg, exp.totalExceptions).(prometheus.Counter) + exp.totalEvents = util.MustRegisterOrGet(reg, exp.totalEvents).(prometheus.Counter) return exp } diff --git a/internal/component/faro/receiver/handler.go b/internal/component/faro/receiver/handler.go index 95c55c732..be82d07a7 100644 --- a/internal/component/faro/receiver/handler.go +++ b/internal/component/faro/receiver/handler.go @@ -10,6 +10,7 @@ import ( "github.com/go-kit/log" "github.com/grafana/alloy/internal/component/faro/receiver/internal/payload" "github.com/grafana/alloy/internal/runtime/logging/level" + "github.com/grafana/alloy/internal/util" "github.com/prometheus/client_golang/prometheus" "github.com/rs/cors" "go.opentelemetry.io/collector/client" @@ -36,7 +37,7 @@ func newHandler(l log.Logger, reg prometheus.Registerer, exporters []exporter) * Name: "faro_receiver_exporter_errors_total", Help: "Total number of errors produced by a receiver exporter", }, []string{"exporter"}) - reg.MustRegister(errorsTotal) + errorsTotal = util.MustRegisterOrGet(reg, errorsTotal).(*prometheus.CounterVec) return &handler{ log: l, diff --git a/internal/component/faro/receiver/server.go b/internal/component/faro/receiver/server.go index 13b2c0d14..15d35248b 100644 --- a/internal/component/faro/receiver/server.go +++ b/internal/component/faro/receiver/server.go @@ -9,6 +9,7 @@ import ( "github.com/go-kit/log" "github.com/gorilla/mux" "github.com/grafana/alloy/internal/runtime/logging/level" + "github.com/grafana/alloy/internal/util" "github.com/grafana/dskit/instrument" "github.com/grafana/dskit/middleware" "github.com/prometheus/client_golang/prometheus" @@ -46,7 +47,11 @@ func newServerMetrics(reg prometheus.Registerer) *serverMetrics { Help: "Current number of inflight requests.", }, []string{"method", "route"}), } - reg.MustRegister(m.requestDuration, m.rxMessageSize, m.txMessageSize, m.inflightRequests) + + m.requestDuration = util.MustRegisterOrGet(reg, m.requestDuration).(*prometheus.HistogramVec) + m.rxMessageSize = util.MustRegisterOrGet(reg, m.rxMessageSize).(*prometheus.HistogramVec) + m.txMessageSize = util.MustRegisterOrGet(reg, m.txMessageSize).(*prometheus.HistogramVec) + m.inflightRequests = util.MustRegisterOrGet(reg, m.inflightRequests).(*prometheus.GaugeVec) return m } diff --git a/internal/component/faro/receiver/sourcemaps.go b/internal/component/faro/receiver/sourcemaps.go index 6b75aaab4..3a2c48f7c 100644 --- a/internal/component/faro/receiver/sourcemaps.go +++ b/internal/component/faro/receiver/sourcemaps.go @@ -18,6 +18,7 @@ import ( "github.com/go-sourcemap/sourcemap" "github.com/grafana/alloy/internal/component/faro/receiver/internal/payload" "github.com/grafana/alloy/internal/runtime/logging/level" + "github.com/grafana/alloy/internal/util" "github.com/grafana/alloy/internal/util/wildcard" "github.com/prometheus/client_golang/prometheus" "github.com/vincent-petithory/dataurl" @@ -68,7 +69,9 @@ func newSourceMapMetrics(reg prometheus.Registerer) *sourceMapMetrics { }, []string{"origin", "status"}), } - reg.MustRegister(m.cacheSize, m.downloads, m.fileReads) + m.cacheSize = util.MustRegisterOrGet(reg, m.cacheSize).(*prometheus.CounterVec) + m.downloads = util.MustRegisterOrGet(reg, m.downloads).(*prometheus.CounterVec) + m.fileReads = util.MustRegisterOrGet(reg, m.fileReads).(*prometheus.CounterVec) return m } diff --git a/internal/component/loki/relabel/metrics.go b/internal/component/loki/relabel/metrics.go index 6f609fb64..0c002f717 100644 --- a/internal/component/loki/relabel/metrics.go +++ b/internal/component/loki/relabel/metrics.go @@ -1,6 +1,7 @@ package relabel import ( + "github.com/grafana/alloy/internal/util" "github.com/prometheus/client_golang/prometheus" prometheus_client "github.com/prometheus/client_golang/prometheus" ) @@ -40,13 +41,11 @@ func newMetrics(reg prometheus.Registerer) *metrics { }) if reg != nil { - reg.MustRegister( - m.entriesProcessed, - m.entriesOutgoing, - m.cacheMisses, - m.cacheHits, - m.cacheSize, - ) + m.entriesProcessed = util.MustRegisterOrGet(reg, m.entriesProcessed).(prometheus_client.Counter) + m.entriesOutgoing = util.MustRegisterOrGet(reg, m.entriesOutgoing).(prometheus_client.Counter) + m.cacheMisses = util.MustRegisterOrGet(reg, m.cacheMisses).(prometheus_client.Counter) + m.cacheHits = util.MustRegisterOrGet(reg, m.cacheHits).(prometheus_client.Counter) + m.cacheSize = util.MustRegisterOrGet(reg, m.cacheSize).(prometheus_client.Gauge) } return &m diff --git a/internal/component/loki/rules/kubernetes/rules.go b/internal/component/loki/rules/kubernetes/rules.go index 0643c4234..9bfd1070b 100644 --- a/internal/component/loki/rules/kubernetes/rules.go +++ b/internal/component/loki/rules/kubernetes/rules.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/alloy/internal/featuregate" lokiClient "github.com/grafana/alloy/internal/loki/client" "github.com/grafana/alloy/internal/runtime/logging/level" + "github.com/grafana/alloy/internal/util" "github.com/grafana/dskit/backoff" "github.com/grafana/dskit/instrument" promListers "github.com/prometheus-operator/prometheus-operator/pkg/client/listers/monitoring/v1" @@ -82,13 +83,11 @@ type metrics struct { } func (m *metrics) Register(r prometheus.Registerer) error { - r.MustRegister( - m.configUpdatesTotal, - m.eventsTotal, - m.eventsFailed, - m.eventsRetried, - m.lokiClientTiming, - ) + m.configUpdatesTotal = util.MustRegisterOrGet(r, m.configUpdatesTotal).(prometheus.Counter) + m.eventsTotal = util.MustRegisterOrGet(r, m.eventsTotal).(*prometheus.CounterVec) + m.eventsFailed = util.MustRegisterOrGet(r, m.eventsFailed).(*prometheus.CounterVec) + m.eventsRetried = util.MustRegisterOrGet(r, m.eventsRetried).(*prometheus.CounterVec) + m.lokiClientTiming = util.MustRegisterOrGet(r, m.lokiClientTiming).(*prometheus.HistogramVec) return nil } diff --git a/internal/component/loki/source/aws_firehose/internal/metrics.go b/internal/component/loki/source/aws_firehose/internal/metrics.go index 86c570112..a9701593e 100644 --- a/internal/component/loki/source/aws_firehose/internal/metrics.go +++ b/internal/component/loki/source/aws_firehose/internal/metrics.go @@ -1,6 +1,7 @@ package internal import ( + "github.com/grafana/alloy/internal/util" "github.com/prometheus/client_golang/prometheus" ) @@ -45,15 +46,11 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Help: "Number of errors while processing AWS Firehose static labels", }, []string{"reason", "tenant_id"}) - if reg != nil { - reg.MustRegister( - m.errorsAPIRequest, - m.recordsReceived, - m.errorsRecord, - m.batchSize, - m.invalidStaticLabelsCount, - ) - } + m.errorsAPIRequest = util.MustRegisterOrGet(reg, m.errorsAPIRequest).(*prometheus.CounterVec) + m.errorsRecord = util.MustRegisterOrGet(reg, m.errorsRecord).(*prometheus.CounterVec) + m.recordsReceived = util.MustRegisterOrGet(reg, m.recordsReceived).(*prometheus.CounterVec) + m.batchSize = util.MustRegisterOrGet(reg, m.batchSize).(*prometheus.HistogramVec) + m.invalidStaticLabelsCount = util.MustRegisterOrGet(reg, m.invalidStaticLabelsCount).(*prometheus.CounterVec) return &m } diff --git a/internal/component/loki/source/cloudflare/internal/cloudflaretarget/metrics.go b/internal/component/loki/source/cloudflare/internal/cloudflaretarget/metrics.go index f16b3a8b2..0f13f0663 100644 --- a/internal/component/loki/source/cloudflare/internal/cloudflaretarget/metrics.go +++ b/internal/component/loki/source/cloudflare/internal/cloudflaretarget/metrics.go @@ -5,7 +5,10 @@ package cloudflaretarget // read from the Cloudflare Logpull API and forward entries to other loki // components. -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/grafana/alloy/internal/util" + "github.com/prometheus/client_golang/prometheus" +) // Metrics holds a set of cloudflare metrics. type Metrics struct { @@ -31,10 +34,8 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { }) if reg != nil { - reg.MustRegister( - m.Entries, - m.LastEnd, - ) + m.Entries = util.MustRegisterOrGet(reg, m.Entries).(prometheus.Counter) + m.LastEnd = util.MustRegisterOrGet(reg, m.LastEnd).(prometheus.Gauge) } return &m diff --git a/internal/component/loki/source/docker/internal/dockertarget/metrics.go b/internal/component/loki/source/docker/internal/dockertarget/metrics.go index cbc6fb9ef..917a83aab 100644 --- a/internal/component/loki/source/docker/internal/dockertarget/metrics.go +++ b/internal/component/loki/source/docker/internal/dockertarget/metrics.go @@ -4,7 +4,10 @@ package dockertarget // The dockertarget package is used to configure and run the targets that can // read logs from Docker containers and forward them to other loki components. -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/grafana/alloy/internal/util" + "github.com/prometheus/client_golang/prometheus" +) // Metrics holds a set of Docker target metrics. type Metrics struct { @@ -30,10 +33,8 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { }) if reg != nil { - reg.MustRegister( - m.dockerEntries, - m.dockerErrors, - ) + m.dockerEntries = util.MustRegisterOrGet(reg, m.dockerEntries).(prometheus.Counter) + m.dockerErrors = util.MustRegisterOrGet(reg, m.dockerErrors).(prometheus.Counter) } return &m diff --git a/internal/component/loki/source/file/metrics.go b/internal/component/loki/source/file/metrics.go index 4f8be70fb..9f086d80a 100644 --- a/internal/component/loki/source/file/metrics.go +++ b/internal/component/loki/source/file/metrics.go @@ -4,7 +4,10 @@ package file // The metrics struct provides a common set of metrics that are reused between all // implementations of the reader interface. -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/grafana/alloy/internal/util" + "github.com/prometheus/client_golang/prometheus" +) // metrics hold the set of file-based metrics. type metrics struct { @@ -47,13 +50,11 @@ func newMetrics(reg prometheus.Registerer) *metrics { }) if reg != nil { - reg.MustRegister( - m.readBytes, - m.totalBytes, - m.readLines, - m.encodingFailures, - m.filesActive, - ) + m.readBytes = util.MustRegisterOrGet(reg, m.readBytes).(*prometheus.GaugeVec) + m.totalBytes = util.MustRegisterOrGet(reg, m.totalBytes).(*prometheus.GaugeVec) + m.readLines = util.MustRegisterOrGet(reg, m.readLines).(*prometheus.CounterVec) + m.encodingFailures = util.MustRegisterOrGet(reg, m.encodingFailures).(*prometheus.CounterVec) + m.filesActive = util.MustRegisterOrGet(reg, m.filesActive).(prometheus.Gauge) } return &m diff --git a/internal/component/loki/source/gcplog/internal/gcplogtarget/metrics.go b/internal/component/loki/source/gcplog/internal/gcplogtarget/metrics.go index 915ed5fb9..709ebbb89 100644 --- a/internal/component/loki/source/gcplog/internal/gcplogtarget/metrics.go +++ b/internal/component/loki/source/gcplog/internal/gcplogtarget/metrics.go @@ -5,7 +5,10 @@ package gcplogtarget // logs like bucket logs, load balancer logs, and Kubernetes cluster logs // from GCP. -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/grafana/alloy/internal/util" + "github.com/prometheus/client_golang/prometheus" +) // Metrics stores gcplog entry metrics. type Metrics struct { @@ -52,12 +55,10 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Help: "Number of parsing errors while receiving gcplog messages", }, []string{"reason"}) - reg.MustRegister( - m.gcplogEntries, - m.gcplogErrors, - m.gcplogTargetLastSuccessScrape, - m.gcpPushEntries, - m.gcpPushErrors, - ) + m.gcplogEntries = util.MustRegisterOrGet(reg, m.gcplogEntries).(*prometheus.CounterVec) + m.gcplogErrors = util.MustRegisterOrGet(reg, m.gcplogErrors).(*prometheus.CounterVec) + m.gcplogTargetLastSuccessScrape = util.MustRegisterOrGet(reg, m.gcplogTargetLastSuccessScrape).(*prometheus.GaugeVec) + m.gcpPushEntries = util.MustRegisterOrGet(reg, m.gcpPushEntries).(*prometheus.CounterVec) + m.gcpPushErrors = util.MustRegisterOrGet(reg, m.gcpPushErrors).(*prometheus.CounterVec) return &m } diff --git a/internal/component/loki/source/gelf/internal/target/metrics.go b/internal/component/loki/source/gelf/internal/target/metrics.go index 4bfea06f5..3b22c8eda 100644 --- a/internal/component/loki/source/gelf/internal/target/metrics.go +++ b/internal/component/loki/source/gelf/internal/target/metrics.go @@ -4,7 +4,10 @@ package target // configure and run the targets that can read gelf entries and forward them // to other loki components. -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/grafana/alloy/internal/util" + "github.com/prometheus/client_golang/prometheus" +) // Metrics holds a set of gelf metrics. type Metrics struct { @@ -30,10 +33,8 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { }) if reg != nil { - reg.MustRegister( - m.gelfEntries, - m.gelfErrors, - ) + m.gelfEntries = util.MustRegisterOrGet(reg, m.gelfEntries).(prometheus.Counter) + m.gelfErrors = util.MustRegisterOrGet(reg, m.gelfErrors).(prometheus.Counter) } return &m diff --git a/internal/component/loki/source/heroku/internal/herokutarget/metrics.go b/internal/component/loki/source/heroku/internal/herokutarget/metrics.go index ec145b500..dca5f682c 100644 --- a/internal/component/loki/source/heroku/internal/herokutarget/metrics.go +++ b/internal/component/loki/source/heroku/internal/herokutarget/metrics.go @@ -4,7 +4,10 @@ package herokutarget // configure and run the targets that can read heroku entries and forward them // to other loki components. -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/grafana/alloy/internal/util" + "github.com/prometheus/client_golang/prometheus" +) type Metrics struct { herokuEntries prometheus.Counter @@ -24,6 +27,7 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Help: "Number of parsing errors while receiving Heroku messages", }) - reg.MustRegister(m.herokuEntries, m.herokuErrors) + m.herokuEntries = util.MustRegisterOrGet(reg, m.herokuEntries).(prometheus.Counter) + m.herokuErrors = util.MustRegisterOrGet(reg, m.herokuErrors).(prometheus.Counter) return &m } diff --git a/internal/component/loki/source/syslog/internal/syslogtarget/metrics.go b/internal/component/loki/source/syslog/internal/syslogtarget/metrics.go index f198138e7..e88447c1f 100644 --- a/internal/component/loki/source/syslog/internal/syslogtarget/metrics.go +++ b/internal/component/loki/source/syslog/internal/syslogtarget/metrics.go @@ -4,7 +4,10 @@ package syslogtarget // configure and run the targets that can read syslog entries and forward them // to other loki components. -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/grafana/alloy/internal/util" + "github.com/prometheus/client_golang/prometheus" +) // Metrics holds a set of syslog metrics. type Metrics struct { @@ -35,11 +38,9 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { }) if reg != nil { - reg.MustRegister( - m.syslogEntries, - m.syslogParsingErrors, - m.syslogEmptyMessages, - ) + m.syslogEntries = util.MustRegisterOrGet(reg, m.syslogEntries).(prometheus.Counter) + m.syslogParsingErrors = util.MustRegisterOrGet(reg, m.syslogParsingErrors).(prometheus.Counter) + m.syslogEmptyMessages = util.MustRegisterOrGet(reg, m.syslogEmptyMessages).(prometheus.Counter) } return &m diff --git a/internal/component/otelcol/exporter/loki/internal/convert/metrics.go b/internal/component/otelcol/exporter/loki/internal/convert/metrics.go index 923d402dd..6e4298076 100644 --- a/internal/component/otelcol/exporter/loki/internal/convert/metrics.go +++ b/internal/component/otelcol/exporter/loki/internal/convert/metrics.go @@ -1,6 +1,7 @@ package convert import ( + "github.com/grafana/alloy/internal/util" "github.com/prometheus/client_golang/prometheus" prometheus_client "github.com/prometheus/client_golang/prometheus" ) @@ -28,11 +29,9 @@ func newMetrics(reg prometheus.Registerer) *metrics { }) if reg != nil { - reg.MustRegister( - m.entriesTotal, - m.entriesFailed, - m.entriesProcessed, - ) + m.entriesTotal = util.MustRegisterOrGet(reg, m.entriesTotal).(prometheus_client.Counter) + m.entriesFailed = util.MustRegisterOrGet(reg, m.entriesFailed).(prometheus_client.Counter) + m.entriesProcessed = util.MustRegisterOrGet(reg, m.entriesProcessed).(prometheus_client.Counter) } return &m diff --git a/internal/component/pyroscope/ebpf/metrics.go b/internal/component/pyroscope/ebpf/metrics.go index 6468c2e04..6e88cf97a 100644 --- a/internal/component/pyroscope/ebpf/metrics.go +++ b/internal/component/pyroscope/ebpf/metrics.go @@ -5,6 +5,7 @@ package ebpf import ( + "github.com/grafana/alloy/internal/util" ebpfmetrics "github.com/grafana/pyroscope/ebpf/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -49,14 +50,12 @@ func newMetrics(reg prometheus.Registerer) *metrics { } if reg != nil { - reg.MustRegister( - m.targetsActive, - m.profilingSessionsTotal, - m.profilingSessionsFailingTotal, - m.pprofsTotal, - m.pprofBytesTotal, - m.pprofSamplesTotal, - ) + m.targetsActive = util.MustRegisterOrGet(reg, m.targetsActive).(prometheus.Gauge) + m.profilingSessionsTotal = util.MustRegisterOrGet(reg, m.profilingSessionsTotal).(prometheus.Counter) + m.profilingSessionsFailingTotal = util.MustRegisterOrGet(reg, m.profilingSessionsFailingTotal).(prometheus.Counter) + m.pprofsTotal = util.MustRegisterOrGet(reg, m.pprofsTotal).(*prometheus.CounterVec) + m.pprofBytesTotal = util.MustRegisterOrGet(reg, m.pprofBytesTotal).(*prometheus.CounterVec) + m.pprofSamplesTotal = util.MustRegisterOrGet(reg, m.pprofSamplesTotal).(*prometheus.CounterVec) } return m diff --git a/internal/component/pyroscope/write/metrics.go b/internal/component/pyroscope/write/metrics.go index b7f50021a..f6df6ea93 100644 --- a/internal/component/pyroscope/write/metrics.go +++ b/internal/component/pyroscope/write/metrics.go @@ -1,6 +1,9 @@ package write -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/grafana/alloy/internal/util" + "github.com/prometheus/client_golang/prometheus" +) type metrics struct { sentBytes *prometheus.CounterVec @@ -35,13 +38,11 @@ func newMetrics(reg prometheus.Registerer) *metrics { } if reg != nil { - reg.MustRegister( - m.sentBytes, - m.droppedBytes, - m.sentProfiles, - m.droppedProfiles, - m.retries, - ) + m.sentBytes = util.MustRegisterOrGet(reg, m.sentBytes).(*prometheus.CounterVec) + m.droppedBytes = util.MustRegisterOrGet(reg, m.droppedBytes).(*prometheus.CounterVec) + m.sentProfiles = util.MustRegisterOrGet(reg, m.sentProfiles).(*prometheus.CounterVec) + m.droppedProfiles = util.MustRegisterOrGet(reg, m.droppedProfiles).(*prometheus.CounterVec) + m.retries = util.MustRegisterOrGet(reg, m.retries).(*prometheus.CounterVec) } return m diff --git a/internal/component/remote/vault/metrics.go b/internal/component/remote/vault/metrics.go index 91179b502..046a34fb2 100644 --- a/internal/component/remote/vault/metrics.go +++ b/internal/component/remote/vault/metrics.go @@ -1,6 +1,9 @@ package vault -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/grafana/alloy/internal/util" + "github.com/prometheus/client_golang/prometheus" +) type metrics struct { authTotal prometheus.Counter @@ -32,13 +35,11 @@ func newMetrics(r prometheus.Registerer) *metrics { }) if r != nil { - r.MustRegister( - m.authTotal, - m.secretReadTotal, + m.authTotal = util.MustRegisterOrGet(r, m.authTotal).(prometheus.Counter) + m.secretReadTotal = util.MustRegisterOrGet(r, m.secretReadTotal).(prometheus.Counter) - m.authLeaseRenewalTotal, - m.secretLeaseRenewalTotal, - ) + m.authLeaseRenewalTotal = util.MustRegisterOrGet(r, m.authLeaseRenewalTotal).(prometheus.Counter) + m.secretLeaseRenewalTotal = util.MustRegisterOrGet(r, m.secretLeaseRenewalTotal).(prometheus.Counter) } return &m } diff --git a/internal/static/metrics/wal/wal.go b/internal/static/metrics/wal/wal.go index 4af3ec9ea..aa3a4be43 100644 --- a/internal/static/metrics/wal/wal.go +++ b/internal/static/metrics/wal/wal.go @@ -15,6 +15,7 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" + "github.com/grafana/alloy/internal/util" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" @@ -84,15 +85,13 @@ func newStorageMetrics(r prometheus.Registerer) *storageMetrics { }) if r != nil { - r.MustRegister( - m.numActiveSeries, - m.numDeletedSeries, - m.totalOutOfOrderSamples, - m.totalCreatedSeries, - m.totalRemovedSeries, - m.totalAppendedSamples, - m.totalAppendedExemplars, - ) + m.numActiveSeries = util.MustRegisterOrGet(r, m.numActiveSeries).(prometheus.Gauge) + m.numDeletedSeries = util.MustRegisterOrGet(r, m.numDeletedSeries).(prometheus.Gauge) + m.totalOutOfOrderSamples = util.MustRegisterOrGet(r, m.totalOutOfOrderSamples).(prometheus.Counter) + m.totalCreatedSeries = util.MustRegisterOrGet(r, m.totalCreatedSeries).(prometheus.Counter) + m.totalRemovedSeries = util.MustRegisterOrGet(r, m.totalRemovedSeries).(prometheus.Counter) + m.totalAppendedSamples = util.MustRegisterOrGet(r, m.totalAppendedSamples).(prometheus.Counter) + m.totalAppendedExemplars = util.MustRegisterOrGet(r, m.totalAppendedExemplars).(prometheus.Counter) } return &m From b07b6172ff789bade9fb11060e5bfc2e8478207c Mon Sep 17 00:00:00 2001 From: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:54:55 -0800 Subject: [PATCH 04/12] Add new OpenShift topic to Alloy docs (#2265) * Create new openshift topic * Add new info from review * Fix broken link * Updtae URL links --- docs/sources/set-up/install/openshift.md | 166 +++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 docs/sources/set-up/install/openshift.md diff --git a/docs/sources/set-up/install/openshift.md b/docs/sources/set-up/install/openshift.md new file mode 100644 index 000000000..3f4f16251 --- /dev/null +++ b/docs/sources/set-up/install/openshift.md @@ -0,0 +1,166 @@ +--- +canonical: https://grafana.com/docs/alloy/latest/set-up/install/openshift/ +description: Learn how to deploy Grafana Alloy on OpenShift +menuTitle: OpenShift +title: Deploy Grafana Alloy on OpenShift +weight: 530 +--- + +# Deploy {{% param "FULL_PRODUCT_NAME" %}} on OpenShift + +You can deploy {{< param "PRODUCT_NAME" >}} on the Red Hat OpenShift Container Platform (OCP). + +## Before you begin + +* These steps assume you have a working OCP environment. +* You can adapt the suggested policies and configuration to meet your specific needs and [security][] policies. + +## Configure RBAC + +You must configure Role-Based Access Control (RBAC) to allow secure access to Kubernetes and OCP resources. + +1. Download the [rbac.yaml][] configuration file. This configuration file defines the OCP verbs and permissions for {{< param "PRODUCT_NAME" >}}. +1. Review the `rbac.yaml` file and adapt as needed for your local environment. Refer to [Managing Role-based Access Control (RBAC)][rbac] topic in the OCP documentation for more information about updating and managing your RBAC configurations. + +## Run {{% param "PRODUCT_NAME" %}} as a non-root user + +You must configure {{< param "PRODUCT_NAME" >}} to [run as a non-root user][nonroot]. +This ensures that {{< param "PRODUCT_NAME" >}} complies with your OCP security policies. + +## Apply security context constraints + +OCP uses Security Context Constraints (SCC) to control Pod permissions. +Refer to [Managing security context constraints][scc] for more information about how you can define and enforce these permissions. +This ensures that the pods running {{< param "PRODUCT_NAME" >}} comply with OCP security policies. + +{{< admonition type="note" >}} +The security context is only configured at the container level, not at the container and deployment level. +{{< /admonition >}} + +You can apply the following SCCs when you deploy {{< param "PRODUCT_NAME" >}}. + +{{< admonition type="note" >}} +Not all of these SCCs are required for each use case. +You can adapt the SCCs to meet your local requirements and needs. +{{< /admonition >}} + +* `RunAsUser`: Specifies the user ID under which {{< param "PRODUCT_NAME" >}} runs. + You must configure this constraint to allow a non-root user ID. +* `SELinuxContext`: Configures the SELinux context for containers. + If you run {{< param "PRODUCT_NAME" >}} as root, you must configure this constraint to make sure that SELinux policies don't block {{< param "PRODUCT_NAME" >}}. + This SCC is generally not required to deploy {{< param "PRODUCT_NAME" >}} as a non-root user. +* `FSGroup`: Specifies the fsGroup IDs for file system access. + You must configure this constraint to give {{< param "PRODUCT_NAME" >}} group access to the files it needs. +* `Volumes`: Specifies the persistent volumes used for storage. + You must configure this constraint to give {{< param "PRODUCT_NAME" >}} access to the volumes it needs. + +## Example DaemonSet configuration + +The following example shows a DaemonSet configuration that deploys {{< param "PRODUCT_NAME" >}} as a non-root user: + +```yaml +apiVersion: aapps/v1 +kind: DaemonSet +metadata: + name: alloy-logs + namespace: monitoring +spec: + selector: + matchLabels: + app: alloy-logs + template: + metadata: + labels: + app: alloy-logs + spec: + containers: + - name: alloy-logs + image: grafana/alloy: + ports: + - containerPort: 12345 + # The security context configuration + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + runAsUser: 473 + runAsGroup: 473 + fsGroup: 1000 + volumes: + - name: log-volume + emptyDir: {} +``` + +Replace the following: + +* _``_: Set to the specific {{< param "PRODUCT_NAME" >}} version you are deploying. For example, `1.5.1`. + +{{< admonition type="note" >}} +This example uses the simplest volume type, `emptyDir`. In this example configuration, if your node restarts, your data will be lost. Make sure you set the volume type to a persistent storage location for production environments. Refer to [Using volumes to persist container data](https://docs.openshift.com/container-platform/latest/nodes/containers/nodes-containers-volumes.html) in the OpenShift documentation for more information. +{{< /admonition >}} + +## Example SSC definition + +The following example shows an SSC definition that deploys {{< param "PRODUCT_NAME" >}} as a non-root user: + +```yaml +kind: SecurityContextConstraints +apiVersion: security.openshift.io/v1 +metadata: + name: scc-alloy +runAsUser: + type: MustRunAs + uid: 473 +fsGroup: + type: MustRunAs + uid: 1000 +volumes: +- '*' +users: +- my-admin-user +groups: +- my-admin-group +seLinuxContext: + type: MustRunAs + user: + role: + type: + level: +``` + +Replace the following: + +* _``_: The user for your SELinux context. +* _``_: The role for your SELinux context. +* _``_: The container type for your SELinux context. +* _``_: The level for your SELinux context. + +Refer to [SELinux Contexts][selinux] in the RedHat documentation for more information on the SELinux context configuration. + +{{< admonition type="note" >}} +This example sets `volumes:` to `*`. In a production environment, you should set `volumes:` to only the volumes that are necessary for the deployment. For example: + +```yaml +volumes: + - configMap + - downwardAPI + - emptyDir + - persistentVolumeClaim + - secret +``` + +{{< /admonition >}} + +Refer to [Deploy {{< param "FULL_PRODUCT_NAME" >}}][deploy] for more information about deploying {{< param "PRODUCT_NAME" >}} in your environment. + +## Next steps + +* [Configure {{< param "PRODUCT_NAME" >}}][Configure] + +[rbac.yaml]: https://github.com/grafana/alloy/blob/main/operations/helm/charts/alloy/templates/rbac.yaml +[rbac]: https://docs.openshift.com/container-platform/latest/authentication/using-rbac.html +[security]: https://grafana.com/docs/grafana-cloud/monitor-infrastructure/kubernetes-monitoring/configuration/troubleshooting/#openshift-support +[nonroot]: ../../../configure/nonroot/ +[scc]: https://docs.openshift.com/container-platform/latest/authentication/managing-security-context-constraints.html +[Configure]: ../../../configure/linux/ +[deploy]: ../../deploy/ +[selinux]: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/6/html/security-enhanced_linux/chap-security-enhanced_linux-selinux_contexts#chap-Security-Enhanced_Linux-SELinux_Contexts From 14413334e2b87b91f8847ba1bb6c5590f87e08da Mon Sep 17 00:00:00 2001 From: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> Date: Thu, 12 Dec 2024 11:54:22 -0800 Subject: [PATCH 05/12] Remove dead link in OpenShift topic (#2278) --- docs/sources/set-up/install/openshift.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/sources/set-up/install/openshift.md b/docs/sources/set-up/install/openshift.md index 3f4f16251..7bb8b1f29 100644 --- a/docs/sources/set-up/install/openshift.md +++ b/docs/sources/set-up/install/openshift.md @@ -13,7 +13,7 @@ You can deploy {{< param "PRODUCT_NAME" >}} on the Red Hat OpenShift Container P ## Before you begin * These steps assume you have a working OCP environment. -* You can adapt the suggested policies and configuration to meet your specific needs and [security][] policies. +* You can adapt the suggested policies and configuration to meet your specific needs and security policies. ## Configure RBAC @@ -158,7 +158,6 @@ Refer to [Deploy {{< param "FULL_PRODUCT_NAME" >}}][deploy] for more information [rbac.yaml]: https://github.com/grafana/alloy/blob/main/operations/helm/charts/alloy/templates/rbac.yaml [rbac]: https://docs.openshift.com/container-platform/latest/authentication/using-rbac.html -[security]: https://grafana.com/docs/grafana-cloud/monitor-infrastructure/kubernetes-monitoring/configuration/troubleshooting/#openshift-support [nonroot]: ../../../configure/nonroot/ [scc]: https://docs.openshift.com/container-platform/latest/authentication/managing-security-context-constraints.html [Configure]: ../../../configure/linux/ From 5d5c055d8ce32e7159653bbfa27954fcda93d307 Mon Sep 17 00:00:00 2001 From: Robby Milo Date: Fri, 13 Dec 2024 14:42:40 +0100 Subject: [PATCH 06/12] add docs deploy preview (#2268) * add docs deploy preview * revert codeowners change --- .github/workflows/deploy-pr-preview.yml | 23 +++++++++++++++++++++++ CODEOWNERS | 1 + 2 files changed, 24 insertions(+) create mode 100644 .github/workflows/deploy-pr-preview.yml diff --git a/.github/workflows/deploy-pr-preview.yml b/.github/workflows/deploy-pr-preview.yml new file mode 100644 index 000000000..fadcde280 --- /dev/null +++ b/.github/workflows/deploy-pr-preview.yml @@ -0,0 +1,23 @@ +name: Deploy pr preview + +on: + pull_request: + types: + - opened + - synchronize + - closed + paths: + - "docs/sources/**" + +jobs: + deploy-pr-preview: + uses: grafana/writers-toolkit/.github/workflows/deploy-preview.yml@main + with: + sha: ${{ github.event.pull_request.head.sha }} + branch: ${{ github.head_ref }} + event_number: ${{ github.event.number }} + title: ${{ github.event.pull_request.title }} + repo: alloy + website_directory: content/docs/alloy/latest + relative_prefix: /docs/alloy/latest/ + index_file: true diff --git a/CODEOWNERS b/CODEOWNERS index 750a939cd..3d3b770f0 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -9,6 +9,7 @@ * @grafana/grafana-alloy-maintainers #`make docs` procedure and related workflows are owned by @jdbaldry. +/.github/workflows/deploy-pr-preview.yml @jdbaldry /.github/workflows/publish-technical-documentation-next.yml @jdbaldry /.github/workflows/publish-technical-documentation-release.yml @jdbaldry /.github/workflows/update-make-docs.yml @jdbaldry From 61a9a09f7a6fe850a1f0f2f081af36b18209c404 Mon Sep 17 00:00:00 2001 From: Ravishankar Date: Fri, 13 Dec 2024 22:10:32 +0530 Subject: [PATCH 07/12] feat: Add ignore_older option for local.file_match (#2245) * feat: Add ignore_older option for local.file_match * Review comments * Change the config name * Push the check for each file instead of dir Doc update * Add default value to docs * Update docs/sources/reference/components/local/local.file_match.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Update docs/sources/reference/components/local/local.file_match.md --------- Co-authored-by: William Dumont Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- CHANGELOG.md | 5 ++-- .../components/local/local.file_match.md | 11 ++++--- internal/component/local/file_match/file.go | 10 ++++--- .../component/local/file_match/file_test.go | 29 +++++++++++++++++++ internal/component/local/file_match/watch.go | 12 ++++++-- 5 files changed, 55 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9da7c5266..6bf82b5d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ Main (unreleased) - Add perf_schema quantile columns to collector - Add three new stdlib functions to_base64, from_URLbase64 and to_URLbase64 (@ravishankar15) +- Add `ignore_older_than` option for local.file_match (@ravishankar15) - Use a forked `github.com/goccy/go-json` module which reduces the memory consumption of an Alloy instance by 20MB. If Alloy is running certain otelcol components, this reduction will not apply. (@ptodev) @@ -103,7 +104,7 @@ v1.5.1 - Fixed a crash when updating the configuration of `remote.http`. (@kinolaev) -- Fixed an issue in the `otelcol.processor.attribute` component where the actions `delete` and `hash` could not be used with the `pattern` argument. (@wildum) +- Fixed an issue in the `otelcol.processor.attribute` component where the actions `delete` and `hash` could not be used with the `pattern` argument. (@wildum) - Fixed an issue in the `prometheus.exporter.postgres` component that would leak goroutines when the target was not reachable (@dehaansa) @@ -303,7 +304,7 @@ v1.4.0 - Add the label `alloy_cluster` in the metric `alloy_config_hash` when the flag `cluster.name` is set to help differentiate between configs from the same alloy cluster or different alloy clusters. (@wildum) - + - Add support for discovering the cgroup path(s) of a process in `process.discovery`. (@mahendrapaipuri) ### Bugfixes diff --git a/docs/sources/reference/components/local/local.file_match.md b/docs/sources/reference/components/local/local.file_match.md index 70f036f1b..fd7d4048b 100644 --- a/docs/sources/reference/components/local/local.file_match.md +++ b/docs/sources/reference/components/local/local.file_match.md @@ -24,16 +24,19 @@ local.file_match "LABEL" { The following arguments are supported: -Name | Type | Description | Default | Required ---------------- | ------------------- | ------------------------------------------------------------------------------------------ |---------| -------- -`path_targets` | `list(map(string))` | Targets to expand; looks for glob patterns on the `__path__` and `__path_exclude__` keys. | | yes -`sync_period` | `duration` | How often to sync filesystem and targets. | `"10s"` | no +Name | Type | Description | Default | Required +--------------- | ------------------- | ------------------------------------------------------------------------------------------ |---------| -------- +`path_targets` | `list(map(string))` | Targets to expand; looks for glob patterns on the `__path__` and `__path_exclude__` keys. | | yes +`sync_period` | `duration` | How often to sync filesystem and targets. | `"10s"` | no +`ignore_older_than` | `duration` | Ignores files which are modified before this duration. | `"0s"` | no `path_targets` uses [doublestar][] style paths. * `/tmp/**/*.log` will match all subfolders of `tmp` and include any files that end in `*.log`. * `/tmp/apache/*.log` will match only files in `/tmp/apache/` that end in `*.log`. * `/tmp/**` will match all subfolders of `tmp`, `tmp` itself, and all files. +`local.file_match` doesn't ignore files when `ignore_older_than` is set to the default, `0s`. + ## Exported fields diff --git a/internal/component/local/file_match/file.go b/internal/component/local/file_match/file.go index e5b9766e2..102f81bee 100644 --- a/internal/component/local/file_match/file.go +++ b/internal/component/local/file_match/file.go @@ -26,8 +26,9 @@ func init() { // Arguments holds values which are used to configure the local.file_match // component. type Arguments struct { - PathTargets []discovery.Target `alloy:"path_targets,attr"` - SyncPeriod time.Duration `alloy:"sync_period,attr,optional"` + PathTargets []discovery.Target `alloy:"path_targets,attr"` + SyncPeriod time.Duration `alloy:"sync_period,attr,optional"` + IgnoreOlderThan time.Duration `alloy:"ignore_older_than,attr,optional"` } var _ component.Component = (*Component)(nil) @@ -80,8 +81,9 @@ func (c *Component) Update(args component.Arguments) error { c.watches = c.watches[:0] for _, v := range c.args.PathTargets { c.watches = append(c.watches, watch{ - target: v, - log: c.opts.Logger, + target: v, + log: c.opts.Logger, + ignoreOlderThan: c.args.IgnoreOlderThan, }) } diff --git a/internal/component/local/file_match/file_test.go b/internal/component/local/file_match/file_test.go index bec538ce2..63645315b 100644 --- a/internal/component/local/file_match/file_test.go +++ b/internal/component/local/file_match/file_test.go @@ -63,6 +63,35 @@ func TestDirectoryFile(t *testing.T) { require.True(t, contains(foundFiles, "t1.txt")) } +func TestFileIgnoreOlder(t *testing.T) { + dir := path.Join(os.TempDir(), "alloy_testing", "t1") + err := os.MkdirAll(dir, 0755) + require.NoError(t, err) + writeFile(t, dir, "t1.txt") + t.Cleanup(func() { + os.RemoveAll(dir) + }) + c := createComponent(t, dir, []string{path.Join(dir, "*.txt")}, nil) + ct := context.Background() + ct, ccl := context.WithTimeout(ct, 5*time.Second) + defer ccl() + c.args.SyncPeriod = 10 * time.Millisecond + c.args.IgnoreOlderThan = 100 * time.Millisecond + c.Update(c.args) + go c.Run(ct) + + foundFiles := c.getWatchedFiles() + require.Len(t, foundFiles, 1) + require.True(t, contains(foundFiles, "t1.txt")) + time.Sleep(150 * time.Millisecond) + + writeFile(t, dir, "t2.txt") + ct.Done() + foundFiles = c.getWatchedFiles() + require.Len(t, foundFiles, 1) + require.True(t, contains(foundFiles, "t2.txt")) +} + func TestAddingFile(t *testing.T) { dir := path.Join(os.TempDir(), "alloy_testing", "t2") err := os.MkdirAll(dir, 0755) diff --git a/internal/component/local/file_match/watch.go b/internal/component/local/file_match/watch.go index 709d82115..04d8e456c 100644 --- a/internal/component/local/file_match/watch.go +++ b/internal/component/local/file_match/watch.go @@ -3,6 +3,7 @@ package file_match import ( "os" "path/filepath" + "time" "github.com/bmatcuk/doublestar" "github.com/go-kit/log" @@ -14,8 +15,9 @@ import ( // watch handles a single discovery.target for file watching. type watch struct { - target discovery.Target - log log.Logger + target discovery.Target + log log.Logger + ignoreOlderThan time.Duration } func (w *watch) getPaths() ([]discovery.Target, error) { @@ -48,9 +50,15 @@ func (w *watch) getPaths() ([]discovery.Target, error) { } continue } + if fi.IsDir() { continue } + + if w.ignoreOlderThan != 0 && fi.ModTime().Before(time.Now().Add(-w.ignoreOlderThan)) { + continue + } + dt := discovery.Target{} for dk, v := range w.target { dt[dk] = v From 70dbba82e223bb921edc0e08c55742b2589e6207 Mon Sep 17 00:00:00 2001 From: Jack Baldry Date: Fri, 13 Dec 2024 18:31:20 +0000 Subject: [PATCH 08/12] Don't run the workflow on forks (#2284) It's expected that it can't succeed on forks. We aren't supporting that use case yet. --- .github/workflows/deploy-pr-preview.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-pr-preview.yml b/.github/workflows/deploy-pr-preview.yml index fadcde280..983b5d57e 100644 --- a/.github/workflows/deploy-pr-preview.yml +++ b/.github/workflows/deploy-pr-preview.yml @@ -11,6 +11,7 @@ on: jobs: deploy-pr-preview: + if: github.repository == 'grafana/alloy' uses: grafana/writers-toolkit/.github/workflows/deploy-preview.yml@main with: sha: ${{ github.event.pull_request.head.sha }} From 8c2ac3c7eef6134ddad06cfc05e4310e09f7edfb Mon Sep 17 00:00:00 2001 From: Ravishankar Date: Tue, 17 Dec 2024 19:58:26 +0530 Subject: [PATCH 09/12] Add livedebugging support for discover.relabel (#2291) * Add livedebugging support for discover.relabel * Update docs * Update docs/sources/reference/components/discovery/discovery.relabel.md Co-authored-by: William Dumont --------- Co-authored-by: William Dumont --- CHANGELOG.md | 3 ++- docs/sources/troubleshoot/debug.md | 1 + .../component/discovery/relabel/relabel.go | 24 ++++++++++++++++--- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bf82b5d5..9fc413ca4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,10 +50,11 @@ Main (unreleased) - Add three new stdlib functions to_base64, from_URLbase64 and to_URLbase64 (@ravishankar15) - Add `ignore_older_than` option for local.file_match (@ravishankar15) +- Add livedebugging support for `discover.relabel` (@ravishankar15) - Use a forked `github.com/goccy/go-json` module which reduces the memory consumption of an Alloy instance by 20MB. If Alloy is running certain otelcol components, this reduction will not apply. (@ptodev) - + - Update `prometheus.write.queue` library for performance increases in cpu. (@mattdurham) ### Bugfixes diff --git a/docs/sources/troubleshoot/debug.md b/docs/sources/troubleshoot/debug.md index 0e674aa1a..5b8518515 100644 --- a/docs/sources/troubleshoot/debug.md +++ b/docs/sources/troubleshoot/debug.md @@ -111,6 +111,7 @@ Supported components: * `otelcol.receiver.*` * `prometheus.relabel` {{< /admonition >}} +* `discovery.relabel` ## Debug using the UI diff --git a/internal/component/discovery/relabel/relabel.go b/internal/component/discovery/relabel/relabel.go index 551f361d7..6613c6c80 100644 --- a/internal/component/discovery/relabel/relabel.go +++ b/internal/component/discovery/relabel/relabel.go @@ -2,12 +2,14 @@ package relabel import ( "context" + "fmt" "sync" "github.com/grafana/alloy/internal/component" alloy_relabel "github.com/grafana/alloy/internal/component/common/relabel" "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/featuregate" + "github.com/grafana/alloy/internal/service/livedebugging" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/relabel" ) @@ -46,13 +48,23 @@ type Component struct { mut sync.RWMutex rcs []*relabel.Config + + debugDataPublisher livedebugging.DebugDataPublisher } var _ component.Component = (*Component)(nil) +var _ component.LiveDebugging = (*Component)(nil) // New creates a new discovery.relabel component. func New(o component.Options, args Arguments) (*Component, error) { - c := &Component{opts: o} + debugDataPublisher, err := o.GetServiceData(livedebugging.ServiceName) + if err != nil { + return nil, err + } + c := &Component{ + opts: o, + debugDataPublisher: debugDataPublisher.(livedebugging.DebugDataPublisher), + } // Call to Update() to set the output once at the start if err := c.Update(args); err != nil { @@ -81,9 +93,13 @@ func (c *Component) Update(args component.Arguments) error { for _, t := range newArgs.Targets { lset := componentMapToPromLabels(t) - lset, keep := relabel.Process(lset, relabelConfigs...) + relabelled, keep := relabel.Process(lset, relabelConfigs...) if keep { - targets = append(targets, promLabelsToComponent(lset)) + targets = append(targets, promLabelsToComponent(relabelled)) + } + componentID := livedebugging.ComponentID(c.opts.ID) + if c.debugDataPublisher.IsActive(componentID) { + c.debugDataPublisher.Publish(componentID, fmt.Sprintf("%s => %s", lset.String(), relabelled.String())) } } @@ -95,6 +111,8 @@ func (c *Component) Update(args component.Arguments) error { return nil } +func (c *Component) LiveDebugging(_ int) {} + func componentMapToPromLabels(ls discovery.Target) labels.Labels { res := make([]labels.Label, 0, len(ls)) for k, v := range ls { From d8ba0093e7945ed4629793f2be2f2bc1e5ac273b Mon Sep 17 00:00:00 2001 From: Ravishankar Date: Wed, 18 Dec 2024 15:59:35 +0530 Subject: [PATCH 10/12] Live Debugging button should appear in UI only for supported components (#2293) * Live Debugging button should appear in UI only for supported components * Update string for unavailable components --- CHANGELOG.md | 1 + internal/component/component_provider.go | 43 ++++++++++--------- internal/runtime/alloy_components.go | 4 ++ .../src/features/component/ComponentView.tsx | 29 +++++++++---- .../web/ui/src/features/component/types.ts | 5 +++ 5 files changed, 53 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fc413ca4..8279c6719 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ Main (unreleased) - Change processlist query to support ONLY_FULL_GROUP_BY sql_mode - Add perf_schema quantile columns to collector +- Live Debugging button should appear in UI only for supported components (@ravishankar15) - Add three new stdlib functions to_base64, from_URLbase64 and to_URLbase64 (@ravishankar15) - Add `ignore_older_than` option for local.file_match (@ravishankar15) - Add livedebugging support for `discover.relabel` (@ravishankar15) diff --git a/internal/component/component_provider.go b/internal/component/component_provider.go index 1f2c98144..bc685ae59 100644 --- a/internal/component/component_provider.go +++ b/internal/component/component_provider.go @@ -123,9 +123,10 @@ type Info struct { ComponentName string // Name of the component. Health Health // Current component health. - Arguments Arguments // Current arguments value of the component. - Exports Exports // Current exports value of the component. - DebugInfo interface{} // Current debug info of the component. + Arguments Arguments // Current arguments value of the component. + Exports Exports // Current exports value of the component. + DebugInfo interface{} // Current debug info of the component. + LiveDebuggingEnabled bool } // MarshalJSON returns a JSON representation of cd. The format of the @@ -139,19 +140,20 @@ func (info *Info) MarshalJSON() ([]byte, error) { } componentDetailJSON struct { - Name string `json:"name"` - Type string `json:"type,omitempty"` - LocalID string `json:"localID"` - ModuleID string `json:"moduleID"` - Label string `json:"label,omitempty"` - References []string `json:"referencesTo"` - ReferencedBy []string `json:"referencedBy"` - Health *componentHealthJSON `json:"health"` - Original string `json:"original"` - Arguments json.RawMessage `json:"arguments,omitempty"` - Exports json.RawMessage `json:"exports,omitempty"` - DebugInfo json.RawMessage `json:"debugInfo,omitempty"` - CreatedModuleIDs []string `json:"createdModuleIDs,omitempty"` + Name string `json:"name"` + Type string `json:"type,omitempty"` + LocalID string `json:"localID"` + ModuleID string `json:"moduleID"` + Label string `json:"label,omitempty"` + References []string `json:"referencesTo"` + ReferencedBy []string `json:"referencedBy"` + Health *componentHealthJSON `json:"health"` + Original string `json:"original"` + Arguments json.RawMessage `json:"arguments,omitempty"` + Exports json.RawMessage `json:"exports,omitempty"` + DebugInfo json.RawMessage `json:"debugInfo,omitempty"` + CreatedModuleIDs []string `json:"createdModuleIDs,omitempty"` + LiveDebuggingEnabled bool `json:"liveDebuggingEnabled"` } ) @@ -196,10 +198,11 @@ func (info *Info) MarshalJSON() ([]byte, error) { Message: info.Health.Message, UpdatedTime: info.Health.UpdateTime, }, - Arguments: arguments, - Exports: exports, - DebugInfo: debugInfo, - CreatedModuleIDs: info.ModuleIDs, + Arguments: arguments, + Exports: exports, + DebugInfo: debugInfo, + CreatedModuleIDs: info.ModuleIDs, + LiveDebuggingEnabled: info.LiveDebuggingEnabled, }) } diff --git a/internal/runtime/alloy_components.go b/internal/runtime/alloy_components.go index fef0061a8..439d96949 100644 --- a/internal/runtime/alloy_components.go +++ b/internal/runtime/alloy_components.go @@ -128,6 +128,10 @@ func (f *Runtime) getComponentDetail(cn controller.ComponentNode, graph *dag.Gra componentInfo.DebugInfo = builtinComponent.DebugInfo() } } + + _, liveDebuggingEnabled := componentInfo.Component.(component.LiveDebugging) + componentInfo.LiveDebuggingEnabled = liveDebuggingEnabled + return componentInfo } diff --git a/internal/web/ui/src/features/component/ComponentView.tsx b/internal/web/ui/src/features/component/ComponentView.tsx index 81b81c1d6..5aa29c137 100644 --- a/internal/web/ui/src/features/component/ComponentView.tsx +++ b/internal/web/ui/src/features/component/ComponentView.tsx @@ -23,6 +23,7 @@ export const ComponentView: FC = (props) => { const referencedBy = props.component.referencedBy.filter((id) => props.info[id] !== undefined).map((id) => props.info[id]); const referencesTo = props.component.referencesTo.filter((id) => props.info[id] !== undefined).map((id) => props.info[id]); + const liveDebuggingEnabled = props.component.liveDebuggingEnabled; const argsPartition = partitionBody(props.component.arguments, 'Arguments'); const exportsPartition = props.component.exports && partitionBody(props.component.exports, 'Exports'); @@ -47,6 +48,24 @@ export const ComponentView: FC = (props) => { ); } + function liveDebuggingButton(): ReactElement | string { + if (useRemotecfg) { + return 'Live debugging is not yet available for remote components'; + } + + if (!liveDebuggingEnabled) { + return 'Live debugging is not yet available for this component'; + } + + return ( + + ); + } + return (
- {useRemotecfg ? ( - 'Live debugging is not yet available for remote components' - ) : ( - - )} + {liveDebuggingButton()} {props.component.health.message && (
diff --git a/internal/web/ui/src/features/component/types.ts b/internal/web/ui/src/features/component/types.ts index 4678b4ca1..b8e8162ec 100644 --- a/internal/web/ui/src/features/component/types.ts +++ b/internal/web/ui/src/features/component/types.ts @@ -42,6 +42,11 @@ export interface ComponentInfo { * IDs of components which this component is referencing. */ referencesTo: string[]; + + /** + * Used to indicate if live debugging is available for the component + */ + liveDebuggingEnabled: boolean; } /** From 3195e76555c0d913b26d47bd2b1f618d09d34427 Mon Sep 17 00:00:00 2001 From: Paulin Todev Date: Thu, 19 Dec 2024 11:02:04 +0200 Subject: [PATCH 11/12] Upgrade goccy/go-json to v0.10.4 (#2300) --- CHANGELOG.md | 2 +- go.mod | 5 +---- go.sum | 5 +++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8279c6719..685f79921 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,7 +53,7 @@ Main (unreleased) - Add `ignore_older_than` option for local.file_match (@ravishankar15) - Add livedebugging support for `discover.relabel` (@ravishankar15) -- Use a forked `github.com/goccy/go-json` module which reduces the memory consumption of an Alloy instance by 20MB. +- Upgrade `github.com/goccy/go-json` to v0.10.4, which reduces the memory consumption of an Alloy instance by 20MB. If Alloy is running certain otelcol components, this reduction will not apply. (@ptodev) - Update `prometheus.write.queue` library for performance increases in cpu. (@mattdurham) diff --git a/go.mod b/go.mod index 0c662da96..4dbf6351b 100644 --- a/go.mod +++ b/go.mod @@ -518,7 +518,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/go-zookeeper/zk v1.0.3 // indirect github.com/gobwas/glob v0.2.3 // indirect - github.com/goccy/go-json v0.10.3 // indirect + github.com/goccy/go-json v0.10.4 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/googleapis v1.4.1 // indirect @@ -945,6 +945,3 @@ exclude ( ) replace github.com/prometheus/procfs => github.com/prometheus/procfs v0.12.0 - -// TODO(ptodev): Remove when this PR has been merged: https://github.com/goccy/go-json/pull/490 -replace github.com/goccy/go-json => github.com/grafana/go-json v0.0.0-20241106155216-71a03f133f5c diff --git a/go.sum b/go.sum index 77ad0b47c..6076b1a75 100644 --- a/go.sum +++ b/go.sum @@ -1619,6 +1619,9 @@ github.com/goburrow/modbus v0.1.0/go.mod h1:Kx552D5rLIS8E7TyUwQ/UdHEqvX5T8tyiGBT github.com/goburrow/serial v0.1.0/go.mod h1:sAiqG0nRVswsm1C97xsttiYCzSLBmUZ/VSlVLZJ8haA= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= +github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -1853,8 +1856,6 @@ github.com/grafana/dskit v0.0.0-20240104111617-ea101a3b86eb h1:AWE6+kvtE18HP+lRW github.com/grafana/dskit v0.0.0-20240104111617-ea101a3b86eb/go.mod h1:kkWM4WUV230bNG3urVRWPBnSJHs64y/0RmWjftnnn0c= github.com/grafana/go-gelf/v2 v2.0.1 h1:BOChP0h/jLeD+7F9mL7tq10xVkDG15he3T1zHuQaWak= github.com/grafana/go-gelf/v2 v2.0.1/go.mod h1:lexHie0xzYGwCgiRGcvZ723bSNyNI8ZRD4s0CLobh90= -github.com/grafana/go-json v0.0.0-20241106155216-71a03f133f5c h1:yKBKEC347YZpgii1KazRCfxHsTaxMqWZzoivM1OTT50= -github.com/grafana/go-json v0.0.0-20241106155216-71a03f133f5c/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/grafana/go-offsets-tracker v0.1.7 h1:2zBQ7iiGzvyXY7LA8kaaSiEqH/Yx82UcfRabbY5aOG4= github.com/grafana/go-offsets-tracker v0.1.7/go.mod h1:qcQdu7zlUKIFNUdBJlLyNHuJGW0SKWKjkrN6jtt+jds= github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85/go.mod h1:crI9WX6p0IhrqB+DqIUHulRW853PaNFf7o4UprV//3I= From 2b42ec43b639a83376b2dd1c4a5ac43117637f8f Mon Sep 17 00:00:00 2001 From: Christian Simon Date: Thu, 19 Dec 2024 15:42:01 +0000 Subject: [PATCH 12/12] fix(pyroscope.receive_http): Handle metrics registry gracefully (#2302) This component did not handle duplciate registrations well. This fixes #2285. --- .../pyroscope/receive_http/receive_http.go | 28 +++++++++---- .../receive_http/receive_http_test.go | 42 +++++++++++++++++++ 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/internal/component/pyroscope/receive_http/receive_http.go b/internal/component/pyroscope/receive_http/receive_http.go index 35a124027..c17868b42 100644 --- a/internal/component/pyroscope/receive_http/receive_http.go +++ b/internal/component/pyroscope/receive_http/receive_http.go @@ -10,6 +10,7 @@ import ( "sync" "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" "github.com/grafana/alloy/internal/component" @@ -18,6 +19,7 @@ import ( "github.com/grafana/alloy/internal/component/pyroscope/write" "github.com/grafana/alloy/internal/featuregate" "github.com/grafana/alloy/internal/runtime/logging/level" + "github.com/grafana/alloy/internal/util" ) const ( @@ -53,16 +55,21 @@ func (a *Arguments) SetToDefault() { } type Component struct { - opts component.Options - server *fnet.TargetServer - appendables []pyroscope.Appendable - mut sync.Mutex + opts component.Options + server *fnet.TargetServer + uncheckedCollector *util.UncheckedCollector + appendables []pyroscope.Appendable + mut sync.Mutex } func New(opts component.Options, args Arguments) (*Component, error) { + uncheckedCollector := util.NewUncheckedCollector(nil) + opts.Registerer.MustRegister(uncheckedCollector) + c := &Component{ - opts: opts, - appendables: args.ForwardTo, + opts: opts, + uncheckedCollector: uncheckedCollector, + appendables: args.ForwardTo, } if err := c.Update(args); err != nil { @@ -116,7 +123,14 @@ func (c *Component) Update(args component.Arguments) error { c.shutdownServer() - srv, err := fnet.NewTargetServer(c.opts.Logger, "pyroscope_receive_http", c.opts.Registerer, newArgs.Server) + // [server.Server] registers new metrics every time it is created. To + // avoid issues with re-registering metrics with the same name, we create a + // new registry for the server every time we create one, and pass it to an + // unchecked collector to bypass uniqueness checking. + serverRegistry := prometheus.NewRegistry() + c.uncheckedCollector.SetCollector(serverRegistry) + + srv, err := fnet.NewTargetServer(c.opts.Logger, "pyroscope_receive_http", serverRegistry, newArgs.Server) if err != nil { return fmt.Errorf("failed to create server: %w", err) } diff --git a/internal/component/pyroscope/receive_http/receive_http_test.go b/internal/component/pyroscope/receive_http/receive_http_test.go index 71929abce..5c507016c 100644 --- a/internal/component/pyroscope/receive_http/receive_http_test.go +++ b/internal/component/pyroscope/receive_http/receive_http_test.go @@ -287,3 +287,45 @@ func testOptions(t *testing.T) component.Options { Registerer: prometheus.NewRegistry(), } } + +// TestUpdateArgs verifies that the component can be updated with new arguments. This explictly also makes sure that the server is restarted when the server configuration changes. And there are no metric registration conflicts. +func TestUpdateArgs(t *testing.T) { + ports, err := freeport.GetFreePorts(2) + require.NoError(t, err) + + forwardTo := []pyroscope.Appendable{testAppendable(nil)} + + args := Arguments{ + Server: &fnet.ServerConfig{ + HTTP: &fnet.HTTPConfig{ + ListenAddress: "localhost", + ListenPort: ports[0], + }, + }, + ForwardTo: forwardTo, + } + + comp, err := New(testOptions(t), args) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + go func() { + require.NoError(t, comp.Run(ctx)) + }() + + waitForServerReady(t, ports[0]) + + comp.Update(Arguments{ + Server: &fnet.ServerConfig{ + HTTP: &fnet.HTTPConfig{ + ListenAddress: "localhost", + ListenPort: ports[1], + }, + }, + ForwardTo: forwardTo, + }) + + waitForServerReady(t, ports[1]) +}