From be47c3a014b98beec1967608d372dd80b869f794 Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Mon, 23 Oct 2023 16:07:17 +0300 Subject: [PATCH] loki.source.cloudflare: port latest Promtail changes (#5571) Signed-off-by: Paschalis Tsilias --- component/loki/source/cloudflare/cloudflare.go | 5 +++++ .../cloudflare/internal/cloudflaretarget/client.go | 7 ++++--- .../cloudflare/internal/cloudflaretarget/fields.go | 11 ++++++----- .../internal/cloudflaretarget/fields_test.go | 5 +++++ .../cloudflare/internal/cloudflaretarget/metrics.go | 7 ++++--- .../cloudflare/internal/cloudflaretarget/target.go | 9 +++++---- .../internal/cloudflaretarget/target_test.go | 7 ++++--- .../cloudflare/internal/cloudflaretarget/util_test.go | 7 ++++--- .../reference/components/loki.source.cloudflare.md | 11 ++++++----- go.mod | 3 +-- go.sum | 4 ---- 11 files changed, 44 insertions(+), 32 deletions(-) diff --git a/component/loki/source/cloudflare/cloudflare.go b/component/loki/source/cloudflare/cloudflare.go index f1435bfae6ef..98a24965270b 100644 --- a/component/loki/source/cloudflare/cloudflare.go +++ b/component/loki/source/cloudflare/cloudflare.go @@ -1,5 +1,10 @@ package cloudflare +// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868). +// The cloudflaretarget package is used to configure and run a target that can +// read from the Cloudflare Logpull API and forward entries to other loki +// components. + import ( "context" "fmt" diff --git a/component/loki/source/cloudflare/internal/cloudflaretarget/client.go b/component/loki/source/cloudflare/internal/cloudflaretarget/client.go index 8fa2ac3e11b7..00a8f0f27a5f 100644 --- a/component/loki/source/cloudflare/internal/cloudflaretarget/client.go +++ b/component/loki/source/cloudflare/internal/cloudflaretarget/client.go @@ -1,8 +1,9 @@ package cloudflaretarget -// This code is copied from Promtail. The cloudflaretarget package is used to -// configure and run a target that can read from the Cloudflare Logpull API and -// forward entries to other loki components. +// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868). +// The cloudflaretarget package is used to configure and run a target that can +// read from the Cloudflare Logpull API and forward entries to other loki +// components. import ( "context" diff --git a/component/loki/source/cloudflare/internal/cloudflaretarget/fields.go b/component/loki/source/cloudflare/internal/cloudflaretarget/fields.go index cd5edadb6f73..84202780161a 100644 --- a/component/loki/source/cloudflare/internal/cloudflaretarget/fields.go +++ b/component/loki/source/cloudflare/internal/cloudflaretarget/fields.go @@ -1,8 +1,9 @@ package cloudflaretarget -// This code is copied from Promtail. The cloudflaretarget package is used to -// configure and run a target that can read from the Cloudflare Logpull API and -// forward entries to other loki components. +// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868). +// The cloudflaretarget package is used to configure and run a target that can +// read from the Cloudflare Logpull API and forward entries to other loki +// components. import ( "fmt" @@ -38,9 +39,9 @@ var ( "OriginResponseHTTPExpires", "OriginResponseHTTPLastModified", }...) allFields = append(extendedFields, []string{ - "BotScore", "BotScoreSrc", "ClientRequestBytes", "ClientSrcPort", "ClientXRequestedWith", "CacheTieredFill", "EdgeResponseCompressionRatio", "EdgeServerIP", "FirewallMatchesSources", + "BotScore", "BotScoreSrc", "BotTags", "ClientRequestBytes", "ClientSrcPort", "ClientXRequestedWith", "CacheTieredFill", "EdgeResponseCompressionRatio", "EdgeServerIP", "FirewallMatchesSources", "FirewallMatchesActions", "FirewallMatchesRuleIDs", "OriginResponseBytes", "OriginResponseTime", "ClientDeviceType", "WAFFlags", "WAFMatchedVar", "EdgeColoID", - "RequestHeaders", "ResponseHeaders", + "RequestHeaders", "ResponseHeaders", "ClientRequestSource", }...) ) diff --git a/component/loki/source/cloudflare/internal/cloudflaretarget/fields_test.go b/component/loki/source/cloudflare/internal/cloudflaretarget/fields_test.go index 6292fe243bc8..c170d7657beb 100644 --- a/component/loki/source/cloudflare/internal/cloudflaretarget/fields_test.go +++ b/component/loki/source/cloudflare/internal/cloudflaretarget/fields_test.go @@ -1,5 +1,10 @@ package cloudflaretarget +// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868). +// The cloudflaretarget package is used to configure and run a target that can +// read from the Cloudflare Logpull API and forward entries to other loki +// components. + import ( "testing" diff --git a/component/loki/source/cloudflare/internal/cloudflaretarget/metrics.go b/component/loki/source/cloudflare/internal/cloudflaretarget/metrics.go index 71c1b766d9d0..f16b3a8b23c5 100644 --- a/component/loki/source/cloudflare/internal/cloudflaretarget/metrics.go +++ b/component/loki/source/cloudflare/internal/cloudflaretarget/metrics.go @@ -1,8 +1,9 @@ package cloudflaretarget -// This code is copied from Promtail. The cloudflaretarget package is used to -// configure and run a target that can read from the Cloudflare Logpull API and -// forward entries to other loki components. +// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868). +// The cloudflaretarget package is used to configure and run a target that can +// read from the Cloudflare Logpull API and forward entries to other loki +// components. import "github.com/prometheus/client_golang/prometheus" diff --git a/component/loki/source/cloudflare/internal/cloudflaretarget/target.go b/component/loki/source/cloudflare/internal/cloudflaretarget/target.go index 004565bfa7fe..d6401d91bb3a 100644 --- a/component/loki/source/cloudflare/internal/cloudflaretarget/target.go +++ b/component/loki/source/cloudflare/internal/cloudflaretarget/target.go @@ -1,8 +1,9 @@ package cloudflaretarget -// This code is copied from Promtail. The cloudflaretarget package is used to -// configure and run a target that can read from the Cloudflare Logpull API and -// forward entries to other loki components. +// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868). +// The cloudflaretarget package is used to configure and run a target that can +// read from the Cloudflare Logpull API and forward entries to other loki +// components. import ( "context" @@ -12,11 +13,11 @@ import ( "time" "github.com/buger/jsonparser" - "github.com/cloudflare/cloudflare-go" "github.com/go-kit/log" "github.com/grafana/agent/component/common/loki" "github.com/grafana/agent/component/common/loki/positions" "github.com/grafana/agent/pkg/flow/logging/level" + "github.com/grafana/cloudflare-go" "github.com/grafana/dskit/backoff" "github.com/grafana/dskit/concurrency" "github.com/grafana/dskit/multierror" diff --git a/component/loki/source/cloudflare/internal/cloudflaretarget/target_test.go b/component/loki/source/cloudflare/internal/cloudflaretarget/target_test.go index 73e48e622001..e7e935fc337d 100644 --- a/component/loki/source/cloudflare/internal/cloudflaretarget/target_test.go +++ b/component/loki/source/cloudflare/internal/cloudflaretarget/target_test.go @@ -1,8 +1,9 @@ package cloudflaretarget -// This code is copied from Promtail. The cloudflaretarget package is used to -// configure and run a target that can read from the Cloudflare Logpull API and -// forward entries to other loki components. +// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868). +// The cloudflaretarget package is used to configure and run a target that can +// read from the Cloudflare Logpull API and forward entries to other loki +// components. import ( "context" diff --git a/component/loki/source/cloudflare/internal/cloudflaretarget/util_test.go b/component/loki/source/cloudflare/internal/cloudflaretarget/util_test.go index 1334c27ae592..5b16d032116e 100644 --- a/component/loki/source/cloudflare/internal/cloudflaretarget/util_test.go +++ b/component/loki/source/cloudflare/internal/cloudflaretarget/util_test.go @@ -1,8 +1,9 @@ package cloudflaretarget -// This code is copied from Promtail. The cloudflaretarget package is used to -// configure and run a target that can read from the Cloudflare Logpull API and -// forward entries to other loki components. +// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868). +// The cloudflaretarget package is used to configure and run a target that can +// read from the Cloudflare Logpull API and forward entries to other loki +// components. import ( "context" diff --git a/docs/sources/flow/reference/components/loki.source.cloudflare.md b/docs/sources/flow/reference/components/loki.source.cloudflare.md index 4cd5eca094f7..a24508a7b236 100644 --- a/docs/sources/flow/reference/components/loki.source.cloudflare.md +++ b/docs/sources/flow/reference/components/loki.source.cloudflare.md @@ -71,7 +71,7 @@ plus any extra fields provided via `additional_fields` argument. * `all` includes all `extended` fields and adds: ``` - "BotScore", "BotScoreSrc", "ClientRequestBytes", "ClientSrcPort", "ClientXRequestedWith", "CacheTieredFill", "EdgeResponseCompressionRatio", "EdgeServerIP", "FirewallMatchesSources", "FirewallMatchesActions", "FirewallMatchesRuleIDs", "OriginResponseBytes", "OriginResponseTime", "ClientDeviceType", "WAFFlags", "WAFMatchedVar", "EdgeColoID", "RequestHeaders", "ResponseHeaders" + "BotScore", "BotScoreSrc", "BotTags", "ClientRequestBytes", "ClientSrcPort", "ClientXRequestedWith", "CacheTieredFill", "EdgeResponseCompressionRatio", "EdgeServerIP", "FirewallMatchesSources", "FirewallMatchesActions", "FirewallMatchesRuleIDs", "OriginResponseBytes", "OriginResponseTime", "ClientDeviceType", "WAFFlags", "WAFMatchedVar", "EdgeColoID", "RequestHeaders", "ResponseHeaders", "ClientRequestSource"` ``` plus any extra fields provided via `additional_fields` argument (this is still relevant in this case if new fields are made available via Cloudflare API but are not yet included in `all`). @@ -113,6 +113,7 @@ change the log line format. A sample log looks like this: "ClientRequestReferer": "https://www.foo.com/foo/168855/?offset=8625", "ClientRequestURI": "/foo/15248108/", "ClientRequestUserAgent": "some bot", + "ClientRequestSource": "1" "ClientSSLCipher": "ECDHE-ECDSA-AES128-GCM-SHA256", "ClientSSLProtocol": "TLSv1.2", "ClientSrcPort": 39816, @@ -144,10 +145,10 @@ change the log line format. A sample log looks like this: "OriginSSLProtocol": "TLSv1.2", "ParentRayID": "00", "RayID": "6b0a...", - "RequestHeaders": [], - "ResponseHeaders": [ - "x-foo": "bar" - ], + "RequestHeaders": [], + "ResponseHeaders": [ + "x-foo": "bar" + ], "SecurityLevel": "med", "WAFAction": "unknown", "WAFFlags": "0", diff --git a/go.mod b/go.mod index da5014df041c..5880cb1c49e7 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/burningalchemist/sql_exporter v0.0.0-20221222155641-2ff59aa75200 github.com/cespare/xxhash/v2 v2.2.0 github.com/cilium/ebpf v0.11.0 // indirect - github.com/cloudflare/cloudflare-go v0.27.0 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/coreos/go-systemd/v22 v22.5.0 github.com/davidmparrott/kafka_exporter/v2 v2.0.1 @@ -683,7 +682,7 @@ replace ( github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v36.2.0+incompatible github.com/Azure/azure-storage-blob-go => github.com/MasslessParticle/azure-storage-blob-go v0.14.1-0.20220216145902-b5e698eff68e github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab - github.com/cloudflare/cloudflare-go => github.com/cyriltovena/cloudflare-go v0.27.1-0.20211118103540-ff77400bcb93 + github.com/cloudflare/cloudflare-go => github.com/grafana/cloudflare-go v0.0.0-20230110200409-c627cf6792f2 github.com/go-kit/log => github.com/dannykopping/go-kit-log v0.2.2-0.20221002180827-5591c1641b6b github.com/gocql/gocql => github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85 github.com/hashicorp/consul => github.com/hashicorp/consul v1.5.1 diff --git a/go.sum b/go.sum index 4ee6def18ea1..915a4e9b1647 100644 --- a/go.sum +++ b/go.sum @@ -534,8 +534,6 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/cyriltovena/cloudflare-go v0.27.1-0.20211118103540-ff77400bcb93 h1:PEBeRA25eDfHWkXNJs0HOnMhjIuKMcxKg/Z3VeuoRbU= -github.com/cyriltovena/cloudflare-go v0.27.1-0.20211118103540-ff77400bcb93/go.mod h1:sPWL/lIC6biLEdyGZwBQ1rGQKF1FhM7N60fuNiFdYTI= github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE= github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec= github.com/dannykopping/go-kit-log v0.2.2-0.20221002180827-5591c1641b6b h1:G8g9mAKEj9O3RsU6Hd/ow6lIcHarlcUl5omV6sFKEOU= @@ -2208,7 +2206,6 @@ github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVK github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/vertica/vertica-sql-go v1.3.0 h1:oZL8PgwrpALegtTFhImsaJvg5oQd2G7v7Uai97X9Xp8= github.com/vertica/vertica-sql-go v1.3.0/go.mod h1:jnn2GFuv+O2Jcjktb7zyc4Utlbu9YVqpHH/lx63+1M4= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= @@ -2818,7 +2815,6 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=