imporve tracesToProfiles and tracesToLogsV2

Signed-off-by: Weifeng Wang <[email protected]> imporve tracesToProfiles and tracesToLogsV2 Signed-off-by: Weifeng Wang <[email protected]> imporve tracesToProfiles and tracesToLogsV2 Signed-off-by: Weifeng Wang <[email protected]> imporve tracesToProfiles and tracesToLogsV2 Signed-off-by: Weifeng Wang <[email protected]>
qclaogui · Mar 7, 2024 · 4d382ca · 4d382ca
1 parent 6444059
commit 4d382ca
Show file tree

Hide file tree

Showing 30 changed files with 322 additions and 359 deletions.
diff --git a/kubernetes/common/grafana-agent/configs/config.river b/kubernetes/common/grafana-agent/configs/config.river
@@ -7,79 +7,3 @@ logging {
 	level  = coalesce(env("AGENT_LOG_LEVEL"), "info")
 	format = "logfmt"
 }
-
-module.file "lgtmp" {
-	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/lgtmp.river"
-
-	arguments {
-		cluster           = coalesce(env("CLUSTER"), "k3d-k3s-codelab")
-		logs_endpoint     = coalesce(env("LOGS_ENDPOINT"), "http://nginx.gateway.svc:3100")
-		metrics_endpoint  = coalesce(env("METRICS_ENDPOINT"), "http://nginx.gateway.svc:8080")
-		profiles_endpoint = coalesce(env("PROFILES_ENDPOINT"), "http://nginx.gateway.svc:4040")
-		traces_endpoint   = coalesce(env("TRACES_ENDPOINT"), "nginx.gateway.svc:4317")
-	}
-}
-
-/********************************************
- * Metrics
- ********************************************/
-module.file "metrics_primary" {
-	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/metrics.river"
-
-	arguments {
-		forward_to = [module.file.lgtmp.exports.metrics_receiver]
-		clustering = true
-	}
-}
-
-/********************************************
- * Logs
- ********************************************/
-module.git "logs_primary" {
-	repository = "https://github.com/grafana/agent-modules.git"
-	revision   = "main"
-	path       = "modules/kubernetes/logs/all.river"
-
-	arguments {
-		forward_to = [module.file.lgtmp.exports.logs_receiver]
-	}
-}
-
-/********************************************
- * Traces
- ********************************************/
-module.file "traces_primary" {
-	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/traces.river"
-
-	arguments {
-		metrics_forward_to = [module.file.lgtmp.exports.metrics_receiver]
-		logs_forward_to    = [module.file.lgtmp.exports.logs_receiver]
-		traces_forward_to  = [module.file.lgtmp.exports.traces_receiver]
-		cluster            = coalesce(env("CLUSTER"), "k3d-k3s-codelab")
-	}
-}
-
-/********************************************
- * Profiles
- ********************************************/
-module.file "profiles_primary" {
-	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/profiles.river"
-
-	arguments {
-		forward_to = [module.file.lgtmp.exports.profiles_receiver]
-		clustering = true
-	}
-}
-
-/********************************************
- * Agent Integrations
- ********************************************/
-module.file "agent_integrations" {
-	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/integrations.river"
-
-	arguments {
-		name       = "agent-integrations"
-		namespace  = "monitoring-system"
-		forward_to = [module.file.lgtmp.exports.metrics_receiver]
-	}
-}
diff --git a/kubernetes/common/grafana-agent/configs/modules/traces.river b/kubernetes/common/grafana-agent/configs/modules/traces.river
@@ -1,6 +1,10 @@
 /*
 Module: traces
 */
+
+/********************************************
+ * ARGUMENTS
+ ********************************************/
 argument "traces_forward_to" {
 	optional = false
 }
@@ -28,6 +32,13 @@ argument "otlp_grpc_endpoint" {
 	default  = "0.0.0.0:4317"
 }
 
+/********************************************
+ * EXPORTS
+ ********************************************/
+export "agent_traces_input" {
+	value = otelcol.processor.batch.default.input
+}
+
 /********************************************
  * Jaeger for Metrics Logs Traces
  ********************************************/
@@ -52,9 +63,9 @@ otelcol.receiver.jaeger "default" {
 	}
 
 	output {
-		metrics = [otelcol.processor.batch.default.input]
-		logs    = [otelcol.processor.batch.default.input]
-		traces  = [otelcol.processor.batch.default.input]
+		metrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]
+		logs    = [otelcol.processor.resourcedetection.default.input]
+		traces  = [otelcol.processor.resourcedetection.default.input]
 	}
 }
 
@@ -72,15 +83,114 @@ otelcol.receiver.otlp "default" {
 	}
 
 	output {
-		metrics = [otelcol.processor.batch.default.input]
-		logs    = [otelcol.processor.batch.default.input]
+		metrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]
+		logs    = [otelcol.processor.resourcedetection.default.input]
 		traces  = [
-			otelcol.processor.batch.default.input,
+			otelcol.processor.resourcedetection.default.input,
 			otelcol.connector.spanlogs.autologging.input,
 		]
 	}
 }
 
+otelcol.processor.resourcedetection "default" {
+	detectors = ["env"]
+
+	output {
+		logs    = [otelcol.processor.k8sattributes.default.input]
+		metrics = [otelcol.processor.k8sattributes.default.input]
+		traces  = [otelcol.processor.k8sattributes.default.input]
+	}
+}
+
+otelcol.processor.transform "add_metric_datapoint_attributes" {
+	error_mode = "ignore"
+
+	metric_statements {
+		context    = "datapoint"
+		statements = [
+			`set(attributes["deployment.environment"], resource.attributes["deployment.environment"])`,
+			`set(attributes["service.version"], resource.attributes["service.version"])`,
+		]
+	}
+
+	output {
+		metrics = [otelcol.processor.k8sattributes.default.input]
+	}
+}
+
+otelcol.processor.k8sattributes "default" {
+	extract {
+		metadata = [
+			"k8s.namespace.name",
+			"k8s.pod.name",
+			"k8s.deployment.name",
+			"k8s.statefulset.name",
+			"k8s.daemonset.name",
+			"k8s.cronjob.name",
+			"k8s.job.name",
+			"k8s.node.name",
+			"k8s.pod.uid",
+			"k8s.pod.start_time",
+		]
+	}
+
+	pod_association {
+		source {
+			from = "connection"
+		}
+	}
+
+	output {
+		metrics = [otelcol.processor.transform.add_resource_attributes.input]
+		logs    = [otelcol.processor.transform.add_resource_attributes.input]
+		traces  = [otelcol.processor.transform.add_resource_attributes.input]
+	}
+}
+
+otelcol.processor.transform "add_resource_attributes" {
+	error_mode = "ignore"
+
+	metric_statements {
+		context    = "resource"
+		statements = [
+			`set(attributes["k8s.cluster.name"], "k3d-k3s-codelab") where attributes["k8s.cluster.name"] == nil`,
+		]
+	}
+
+	log_statements {
+		context    = "resource"
+		statements = [
+			`set(attributes["pod"], attributes["k8s.pod.name"])`,
+			`set(attributes["namespace"], attributes["k8s.namespace.name"])`,
+			`set(attributes["loki.resource.labels"], "pod, namespace, cluster, job")`,
+			`set(attributes["k8s.cluster.name"], "k3d-k3s-codelab") where attributes["k8s.cluster.name"] == nil`,
+		]
+	}
+
+	trace_statements {
+		context    = "resource"
+		statements = [
+			`set(attributes["k8s.cluster.name"], "k3d-k3s-codelab") where attributes["k8s.cluster.name"] == nil`,
+		]
+	}
+
+	output {
+		metrics = [otelcol.processor.filter.default.input]
+		logs    = [otelcol.processor.filter.default.input]
+		traces  = [otelcol.processor.filter.default.input]
+	}
+}
+
+otelcol.processor.filter "default" {
+	error_mode = "ignore"
+
+	output {
+		metrics = [otelcol.processor.batch.default.input]
+		logs    = [otelcol.processor.batch.default.input]
+		traces  = [otelcol.processor.batch.default.input]
+	}
+}
+
 otelcol.processor.batch "default" {
 	send_batch_size     = 16384
 	send_batch_max_size = 0
@@ -98,14 +208,6 @@ otelcol.processor.memory_limiter "default" {
 	limit_percentage       = 50
 	spike_limit_percentage = 30
 
-	output {
-		metrics = [otelcol.processor.k8sattributes.default.input]
-		logs    = [otelcol.processor.k8sattributes.default.input]
-		traces  = [otelcol.processor.k8sattributes.default.input]
-	}
-}
-
-otelcol.processor.k8sattributes "default" {
 	output {
 		metrics = [otelcol.exporter.prometheus.tracesmetrics.input]
 		logs    = [otelcol.exporter.loki.traceslogs.input]

diff --git a/kubernetes/common/grafana-agent/manifests/k8s-all-in-one.yaml b/kubernetes/common/grafana-agent/manifests/k8s-all-in-one.yaml
@@ -136,33 +136,7 @@ data:
   config.river: "/*\nThe following example shows using the default all logs processing
     module, for\na single tenant and specifying the destination url/credentials via
     environment\nvariables.\n*/\nlogging {\n\tlevel  = coalesce(env(\"AGENT_LOG_LEVEL\"),
-    \"info\")\n\tformat = \"logfmt\"\n}\n\nmodule.file \"lgtmp\" {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"),
-    \"/etc/agent-modules\") + \"/lgtmp.river\"\n\n\targuments {\n\t\tcluster           =
-    coalesce(env(\"CLUSTER\"), \"k3d-k3s-codelab\")\n\t\tlogs_endpoint     = coalesce(env(\"LOGS_ENDPOINT\"),
-    \"http://nginx.gateway.svc:3100\")\n\t\tmetrics_endpoint  = coalesce(env(\"METRICS_ENDPOINT\"),
-    \"http://nginx.gateway.svc:8080\")\n\t\tprofiles_endpoint = coalesce(env(\"PROFILES_ENDPOINT\"),
-    \"http://nginx.gateway.svc:4040\")\n\t\ttraces_endpoint   = coalesce(env(\"TRACES_ENDPOINT\"),
-    \"nginx.gateway.svc:4317\")\n\t}\n}\n\n/********************************************\n
-    * Metrics\n ********************************************/\nmodule.file \"metrics_primary\"
-    {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\")
-    + \"/metrics.river\"\n\n\targuments {\n\t\tforward_to = [module.file.lgtmp.exports.metrics_receiver]\n\t\tclustering
-    = true\n\t}\n}\n\n/********************************************\n * Logs\n ********************************************/\nmodule.git
-    \"logs_primary\" {\n\trepository = \"https://github.com/grafana/agent-modules.git\"\n\trevision
-    \  = \"main\"\n\tpath       = \"modules/kubernetes/logs/all.river\"\n\n\targuments
-    {\n\t\tforward_to = [module.file.lgtmp.exports.logs_receiver]\n\t}\n}\n\n/********************************************\n
-    * Traces\n ********************************************/\nmodule.file \"traces_primary\"
-    {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\")
-    + \"/traces.river\"\n\n\targuments {\n\t\tmetrics_forward_to = [module.file.lgtmp.exports.metrics_receiver]\n\t\tlogs_forward_to
-    \   = [module.file.lgtmp.exports.logs_receiver]\n\t\ttraces_forward_to  = [module.file.lgtmp.exports.traces_receiver]\n\t\tcluster
-    \           = coalesce(env(\"CLUSTER\"), \"k3d-k3s-codelab\")\n\t}\n}\n\n/********************************************\n
-    * Profiles\n ********************************************/\nmodule.file \"profiles_primary\"
-    {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\")
-    + \"/profiles.river\"\n\n\targuments {\n\t\tforward_to = [module.file.lgtmp.exports.profiles_receiver]\n\t\tclustering
-    = true\n\t}\n}\n\n/********************************************\n * Agent Integrations\n
-    ********************************************/\nmodule.file \"agent_integrations\"
-    {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\")
-    + \"/integrations.river\"\n\n\targuments {\n\t\tname       = \"agent-integrations\"\n\t\tnamespace
-    \ = \"monitoring-system\"\n\t\tforward_to = [module.file.lgtmp.exports.metrics_receiver]\n\t}\n}\n"
+    \"info\")\n\tformat = \"logfmt\"\n}\n"
 kind: ConfigMap
 metadata:
   name: agent-config
@@ -561,31 +535,56 @@ data:
     = false\n\t\t}\n\n\t\tprofile.process_cpu {\n\t\t\tenabled = false\n\t\t}\n\n\t\tprofile.goroutine
     {\n\t\t\tenabled = false\n\t\t}\n\n\t\tprofile.block {\n\t\t\tenabled = false\n\t\t}\n\n\t\tprofile.mutex
     {\n\t\t\tenabled = false\n\t\t}\n\n\t\tprofile.fgprof {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n"
-  traces.river: "/*\nModule: traces\n*/\nargument \"traces_forward_to\" {\n\toptional
-    = false\n}\n\nargument \"logs_forward_to\" {\n\toptional = false\n}\n\nargument
+  traces.river: "/*\nModule: traces\n*/\n\n/********************************************\n
+    * ARGUMENTS\n ********************************************/\nargument \"traces_forward_to\"
+    {\n\toptional = false\n}\n\nargument \"logs_forward_to\" {\n\toptional = false\n}\n\nargument
     \"metrics_forward_to\" {\n\toptional = false\n}\n\nargument \"cluster\" {\n\toptional
     = true\n\tdefault  = \"k3d-k3s-codelab\"\n}\n\nargument \"otlp_http_endpoint\"
     {\n\toptional = true\n\tdefault  = \"0.0.0.0:4318\"\n}\n\nargument \"otlp_grpc_endpoint\"
     {\n\toptional = true\n\tdefault  = \"0.0.0.0:4317\"\n}\n\n/********************************************\n
+    * EXPORTS\n ********************************************/\nexport \"agent_traces_input\"
+    {\n\tvalue = otelcol.processor.batch.default.input\n}\n\n/********************************************\n
     * Jaeger for Metrics Logs Traces\n ********************************************/\n\notelcol.receiver.jaeger
     \"default\" {\n\tprotocols {\n\t\tgrpc {\n\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t}\n\n\t\tthrift_http
     {\n\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t}\n\n\t\tthrift_binary {\n\t\t\tendpoint
     = \"0.0.0.0:6832\"\n\t\t}\n\n\t\tthrift_compact {\n\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t}\n\t}\n\n\toutput
-    {\n\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\tlogs    = [otelcol.processor.batch.default.input]\n\t\ttraces
-    \ = [otelcol.processor.batch.default.input]\n\t}\n}\n\n/********************************************\n
+    {\n\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\t\tlogs
+    \   = [otelcol.processor.resourcedetection.default.input]\n\t\ttraces  = [otelcol.processor.resourcedetection.default.input]\n\t}\n}\n\n/********************************************\n
     * Otelcol for Metrics Logs Traces\n ********************************************/\n//
     https://grafana.com/docs/agent/latest/flow/reference/components/otelcol.receiver.otlp/\notelcol.receiver.otlp
     \"default\" {\n\tgrpc {\n\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t}\n\n\thttp
     {\n\t\tendpoint = argument.otlp_http_endpoint.value\n\t}\n\n\toutput {\n\t\tmetrics
-    = [otelcol.processor.batch.default.input]\n\t\tlogs    = [otelcol.processor.batch.default.input]\n\t\ttraces
-    \ = [\n\t\t\totelcol.processor.batch.default.input,\n\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t]\n\t}\n}\n\notelcol.processor.batch
+    = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\t\tlogs
+    \   = [otelcol.processor.resourcedetection.default.input]\n\t\ttraces  = [\n\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t]\n\t}\n}\n\notelcol.processor.resourcedetection
+    \"default\" {\n\tdetectors = [\"env\"]\n\n\toutput {\n\t\tlogs    = [otelcol.processor.k8sattributes.default.input]\n\t\tmetrics
+    = [otelcol.processor.k8sattributes.default.input]\n\t\ttraces  = [otelcol.processor.k8sattributes.default.input]\n\t}\n}\n\notelcol.processor.transform
+    \"add_metric_datapoint_attributes\" {\n\terror_mode = \"ignore\"\n\n\tmetric_statements
+    {\n\t\tcontext    = \"datapoint\"\n\t\tstatements = [\n\t\t\t`set(attributes[\"deployment.environment\"],
+    resource.attributes[\"deployment.environment\"])`,\n\t\t\t`set(attributes[\"service.version\"],
+    resource.attributes[\"service.version\"])`,\n\t\t]\n\t}\n\n\toutput {\n\t\tmetrics
+    = [otelcol.processor.k8sattributes.default.input]\n\t}\n}\n\notelcol.processor.k8sattributes
+    \"default\" {\n\textract {\n\t\tmetadata = [\n\t\t\t\"k8s.namespace.name\",\n\t\t\t\"k8s.pod.name\",\n\t\t\t\"k8s.deployment.name\",\n\t\t\t\"k8s.statefulset.name\",\n\t\t\t\"k8s.daemonset.name\",\n\t\t\t\"k8s.cronjob.name\",\n\t\t\t\"k8s.job.name\",\n\t\t\t\"k8s.node.name\",\n\t\t\t\"k8s.pod.uid\",\n\t\t\t\"k8s.pod.start_time\",\n\t\t]\n\t}\n\n\tpod_association
+    {\n\t\tsource {\n\t\t\tfrom = \"connection\"\n\t\t}\n\t}\n\n\toutput {\n\t\tmetrics
+    = [otelcol.processor.transform.add_resource_attributes.input]\n\t\tlogs    = [otelcol.processor.transform.add_resource_attributes.input]\n\t\ttraces
+    \ = [otelcol.processor.transform.add_resource_attributes.input]\n\t}\n}\n\notelcol.processor.transform
+    \"add_resource_attributes\" {\n\terror_mode = \"ignore\"\n\n\tmetric_statements
+    {\n\t\tcontext    = \"resource\"\n\t\tstatements = [\n\t\t\t`set(attributes[\"k8s.cluster.name\"],
+    \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t]\n\t}\n\n\tlog_statements
+    {\n\t\tcontext    = \"resource\"\n\t\tstatements = [\n\t\t\t`set(attributes[\"pod\"],
+    attributes[\"k8s.pod.name\"])`,\n\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t`set(attributes[\"loki.resource.labels\"],
+    \"pod, namespace, cluster, job\")`,\n\t\t\t`set(attributes[\"k8s.cluster.name\"],
+    \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t]\n\t}\n\n\ttrace_statements
+    {\n\t\tcontext    = \"resource\"\n\t\tstatements = [\n\t\t\t`set(attributes[\"k8s.cluster.name\"],
+    \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t]\n\t}\n\n\toutput
+    {\n\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\tlogs    = [otelcol.processor.filter.default.input]\n\t\ttraces
+    \ = [otelcol.processor.filter.default.input]\n\t}\n}\n\notelcol.processor.filter
+    \"default\" {\n\terror_mode = \"ignore\"\n\n\toutput {\n\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\tlogs
+    \   = [otelcol.processor.batch.default.input]\n\t\ttraces  = [otelcol.processor.batch.default.input]\n\t}\n}\n\notelcol.processor.batch
     \"default\" {\n\tsend_batch_size     = 16384\n\tsend_batch_max_size = 0\n\ttimeout
     \            = \"2s\"\n\n\toutput {\n\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\tlogs
     \   = [otelcol.processor.memory_limiter.default.input]\n\t\ttraces  = [otelcol.processor.memory_limiter.default.input]\n\t}\n}\n\notelcol.processor.memory_limiter
     \"default\" {\n\tcheck_interval         = \"1s\"\n\tlimit_percentage       = 50\n\tspike_limit_percentage
-    = 30\n\n\toutput {\n\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\tlogs
-    \   = [otelcol.processor.k8sattributes.default.input]\n\t\ttraces  = [otelcol.processor.k8sattributes.default.input]\n\t}\n}\n\notelcol.processor.k8sattributes
-    \"default\" {\n\toutput {\n\t\tmetrics = [otelcol.exporter.prometheus.tracesmetrics.input]\n\t\tlogs
+    = 30\n\n\toutput {\n\t\tmetrics = [otelcol.exporter.prometheus.tracesmetrics.input]\n\t\tlogs
     \   = [otelcol.exporter.loki.traceslogs.input]\n\t\ttraces  = argument.traces_forward_to.value\n\t}\n}\n\notelcol.exporter.prometheus
     \"tracesmetrics\" {\n\tforward_to = argument.metrics_forward_to.value\n}\n\notelcol.exporter.loki
     \"traceslogs\" {\n\tforward_to = [loki.process.traceslogs.receiver]\n}\n\n// The
@@ -614,7 +613,7 @@ data:
     {\n\t\tvalue = \"anonymous\"\n\t}\n\n\tforward_to = argument.logs_forward_to.value\n}\n"
 kind: ConfigMap
 metadata:
-  name: agent-modules-k2kfd9mm27
+  name: agent-modules-bcccgk262t
   namespace: monitoring-system
 ---
 apiVersion: v1
@@ -754,6 +753,7 @@ spec:
         profiles.grafana.com/goroutine.scrape: "true"
         profiles.grafana.com/memory.port_name: http-metrics
         profiles.grafana.com/memory.scrape: "true"
+        pyroscope.io/service_name: grafana-agent
       labels:
         app.kubernetes.io/instance: grafana-agent
         app.kubernetes.io/name: grafana-agent
@@ -838,7 +838,7 @@ spec:
           path: /var/log
         name: varlog
       - configMap:
-          name: agent-modules-k2kfd9mm27
+          name: agent-modules-bcccgk262t
         name: agent-modules
 ---
 apiVersion: monitoring.coreos.com/v1

diff --git a/kubernetes/common/grafana-agent/values-k3d-k3s.yaml b/kubernetes/common/grafana-agent/values-k3d-k3s.yaml
@@ -59,6 +59,7 @@ controller:
     logs.agent.grafana.com/scrape: "true"
     logs.agent.grafana.com/scrub-level: "debug" # Boolean whether or not the level should be dropped from the log message (as it is a label).
     # logs.agent.grafana.com/tenant: ".*"       # Allow a pod to override the tenant for its logs.
+    pyroscope.io/service_name: grafana-agent
     profiles.grafana.com/memory.scrape: "true"
     profiles.grafana.com/memory.port_name: "http-metrics"
     profiles.grafana.com/cpu.scrape: "true"