diff --git a/docker-compose/common/config/agent-flow/logs.river b/docker-compose/common/config/agent-flow/logs.river index 2bf99ebe..9e36d42b 100644 --- a/docker-compose/common/config/agent-flow/logs.river +++ b/docker-compose/common/config/agent-flow/logs.river @@ -45,6 +45,12 @@ discovery.relabel "containers" { target_label = "pod" } + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + regex = "(.*)" + target_label = "app" + } + rule { source_labels = ["__meta_docker_container_name"] regex = "/(.*)" @@ -95,14 +101,19 @@ otelcol.receiver.otlp "containers" { } otelcol.processor.batch "containers" { + send_batch_size = 16384 + send_batch_max_size = 0 + timeout = "2s" + output { logs = [otelcol.processor.memory_limiter.containers.input] } } otelcol.processor.memory_limiter "containers" { - check_interval = "1s" - limit = "256MiB" + check_interval = "1s" + limit_percentage = 50 + spike_limit_percentage = 30 output { logs = [otelcol.exporter.loki.containers.input] @@ -110,5 +121,5 @@ otelcol.processor.memory_limiter "containers" { } otelcol.exporter.loki "containers" { - forward_to = [module.file.docker_compose.exports.logs_receiver] + forward_to = [loki.process.containers.receiver] } diff --git a/docker-compose/common/config/loki/datasources.yaml b/docker-compose/common/config/loki/datasources.yaml index 7c992e9d..d158b0c1 100644 --- a/docker-compose/common/config/loki/datasources.yaml +++ b/docker-compose/common/config/loki/datasources.yaml @@ -21,31 +21,6 @@ datasources: uid: logs url: http://gateway:3100 basicAuth: false - isDefault: true # Note: default datasources - version: 1 - editable: true - -# Tempo for traces -- name: Traces - type: tempo - access: proxy - orgId: 1 - uid: traces - url: http://gateway:3200 - basicAuth: false - isDefault: false - version: 1 - editable: true - apiVersion: 1 - -# Pyroscope for profiles -- name: Profiles - type: grafana-pyroscope-datasource - access: proxy - orgId: 1 - uid: profiles - url: http://gateway:4040 - basicAuth: false - isDefault: false # Note: default datasources + isDefault: true version: 1 editable: true diff --git a/docker-compose/common/config/mimir/datasources.yaml b/docker-compose/common/config/mimir/datasources.yaml index 7030f105..c02fb02a 100644 --- a/docker-compose/common/config/mimir/datasources.yaml +++ b/docker-compose/common/config/mimir/datasources.yaml @@ -24,28 +24,3 @@ datasources: isDefault: false version: 1 editable: true - -# Tempo for traces -- name: Traces - type: tempo - access: proxy - orgId: 1 - uid: traces - url: http://gateway:3200 - basicAuth: false - isDefault: false - version: 1 - editable: true - apiVersion: 1 - -# Pyroscope for profiles -- name: Profiles - type: grafana-pyroscope-datasource - access: proxy - orgId: 1 - uid: profiles - url: http://gateway:4040 - basicAuth: false - isDefault: false # Note: default datasources - version: 1 - editable: true diff --git a/docker-compose/common/config/pyroscope/datasources.yaml b/docker-compose/common/config/pyroscope/datasources.yaml index edfcd086..e61e4dde 100644 --- a/docker-compose/common/config/pyroscope/datasources.yaml +++ b/docker-compose/common/config/pyroscope/datasources.yaml @@ -1,43 +1,10 @@ apiVersion: 1 -datasources: -# Mimir for metrics -- name: Metrics - type: prometheus - uid: metrics - access: proxy - orgId: 1 - url: http://gateway:8080/prometheus - basicAuth: false - isDefault: false - version: 1 - editable: true - -# Loki for logs -- name: Logs - type: loki - access: proxy - orgId: 1 - uid: logs - url: http://gateway:3100 - basicAuth: false - isDefault: false - version: 1 - editable: true - -# Tempo for traces -- name: Traces - type: tempo - access: proxy - orgId: 1 - uid: traces - url: http://gateway:3200 - basicAuth: false - isDefault: false - version: 1 - editable: true - apiVersion: 1 +deleteDatasources: +- name: Profiles + uid: profiles +datasources: # Pyroscope for profiles - name: Profiles type: grafana-pyroscope-datasource @@ -46,6 +13,6 @@ datasources: uid: profiles url: http://gateway:4040 basicAuth: false - isDefault: true # Note: default datasources + isDefault: true version: 1 editable: true diff --git a/docker-compose/common/config/pyroscope/microservices-mode-profiles.yaml b/docker-compose/common/config/pyroscope/microservices-mode-profiles.yaml index 606df64c..8dc64e91 100644 --- a/docker-compose/common/config/pyroscope/microservices-mode-profiles.yaml +++ b/docker-compose/common/config/pyroscope/microservices-mode-profiles.yaml @@ -1,6 +1,10 @@ analytics: reporting_enabled: false +# https://grafana.com/docs/pyroscope/latest/configure-server/configure-disk-storage/#configure-pyroscope-disk-storage +pyroscopedb: + max_block_duration: 5m + # https://grafana.com/docs/pyroscope/latest/configure-server/reference-configuration-parameters/#use-environment-variables-in-the-configuration storage: backend: s3 diff --git a/docker-compose/common/config/tempo/datasources.yaml b/docker-compose/common/config/tempo/datasources.yaml index 106f2c43..d9a14053 100644 --- a/docker-compose/common/config/tempo/datasources.yaml +++ b/docker-compose/common/config/tempo/datasources.yaml @@ -7,8 +7,6 @@ deleteDatasources: uid: logs - name: Traces uid: traces -- name: Profiles - uid: profiles datasources: # Mimir for metrics @@ -78,15 +76,3 @@ datasources: query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags, status_code="STATUS_CODE_ERROR"}[$$__rate_interval]))' - name: '(D) Duration' query: 'histogram_quantile(0.9, sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[$$__rate_interval])) by (le))' - -# Pyroscope for profiles -- name: Profiles - type: grafana-pyroscope-datasource - access: proxy - orgId: 1 - uid: profiles - url: http://gateway:4040 - basicAuth: false - isDefault: false # Note: default datasources - version: 1 - editable: true diff --git a/docker-compose/monolithic-mode/profiles/docker-compose.yaml b/docker-compose/monolithic-mode/profiles/docker-compose.yaml index c841bb10..95f00889 100644 --- a/docker-compose/monolithic-mode/profiles/docker-compose.yaml +++ b/docker-compose/monolithic-mode/profiles/docker-compose.yaml @@ -47,8 +47,6 @@ services: - -target=all - -log.level=warn - -config.expand-env=true - # ports: - # - "4040:4040" healthcheck: test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:4040/ready || exit 1" ] interval: 15s diff --git a/kubernetes/microservices-mode/profiles/configs/pyroscope.yaml b/kubernetes/microservices-mode/profiles/configs/pyroscope.yaml index 36c6e759..e9eedf70 100644 --- a/kubernetes/microservices-mode/profiles/configs/pyroscope.yaml +++ b/kubernetes/microservices-mode/profiles/configs/pyroscope.yaml @@ -1,6 +1,10 @@ analytics: reporting_enabled: false +# https://grafana.com/docs/pyroscope/latest/configure-server/configure-disk-storage/#configure-pyroscope-disk-storage +pyroscopedb: + max_block_duration: 5m + storage: backend: s3 s3: diff --git a/kubernetes/monolithic-mode/profiles/configs/pyroscope.yaml b/kubernetes/monolithic-mode/profiles/configs/pyroscope.yaml index 50b42da9..75e54687 100644 --- a/kubernetes/monolithic-mode/profiles/configs/pyroscope.yaml +++ b/kubernetes/monolithic-mode/profiles/configs/pyroscope.yaml @@ -1,6 +1,10 @@ analytics: reporting_enabled: false +# https://grafana.com/docs/pyroscope/latest/configure-server/configure-disk-storage/#configure-pyroscope-disk-storage +pyroscopedb: + max_block_duration: 5m + # https://grafana.com/docs/pyroscope/latest/configure-server/reference-configuration-parameters/#use-environment-variables-in-the-configuration storage: backend: s3