Add customized usage examples

Signed-off-by: Weifeng Wang <[email protected]> Update README.md Signed-off-by: Weifeng Wang <[email protected]>
qclaogui · Mar 26, 2024 · bdec5a2 · bdec5a2
1 parent 5e8fbdd
commit bdec5a2
Show file tree

Hide file tree

Showing 9 changed files with 385 additions and 201 deletions.
diff --git a/Makefile b/Makefile
@@ -88,8 +88,8 @@ up-monolithic-mode-all-in-one: ## Run monolithic-mode all-in-one
 	$(info ******************** run monolithic-mode all-in-one ********************)
 	docker compose \
 		--project-name all-in-one \
-		--project-directory . \
-		--file ./compose.yaml \
+		--project-directory docker-compose/monolithic-mode/all-in-one \
+		--file docker-compose/monolithic-mode/all-in-one/compose.yaml \
 		--env-file docker-compose/common/config/.env \
 		up -d --remove-orphans
 	@$(call echo_info, "Go to http://localhost:3000/explore for the metrics.")

diff --git a/README.md b/README.md
@@ -27,13 +27,14 @@ An simple use case `compose.yaml` like so:
 ```yaml
 # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later.
 include:
-# - path: https://github.com/qclaogui/codelab-monitoring.git # All in one(Logs Traces Metrics Profiles)
 - path: https://github.com/qclaogui/codelab-monitoring.git#main:docker-compose/monolithic-mode/logs/compose.yaml # Just Metrics and Logs
 
-# https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md
 services:
   github-exporter:
-    image: githubexporter/github-exporter:latest
+    # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md
+    labels:
+      metrics.agent.grafana.com/scrape: false
+    image: githubexporter/github-exporter:1.1.0
     environment:
     - REPOS=qclaogui/codelab-monitoring
 ```

diff --git a/compose.override.yaml b/compose.override.yaml
@@ -0,0 +1,84 @@
+
+# https://docs.docker.com/compose/multiple-compose-files/include/#include-and-overrides
+
+# https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md
+x-labels: &profiles-labels
+  profiles.agent.grafana.com/cpu.scrape: true
+  profiles.agent.grafana.com/memory.scrape: true
+  profiles.agent.grafana.com/goroutine.scrape: true
+
+x-environment: &jaeger-environment
+  JAEGER_AGENT_HOST: grafana-agent
+  JAEGER_AGENT_PORT: 6831
+  JAEGER_SAMPLER_TYPE: const
+  JAEGER_SAMPLER_PARAM: 1
+
+services:
+  # override compose.yaml included service grafana-agent's entrypoint to use monolithic-mode-all.river. default(metrics.river)
+  grafana-agent:
+    entrypoint:
+      - /bin/grafana-agent
+      - run
+      - /etc/agent-config/monolithic-mode-all.river # Note: Agent use monolithic-mode-all.river
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --cluster.enabled=true
+      - --cluster.join-addresses=grafana-agent-cluster:12345
+      - --disable-reporting=true
+
+  # override compose.yaml included service loki's labels and environment to enabled profiles and traces data collection
+  loki:
+    labels:
+      <<: *profiles-labels
+      profiles.agent.grafana.com/service_name: loki
+      logs.agent.grafana.com/log-format: json
+    environment:
+      <<: *jaeger-environment
+      JAEGER_TAGS: app=loki
+
+  # override compose.yaml included service grafana's labels and environment to enabled profiles data collection
+  grafana:
+    labels:
+      <<: *profiles-labels
+      profiles.agent.grafana.com/service_name: grafana
+      profiles.agent.grafana.com/port: 6060
+      logs.agent.grafana.com/log-format: logfmt
+    environment:
+      - GF_LOG_LEVEL=warn
+      - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin_password}
+      - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor tracesEmbeddedFlameGraph traceqlSearch correlations metricsSummary traceToMetrics traceToProfiles
+      - GF_DIAGNOSTICS_PROFILING_ENABLED=true
+      - GF_DIAGNOSTICS_PROFILING_ADDR=0.0.0.0
+      - GF_DIAGNOSTICS_PROFILING_PORT=6060
+
+  # override compose.yaml included service tempo's labels and environment to enabled profiles and traces data collection
+  tempo:
+    labels:
+      <<: *profiles-labels
+      profiles.agent.grafana.com/service_name: tempo
+      logs.agent.grafana.com/log-format: logfmt
+    depends_on:
+      minio:
+        condition: service_healthy
+      mimir:
+        condition: service_healthy
+    environment:
+      <<: *jaeger-environment
+      JAEGER_TAGS: app=tempo
+
+  # override compose.yaml included service mimir's labels and environment to enabled profiles and traces data collection
+  mimir:
+    labels:
+      <<: *profiles-labels
+      profiles.agent.grafana.com/service_name: mimir
+      logs.agent.grafana.com/log-format: logfmt
+    environment:
+      <<: *jaeger-environment
+      JAEGER_TAGS: app=mimir
+
+  # override compose.yaml included service pyroscope's labels and environment to enabled traces data collection
+  pyroscope:
+    labels:
+      - logs.agent.grafana.com/log-format=logfmt
+    environment:
+      <<: *jaeger-environment
+      JAEGER_TAGS: app=pyroscope
diff --git a/compose.yaml b/compose.yaml
@@ -1,56 +1,31 @@
 version: '3.9'
 
 # ============================================================================ #
-#                  Monolithic Mode - All in one
+#                  Highly Customized Usage Examples
 # ============================================================================ #
 
+# ```shell
+#   git clone https://github.com/qclaogui/codelab-monitoring.git && cd "$(basename "$_" .git)"
+#
+#   COMPOSE_EXPERIMENTAL_GIT_REMOTE=true docker compose up -d --remove-orphans
+# ```
+
 # Note: 
 # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later.
+# docs: https://docs.docker.com/compose/multiple-compose-files/include/#include-and-overrides
 include:
   - path: ./docker-compose/common/compose-include/minio.yaml
   - path: ./docker-compose/common/compose-include/memcached.yaml
   - path: ./docker-compose/common/compose-include/load-rules-to-mimir.yaml
-
-# https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md
-x-labels: &profiles-labels
-  profiles.agent.grafana.com/cpu.scrape: true
-  profiles.agent.grafana.com/memory.scrape: true
-  profiles.agent.grafana.com/goroutine.scrape: true
-
-x-environment: &jaeger-environment
-  JAEGER_AGENT_HOST: grafana-agent
-  JAEGER_AGENT_PORT: 6831
-  JAEGER_SAMPLER_TYPE: const
-  JAEGER_SAMPLER_PARAM: 1
+  #  Grafana LGTMP Stack Components
+  - path: ./docker-compose/common/compose-include/loki.yaml
+  - path: ./docker-compose/common/compose-include/grafana-agent.yaml
+  - path: ./docker-compose/common/compose-include/grafana.yaml
+  - path: ./docker-compose/common/compose-include/tempo.yaml
+  - path: ./docker-compose/common/compose-include/mimir.yaml
+  - path: ./docker-compose/common/compose-include/pyroscope.yaml
 
 services:
-  grafana-agent:
-    labels:
-      logs.agent.grafana.com/log-format: logfmt
-    depends_on:
-      gateway:
-        condition: service_healthy
-    image: ${AGENT_IMAGE:-docker.io/grafana/agent:v0.40.3}
-    volumes:
-      - ./docker-compose/common/config/agent-flow:/etc/agent-config
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-      - /:/rootfs:ro
-      - /sys:/sys:ro
-      - /var/lib/docker:/var/lib/docker:ro
-    entrypoint:
-      - /bin/grafana-agent
-      - run
-      - /etc/agent-config/monolithic-mode-all.river
-      - --server.http.listen-addr=0.0.0.0:12345
-      - --cluster.enabled=true
-      - --disable-reporting=true
-    environment:
-      - AGENT_MODE=flow
-      - AGENT_CONFIG_FOLDER=/etc/agent-config
-    # expose 12345 port so we can directly access grafana-agent inside container
-    ports:
-      - "12345:12345"
-
   gateway:
     labels:
       metrics.agent.grafana.com/scrape: false
@@ -77,151 +52,3 @@ services:
       interval: 10s
       timeout: 5s
       retries: 10
-
-  loki:
-    labels:
-      <<: *profiles-labels
-      profiles.agent.grafana.com/service_name: loki
-      logs.agent.grafana.com/log-format: json
-    depends_on:
-      minio:
-        condition: service_healthy
-    image: ${LOKI_IMAGE:-docker.io/grafana/loki:2.9.6}
-    volumes:
-      - ./docker-compose/common/config/loki:/etc/loki
-    command:
-      - -config.file=/etc/loki/monolithic-mode-logs.yaml
-      - -target=all
-      - -config.expand-env=true
-    environment:
-      <<: *jaeger-environment
-      JAEGER_TAGS: app=loki
-    healthcheck:
-      test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1" ]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    # expose 33100 port so we can directly access loki inside container
-    ports:
-      - "33100:3100"
-    networks:
-      default:
-        aliases:
-          - loki-memberlist
-
-  tempo:
-    labels:
-      <<: *profiles-labels
-      profiles.agent.grafana.com/service_name: tempo
-      logs.agent.grafana.com/log-format: logfmt
-    depends_on:
-      minio:
-        condition: service_healthy
-      mimir:
-        condition: service_healthy
-    image: ${TEMPO_IMAGE:-docker.io/grafana/tempo:2.4.1}
-    volumes:
-      - ./docker-compose/common/config/tempo:/etc/tempo
-    command:
-      - -config.file=/etc/tempo/monolithic-mode-traces.yaml
-      - -target=all
-      - -config.expand-env=true
-    environment:
-      <<: *jaeger-environment
-      JAEGER_TAGS: app=tempo
-    healthcheck:
-      test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3200/ready || exit 1" ]
-      interval: 10s
-      timeout: 5s
-      retries: 10
-    # expose 33200 port so we can directly access tempo inside container
-    ports:
-      - "33200:3200"
-
-  mimir:
-    labels:
-      <<: *profiles-labels
-      profiles.agent.grafana.com/service_name: mimir
-      logs.agent.grafana.com/log-format: logfmt
-    depends_on:
-      minio:
-        condition: service_healthy
-    image: ${MIMIR_IMAGE:-docker.io/grafana/mimir:2.11.0}
-    volumes:
-      - ./docker-compose/common/config/mimir:/etc/mimir
-    command:
-      - -config.file=/etc/mimir/monolithic-mode-metrics.yaml
-      - -target=all
-      - -config.expand-env=true
-      - -ruler.max-rules-per-rule-group=50
-    environment:
-      <<: *jaeger-environment
-      JAEGER_TAGS: app=mimir
-    healthcheck:
-      test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:8080/ready || exit 1" ]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    # expose 38080 port so we can directly access mimir inside container
-    ports:
-      - "38080:8080"
-    networks:
-      default:
-        aliases:
-          - mimir-memberlist
-
-  pyroscope:
-    labels:
-      - logs.agent.grafana.com/log-format=logfmt
-    depends_on:
-      minio:
-        condition: service_healthy
-    image: ${PYROSCOPE_IMAGE:-docker.io/grafana/pyroscope:1.5.0}
-    container_name: pyroscope
-    volumes:
-      - ./docker-compose/common/config/pyroscope/monolithic-mode-profiles.yaml:/etc/config.yaml
-    command:
-      - -config.file=/etc/config.yaml
-      - -config.expand-env=true
-    environment:
-      <<: *jaeger-environment
-      JAEGER_TAGS: app=pyroscope
-    healthcheck:
-      test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:4040/ready || exit 1" ]
-      interval: 15s
-      timeout: 10s
-      retries: 5
-    # expose 34040 port so we can directly access pyroscope inside container
-    ports:
-      - "34040:4040"
-
-  grafana:
-    labels:
-      <<: *profiles-labels
-      profiles.agent.grafana.com/service_name: grafana
-      profiles.agent.grafana.com/port: 6060
-      logs.agent.grafana.com/log-format: logfmt
-    image: ${GRAFANA_IMAGE:-docker.io/grafana/grafana:10.4.1}
-    command:
-      - --config=/etc/grafana-config/grafana.ini
-    volumes:
-      - ./docker-compose/common/config/grafana/grafana.ini:/etc/grafana-config/grafana.ini
-      - ./docker-compose/common/config/grafana/dashboards:/var/lib/grafana/dashboards
-      - ./docker-compose/common/config/grafana/provisioning:/etc/grafana/provisioning
-      - ./monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin
-      - ./monitoring-mixins/go-runtime-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/go-runtime-mixin
-      - ./monitoring-mixins/loki-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/loki-mixin
-      - ./monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin
-      - ./monitoring-mixins/pyroscope-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/pyroscope-mixin
-      - ./monitoring-mixins/memcached-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/memcached-mixin
-      # - ./monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
-    environment:
-      - GF_LOG_LEVEL=warn
-      - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin_password}
-      - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor tracesEmbeddedFlameGraph traceqlSearch correlations metricsSummary traceToMetrics traceToProfiles
-      - GF_DIAGNOSTICS_PROFILING_ENABLED=true
-      - GF_DIAGNOSTICS_PROFILING_ADDR=0.0.0.0
-      - GF_DIAGNOSTICS_PROFILING_PORT=6060
-    ports:
-      - "3000:3000"
-
diff --git a/docker-compose/common/compose-include/pyroscope.yaml b/docker-compose/common/compose-include/pyroscope.yaml
@@ -0,0 +1,22 @@
+services:
+  pyroscope:
+    depends_on:
+      minio:
+        condition: service_healthy
+    image: ${PYROSCOPE_IMAGE:-docker.io/grafana/pyroscope:1.5.0}
+    container_name: pyroscope
+    volumes:
+      - ../config/pyroscope:/etc/pyroscope
+    command:
+      - -config.file=/etc/pyroscope/monolithic-mode-profiles.yaml
+      - -target=all
+      - -config.expand-env=true
+      # - -runtime-config.file=/etc/pyroscope/overrides.yaml
+    healthcheck:
+      test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:4040/ready || exit 1" ]
+      interval: 15s
+      timeout: 10s
+      retries: 5
+    # expose 34040 port so we can directly access pyroscope inside container
+    ports:
+      - "34040:4040"
diff --git a/docker-compose/common/compose-include/tempo.yaml b/docker-compose/common/compose-include/tempo.yaml
@@ -0,0 +1,21 @@
+services:
+  tempo:
+    depends_on:
+      minio:
+        condition: service_healthy
+    image: ${TEMPO_IMAGE:-docker.io/grafana/tempo:2.4.1}
+    restart: always
+    volumes:
+      - ../config/tempo:/etc/tempo
+    command:
+      - -config.file=/etc/tempo/monolithic-mode-traces.yaml
+      - -target=all
+      - -config.expand-env=true
+    healthcheck:
+      test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3200/ready || exit 1" ]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+    # expose 33200 port so we can directly access tempo inside container
+    ports:
+      - "33200:3200"
diff --git a/docker-compose/monolithic-mode/all-in-one/README.md b/docker-compose/monolithic-mode/all-in-one/README.md
@@ -1,3 +1,3 @@
 # Monolithic mode (单体模式) - All in one
 
-[compose.yaml](../../../compose.yaml) //TODO(qc)
+//TODO(qc)