fix(otel): ensuring service will start

Pocket · Oct 8, 2024 · e5e07e3 · e5e07e3
1 parent e3b7d37
commit e5e07e3
Show file tree

Hide file tree

Showing 6 changed files with 36 additions and 28 deletions.
diff --git a/infrastructure/otel-collector/src/config/index.ts b/infrastructure/otel-collector/src/config/index.ts
@@ -27,7 +27,7 @@ export const config = {
     env_code: isDev ? 'dev' : 'prod',
   },
   healthCheck: {
-    command: ['CMD-SHELL', 'curl -f http://localhost/status || exit 1'],
+    command: ['CMD-SHELL', 'curl -f http://localhost:3000/status || exit 1'],
     interval: 15,
     retries: 3,
     timeout: 5,

diff --git a/infrastructure/otel-collector/src/main.ts b/infrastructure/otel-collector/src/main.ts
@@ -91,8 +91,7 @@ class OTELCollector extends TerraformStack {
       containerConfigs: [
         {
           healthCheck: config.healthCheck,
-          name: 'otel-collector',
-          containerImage: 'pocket/opentelemetry-collector-contrib',
+          name: 'app',
           essential: true,
           portMappings: [{ containerPort: 3000, hostPort: 3000 }],
           logMultilinePattern: '^\\S.+',
@@ -109,7 +108,6 @@ class OTELCollector extends TerraformStack {
               valueFrom: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:Shared/GCP_SA_TRACES:::`,
             },
           ],
-          repositoryCredentialsParam: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:Shared/DockerHub`,
         },
       ],
       codeDeploy: {

diff --git a/servers/otel-collector/Dockerfile b/servers/otel-collector/Dockerfile
@@ -1,13 +1,10 @@
 # Use an alpine image as our base image, this allows us to define our own entry point script and have access to sh or bash, which the otel image does not have
 FROM alpine:3
+RUN apk add nginx curl
 
 # Copy the otelcol-contrib binary from the official image
 COPY --from=otel/opentelemetry-collector-contrib:0.111.0 /otelcol-contrib /otelcol-contrib
-
 COPY otel-collector-config.yaml /etc/otelcol-contrib/config.yaml
-
-
-RUN apk add nginx curl
 COPY httpd.conf /etc/nginx/http.d/default.conf
 
 COPY entrypoint.sh /entrypoint.sh

diff --git a/servers/otel-collector/entrypoint.sh b/servers/otel-collector/entrypoint.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
 set -e
 echo "$GOOGLE_APPLICATION_CREDENTIALS_JSON" > /etc/otelcol-contrib/key.json
-nginx
+nginx &
 exec "$@"
diff --git a/servers/otel-collector/httpd.conf b/servers/otel-collector/httpd.conf
@@ -5,7 +5,8 @@ server {
 
       # Everything is a 200
       location /status {
-          proxy_pass http://localhost:13133;
+          default_type application/json;
+          return 200 '{"status": "ok"}';
       }
 
       location / {

diff --git a/servers/otel-collector/otel-collector-config.yaml b/servers/otel-collector/otel-collector-config.yaml
@@ -6,13 +6,7 @@ receivers:
       # grpc:
       #   endpoint: "0.0.0.0:4317" # gRPC port for receiving traces, metrics, and logs over gRPC
       http:
-        endpoint: "localhost:4318" # http port for receiving traces, metrics, and logs over http
-
-extensions:
-  health_check:
-    endpoint: localhost:13133
-
-
+        endpoint: '0.0.0.0:4318' # http port for receiving traces, metrics, and logs over http
 
 exporters:
   googlecloud:
@@ -23,19 +17,19 @@ exporters:
 processors:
   memory_limiter:
     check_interval: 1s
-    limit_percentage: 30
+    limit_percentage: 70
     spike_limit_percentage: 10
   batch:
     # batch metrics before sending to reduce API usage
     send_batch_max_size: 10100
     send_batch_size: 10000
     timeout: 5s
   resourcedetection/ecs:
-      detectors: [env, ecs]
-      timeout: 2s
-      # Ensures that anything coming from the ECS detector in code will not override any existing attributes.
-      # Otherwise, note, that the ECS detector will be that of where the collector is running, not the service.
-      override: false
+    detectors: [env, ecs]
+    timeout: 2s
+    # Ensures that anything coming from the ECS detector in code will not override any existing attributes.
+    # Otherwise, note, that the ECS detector will be that of where the collector is running, not the service.
+    override: false
   resource/cleanup_for_google:
     attributes:
       - key: service.namespace
@@ -61,13 +55,31 @@ service:
   pipelines:
     traces:
       receivers: [otlp]
-      processors: [resourcedetection/ecs, resource/cleanup_for_google, memory_limiter, batch]
-      exporters: [debug]
+      processors:
+        [
+          resourcedetection/ecs,
+          resource/cleanup_for_google,
+          memory_limiter,
+          batch,
+        ]
+      exporters: [googlecloud]
     metrics:
       receivers: [otlp]
-      processors: [resourcedetection/ecs, resource/cleanup_for_google, memory_limiter, batch]
+      processors:
+        [
+          resourcedetection/ecs,
+          resource/cleanup_for_google,
+          memory_limiter,
+          batch,
+        ]
       exporters: [googlecloud]
     logs:
       receivers: [otlp]
-      processors: [resourcedetection/ecs, resource/cleanup_for_google, memory_limiter, batch]
-      exporters: [debug]
+      processors:
+        [
+          resourcedetection/ecs,
+          resource/cleanup_for_google,
+          memory_limiter,
+          batch,
+        ]
+      exporters: [googlecloud]