feat(otel): adding in basic tracing via otel

.github/workflows/docker-images.yml

@@ -0,0 +1,29 @@
+name: Docker Images
+on:
+    pull_request:
+    push:
+        branches:
+            - main
+    schedule: # Rebuild images nightly
+      - cron: '0 0 * * *'
+
+jobs:
+  otel:
+      runs-on: ubuntu-latest
+      steps:
+        - name: Set up QEMU
+          uses: docker/setup-qemu-action@v3
+        - name: Set up Docker Buildx
+          uses: docker/setup-buildx-action@v3
+        - name: Login to Docker Hub
+          uses: docker/login-action@v3
+          with:
+              username: ${{ secrets.DOCKERHUB_USERNAME }}
+              password: ${{ secrets.DOCKERHUB_TOKEN }}
+        - name: Build Base
+          uses: docker/build-push-action@v6
+          with:
+            push: ${{ github.ref == 'refs/heads/main' && true || false}}
+            platforms: linux/amd64,linux/arm64
+            tags: pocket/opentelemetry-collector-contrib
+            context: "{{defaultContext}}:images/otel"

docker-compose.yml

@@ -96,3 +96,25 @@ services:
       retries: 30
     ports:
       - '9090:9090'
+
+  otlpcollector:
+    image: grafana/otel-lgtm
+    ports:
+      # there are lots more ports available, but these are the only one's
+      # currently being used
+      - 4317:4317 # OTLP gRPC receiver
+      - 4318:4318 # OTLP http receiver
+      - 3000:3000 # Grafana
+
+  # Uncomment this if you want to run the collector locally and send to GCP instead of the one used for local development
+  # Make sure to have a service json in GOOGLE_APPLICATION_CREDENTIALS_JSON in the .env file
+  # otlpcollector:
+  #   image: pocket/opentelemetry-collector-contrib
+  #   env_file:
+  #     - .env
+  #   ports:
+  #     # there are lots more ports available, but these are the only one's
+  #     # currently being used
+  #     - 4317:4317 # OTLP gRPC receiver
+  #     - 4318:4318 # OTLP http receiver
+  #     - 3000:3000 # Grafana

images/otel/Dockerfile

@@ -0,0 +1,17 @@
+# Use an alpine image as our base image, this allows us to define our own entry point script and have access to sh or bash, which the otel image does not have
+FROM alpine:3
+
+# Copy the otelcol-contrib binary from the official image
+COPY --from=otel/opentelemetry-collector-contrib:0.111.0 /otelcol-contrib /otelcol-contrib
+
+COPY otel-collector-config.yaml /etc/otelcol-contrib/config.yaml
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod a+x /entrypoint.sh
+
+#Set the ENV variable for the google credentials that will be used by the google cloud exporter and mounted via an environment variable that will be saved to this file via our entrypoint script
+ENV GOOGLE_APPLICATION_CREDENTIALS=/etc/otelcol-contrib/key.json
+WORKDIR /
+
+ENTRYPOINT [ "./entrypoint.sh" ]
+
+CMD [ "/otelcol-contrib", "--config", "/etc/otelcol-contrib/config.yaml" ]

images/otel/entrypoint.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e
+echo "$GOOGLE_APPLICATION_CREDENTIALS_JSON" > /etc/otelcol-contrib/key.json
+exec "$@"

images/otel/otel-collector-config.yaml

@@ -0,0 +1,63 @@
+# Note this is only built nightly and will take effect on the next deploy after being built.
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: "0.0.0.0:4317" # gRPC port for receiving traces, metrics, and logs over gRPC
+      http:
+        endpoint: "0.0.0.0:4318" # http port for receiving traces, metrics, and logs over http
+exporters:
+  googlecloud:
+    log:
+      default_log_name: opentelemetry.io/collector-exported-log
+  # debug:
+  #   verbosity: detailed
+processors:
+  memory_limiter:
+    check_interval: 1s
+    limit_percentage: 65
+    spike_limit_percentage: 20
+  batch:
+    # batch metrics before sending to reduce API usage
+    send_batch_max_size: 200
+    send_batch_size: 200
+    timeout: 5s
+  resourcedetection/ecs:
+      detectors: [env, ecs]
+      timeout: 2s
+      override: false
+  resource/cleanup_for_google:
+    attributes:
+      - key: service.namespace
+        value: pocket
+        action: insert
+      # We must set the region to us-east1, as the google exporter uses this to know where to send data to.
+      - key: cloud.region
+        value: us-east1
+        action: upsert
+      - key: cloud.availability_zone
+        value: us-east1-b
+        action: upsert
+      # set this to the ecs task id
+      - key: host.id
+        from_attribute: aws.ecs.task.id
+        action: upsert
+      #  https://opentelemetry.io/docs/specs/semconv/resource/deployment-environment/
+      - key: deployment.environment.name
+        value: ${env:DEPLOYMENT_ENVIRONMENT_NAME}
+        action: upsert
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [resourcedetection/ecs, resource/cleanup_for_google, memory_limiter, batch]
+      exporters: [googlecloud]
+    metrics:
+      receivers: [otlp]
+      processors: [resourcedetection/ecs, resource/cleanup_for_google, memory_limiter, batch]
+      exporters: [googlecloud]
+    logs:
+      receivers: [otlp]
+      processors: [resourcedetection/ecs, resource/cleanup_for_google, memory_limiter, batch]
+      exporters: [googlecloud]

infrastructure/client-api/src/main.ts

@@ -266,6 +266,40 @@ class ClientAPI extends TerraformStack {
             startPeriod: 0,
           },
         },
+        {
+          name: 'otel-collector',
+          containerImage: 'pocket/opentelemetry-collector-contrib',
+          essential: true,
+          logMultilinePattern: '^\\S.+',
+          logGroup: this.createCustomLogGroup('otel-collector'),
+          portMappings: [
+            {
+              hostPort: 4138,
+              containerPort: 4138,
+            },
+            {
+              hostPort: 4137,
+              containerPort: 4137,
+            },
+            {
+              hostPort: 55681,
+              containerPort: 55681,
+            },
+          ],
+          envVars: [
+            {
+              name: 'DEPLOYMENT_ENVIRONMENT_NAME',
+              value: config.tags.env_code,
+            },
+          ],
+          secretEnvVars: [
+            {
+              name: 'GOOGLE_APPLICATION_CREDENTIALS_JSON',
+              valueFrom: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:Shared/GCP_SA_TRACES:::`,
+            },
+          ],
+          repositoryCredentialsParam: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:Shared/DockerHub`,
+        },
       ],
       codeDeploy: {
         useCodeDeploy: true,
@@ -309,19 +343,7 @@ class ClientAPI extends TerraformStack {
             effect: 'Allow',
           },
         ],
-        taskRolePolicyStatements: [
-          {
-            actions: [
-              'xray:PutTraceSegments',
-              'xray:PutTelemetryRecords',
-              'xray:GetSamplingRules',
-              'xray:GetSamplingTargets',
-              'xray:GetSamplingStatisticSummaries',
-            ],
-            resources: ['*'],
-            effect: 'Allow',
-          },
-        ],
+        taskRolePolicyStatements: [],
         taskExecutionDefaultAttachmentArn:
           'arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy',
       },

infrastructure/list-api/src/main.ts

@@ -304,6 +304,40 @@ class ListAPI extends TerraformStack {
           logGroup: this.createCustomLogGroup('app'),
           logMultilinePattern: '^\\S.+',
         },
+        {
+          name: 'otel-collector',
+          containerImage: 'pocket/opentelemetry-collector-contrib',
+          essential: true,
+          logMultilinePattern: '^\\S.+',
+          logGroup: this.createCustomLogGroup('otel-collector'),
+          portMappings: [
+            {
+              hostPort: 4138,
+              containerPort: 4138,
+            },
+            {
+              hostPort: 4137,
+              containerPort: 4137,
+            },
+            {
+              hostPort: 55681,
+              containerPort: 55681,
+            },
+          ],
+          envVars: [
+            {
+              name: 'DEPLOYMENT_ENVIRONMENT_NAME',
+              value: config.tags.env_code,
+            },
+          ],
+          secretEnvVars: [
+            {
+              name: 'GOOGLE_APPLICATION_CREDENTIALS_JSON',
+              valueFrom: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:Shared/GCP_SA_TRACES:::`,
+            },
+          ],
+          repositoryCredentialsParam: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:Shared/DockerHub`,
+        },
       ],
       codeDeploy: {
         useCodeDeploy: true,
@@ -349,11 +383,6 @@ class ListAPI extends TerraformStack {
               'logs:CreateLogStream',
               'logs:DescribeLogStreams',
               'logs:DescribeLogGroups',
-              'xray:PutTraceSegments',
-              'xray:PutTelemetryRecords',
-              'xray:GetSamplingRules',
-              'xray:GetSamplingTargets',
-              'xray:GetSamplingStatisticSummaries',
             ],
             resources: ['*'],
             effect: 'Allow',

infrastructure/parser-graphql-wrapper/src/main.ts

@@ -278,6 +278,40 @@ class ParserGraphQLWrapper extends TerraformStack {
             },
           ],
         },
+        {
+          name: 'otel-collector',
+          containerImage: 'pocket/opentelemetry-collector-contrib',
+          essential: true,
+          logMultilinePattern: '^\\S.+',
+          logGroup: this.createCustomLogGroup('otel-collector'),
+          portMappings: [
+            {
+              hostPort: 4138,
+              containerPort: 4138,
+            },
+            {
+              hostPort: 4137,
+              containerPort: 4137,
+            },
+            {
+              hostPort: 55681,
+              containerPort: 55681,
+            },
+          ],
+          envVars: [
+            {
+              name: 'DEPLOYMENT_ENVIRONMENT_NAME',
+              value: config.tags.env_code,
+            },
+          ],
+          secretEnvVars: [
+            {
+              name: 'GOOGLE_APPLICATION_CREDENTIALS_JSON',
+              valueFrom: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:Shared/GCP_SA_TRACES`,
+            },
+          ],
+          repositoryCredentialsParam: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:Shared/DockerHub`,
+        },
       ],
       codeDeploy: {
         useCodeDeploy: true,
@@ -326,11 +360,6 @@ class ParserGraphQLWrapper extends TerraformStack {
               'logs:CreateLogStream',
               'logs:DescribeLogStreams',
               'logs:DescribeLogGroups',
-              'xray:PutTraceSegments',
-              'xray:PutTelemetryRecords',
-              'xray:GetSamplingRules',
-              'xray:GetSamplingTargets',
-              'xray:GetSamplingStatisticSummaries',
             ],
             resources: ['*'],
             effect: 'Allow',

infrastructure/user-api/src/main.ts

@@ -186,10 +186,52 @@ class UserAPI extends TerraformStack {
               name: 'SALT_2',
               valueFrom: `${intMaskSecretArn}:salt2::`,
             },
+            {
+              name: 'UNLEASH_ENDPOINT',
+              valueFrom: `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Shared/${config.environment}/UNLEASH_ENDPOINT`,
+            },
+            {
+              name: 'UNLEASH_KEY',
+              valueFrom: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:${config.name}/${config.environment}/UNLEASH_KEY`,
+            },
           ],
           logGroup: this.createCustomLogGroup('app'),
           logMultilinePattern: '^\\S.+',
         },
+        {
+          name: 'otel-collector',
+          containerImage: 'pocket/opentelemetry-collector-contrib',
+          essential: true,
+          logMultilinePattern: '^\\S.+',
+          logGroup: this.createCustomLogGroup('otel-collector'),
+          portMappings: [
+            {
+              hostPort: 4138,
+              containerPort: 4138,
+            },
+            {
+              hostPort: 4137,
+              containerPort: 4137,
+            },
+            {
+              hostPort: 55681,
+              containerPort: 55681,
+            },
+          ],
+          envVars: [
+            {
+              name: 'DEPLOYMENT_ENVIRONMENT_NAME',
+              value: config.tags.env_code,
+            },
+          ],
+          secretEnvVars: [
+            {
+              name: 'GOOGLE_APPLICATION_CREDENTIALS_JSON',
+              valueFrom: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:Shared/GCP_SA_TRACES:::`,
+            },
+          ],
+          repositoryCredentialsParam: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:Shared/DockerHub`,
+        },
       ],
       codeDeploy: {
         useCodeDeploy: true,
@@ -232,6 +274,8 @@ class UserAPI extends TerraformStack {
             resources: [
               `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/${config.name}/${config.environment}`,
               `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/${config.name}/${config.environment}/*`,
+              `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Shared/${config.environment}/*`,
+              `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Shared/${config.environment}`,
             ],
             effect: 'Allow',
           },
@@ -244,11 +288,6 @@ class UserAPI extends TerraformStack {
               'logs:CreateLogStream',
               'logs:DescribeLogStreams',
               'logs:DescribeLogGroups',
-              'xray:PutTraceSegments',
-              'xray:PutTelemetryRecords',
-              'xray:GetSamplingRules',
-              'xray:GetSamplingTargets',
-              'xray:GetSamplingStatisticSummaries',
             ],
             resources: ['*'],
             effect: 'Allow',