feat(otel): adding in basic tracing via otel (#800)

* Revert "Revert "feat(otel): adding in basic tracing via otel (#798)" (#799)"

This reverts commit 57c2a13.

* fix(collector): move to an otel collector

docker-compose.yml

@@ -96,3 +96,25 @@ services:
       retries: 30
     ports:
       - '9090:9090'
+
+  otlpcollector:
+    image: grafana/otel-lgtm
+    ports:
+      # there are lots more ports available, but these are the only one's
+      # currently being used
+      - 4317:4317 # OTLP gRPC receiver
+      - 4318:4318 # OTLP http receiver
+      - 3000:3000 # Grafana
+
+  # Uncomment this if you want to run the collector locally and send to GCP instead of the one used for local development
+  # Make sure to have a service json in GOOGLE_APPLICATION_CREDENTIALS_JSON in the .env file
+  # otlpcollector:
+  #   image: pocket/opentelemetry-collector-contrib
+  #   env_file:
+  #     - .env
+  #   ports:
+  #     # there are lots more ports available, but these are the only one's
+  #     # currently being used
+  #     - 4317:4317 # OTLP gRPC receiver
+  #     - 4318:4318 # OTLP http receiver
+  #     - 3000:3000 # Grafana

infrastructure/client-api/src/config/index.ts

@@ -47,6 +47,8 @@ export const config = {
     },
   },
   tracing: {
-    host: 'localhost',
+    url: isDev
+      ? 'https://otel-collector.getpocket.dev:443'
+      : 'https://otel-collector.readitlater.com:443',
   },
 };

infrastructure/client-api/src/main.ts

@@ -235,8 +235,8 @@ class ClientAPI extends TerraformStack {
               value: config.isProd ? 'production' : 'development',
             },
             {
-              name: 'OTLP_COLLECTOR_HOST',
-              value: `${config.tracing.host}`,
+              name: 'OTLP_COLLECTOR_URL',
+              value: `${config.tracing.url}`,
             },
             {
               name: 'REDIS_ENDPOINT',
@@ -309,19 +309,7 @@ class ClientAPI extends TerraformStack {
             effect: 'Allow',
           },
         ],
-        taskRolePolicyStatements: [
-          {
-            actions: [
-              'xray:PutTraceSegments',
-              'xray:PutTelemetryRecords',
-              'xray:GetSamplingRules',
-              'xray:GetSamplingTargets',
-              'xray:GetSamplingStatisticSummaries',
-            ],
-            resources: ['*'],
-            effect: 'Allow',
-          },
-        ],
+        taskRolePolicyStatements: [],
         taskExecutionDefaultAttachmentArn:
           'arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy',
       },

infrastructure/list-api/src/config/index.ts

@@ -50,4 +50,9 @@ export const config = {
       userEvents: `PocketEventBridge-${environment}-UserEventTopic`,
     },
   },
+  tracing: {
+    url: isDev
+      ? 'https://otel-collector.getpocket.dev:443'
+      : 'https://otel-collector.readitlater.com:443',
+  },
 };

infrastructure/list-api/src/main.ts

@@ -230,6 +230,10 @@ class ListAPI extends TerraformStack {
               name: 'EVENT_BUS_NAME',
               value: config.envVars.eventBusName,
             },
+            {
+              name: 'OTLP_COLLECTOR_URL',
+              value: config.tracing.url,
+            },
           ],
           secretEnvVars: [
             {
@@ -349,11 +353,6 @@ class ListAPI extends TerraformStack {
               'logs:CreateLogStream',
               'logs:DescribeLogStreams',
               'logs:DescribeLogGroups',
-              'xray:PutTraceSegments',
-              'xray:PutTelemetryRecords',
-              'xray:GetSamplingRules',
-              'xray:GetSamplingTargets',
-              'xray:GetSamplingStatisticSummaries',
             ],
             resources: ['*'],
             effect: 'Allow',

infrastructure/parser-graphql-wrapper/src/config/index.ts

@@ -42,4 +42,9 @@ export const config = {
     databaseName: 'readitla_shares',
     masterUsername: 'share_urls',
   },
+  tracing: {
+    url: isDev
+      ? 'https://otel-collector.getpocket.dev:443'
+      : 'https://otel-collector.readitlater.com:443',
+  },
 };

infrastructure/parser-graphql-wrapper/src/main.ts

@@ -176,6 +176,10 @@ class ParserGraphQLWrapper extends TerraformStack {
               name: 'ITEM_SUMMARY_TABLE',
               value: dynamodb.itemSummaryTable.dynamodb.name,
             },
+            {
+              name: 'OTLP_COLLECTOR_URL',
+              value: config.tracing.url,
+            },
           ],
           healthCheck: {
             command: [
@@ -326,11 +330,6 @@ class ParserGraphQLWrapper extends TerraformStack {
               'logs:CreateLogStream',
               'logs:DescribeLogStreams',
               'logs:DescribeLogGroups',
-              'xray:PutTraceSegments',
-              'xray:PutTelemetryRecords',
-              'xray:GetSamplingRules',
-              'xray:GetSamplingTargets',
-              'xray:GetSamplingStatisticSummaries',
             ],
             resources: ['*'],
             effect: 'Allow',

infrastructure/user-api/src/config/index.ts

@@ -37,6 +37,8 @@ export const config = {
     env_code: isDev ? 'dev' : 'prod',
   },
   tracing: {
-    host: 'localhost',
+    url: isDev
+      ? 'https://otel-collector.getpocket.dev:443'
+      : 'https://otel-collector.readitlater.com:443',
   },
 };

infrastructure/user-api/src/main.ts

@@ -112,6 +112,10 @@ class UserAPI extends TerraformStack {
             startPeriod: 0,
           },
           envVars: [
+            {
+              name: 'OTLP_COLLECTOR_URL',
+              value: config.tracing.url,
+            },
             {
               name: 'NODE_ENV',
               value: process.env.NODE_ENV ?? 'development',
@@ -186,6 +190,14 @@ class UserAPI extends TerraformStack {
               name: 'SALT_2',
               valueFrom: `${intMaskSecretArn}:salt2::`,
             },
+            {
+              name: 'UNLEASH_ENDPOINT',
+              valueFrom: `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Shared/${config.environment}/UNLEASH_ENDPOINT`,
+            },
+            {
+              name: 'UNLEASH_KEY',
+              valueFrom: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:${config.name}/${config.environment}/UNLEASH_KEY`,
+            },
           ],
           logGroup: this.createCustomLogGroup('app'),
           logMultilinePattern: '^\\S.+',
@@ -232,6 +244,8 @@ class UserAPI extends TerraformStack {
             resources: [
               `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/${config.name}/${config.environment}`,
               `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/${config.name}/${config.environment}/*`,
+              `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Shared/${config.environment}/*`,
+              `arn:aws:ssm:${region.name}:${caller.accountId}:parameter/Shared/${config.environment}`,
             ],
             effect: 'Allow',
           },
@@ -244,11 +258,6 @@ class UserAPI extends TerraformStack {
               'logs:CreateLogStream',
               'logs:DescribeLogStreams',
               'logs:DescribeLogGroups',
-              'xray:PutTraceSegments',
-              'xray:PutTelemetryRecords',
-              'xray:GetSamplingRules',
-              'xray:GetSamplingTargets',
-              'xray:GetSamplingStatisticSummaries',
             ],
             resources: ['*'],
             effect: 'Allow',

infrastructure/user-list-search/apollo_ecs.tf

@@ -110,13 +110,17 @@ module "apollo" {
     {
       name  = "CORPUS_SEARCH_ENDPOINT"
       value = module.corpus_embeddings.opensearch_endpoint
+    },
+    {
+      name  = "OTLP_COLLECTOR_URL"
+      value = local.workspace.otlpCollectorUrl
     }
   ]
 }
 
 resource "aws_ecs_task_definition" "apollo" {
   family                = "${local.prefix}-Apollo"
-  container_definitions = "[${module.apollo.json_map_encoded}, ${module.otel.json_map_encoded}]"
+  container_definitions = "[${module.apollo.json_map_encoded}]"
 
   task_role_arn      = aws_iam_role.ecs_task_role.arn
   execution_role_arn = aws_iam_role.ecs_execution_role.arn

infrastructure/user-list-search/ecs.tf

@@ -101,47 +101,3 @@ resource "aws_security_group" "ecs_security_group" {
   depends_on = [aws_security_group.alb_security_group]
 }
 
-resource "aws_cloudwatch_log_group" "xray" {
-  name              = "/ecs/${local.name}/${local.env}/xray"
-  retention_in_days = 30
-}
-
-module "otel" {
-  source  = "cloudposse/ecs-container-definition/aws"
-  version = "0.61.1"
-
-  essential       = true
-  container_name  = "aws-otel-collector"
-  container_image = "amazon/aws-otel-collector"
-  command = [
-    "--config=/etc/ecs/ecs-xray.yaml",
-  ]
-
-  repository_credentials = {
-    credentialsParameter : local.container_credential
-  }
-
-  log_configuration = {
-    logDriver     = "awslogs"
-    secretOptions = []
-    options = {
-      awslogs-region        = data.aws_region.current.name
-      awslogs-group         = aws_cloudwatch_log_group.xray.name
-      awslogs-stream-prefix = "ecs"
-    }
-  }
-
-  port_mappings = [
-    {
-      containerPort = 4138
-      hostPort      = 4138
-    },
-    {
-      containerPort = 4137
-      hostPort      = 4137
-    }
-  ]
-  container_cpu                = null
-  container_memory             = null
-  container_memory_reservation = null
-}

infrastructure/user-list-search/locals.tf

@@ -29,6 +29,7 @@ locals {
   }
   private_subnet_ids = split(",", data.aws_ssm_parameter.private_subnets.value)
   secret_path        = "arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:${local.name}/${local.env}/"
+  secret_path_shared = "arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:Shared/"
   ssm_path           = "arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter/${local.name}/${local.env}/"
   ssm_path_shared    = "arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter/Shared/${local.env}/"
 
@@ -80,6 +81,7 @@ locals {
       sns_topic_corpus_events     = "PocketEventBridge-Dev-CorpusEventsTopic"
       sns_topic_collection_events = "PocketEventBridge-Dev-CollectionEventTopic"
       userApiUri                  = "https://user-list-search.getpocket.dev"
+      otlpCollectorUrl            = "https://otel-collector.getpocket.dev:443"
     }
 
     UserListSearch-Prod = {
@@ -96,6 +98,7 @@ locals {
       sns_topic_corpus_events     = "PocketEventBridge-Prod-CorpusEventsTopic"
       sns_topic_collection_events = "PocketEventBridge-Prod-CollectionEventTopic"
       userApiUri                  = "https://user-list-search.readitlater.com"
+      otlpCollectorUrl            = "https://otel-collector.readitlater.com:443"
     }
   }
 

infrastructure/user-list-search/queue_users_ecs.tf

@@ -72,7 +72,7 @@ module "queue_users" {
 
 resource "aws_ecs_task_definition" "queue_users" {
   family                = "${local.prefix}-QueueUsers"
-  container_definitions = "[${module.queue_users.json_map_encoded}, ${module.otel.json_map_encoded}]"
+  container_definitions = "[${module.queue_users.json_map_encoded}]"
 
   task_role_arn      = aws_iam_role.ecs_task_role.arn
   execution_role_arn = aws_iam_role.ecs_execution_role.arn

infrastructure/v3-proxy-api/src/config/index.ts

@@ -38,4 +38,9 @@ export const config = {
     component_code: `pocket-${name.toLowerCase()}`,
     env_code: isDev ? 'dev' : 'prod',
   },
+  tracing: {
+    url: isDev
+      ? 'https://otel-collector.getpocket.dev:443'
+      : 'https://otel-collector.readitlater.com:443',
+  },
 };

infrastructure/v3-proxy-api/src/main.ts

@@ -112,6 +112,10 @@ class Stack extends TerraformStack {
               name: 'ENVIRONMENT',
               value: process.env.NODE_ENV ?? 'development', // this gives us a nice lowercase production and development
             },
+            {
+              name: 'OTLP_COLLECTOR_URL',
+              value: config.tracing.url,
+            },
           ],
           secretEnvVars: [
             {
@@ -128,25 +132,6 @@ class Stack extends TerraformStack {
             },
           ],
         },
-        {
-          name: 'aws-otel-collector',
-          command: ['--config=/etc/ecs/ecs-xray.yaml'],
-          containerImage: 'amazon/aws-otel-collector',
-          essential: true,
-          logMultilinePattern: '^\\S.+',
-          logGroup: this.createCustomLogGroup('aws-otel-collector'),
-          portMappings: [
-            {
-              hostPort: 4138,
-              containerPort: 4138,
-            },
-            {
-              hostPort: 4137,
-              containerPort: 4137,
-            },
-          ],
-          repositoryCredentialsParam: `arn:aws:secretsmanager:${region.name}:${caller.accountId}:secret:Shared/DockerHub`,
-        },
       ],
       codeDeploy: {
         useCodeDeploy: true,
@@ -194,19 +179,7 @@ class Stack extends TerraformStack {
             effect: 'Allow',
           },
         ],
-        taskRolePolicyStatements: [
-          {
-            actions: [
-              'xray:PutTraceSegments',
-              'xray:PutTelemetryRecords',
-              'xray:GetSamplingRules',
-              'xray:GetSamplingTargets',
-              'xray:GetSamplingStatisticSummaries',
-            ],
-            resources: ['*'],
-            effect: 'Allow',
-          },
-        ],
+        taskRolePolicyStatements: [],
         taskExecutionDefaultAttachmentArn:
           'arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy',
       },

package.json

@@ -28,11 +28,11 @@
   },
   "devDependencies": {
     "@commitlint/cli": "19.4.1",
-    "@commitlint/config-conventional": "^19.4.1",
+    "@commitlint/config-conventional": "^19.5.0",
     "@pocket-tools/eslint-config": "workspace:*",
     "syncpack": "^13.0.0",
     "tsconfig": "workspace:*",
-    "turbo": "^2.1.1"
+    "turbo": "^2.1.3"
   },
   "packageManager": "[email protected]",
   "engines": {

packages/feature-flags-client/package.json

@@ -24,7 +24,6 @@
     "test:watch": "pnpm run test -- --watch"
   },
   "dependencies": {
-    "@pocket-tools/ts-logger": "workspace:*",
     "tslib": "2.7.0",
     "unleash-client": "6.1.1"
   },
@@ -39,4 +38,4 @@
     "tsup": "8.2.4",
     "typescript": "5.5.4"
   }
-}
+}