From f0f137375156bc495c5baefb4c686f7571b75b52 Mon Sep 17 00:00:00 2001
From: TzeYiing <ty@tzeyiing.com>
Date: Wed, 6 Nov 2024 13:46:39 +0800
Subject: [PATCH 1/6] chore: re-enable compilation checks, update otel and
 protobuf

---
 .github/workflows/elixir-ci.yml               |   4 +-
 .tool-versions                                |   1 +
 DEVELOPMENT.md                                |   3 +-
 Makefile                                      |   7 +
 .../collector/logs/v1/logs_service.pb.ex      |  13 +-
 .../metrics/v1/metrics_service.pb.ex          |  13 +-
 .../v1development/profiles_service.pb.ex      |  50 ++++
 .../collector/trace/v1/trace_service.pb.ex    |  27 ++-
 .../proto/common/v1/common.pb.ex              |  15 +-
 .../opentelemetry/proto/logs/v1/logs.pb.ex    |  22 +-
 .../proto/metrics/v1/metrics.pb.ex            |  59 +++--
 .../profiles/v1development/profiles.pb.ex     | 218 ++++++++++++++++++
 .../proto/resource/v1/resource.pb.ex          |   3 +-
 .../opentelemetry/proto/trace/v1/trace.pb.ex  |  40 +++-
 mix.exs                                       |  12 +-
 mix.lock                                      |  10 +-
 16 files changed, 420 insertions(+), 77 deletions(-)
 create mode 100644 lib/logflare_grpc/opentelemetry/proto/collector/profiles/v1development/profiles_service.pb.ex
 create mode 100644 lib/logflare_grpc/opentelemetry/proto/profiles/v1development/profiles.pb.ex

diff --git a/.github/workflows/elixir-ci.yml b/.github/workflows/elixir-ci.yml
index c7799d732..147774649 100644
--- a/.github/workflows/elixir-ci.yml
+++ b/.github/workflows/elixir-ci.yml
@@ -33,8 +33,8 @@ jobs:
             run: mix lint --only warnings
           - name: Code Quality - Formatting
             run: mix test.format
-          # - name: Compilation Warnings
-          #   run: mix test.compile
+          - name: Compilation Warnings
+            run: mix test.compile
           - name: Tests
             run: mix do ecto.create, ecto.migrate, test
           # - name: Security - Sobelow Code Scan
diff --git a/.tool-versions b/.tool-versions
index 990ef2279..b4454dd25 100644
--- a/.tool-versions
+++ b/.tool-versions
@@ -2,3 +2,4 @@ elixir 1.17.3-otp-27
 nodejs 18.13.0
 rust 1.64.0
 erlang 27.1.2
+protoc 28.0
\ No newline at end of file
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
index 83bcca613..7a5ff4ebb 100644
--- a/DEVELOPMENT.md
+++ b/DEVELOPMENT.md
@@ -2,7 +2,8 @@
 
 ## Dev Env Setup
 
-1. Install dependencies with `asdf` using `asdf install`
+1. Run `make setup`, which will:
+   1. Install dependencies with `asdf` using `asdf install`
 2. Decrypt secrets with `make decrypt.dev`. It will decrypt two files:
    1. Dev secrets - `.dev.env`
    2. Google JWT key - `.gcloud.json`
diff --git a/Makefile b/Makefile
index 3743d5794..a03090141 100644
--- a/Makefile
+++ b/Makefile
@@ -23,6 +23,13 @@ test.only:
 .PHONY: test test.only
 
 setup: setup.node
+	# install dependencies
+	asdf install
+
+	# add protobuf install
+	mix escript.install hex protobuf
+	asdf reshim
+	# run elixir setup
 	mix setup
 
 setup.node:
diff --git a/lib/logflare_grpc/opentelemetry/proto/collector/logs/v1/logs_service.pb.ex b/lib/logflare_grpc/opentelemetry/proto/collector/logs/v1/logs_service.pb.ex
index f40043324..53d6d28fc 100644
--- a/lib/logflare_grpc/opentelemetry/proto/collector/logs/v1/logs_service.pb.ex
+++ b/lib/logflare_grpc/opentelemetry/proto/collector/logs/v1/logs_service.pb.ex
@@ -1,6 +1,7 @@
 defmodule Opentelemetry.Proto.Collector.Logs.V1.ExportLogsServiceRequest do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :resource_logs, 1,
     repeated: true,
@@ -10,7 +11,8 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Logs.V1.ExportLogsServiceResponse do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :partial_success, 1,
     type: Opentelemetry.Proto.Collector.Logs.V1.ExportLogsPartialSuccess,
@@ -19,7 +21,8 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Logs.V1.ExportLogsPartialSuccess do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :rejected_log_records, 1, type: :int64, json_name: "rejectedLogRecords"
   field :error_message, 2, type: :string, json_name: "errorMessage"
@@ -27,9 +30,10 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Logs.V1.LogsService.Service do
   @moduledoc false
+
   use GRPC.Service,
     name: "opentelemetry.proto.collector.logs.v1.LogsService",
-    protoc_gen_elixir_version: "0.11.0"
+    protoc_gen_elixir_version: "0.13.0"
 
   rpc(
     :Export,
@@ -40,5 +44,6 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Logs.V1.LogsService.Stub do
   @moduledoc false
+
   use GRPC.Stub, service: Opentelemetry.Proto.Collector.Logs.V1.LogsService.Service
 end
diff --git a/lib/logflare_grpc/opentelemetry/proto/collector/metrics/v1/metrics_service.pb.ex b/lib/logflare_grpc/opentelemetry/proto/collector/metrics/v1/metrics_service.pb.ex
index 0008c26d6..4d75d1e6e 100644
--- a/lib/logflare_grpc/opentelemetry/proto/collector/metrics/v1/metrics_service.pb.ex
+++ b/lib/logflare_grpc/opentelemetry/proto/collector/metrics/v1/metrics_service.pb.ex
@@ -1,6 +1,7 @@
 defmodule Opentelemetry.Proto.Collector.Metrics.V1.ExportMetricsServiceRequest do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :resource_metrics, 1,
     repeated: true,
@@ -10,7 +11,8 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Metrics.V1.ExportMetricsServiceResponse do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :partial_success, 1,
     type: Opentelemetry.Proto.Collector.Metrics.V1.ExportMetricsPartialSuccess,
@@ -19,7 +21,8 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Metrics.V1.ExportMetricsPartialSuccess do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :rejected_data_points, 1, type: :int64, json_name: "rejectedDataPoints"
   field :error_message, 2, type: :string, json_name: "errorMessage"
@@ -27,9 +30,10 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Metrics.V1.MetricsService.Service do
   @moduledoc false
+
   use GRPC.Service,
     name: "opentelemetry.proto.collector.metrics.v1.MetricsService",
-    protoc_gen_elixir_version: "0.11.0"
+    protoc_gen_elixir_version: "0.13.0"
 
   rpc(
     :Export,
@@ -40,5 +44,6 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Metrics.V1.MetricsService.Stub do
   @moduledoc false
+
   use GRPC.Stub, service: Opentelemetry.Proto.Collector.Metrics.V1.MetricsService.Service
 end
diff --git a/lib/logflare_grpc/opentelemetry/proto/collector/profiles/v1development/profiles_service.pb.ex b/lib/logflare_grpc/opentelemetry/proto/collector/profiles/v1development/profiles_service.pb.ex
new file mode 100644
index 000000000..304235acb
--- /dev/null
+++ b/lib/logflare_grpc/opentelemetry/proto/collector/profiles/v1development/profiles_service.pb.ex
@@ -0,0 +1,50 @@
+defmodule Opentelemetry.Proto.Collector.Profiles.V1development.ExportProfilesServiceRequest do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :resource_profiles, 1,
+    repeated: true,
+    type: Opentelemetry.Proto.Profiles.V1development.ResourceProfiles,
+    json_name: "resourceProfiles"
+end
+
+defmodule Opentelemetry.Proto.Collector.Profiles.V1development.ExportProfilesServiceResponse do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :partial_success, 1,
+    type: Opentelemetry.Proto.Collector.Profiles.V1development.ExportProfilesPartialSuccess,
+    json_name: "partialSuccess"
+end
+
+defmodule Opentelemetry.Proto.Collector.Profiles.V1development.ExportProfilesPartialSuccess do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :rejected_profiles, 1, type: :int64, json_name: "rejectedProfiles"
+  field :error_message, 2, type: :string, json_name: "errorMessage"
+end
+
+defmodule Opentelemetry.Proto.Collector.Profiles.V1development.ProfilesService.Service do
+  @moduledoc false
+
+  use GRPC.Service,
+    name: "opentelemetry.proto.collector.profiles.v1development.ProfilesService",
+    protoc_gen_elixir_version: "0.13.0"
+
+  rpc(
+    :Export,
+    Opentelemetry.Proto.Collector.Profiles.V1development.ExportProfilesServiceRequest,
+    Opentelemetry.Proto.Collector.Profiles.V1development.ExportProfilesServiceResponse
+  )
+end
+
+defmodule Opentelemetry.Proto.Collector.Profiles.V1development.ProfilesService.Stub do
+  @moduledoc false
+
+  use GRPC.Stub,
+    service: Opentelemetry.Proto.Collector.Profiles.V1development.ProfilesService.Service
+end
diff --git a/lib/logflare_grpc/opentelemetry/proto/collector/trace/v1/trace_service.pb.ex b/lib/logflare_grpc/opentelemetry/proto/collector/trace/v1/trace_service.pb.ex
index 52301aae1..a1055a3ff 100644
--- a/lib/logflare_grpc/opentelemetry/proto/collector/trace/v1/trace_service.pb.ex
+++ b/lib/logflare_grpc/opentelemetry/proto/collector/trace/v1/trace_service.pb.ex
@@ -1,46 +1,49 @@
 defmodule Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceRequest do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
 
-  field(:resource_spans, 1,
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :resource_spans, 1,
     repeated: true,
     type: Opentelemetry.Proto.Trace.V1.ResourceSpans,
     json_name: "resourceSpans"
-  )
 end
 
 defmodule Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceResponse do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
 
-  field(:partial_success, 1,
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :partial_success, 1,
     type: Opentelemetry.Proto.Collector.Trace.V1.ExportTracePartialSuccess,
     json_name: "partialSuccess"
-  )
 end
 
 defmodule Opentelemetry.Proto.Collector.Trace.V1.ExportTracePartialSuccess do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
 
-  field(:rejected_spans, 1, type: :int64, json_name: "rejectedSpans")
-  field(:error_message, 2, type: :string, json_name: "errorMessage")
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :rejected_spans, 1, type: :int64, json_name: "rejectedSpans"
+  field :error_message, 2, type: :string, json_name: "errorMessage"
 end
 
 defmodule Opentelemetry.Proto.Collector.Trace.V1.TraceService.Service do
   @moduledoc false
+
   use GRPC.Service,
     name: "opentelemetry.proto.collector.trace.v1.TraceService",
-    protoc_gen_elixir_version: "0.11.0"
+    protoc_gen_elixir_version: "0.13.0"
 
   rpc(
     :Export,
-    stream(Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceRequest),
-    stream(Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceResponse)
+    Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceRequest,
+    Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceResponse
   )
 end
 
 defmodule Opentelemetry.Proto.Collector.Trace.V1.TraceService.Stub do
   @moduledoc false
+
   use GRPC.Stub, service: Opentelemetry.Proto.Collector.Trace.V1.TraceService.Service
 end
diff --git a/lib/logflare_grpc/opentelemetry/proto/common/v1/common.pb.ex b/lib/logflare_grpc/opentelemetry/proto/common/v1/common.pb.ex
index 25b3d8e0c..40bc67c69 100644
--- a/lib/logflare_grpc/opentelemetry/proto/common/v1/common.pb.ex
+++ b/lib/logflare_grpc/opentelemetry/proto/common/v1/common.pb.ex
@@ -1,6 +1,7 @@
 defmodule Opentelemetry.Proto.Common.V1.AnyValue do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   oneof(:value, 0)
 
@@ -24,21 +25,24 @@ end
 
 defmodule Opentelemetry.Proto.Common.V1.ArrayValue do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :values, 1, repeated: true, type: Opentelemetry.Proto.Common.V1.AnyValue
 end
 
 defmodule Opentelemetry.Proto.Common.V1.KeyValueList do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :values, 1, repeated: true, type: Opentelemetry.Proto.Common.V1.KeyValue
 end
 
 defmodule Opentelemetry.Proto.Common.V1.KeyValue do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :key, 1, type: :string
   field :value, 2, type: Opentelemetry.Proto.Common.V1.AnyValue
@@ -46,7 +50,8 @@ end
 
 defmodule Opentelemetry.Proto.Common.V1.InstrumentationScope do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :name, 1, type: :string
   field :version, 2, type: :string
diff --git a/lib/logflare_grpc/opentelemetry/proto/logs/v1/logs.pb.ex b/lib/logflare_grpc/opentelemetry/proto/logs/v1/logs.pb.ex
index f63924653..3034d7d04 100644
--- a/lib/logflare_grpc/opentelemetry/proto/logs/v1/logs.pb.ex
+++ b/lib/logflare_grpc/opentelemetry/proto/logs/v1/logs.pb.ex
@@ -1,6 +1,7 @@
 defmodule Opentelemetry.Proto.Logs.V1.SeverityNumber do
   @moduledoc false
-  use Protobuf, enum: true, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, enum: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :SEVERITY_NUMBER_UNSPECIFIED, 0
   field :SEVERITY_NUMBER_TRACE, 1
@@ -31,15 +32,17 @@ end
 
 defmodule Opentelemetry.Proto.Logs.V1.LogRecordFlags do
   @moduledoc false
-  use Protobuf, enum: true, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
 
-  field :LOG_RECORD_FLAG_UNSPECIFIED, 0
-  field :LOG_RECORD_FLAG_TRACE_FLAGS_MASK, 255
+  use Protobuf, enum: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :LOG_RECORD_FLAGS_DO_NOT_USE, 0
+  field :LOG_RECORD_FLAGS_TRACE_FLAGS_MASK, 255
 end
 
 defmodule Opentelemetry.Proto.Logs.V1.LogsData do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :resource_logs, 1,
     repeated: true,
@@ -49,7 +52,8 @@ end
 
 defmodule Opentelemetry.Proto.Logs.V1.ResourceLogs do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :resource, 1, type: Opentelemetry.Proto.Resource.V1.Resource
 
@@ -63,7 +67,8 @@ end
 
 defmodule Opentelemetry.Proto.Logs.V1.ScopeLogs do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :scope, 1, type: Opentelemetry.Proto.Common.V1.InstrumentationScope
 
@@ -77,7 +82,8 @@ end
 
 defmodule Opentelemetry.Proto.Logs.V1.LogRecord do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :time_unix_nano, 1, type: :fixed64, json_name: "timeUnixNano"
   field :observed_time_unix_nano, 11, type: :fixed64, json_name: "observedTimeUnixNano"
diff --git a/lib/logflare_grpc/opentelemetry/proto/metrics/v1/metrics.pb.ex b/lib/logflare_grpc/opentelemetry/proto/metrics/v1/metrics.pb.ex
index 97c59d3d6..e4bd1734a 100644
--- a/lib/logflare_grpc/opentelemetry/proto/metrics/v1/metrics.pb.ex
+++ b/lib/logflare_grpc/opentelemetry/proto/metrics/v1/metrics.pb.ex
@@ -1,6 +1,7 @@
 defmodule Opentelemetry.Proto.Metrics.V1.AggregationTemporality do
   @moduledoc false
-  use Protobuf, enum: true, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, enum: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :AGGREGATION_TEMPORALITY_UNSPECIFIED, 0
   field :AGGREGATION_TEMPORALITY_DELTA, 1
@@ -9,15 +10,17 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.DataPointFlags do
   @moduledoc false
-  use Protobuf, enum: true, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
 
-  field :FLAG_NONE, 0
-  field :FLAG_NO_RECORDED_VALUE, 1
+  use Protobuf, enum: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :DATA_POINT_FLAGS_DO_NOT_USE, 0
+  field :DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK, 1
 end
 
 defmodule Opentelemetry.Proto.Metrics.V1.MetricsData do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :resource_metrics, 1,
     repeated: true,
@@ -27,7 +30,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.ResourceMetrics do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :resource, 1, type: Opentelemetry.Proto.Resource.V1.Resource
 
@@ -41,7 +45,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.ScopeMetrics do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :scope, 1, type: Opentelemetry.Proto.Common.V1.InstrumentationScope
   field :metrics, 2, repeated: true, type: Opentelemetry.Proto.Metrics.V1.Metric
@@ -50,7 +55,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.Metric do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   oneof(:data, 0)
 
@@ -67,11 +73,13 @@ defmodule Opentelemetry.Proto.Metrics.V1.Metric do
     oneof: 0
 
   field :summary, 11, type: Opentelemetry.Proto.Metrics.V1.Summary, oneof: 0
+  field :metadata, 12, repeated: true, type: Opentelemetry.Proto.Common.V1.KeyValue
 end
 
 defmodule Opentelemetry.Proto.Metrics.V1.Gauge do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :data_points, 1,
     repeated: true,
@@ -81,7 +89,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.Sum do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :data_points, 1,
     repeated: true,
@@ -98,7 +107,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.Histogram do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :data_points, 1,
     repeated: true,
@@ -113,7 +123,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.ExponentialHistogram do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :data_points, 1,
     repeated: true,
@@ -128,7 +139,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.Summary do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :data_points, 1,
     repeated: true,
@@ -138,7 +150,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.NumberDataPoint do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   oneof(:value, 0)
 
@@ -153,7 +166,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.HistogramDataPoint do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :attributes, 9, repeated: true, type: Opentelemetry.Proto.Common.V1.KeyValue
   field :start_time_unix_nano, 2, type: :fixed64, json_name: "startTimeUnixNano"
@@ -170,7 +184,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.ExponentialHistogramDataPoint.Buckets do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :offset, 1, type: :sint32
   field :bucket_counts, 2, repeated: true, type: :uint64, json_name: "bucketCounts"
@@ -178,7 +193,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.ExponentialHistogramDataPoint do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :attributes, 1, repeated: true, type: Opentelemetry.Proto.Common.V1.KeyValue
   field :start_time_unix_nano, 2, type: :fixed64, json_name: "startTimeUnixNano"
@@ -198,7 +214,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.SummaryDataPoint.ValueAtQuantile do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :quantile, 1, type: :double
   field :value, 2, type: :double
@@ -206,7 +223,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.SummaryDataPoint do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :attributes, 7, repeated: true, type: Opentelemetry.Proto.Common.V1.KeyValue
   field :start_time_unix_nano, 2, type: :fixed64, json_name: "startTimeUnixNano"
@@ -224,7 +242,8 @@ end
 
 defmodule Opentelemetry.Proto.Metrics.V1.Exemplar do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   oneof(:value, 0)
 
diff --git a/lib/logflare_grpc/opentelemetry/proto/profiles/v1development/profiles.pb.ex b/lib/logflare_grpc/opentelemetry/proto/profiles/v1development/profiles.pb.ex
new file mode 100644
index 000000000..8988affe4
--- /dev/null
+++ b/lib/logflare_grpc/opentelemetry/proto/profiles/v1development/profiles.pb.ex
@@ -0,0 +1,218 @@
+defmodule Opentelemetry.Proto.Profiles.V1development.AggregationTemporality do
+  @moduledoc false
+
+  use Protobuf, enum: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :AGGREGATION_TEMPORALITY_UNSPECIFIED, 0
+  field :AGGREGATION_TEMPORALITY_DELTA, 1
+  field :AGGREGATION_TEMPORALITY_CUMULATIVE, 2
+end
+
+defmodule Opentelemetry.Proto.Profiles.V1development.ProfilesData do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :resource_profiles, 1,
+    repeated: true,
+    type: Opentelemetry.Proto.Profiles.V1development.ResourceProfiles,
+    json_name: "resourceProfiles"
+end
+
+defmodule Opentelemetry.Proto.Profiles.V1development.ResourceProfiles do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :resource, 1, type: Opentelemetry.Proto.Resource.V1.Resource
+
+  field :scope_profiles, 2,
+    repeated: true,
+    type: Opentelemetry.Proto.Profiles.V1development.ScopeProfiles,
+    json_name: "scopeProfiles"
+
+  field :schema_url, 3, type: :string, json_name: "schemaUrl"
+end
+
+defmodule Opentelemetry.Proto.Profiles.V1development.ScopeProfiles do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :scope, 1, type: Opentelemetry.Proto.Common.V1.InstrumentationScope
+  field :profiles, 2, repeated: true, type: Opentelemetry.Proto.Profiles.V1development.Profile
+  field :schema_url, 3, type: :string, json_name: "schemaUrl"
+end
+
+defmodule Opentelemetry.Proto.Profiles.V1development.Profile do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :sample_type, 1,
+    repeated: true,
+    type: Opentelemetry.Proto.Profiles.V1development.ValueType,
+    json_name: "sampleType"
+
+  field :sample, 2, repeated: true, type: Opentelemetry.Proto.Profiles.V1development.Sample
+
+  field :mapping_table, 3,
+    repeated: true,
+    type: Opentelemetry.Proto.Profiles.V1development.Mapping,
+    json_name: "mappingTable"
+
+  field :location_table, 4,
+    repeated: true,
+    type: Opentelemetry.Proto.Profiles.V1development.Location,
+    json_name: "locationTable"
+
+  field :location_indices, 15, repeated: true, type: :int32, json_name: "locationIndices"
+
+  field :function_table, 5,
+    repeated: true,
+    type: Opentelemetry.Proto.Profiles.V1development.Function,
+    json_name: "functionTable"
+
+  field :attribute_table, 16,
+    repeated: true,
+    type: Opentelemetry.Proto.Common.V1.KeyValue,
+    json_name: "attributeTable"
+
+  field :attribute_units, 17,
+    repeated: true,
+    type: Opentelemetry.Proto.Profiles.V1development.AttributeUnit,
+    json_name: "attributeUnits"
+
+  field :link_table, 18,
+    repeated: true,
+    type: Opentelemetry.Proto.Profiles.V1development.Link,
+    json_name: "linkTable"
+
+  field :string_table, 6, repeated: true, type: :string, json_name: "stringTable"
+  field :drop_frames_strindex, 7, type: :int32, json_name: "dropFramesStrindex"
+  field :keep_frames_strindex, 8, type: :int32, json_name: "keepFramesStrindex"
+  field :time_nanos, 9, type: :int64, json_name: "timeNanos"
+  field :duration_nanos, 10, type: :int64, json_name: "durationNanos"
+
+  field :period_type, 11,
+    type: Opentelemetry.Proto.Profiles.V1development.ValueType,
+    json_name: "periodType"
+
+  field :period, 12, type: :int64
+  field :comment_strindices, 13, repeated: true, type: :int32, json_name: "commentStrindices"
+  field :default_sample_type_strindex, 14, type: :int32, json_name: "defaultSampleTypeStrindex"
+  field :profile_id, 19, type: :bytes, json_name: "profileId"
+  field :attributes, 20, repeated: true, type: Opentelemetry.Proto.Common.V1.KeyValue
+  field :dropped_attributes_count, 21, type: :uint32, json_name: "droppedAttributesCount"
+  field :original_payload_format, 22, type: :string, json_name: "originalPayloadFormat"
+  field :original_payload, 23, type: :bytes, json_name: "originalPayload"
+end
+
+defmodule Opentelemetry.Proto.Profiles.V1development.AttributeUnit do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :attribute_key_strindex, 1, type: :int32, json_name: "attributeKeyStrindex"
+  field :unit_strindex, 2, type: :int32, json_name: "unitStrindex"
+end
+
+defmodule Opentelemetry.Proto.Profiles.V1development.Link do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :trace_id, 1, type: :bytes, json_name: "traceId"
+  field :span_id, 2, type: :bytes, json_name: "spanId"
+end
+
+defmodule Opentelemetry.Proto.Profiles.V1development.ValueType do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :type_strindex, 1, type: :int32, json_name: "typeStrindex"
+  field :unit_strindex, 2, type: :int32, json_name: "unitStrindex"
+
+  field :aggregation_temporality, 3,
+    type: Opentelemetry.Proto.Profiles.V1development.AggregationTemporality,
+    json_name: "aggregationTemporality",
+    enum: true
+end
+
+defmodule Opentelemetry.Proto.Profiles.V1development.Sample do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :location_index, 1, repeated: true, type: :int32, json_name: "locationIndex"
+  field :locations_start_index, 7, type: :int32, json_name: "locationsStartIndex"
+  field :locations_length, 8, type: :int32, json_name: "locationsLength"
+  field :value, 2, repeated: true, type: :int64
+  field :attribute_indices, 10, repeated: true, type: :int32, json_name: "attributeIndices"
+  field :link_index, 12, type: :int32, json_name: "linkIndex"
+  field :timestamps_unix_nano, 13, repeated: true, type: :uint64, json_name: "timestampsUnixNano"
+end
+
+defmodule Opentelemetry.Proto.Profiles.V1development.Label do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :key_strindex, 1, type: :int32, json_name: "keyStrindex"
+  field :str_strindex, 2, type: :int32, json_name: "strStrindex"
+  field :num, 3, type: :int64
+  field :num_unit_strindex, 4, type: :int32, json_name: "numUnitStrindex"
+end
+
+defmodule Opentelemetry.Proto.Profiles.V1development.Mapping do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :id, 1, type: :uint64
+  field :memory_start, 2, type: :uint64, json_name: "memoryStart"
+  field :memory_limit, 3, type: :uint64, json_name: "memoryLimit"
+  field :file_offset, 4, type: :uint64, json_name: "fileOffset"
+  field :filename_strindex, 5, type: :int32, json_name: "filenameStrindex"
+  field :attribute_indices, 12, repeated: true, type: :int32, json_name: "attributeIndices"
+  field :has_functions, 7, type: :bool, json_name: "hasFunctions"
+  field :has_filenames, 8, type: :bool, json_name: "hasFilenames"
+  field :has_line_numbers, 9, type: :bool, json_name: "hasLineNumbers"
+  field :has_inline_frames, 10, type: :bool, json_name: "hasInlineFrames"
+end
+
+defmodule Opentelemetry.Proto.Profiles.V1development.Location do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :id, 1, type: :uint64
+  field :mapping_index, 2, type: :int32, json_name: "mappingIndex"
+  field :address, 3, type: :uint64
+  field :line, 4, repeated: true, type: Opentelemetry.Proto.Profiles.V1development.Line
+  field :is_folded, 5, type: :bool, json_name: "isFolded"
+  field :attribute_indices, 7, repeated: true, type: :int32, json_name: "attributeIndices"
+end
+
+defmodule Opentelemetry.Proto.Profiles.V1development.Line do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :function_index, 1, type: :int32, json_name: "functionIndex"
+  field :line, 2, type: :int64
+  field :column, 3, type: :int64
+end
+
+defmodule Opentelemetry.Proto.Profiles.V1development.Function do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :id, 1, type: :uint64
+  field :name_strindex, 2, type: :int32, json_name: "nameStrindex"
+  field :system_name_strindex, 3, type: :int32, json_name: "systemNameStrindex"
+  field :filename_strindex, 4, type: :int32, json_name: "filenameStrindex"
+  field :start_line, 5, type: :int64, json_name: "startLine"
+end
diff --git a/lib/logflare_grpc/opentelemetry/proto/resource/v1/resource.pb.ex b/lib/logflare_grpc/opentelemetry/proto/resource/v1/resource.pb.ex
index bd75df8e2..0cd889b0b 100644
--- a/lib/logflare_grpc/opentelemetry/proto/resource/v1/resource.pb.ex
+++ b/lib/logflare_grpc/opentelemetry/proto/resource/v1/resource.pb.ex
@@ -1,6 +1,7 @@
 defmodule Opentelemetry.Proto.Resource.V1.Resource do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :attributes, 1, repeated: true, type: Opentelemetry.Proto.Common.V1.KeyValue
   field :dropped_attributes_count, 2, type: :uint32, json_name: "droppedAttributesCount"
diff --git a/lib/logflare_grpc/opentelemetry/proto/trace/v1/trace.pb.ex b/lib/logflare_grpc/opentelemetry/proto/trace/v1/trace.pb.ex
index 52b77c310..311a8e59a 100644
--- a/lib/logflare_grpc/opentelemetry/proto/trace/v1/trace.pb.ex
+++ b/lib/logflare_grpc/opentelemetry/proto/trace/v1/trace.pb.ex
@@ -1,6 +1,18 @@
+defmodule Opentelemetry.Proto.Trace.V1.SpanFlags do
+  @moduledoc false
+
+  use Protobuf, enum: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :SPAN_FLAGS_DO_NOT_USE, 0
+  field :SPAN_FLAGS_TRACE_FLAGS_MASK, 255
+  field :SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK, 256
+  field :SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK, 512
+end
+
 defmodule Opentelemetry.Proto.Trace.V1.Span.SpanKind do
   @moduledoc false
-  use Protobuf, enum: true, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, enum: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :SPAN_KIND_UNSPECIFIED, 0
   field :SPAN_KIND_INTERNAL, 1
@@ -12,7 +24,8 @@ end
 
 defmodule Opentelemetry.Proto.Trace.V1.Status.StatusCode do
   @moduledoc false
-  use Protobuf, enum: true, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, enum: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :STATUS_CODE_UNSET, 0
   field :STATUS_CODE_OK, 1
@@ -21,7 +34,8 @@ end
 
 defmodule Opentelemetry.Proto.Trace.V1.TracesData do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :resource_spans, 1,
     repeated: true,
@@ -31,7 +45,8 @@ end
 
 defmodule Opentelemetry.Proto.Trace.V1.ResourceSpans do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :resource, 1, type: Opentelemetry.Proto.Resource.V1.Resource
 
@@ -45,7 +60,8 @@ end
 
 defmodule Opentelemetry.Proto.Trace.V1.ScopeSpans do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :scope, 1, type: Opentelemetry.Proto.Common.V1.InstrumentationScope
   field :spans, 2, repeated: true, type: Opentelemetry.Proto.Trace.V1.Span
@@ -54,7 +70,8 @@ end
 
 defmodule Opentelemetry.Proto.Trace.V1.Span.Event do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :time_unix_nano, 1, type: :fixed64, json_name: "timeUnixNano"
   field :name, 2, type: :string
@@ -64,23 +81,27 @@ end
 
 defmodule Opentelemetry.Proto.Trace.V1.Span.Link do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :trace_id, 1, type: :bytes, json_name: "traceId"
   field :span_id, 2, type: :bytes, json_name: "spanId"
   field :trace_state, 3, type: :string, json_name: "traceState"
   field :attributes, 4, repeated: true, type: Opentelemetry.Proto.Common.V1.KeyValue
   field :dropped_attributes_count, 5, type: :uint32, json_name: "droppedAttributesCount"
+  field :flags, 6, type: :fixed32
 end
 
 defmodule Opentelemetry.Proto.Trace.V1.Span do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :trace_id, 1, type: :bytes, json_name: "traceId"
   field :span_id, 2, type: :bytes, json_name: "spanId"
   field :trace_state, 3, type: :string, json_name: "traceState"
   field :parent_span_id, 4, type: :bytes, json_name: "parentSpanId"
+  field :flags, 16, type: :fixed32
   field :name, 5, type: :string
   field :kind, 6, type: Opentelemetry.Proto.Trace.V1.Span.SpanKind, enum: true
   field :start_time_unix_nano, 7, type: :fixed64, json_name: "startTimeUnixNano"
@@ -96,7 +117,8 @@ end
 
 defmodule Opentelemetry.Proto.Trace.V1.Status do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :message, 2, type: :string
   field :code, 3, type: Opentelemetry.Proto.Trace.V1.Status.StatusCode, enum: true
diff --git a/mix.exs b/mix.exs
index 0f204c93b..3191853f0 100644
--- a/mix.exs
+++ b/mix.exs
@@ -200,7 +200,7 @@ defmodule Logflare.Mixfile do
       # Postgres Subscribe
       {:cainophile, github: "Logflare/cainophile", ref: "267999b"},
       {:open_api_spex, "~> 3.16"},
-      {:grpc, "~> 0.5.0"},
+      {:grpc, "~> 0.9.0"},
       {:protobuf, "~> 0.10"},
       {:gun, "~> 2.0", override: true},
       {:cowlib, ">=2.12.0", override: true},
@@ -221,11 +221,11 @@ defmodule Logflare.Mixfile do
       {:file_system, "~> 1.0", override: true, only: [:dev, :test]},
 
       # otel
-      {:opentelemetry, "~> 1.3"},
-      {:opentelemetry_api, "~> 1.2"},
-      {:opentelemetry_exporter, "~> 1.6"},
-      {:opentelemetry_phoenix, "~> 1.1"},
-      {:opentelemetry_cowboy, "~> 0.2"},
+      {:opentelemetry, "~> 1.5"},
+      {:opentelemetry_api, "~> 1.4"},
+      {:opentelemetry_exporter, "~> 1.8"},
+      {:opentelemetry_phoenix, "~> 1.2"},
+      {:opentelemetry_cowboy, "~> 0.3"},
       {:live_monaco_editor, "~> 0.1"}
     ]
   end
diff --git a/mix.lock b/mix.lock
index 6b864e5c6..732390646 100644
--- a/mix.lock
+++ b/mix.lock
@@ -57,7 +57,7 @@
   "google_gax": {:git, "https://github.com/Logflare/elixir-google-gax.git", "6772193c438dd72ac253ea34ab13e9c02842cba8", [ref: "6772193"]},
   "goth": {:hex, :goth, "1.4.2", "a598dfbce6fe65db3f5f43b1ab2ce8fbe3b2fe20a7569ad62d71c11c0ddc3f41", [:mix], [{:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.11", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "d51bb6544dc551fe5754ab72e6cf194120b3c06d924282aaa3321a516ed3b98a"},
   "gproc": {:hex, :gproc, "0.9.1", "f1df0364423539cf0b80e8201c8b1839e229e5f9b3ccb944c5834626998f5b8c", [:rebar3], [], "hexpm", "905088e32e72127ed9466f0bac0d8e65704ca5e73ee5a62cb073c3117916d507"},
-  "grpc": {:hex, :grpc, "0.5.0", "a44cb306625a52fa31a2189ce91b40d24e82569568f0cc214c1e1e0faf54f58a", [:mix], [{:cowboy, "~> 2.9", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowlib, "~> 2.11", [hex: :cowlib, repo: "hexpm", optional: false]}, {:gun, "~> 2.0.1", [hex: :grpc_gun, repo: "hexpm", optional: false]}], "hexpm", "17b98593fdb1a65be7b2722821266627b3f2fba29bbbd7d0945389427c0d0d5f"},
+  "grpc": {:hex, :grpc, "0.9.0", "1b930a57272d4356ea65969b984c2eb04f3dab81420e1e28f0e6ec04b8f88515", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:cowboy, "~> 2.10", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowlib, "~> 2.12", [hex: :cowlib, repo: "hexpm", optional: false]}, {:gun, "~> 2.0", [hex: :gun, repo: "hexpm", optional: false]}, {:jason, ">= 0.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mint, "~> 1.5", [hex: :mint, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.11", [hex: :protobuf, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7c059698248738fcf7ad551f1d78f4a3d2e0642a72a5834f2a0b0db4b9f3d2b5"},
   "grpcbox": {:hex, :grpcbox, "0.17.1", "6e040ab3ef16fe699ffb513b0ef8e2e896da7b18931a1ef817143037c454bcce", [:rebar3], [{:acceptor_pool, "~> 1.0.0", [hex: :acceptor_pool, repo: "hexpm", optional: false]}, {:chatterbox, "~> 0.15.1", [hex: :ts_chatterbox, repo: "hexpm", optional: false]}, {:ctx, "~> 0.6.0", [hex: :ctx, repo: "hexpm", optional: false]}, {:gproc, "~> 0.9.1", [hex: :gproc, repo: "hexpm", optional: false]}], "hexpm", "4a3b5d7111daabc569dc9cbd9b202a3237d81c80bf97212fbc676832cb0ceb17"},
   "gun": {:hex, :gun, "2.1.0", "b4e4cbbf3026d21981c447e9e7ca856766046eff693720ba43114d7f5de36e87", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "52fc7fc246bfc3b00e01aea1c2854c70a366348574ab50c57dfe796d24a0101d"},
   "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
@@ -97,10 +97,10 @@
   "oauth2": {:hex, :oauth2, "2.0.1", "70729503e05378697b958919bb2d65b002ba6b28c8112328063648a9348aaa3f", [:mix], [{:hackney, "~> 1.13", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "c64e20d4d105bcdbcbe03170fb530d0eddc3a3e6b135a87528a22c8aecf74c52"},
   "observer_cli": {:hex, :observer_cli, "1.7.4", "3c1bfb6d91bf68f6a3d15f46ae20da0f7740d363ee5bc041191ce8722a6c4fae", [:mix, :rebar3], [{:recon, "~> 2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "50de6d95d814f447458bd5d72666a74624eddb0ef98bdcee61a0153aae0865ff"},
   "open_api_spex": {:hex, :open_api_spex, "3.18.0", "f9952b6bc8a1bf14168f3754981b7c8d72d015112bfedf2588471dd602e1e715", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "37849887ab67efab052376401fac28c0974b273ffaecd98f4532455ca0886464"},
-  "opentelemetry": {:hex, :opentelemetry, "1.4.0", "f928923ed80adb5eb7894bac22e9a198478e6a8f04020ae1d6f289fdcad0b498", [:rebar3], [{:opentelemetry_api, "~> 1.3.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "50b32ce127413e5d87b092b4d210a3449ea80cd8224090fe68d73d576a3faa15"},
-  "opentelemetry_api": {:hex, :opentelemetry_api, "1.3.0", "03e2177f28dd8d11aaa88e8522c81c2f6a788170fe52f7a65262340961e663f9", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "b9e5ff775fd064fa098dba3c398490b77649a352b40b0b730a6b7dc0bdd68858"},
+  "opentelemetry": {:hex, :opentelemetry, "1.5.0", "7dda6551edfc3050ea4b0b40c0d2570423d6372b97e9c60793263ef62c53c3c2", [:rebar3], [{:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "cdf4f51d17b592fc592b9a75f86a6f808c23044ba7cf7b9534debbcc5c23b0ee"},
+  "opentelemetry_api": {:hex, :opentelemetry_api, "1.4.0", "63ca1742f92f00059298f478048dfb826f4b20d49534493d6919a0db39b6db04", [:mix, :rebar3], [], "hexpm", "3dfbbfaa2c2ed3121c5c483162836c4f9027def469c41578af5ef32589fcfc58"},
   "opentelemetry_cowboy": {:hex, :opentelemetry_cowboy, "0.3.0", "0144b211fa6cda0e6211c340cebd1bbd9158e350099ea3bf3d838f993cb4b90e", [:rebar3], [{:cowboy_telemetry, "~> 0.4", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.2", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.0", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4f44537b4c7430018198d480f55bc88a40f7d0582c3ad927a5bab4ceb39e80ea"},
-  "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.7.0", "dec4e90c0667cf11a3642f7fe71982dbc0c6bfbb8725a0b13766830718cf0d98", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.4.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.3.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "d0f25f6439ec43f2561537c3fabbe177b38547cddaa3a692cbb8f4770dbefc1e"},
+  "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.8.0", "5d546123230771ef4174e37bedfd77e3374913304cd6ea3ca82a2add49cd5d56", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.5.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "a1f9f271f8d3b02b81462a6bfef7075fd8457fdb06adff5d2537df5e2264d9af"},
   "opentelemetry_phoenix": {:hex, :opentelemetry_phoenix, "1.2.0", "b8a53ee595b24970571a7d2fcaef3e4e1a021c68e97cac163ca5d9875fad5e9f", [:mix], [{:nimble_options, "~> 0.5 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.0", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:plug, ">= 1.11.0", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "acab991d14ed3efc3f780c5a20cabba27149cf731005b1cc6454c160859debe5"},
   "opentelemetry_process_propagator": {:hex, :opentelemetry_process_propagator, "0.3.0", "ef5b2059403a1e2b2d2c65914e6962e56371570b8c3ab5323d7a8d3444fb7f84", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "7243cb6de1523c473cba5b1aefa3f85e1ff8cc75d08f367104c1e11919c8c029"},
   "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"},
@@ -150,7 +150,7 @@
   "thousand_island": {:hex, :thousand_island, "1.2.0", "4f548ae771ab5f96bc7e199f9824c0c2ce6d365f8c93f5f64dbbb33988e484bf", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "521671fea179672addb6af46455fc2a77be1edda4c0ed351633e0ef37a4b3584"},
   "timber_logfmt": {:git, "https://github.com/Logflare/logfmt-elixir.git", "9766367ccb3014b47b0a621a489261011dcea769", []},
   "timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"},
-  "tls_certificate_check": {:hex, :tls_certificate_check, "1.22.1", "0f450cc1568a67a65ce5e15df53c53f9a098c3da081c5f126199a72505858dc1", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "3092be0babdc0e14c2e900542351e066c0fa5a9cf4b3597559ad1e67f07938c0"},
+  "tls_certificate_check": {:hex, :tls_certificate_check, "1.24.0", "d00e2887551ff8cdae4d0340d90d9fcbc4943c7b5f49d32ed4bc23aff4db9a44", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "90b25a58ee433d91c17f036d4d354bf8859a089bfda60e68a86f8eecae45ef1b"},
   "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"},
   "typed_ecto_schema": {:hex, :typed_ecto_schema, "0.1.1", "220b56d96c3899e6f313b37793bafec09c44a5dd4cd17b4dbd5703ecb75137cc", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "44152174ecfc1b08b2c39eb8a7781dd77d56ea326ffa61782339fa0082fa6a51"},
   "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"},

From 0feed572a5383ac823abd4ba016e6de569c2d8b8 Mon Sep 17 00:00:00 2001
From: TzeYiing <ty@tzeyiing.com>
Date: Wed, 6 Nov 2024 18:52:01 +0800
Subject: [PATCH 2/6] chore: fix failing tests

---
 Makefile                                 |  1 +
 lib/logflare_grpc/trace/server.ex        | 79 +++++++++++++++++++-----
 mix.exs                                  | 12 ++--
 mix.lock                                 |  2 +-
 test/logflare_grpc/trace/server_test.exs | 61 +++++-------------
 5 files changed, 87 insertions(+), 68 deletions(-)

diff --git a/Makefile b/Makefile
index a03090141..72a75fa1d 100644
--- a/Makefile
+++ b/Makefile
@@ -28,6 +28,7 @@ setup: setup.node
 
 	# add protobuf install
 	mix escript.install hex protobuf
+	
 	asdf reshim
 	# run elixir setup
 	mix setup
diff --git a/lib/logflare_grpc/trace/server.ex b/lib/logflare_grpc/trace/server.ex
index 7d9d01a27..c7aaeee40 100644
--- a/lib/logflare_grpc/trace/server.ex
+++ b/lib/logflare_grpc/trace/server.ex
@@ -4,8 +4,14 @@ defmodule LogflareGrpc.Trace.Server do
   """
   alias LogflareGrpc.Trace.Transform
   alias Logflare.Sources
+  alias Logflare.Source
   alias Logflare.Users
+  alias Logflare.Auth
+  alias Logflare.User
+  alias Logflare.Sources
+  alias Logflare.Backends
   alias Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceResponse
+  alias Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceRequest
 
   use GRPC.Server,
     service: Opentelemetry.Proto.Collector.Trace.V1.TraceService.Service,
@@ -13,25 +19,64 @@ defmodule LogflareGrpc.Trace.Server do
 
   require Logger
 
-  def export(events, stream) do
-    with %{"x-api-key" => api_key, "x-source-id" => source_token} <-
-           GRPC.Stream.get_headers(stream),
-         user when not is_nil(user) <- Users.get_by(api_key: api_key),
-         {:ok, source_token} <- Ecto.UUID.cast(source_token),
-         source when not is_nil(source) <- Sources.get_by(user_id: user.id, token: source_token) do
-      events
-      |> Stream.flat_map(fn %{resource_spans: resource_spans} ->
-        Transform.to_log_events(resource_spans, source)
-      end)
-      |> Enum.each(fn le ->
-        Logflare.Logs.ingest(le)
-        GRPC.Server.set_trailers(stream, %{"grpc-status" => "0"})
-        GRPC.Server.send_reply(stream, %ExportTraceServiceResponse{})
-      end)
+  def export(%ExportTraceServiceRequest{resource_spans: spans} = request, stream) do
+    with {:ok, api_key} <- get_access_token(stream),
+         {:ok, source_token} <- get_source_token(stream),
+         {:ok, user} <- verify_user(api_key),
+         {:ok, source} <- get_source(user, source_token) do
+      events = Transform.to_log_events(spans, source)
+      Backends.ingest_logs(events, source)
+      GRPC.Server.set_trailers(stream, %{"grpc-status" => "0"})
+      %ExportTraceServiceResponse{}
     else
-      error ->
-        Logger.error("Invalid GRPC request: #{inspect(error)}")
+      {:error, :unauthorized} ->
         raise GRPC.RPCError, status: :unauthenticated, message: "Invalid API Key or Source ID"
+
+      err ->
+        raise GRPC.RPCError, status: :unknown, message: "Unknown error: #{inspect(err)}"
+    end
+  end
+
+  defp get_access_token(stream) do
+    case GRPC.Stream.get_headers(stream) do
+      %{"authorization" => "Bearer " <> token} -> {:ok, token}
+      %{"x-api-key" => key} -> {:ok, key}
+      _ -> {:error, :unauthorized}
+    end
+  end
+
+  defp get_source_token(stream) do
+    case GRPC.Stream.get_headers(stream) do
+      %{"x-source-token" => token} -> {:ok, token}
+      %{"x-source-id" => token} -> {:ok, token}
+      _ -> {:error, :unauthorized}
+    end
+  end
+
+  defp verify_user(access_token_or_api_key) do
+    with {:ok, %User{} = owner} <-
+           Auth.Cache.verify_access_token(access_token_or_api_key, ["public"]) do
+      {:ok, owner}
+    else
+      {:error, :no_token} = err ->
+        err
+
+      {:error, _} = err ->
+        if user = Users.Cache.get_by_and_preload(api_key: access_token_or_api_key) do
+          {:ok, user}
+        else
+          {:error, :unauthorized}
+        end
+    end
+  end
+
+  defp get_source(user, source_token) do
+    with true <- Sources.valid_source_token_param?(source_token),
+         source = %Source{} <- Sources.get_by_and_preload(user_id: user.id, token: source_token) do
+      {:ok, source}
+    else
+      _ ->
+        {:error, :unauthorized}
     end
   end
 end
diff --git a/mix.exs b/mix.exs
index 3191853f0..99347d57a 100644
--- a/mix.exs
+++ b/mix.exs
@@ -201,7 +201,7 @@ defmodule Logflare.Mixfile do
       {:cainophile, github: "Logflare/cainophile", ref: "267999b"},
       {:open_api_spex, "~> 3.16"},
       {:grpc, "~> 0.9.0"},
-      {:protobuf, "~> 0.10"},
+      {:protobuf, "~> 0.12"},
       {:gun, "~> 2.0", override: true},
       {:cowlib, ">=2.12.0", override: true},
       {:phoenix_live_dashboard, "~> 0.8"},
@@ -221,11 +221,11 @@ defmodule Logflare.Mixfile do
       {:file_system, "~> 1.0", override: true, only: [:dev, :test]},
 
       # otel
-      {:opentelemetry, "~> 1.5"},
-      {:opentelemetry_api, "~> 1.4"},
-      {:opentelemetry_exporter, "~> 1.8"},
-      {:opentelemetry_phoenix, "~> 1.2"},
-      {:opentelemetry_cowboy, "~> 0.3"},
+      {:opentelemetry, "~> 1.3"},
+      {:opentelemetry_api, "~> 1.2"},
+      {:opentelemetry_exporter, "~> 1.6"},
+      {:opentelemetry_phoenix, "~> 1.1"},
+      {:opentelemetry_cowboy, "~> 0.2"},
       {:live_monaco_editor, "~> 0.1"}
     ]
   end
diff --git a/mix.lock b/mix.lock
index 732390646..af7cdfbd6 100644
--- a/mix.lock
+++ b/mix.lock
@@ -125,7 +125,7 @@
   "poison": {:hex, :poison, "5.0.0", "d2b54589ab4157bbb82ec2050757779bfed724463a544b6e20d79855a9e43b24", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "11dc6117c501b80c62a7594f941d043982a1bd05a1184280c0d9166eb4d8d3fc"},
   "postgrex": {:hex, :postgrex, "0.17.3", "c92cda8de2033a7585dae8c61b1d420a1a1322421df84da9a82a6764580c503d", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "946cf46935a4fdca7a81448be76ba3503cff082df42c6ec1ff16a4bdfbfb098d"},
   "proper_case": {:hex, :proper_case, "1.3.1", "5f51cabd2d422a45f374c6061b7379191d585b5154456b371432d0fa7cb1ffda", [:mix], [], "hexpm", "6cc715550cc1895e61608060bbe67aef0d7c9cf55d7ddb013c6d7073036811dd"},
-  "protobuf": {:hex, :protobuf, "0.12.0", "58c0dfea5f929b96b5aa54ec02b7130688f09d2de5ddc521d696eec2a015b223", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "75fa6cbf262062073dd51be44dd0ab940500e18386a6c4e87d5819a58964dc45"},
+  "protobuf": {:hex, :protobuf, "0.13.0", "7a9d9aeb039f68a81717eb2efd6928fdf44f03d2c0dfdcedc7b560f5f5aae93d", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "21092a223e3c6c144c1a291ab082a7ead32821ba77073b72c68515aa51fef570"},
   "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
   "recase": {:hex, :recase, "0.7.0", "3f2f719f0886c7a3b7fe469058ec539cb7bbe0023604ae3bce920e186305e5ae", [:mix], [], "hexpm", "36f5756a9f552f4a94b54a695870e32f4e72d5fad9c25e61bc4a3151c08a4e0c"},
   "recon": {:hex, :recon, "2.5.4", "05dd52a119ee4059fa9daa1ab7ce81bc7a8161a2f12e9d42e9d551ffd2ba901c", [:mix, :rebar3], [], "hexpm", "e9ab01ac7fc8572e41eb59385efeb3fb0ff5bf02103816535bacaedf327d0263"},
diff --git a/test/logflare_grpc/trace/server_test.exs b/test/logflare_grpc/trace/server_test.exs
index 3c99bfd32..bc871e0d3 100644
--- a/test/logflare_grpc/trace/server_test.exs
+++ b/test/logflare_grpc/trace/server_test.exs
@@ -1,13 +1,15 @@
 defmodule LogflareGrpc.Trace.ServerTest do
   use Logflare.DataCase, async: false
 
-  alias Logflare.Logs
   alias LogflareGrpc.Trace.Server
   alias Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceResponse
   alias Opentelemetry.Proto.Collector.Trace.V1.TraceService.Stub
+  alias Logflare.SystemMetrics.AllLogsLogged
 
   setup do
     insert(:plan)
+    start_supervised!(AllLogsLogged)
+
     :ok
   end
 
@@ -22,10 +24,11 @@ defmodule LogflareGrpc.Trace.ServerTest do
     end
 
     defp emulate_request(channel, request) do
-      stream = Stub.export(channel)
-      GRPC.Stub.send_request(stream, request)
-      GRPC.Stub.disconnect(channel)
-      GRPC.Stub.end_stream(stream)
+      Stub.export(channel, request)
+      # GRPC.Stub.send_request(stream, request)
+      # GRPC.Stub.disconnect(channel)
+      # GRPC.Stub.end_stream(stream)
+      # reply
     end
 
     test "returns a success response and starts log event ingestion", %{
@@ -42,20 +45,10 @@ defmodule LogflareGrpc.Trace.ServerTest do
         )
 
       request = TestUtilsGrpc.random_export_service_request()
-      expect(Logs, :ingest, 2, fn _ -> :ok end)
-
-      result =
-        channel
-        |> emulate_request(request)
-        |> GRPC.Stub.recv(timeout: 100, return_headers: true)
-        |> then(&elem(&1, 1))
-        |> Enum.to_list()
-
-      assert [
-               {:ok, %ExportTraceServiceResponse{}},
-               {:ok, %ExportTraceServiceResponse{}},
-               {:trailers, %{"grpc-message" => "", "grpc-status" => "0"}}
-             ] = result
+
+      assert {:ok, %ExportTraceServiceResponse{}} =
+               channel
+               |> emulate_request(request)
     end
 
     test "returns an error if invalid api key", %{source: source, port: port} do
@@ -68,15 +61,10 @@ defmodule LogflareGrpc.Trace.ServerTest do
         )
 
       request = TestUtilsGrpc.random_export_service_request()
-      reject(Logs, :ingest, 1)
 
-      result =
-        channel
-        |> emulate_request(request)
-        |> GRPC.Stub.recv(timeout: 100, return_headers: true)
-        |> then(&elem(&1, 1))
+      {:error, err} = emulate_request(channel, request)
 
-      assert %GRPC.RPCError{message: "Invalid API Key or Source ID"} = result
+      assert %GRPC.RPCError{message: "Invalid API Key or Source ID"} = err
     end
 
     test "returns an error if invalid source ID", %{
@@ -92,13 +80,8 @@ defmodule LogflareGrpc.Trace.ServerTest do
         )
 
       request = TestUtilsGrpc.random_export_service_request()
-      reject(Logs, :ingest, 1)
 
-      result =
-        channel
-        |> emulate_request(request)
-        |> GRPC.Stub.recv(timeout: 100, return_headers: true)
-        |> then(&elem(&1, 1))
+      {:error, result} = emulate_request(channel, request)
 
       assert %GRPC.RPCError{message: "Invalid API Key or Source ID"} = result
     end
@@ -113,13 +96,8 @@ defmodule LogflareGrpc.Trace.ServerTest do
         )
 
       request = TestUtilsGrpc.random_export_service_request()
-      reject(Logs, :ingest, 1)
 
-      result =
-        channel
-        |> emulate_request(request)
-        |> GRPC.Stub.recv(timeout: 100, return_headers: true)
-        |> then(&elem(&1, 1))
+      {:error, result} = emulate_request(channel, request)
 
       assert %GRPC.RPCError{message: "Invalid API Key or Source ID"} = result
     end
@@ -137,13 +115,8 @@ defmodule LogflareGrpc.Trace.ServerTest do
         )
 
       request = TestUtilsGrpc.random_export_service_request()
-      reject(Logs, :ingest, 1)
 
-      result =
-        channel
-        |> emulate_request(request)
-        |> GRPC.Stub.recv(timeout: 100, return_headers: true)
-        |> then(&elem(&1, 1))
+      {:error, result} = emulate_request(channel, request)
 
       assert %GRPC.RPCError{message: "Invalid API Key or Source ID"} = result
     end

From fb03b870f92c15f7751214d770eff27f4c75c57f Mon Sep 17 00:00:00 2001
From: TzeYiing <ty@tzeyiing.com>
Date: Wed, 6 Nov 2024 19:01:58 +0800
Subject: [PATCH 3/6] fix: update grpc server startup config

---
 config/config.exs           | 2 --
 config/test.exs             | 1 -
 lib/logflare/application.ex | 3 ++-
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 8f895f839..e019bbca3 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -125,8 +125,6 @@ config :open_api_spex, :cache_adapter, OpenApiSpex.Plug.PersistentTermCache
 
 config :logflare, Logflare.Cluster.Utils, min_cluster_size: 1
 
-config :grpc, start_server: true
-
 config :logflare, Logflare.AlertsScheduler, init_task: {Logflare.Alerting, :init_alert_jobs, []}
 
 config :opentelemetry,
diff --git a/config/test.exs b/config/test.exs
index 3fcbc09aa..0d669a32b 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -34,7 +34,6 @@ config :logflare, Logflare.Repo,
   pool: Ecto.Adapters.SQL.Sandbox
 
 config :logflare, :postgres_backend_adapter, pool_size: 1
-config :grpc, start_server: false
 
 defmodule LogflareTest.LogFilters do
   def ignore_finch_disconnections(%{meta: %{mfa: {Finch.HTTP2.Pool, :disconnected, _}}}, _opts) do
diff --git a/lib/logflare/application.ex b/lib/logflare/application.ex
index ea5e15071..f22420c31 100644
--- a/lib/logflare/application.ex
+++ b/lib/logflare/application.ex
@@ -104,7 +104,8 @@ defmodule Logflare.Application do
          name: Logflare.Source.V1SourceDynSup},
         LogflareWeb.Endpoint,
         # If we get a log event and the Source.Supervisor is not up it will 500
-        {GRPC.Server.Supervisor, {LogflareGrpc.Endpoint, grpc_port, cred: grpc_creds}},
+        {GRPC.Server.Supervisor,
+         endpoint: LogflareGrpc.Endpoint, port: grpc_port, cred: grpc_creds, start_server: true},
         # Monitor system level metrics
         SystemMetricsSup,
 

From 2d93d2df6c09f8442e61bc34749a9311fab30634 Mon Sep 17 00:00:00 2001
From: TzeYiing <ty@tzeyiing.com>
Date: Wed, 6 Nov 2024 19:24:42 +0800
Subject: [PATCH 4/6] feat: consolidate otel trace transform module into a
 Logs.Processor

---
 .../logs/otel_trace.ex}                       | 35 ++++++++-----------
 lib/logflare_grpc/trace/server.ex             |  7 ++--
 2 files changed, 19 insertions(+), 23 deletions(-)
 rename lib/{logflare_grpc/trace/transform.ex => logflare/logs/otel_trace.ex} (85%)

diff --git a/lib/logflare_grpc/trace/transform.ex b/lib/logflare/logs/otel_trace.ex
similarity index 85%
rename from lib/logflare_grpc/trace/transform.ex
rename to lib/logflare/logs/otel_trace.ex
index f30566f3a..749a2e194 100644
--- a/lib/logflare_grpc/trace/transform.ex
+++ b/lib/logflare/logs/otel_trace.ex
@@ -1,35 +1,30 @@
-defmodule LogflareGrpc.Trace.Transform do
+defmodule Logflare.Logs.OtelTrace do
   @moduledoc """
-  Converts a GRPC Trace to a Logflare Event
-  """
-
-  alias Opentelemetry.Proto.Trace.V1.ResourceSpans
-  alias Opentelemetry.Proto.Common.V1.KeyValue
-  alias Logflare.LogEvent
-  alias Logflare.Source
+  Converts a GRPC Trace to event parameters
 
-  @doc """
   Converts a list of ResourceSpans to a list of Logflare Events
   Important details about the conversation:
   * One ResourceSpans can contain multiple ScopeSpans
   * One ScopeSpans can contain multiple Spans
   * One Span can contain multiple Events
   * A Log Event is created for each Span and each Event
+
   """
-  @spec to_log_events(list(ResourceSpans.t()), Source.t()) :: list(LogEvent.t())
-  def to_log_events(resource_spans, source) do
-    resource_spans
-    |> Enum.map(&to_log_event(&1, source))
-    |> List.flatten()
-  end
+  require Logger
+
+  alias Opentelemetry.Proto.Trace.V1.ResourceSpans
+  alias Opentelemetry.Proto.Common.V1.KeyValue
+  alias Logflare.LogEvent
+  alias Logflare.Source
 
-  defp to_log_event(%ResourceSpans{} = span, source) do
-    span
-    |> handle_resource_span()
-    |> Enum.map(&LogEvent.make(&1, %{source: source}))
+  @behaviour Logflare.Logs.Processor
+
+  def handle_batch(resource_spans, _source) when is_list(resource_spans) do
+    Enum.map(resource_spans, &handle_event/1)
+    |> List.flatten()
   end
 
-  defp handle_resource_span(%{resource: resource, scope_spans: scope_spans}) do
+  defp handle_event(%ResourceSpans{resource: resource, scope_spans: scope_spans} = span) do
     resource = handle_resource(resource)
 
     scope_spans
diff --git a/lib/logflare_grpc/trace/server.ex b/lib/logflare_grpc/trace/server.ex
index c7aaeee40..ad6a5fc0c 100644
--- a/lib/logflare_grpc/trace/server.ex
+++ b/lib/logflare_grpc/trace/server.ex
@@ -2,7 +2,8 @@ defmodule LogflareGrpc.Trace.Server do
   @moduledoc """
   GRPC Server for the Logflare Trace Service
   """
-  alias LogflareGrpc.Trace.Transform
+  alias Logflare.Logs.Processor
+  alias Logflare.Logs.OtelTrace
   alias Logflare.Sources
   alias Logflare.Source
   alias Logflare.Users
@@ -24,8 +25,8 @@ defmodule LogflareGrpc.Trace.Server do
          {:ok, source_token} <- get_source_token(stream),
          {:ok, user} <- verify_user(api_key),
          {:ok, source} <- get_source(user, source_token) do
-      events = Transform.to_log_events(spans, source)
-      Backends.ingest_logs(events, source)
+      Processor.ingest(spans, OtelTrace, source)
+
       GRPC.Server.set_trailers(stream, %{"grpc-status" => "0"})
       %ExportTraceServiceResponse{}
     else

From cf8bc96a3b12d00f11e058c175a1e4f478bef665 Mon Sep 17 00:00:00 2001
From: TzeYiing <ty@tzeyiing.com>
Date: Wed, 6 Nov 2024 19:29:23 +0800
Subject: [PATCH 5/6] chore: compilation warnings

---
 lib/logflare/logs/otel_trace.ex   | 4 +---
 lib/logflare_grpc/trace/server.ex | 5 ++---
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/lib/logflare/logs/otel_trace.ex b/lib/logflare/logs/otel_trace.ex
index 749a2e194..97dccd7d1 100644
--- a/lib/logflare/logs/otel_trace.ex
+++ b/lib/logflare/logs/otel_trace.ex
@@ -14,8 +14,6 @@ defmodule Logflare.Logs.OtelTrace do
 
   alias Opentelemetry.Proto.Trace.V1.ResourceSpans
   alias Opentelemetry.Proto.Common.V1.KeyValue
-  alias Logflare.LogEvent
-  alias Logflare.Source
 
   @behaviour Logflare.Logs.Processor
 
@@ -24,7 +22,7 @@ defmodule Logflare.Logs.OtelTrace do
     |> List.flatten()
   end
 
-  defp handle_event(%ResourceSpans{resource: resource, scope_spans: scope_spans} = span) do
+  defp handle_event(%ResourceSpans{resource: resource, scope_spans: scope_spans}) do
     resource = handle_resource(resource)
 
     scope_spans
diff --git a/lib/logflare_grpc/trace/server.ex b/lib/logflare_grpc/trace/server.ex
index ad6a5fc0c..a33f5f9ca 100644
--- a/lib/logflare_grpc/trace/server.ex
+++ b/lib/logflare_grpc/trace/server.ex
@@ -10,7 +10,6 @@ defmodule LogflareGrpc.Trace.Server do
   alias Logflare.Auth
   alias Logflare.User
   alias Logflare.Sources
-  alias Logflare.Backends
   alias Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceResponse
   alias Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceRequest
 
@@ -20,7 +19,7 @@ defmodule LogflareGrpc.Trace.Server do
 
   require Logger
 
-  def export(%ExportTraceServiceRequest{resource_spans: spans} = request, stream) do
+  def export(%ExportTraceServiceRequest{resource_spans: spans}, stream) do
     with {:ok, api_key} <- get_access_token(stream),
          {:ok, source_token} <- get_source_token(stream),
          {:ok, user} <- verify_user(api_key),
@@ -62,7 +61,7 @@ defmodule LogflareGrpc.Trace.Server do
       {:error, :no_token} = err ->
         err
 
-      {:error, _} = err ->
+      {:error, _} ->
         if user = Users.Cache.get_by_and_preload(api_key: access_token_or_api_key) do
           {:ok, user}
         else

From 98c26fb6627090e90465b1535c0d71309fb13a31 Mon Sep 17 00:00:00 2001
From: TzeYiing <ty@tzeyiing.com>
Date: Wed, 6 Nov 2024 19:52:01 +0800
Subject: [PATCH 6/6] chore: fix failing tests

---
 lib/logflare/logs/otel_trace.ex             |  4 +-
 test/logflare/logs/otel_trace_test.exs      | 60 ++++++++++++++++++++
 test/logflare_grpc/trace/transform_test.exs | 61 ---------------------
 3 files changed, 62 insertions(+), 63 deletions(-)
 create mode 100644 test/logflare/logs/otel_trace_test.exs
 delete mode 100644 test/logflare_grpc/trace/transform_test.exs

diff --git a/lib/logflare/logs/otel_trace.ex b/lib/logflare/logs/otel_trace.ex
index 97dccd7d1..31dad152f 100644
--- a/lib/logflare/logs/otel_trace.ex
+++ b/lib/logflare/logs/otel_trace.ex
@@ -57,7 +57,7 @@ defmodule Logflare.Logs.OtelTrace do
 
     [
       %{
-        "message" => span.name,
+        "event_message" => span.name,
         "metadata" => metadata,
         "span_id" => Base.encode16(span.span_id),
         "parent_span_id" => Base.encode16(span.parent_span_id),
@@ -77,7 +77,7 @@ defmodule Logflare.Logs.OtelTrace do
     metadata = Map.merge(metadata, resource)
 
     %{
-      "message" => event.name,
+      "event_message" => event.name,
       "metadata" => metadata,
       "parent_span_id" => Base.encode16(span_id),
       "trace_id" => Ecto.UUID.cast!(trace_id),
diff --git a/test/logflare/logs/otel_trace_test.exs b/test/logflare/logs/otel_trace_test.exs
new file mode 100644
index 000000000..1824436bc
--- /dev/null
+++ b/test/logflare/logs/otel_trace_test.exs
@@ -0,0 +1,60 @@
+defmodule Logflare.Logs.OtelTraceTest do
+  use Logflare.DataCase
+  alias Logflare.Logs.OtelTrace
+
+  describe "handle_batch/1" do
+    setup do
+      user = build(:user)
+      source = insert(:source, user: user)
+      %{resource_spans: TestUtilsGrpc.random_resource_span(), source: source}
+    end
+
+    test "Creates params from a list with one Resource Span that contains an Event", %{
+      resource_spans: resource_spans,
+      source: source
+    } do
+      batch = OtelTrace.handle_batch(resource_spans, source)
+
+      span =
+        resource_spans
+        |> hd()
+        |> then(fn rs -> rs.scope_spans end)
+        |> hd()
+        |> then(fn ss -> ss.spans end)
+        |> hd()
+
+      le_span = Enum.find(batch, fn params -> params["metadata"]["type"] == "span" end)
+
+      assert le_span["trace_id"] == elem(Ecto.UUID.cast(span.trace_id), 1)
+      assert le_span["span_id"] == Base.encode16(span.span_id)
+      assert le_span["parent_span_id"] == Base.encode16(span.parent_span_id)
+      assert le_span["event_message"] == span.name
+
+      assert elem(DateTime.from_iso8601(le_span["timestamp"]), 1) ==
+               DateTime.from_unix!(span.start_time_unix_nano, :nanosecond)
+
+      assert elem(DateTime.from_iso8601(le_span["start_time"]), 1) ==
+               elem(DateTime.from_unix(span.start_time_unix_nano, :nanosecond), 1)
+
+      assert elem(DateTime.from_iso8601(le_span["end_time"]), 1) ==
+               elem(DateTime.from_unix(span.end_time_unix_nano, :nanosecond), 1)
+
+      event = hd(span.events)
+      le_event = Enum.find(batch, fn params -> params["metadata"]["type"] == "event" end)
+
+      assert le_event["event_message"] == event.name
+      assert le_event["parent_span_id"] == Base.encode16(span.span_id)
+
+      assert elem(DateTime.from_iso8601(le_event["timestamp"]), 1) ==
+               DateTime.from_unix!(event.time_unix_nano, :nanosecond)
+    end
+
+    test "json parsable log event body", %{
+      resource_spans: resource_spans,
+      source: source
+    } do
+      [params | _] = resource_spans |> OtelTrace.handle_batch(source)
+      assert {:ok, _} = Jason.encode(params)
+    end
+  end
+end
diff --git a/test/logflare_grpc/trace/transform_test.exs b/test/logflare_grpc/trace/transform_test.exs
deleted file mode 100644
index a221a831f..000000000
--- a/test/logflare_grpc/trace/transform_test.exs
+++ /dev/null
@@ -1,61 +0,0 @@
-defmodule LogflareGrpc.Trace.TransformTest do
-  use Logflare.DataCase
-  alias LogflareGrpc.Trace.Transform
-
-  describe "to_log_events/1" do
-    setup do
-      user = build(:user)
-      source = insert(:source, user: user)
-      %{resource_spans: TestUtilsGrpc.random_resource_span(), source: source}
-    end
-
-    test "Creates LogEvent from a list with one Resource Span that contains an Event", %{
-      resource_spans: resource_spans,
-      source: source
-    } do
-      log_events = Transform.to_log_events(resource_spans, source)
-      assert Enum.all?(log_events, fn %{source: %{id: id}} -> source.id == id end)
-
-      span =
-        resource_spans
-        |> hd()
-        |> then(fn rs -> rs.scope_spans end)
-        |> hd()
-        |> then(fn ss -> ss.spans end)
-        |> hd()
-
-      le_span = Enum.find(log_events, fn le -> le.body["metadata"]["type"] == "span" end)
-
-      assert le_span.body["trace_id"] == elem(Ecto.UUID.cast(span.trace_id), 1)
-      assert le_span.body["span_id"] == Base.encode16(span.span_id)
-      assert le_span.body["parent_span_id"] == Base.encode16(span.parent_span_id)
-      assert le_span.body["event_message"] == span.name
-
-      assert DateTime.from_unix(le_span.body["timestamp"], :microsecond) ==
-               DateTime.from_unix(span.start_time_unix_nano, :nanosecond)
-
-      assert elem(DateTime.from_iso8601(le_span.body["start_time"]), 1) ==
-               elem(DateTime.from_unix(span.start_time_unix_nano, :nanosecond), 1)
-
-      assert elem(DateTime.from_iso8601(le_span.body["end_time"]), 1) ==
-               elem(DateTime.from_unix(span.end_time_unix_nano, :nanosecond), 1)
-
-      event = hd(span.events)
-      le_event = Enum.find(log_events, fn le -> le.body["metadata"]["type"] == "event" end)
-
-      assert le_event.body["event_message"] == event.name
-      assert le_event.body["parent_span_id"] == Base.encode16(span.span_id)
-
-      assert DateTime.from_unix(le_event.body["timestamp"], :microsecond) ==
-               DateTime.from_unix(event.time_unix_nano, :nanosecond)
-    end
-
-    test "json parsable log event body", %{
-      resource_spans: resource_spans,
-      source: source
-    } do
-      log_events_body = resource_spans |> Transform.to_log_events(source) |> Enum.map(& &1.body)
-      assert {:ok, _} = Jason.encode(log_events_body)
-    end
-  end
-end