feat(EKS Fargate): Add multiline support to EKS Fargate (#3059)

* feat(EKS Fargate): Add multiline support to EKS Fargate

* fix(EKS Fargate): Using the log stream resource attribute as the source identifier for the recombine operator

.changelog/3059.added.txt

@@ -0,0 +1 @@
+feat(EKS Fargate): Add multiline support to EKS Fargate

deploy/helm/sumologic/conf/logs/collector/otelcloudwatch/config.yaml

@@ -26,24 +26,50 @@ processors:
     send_batch_max_size: 2000
     send_batch_size: 1000
     timeout: 1s
-  transform/cloudwatch_logs:
+  transform/set_source_identifier:
     error_mode: ignore
     log_statements:
       - context: log
         statements:
-          - set(attributes["k8s.container.name"], resource.attributes["cloudwatch.log.stream"])
-          - set(attributes["k8s.pod.name"], resource.attributes["cloudwatch.log.stream"])
-          - set(attributes["k8s.namespace.name"], resource.attributes["cloudwatch.log.stream"])
-          - replace_pattern(attributes["k8s.pod.name"], "^.*kube\\.var\\.log\\.containers\\.([0-9a-zA-Z\\-]+)\\_.*", "$$1")
-          - replace_pattern(attributes["k8s.container.name"], "^.*kube\\.var\\.log\\.containers\\.[0-9a-zA-Z\\-]+\\_[a-zA-Z\\-]*\\_([a-zA-Z]*).*", "$$1")
-          - replace_pattern(attributes["k8s.namespace.name"], "^.*kube\\.var\\.log\\.containers\\.[0-9a-zA-Z\\-]+\\_([a-zA-Z\\-]*)_.*", "$$1")
-  transform/parse:
+          - set(attributes["cloudwatch.log.stream"], resource.attributes["cloudwatch.log.stream"])
+  groupbyattrs/stream:
+    keys:
+      - cloudwatch.log.stream
+  transform/parsejson:
+    error_mode: ignore
     log_statements:
       - context: log
         statements:
           - set(body, ParseJSON(body)) where IsMatch(body, "^{") == true
           - merge_maps(attributes, body, "insert")
           - set(body, "") where IsMatch(body, "^{") == true
+  transform/metadata:
+    error_mode: ignore
+    log_statements:
+      - context: log
+        statements:
+          - set(attributes["k8s.container.name"], resource.attributes["cloudwatch.log.stream"])
+          - set(attributes["k8s.pod.name"], resource.attributes["cloudwatch.log.stream"])
+          - set(attributes["k8s.namespace.name"], resource.attributes["cloudwatch.log.stream"])
+          - replace_pattern(attributes["k8s.pod.name"], "^.*kube\\.var\\.log\\.containers\\.([0-9a-zA-Z\\-]+)\\_.*", "$$1")
+          - replace_pattern(attributes["k8s.container.name"], "^.*kube\\.var\\.log\\.containers\\.[0-9a-zA-Z\\-]+\\_[a-zA-Z\\-]*\\_([a-zA-Z]*).*", "$$1")
+          - replace_pattern(attributes["k8s.namespace.name"], "^.*kube\\.var\\.log\\.containers\\.[0-9a-zA-Z\\-]+\\_([a-zA-Z\\-]*)_.*", "$$1")
+  logstransform/cloudwatch:
+    operators:
+      - id: merge-cri-lines
+        combine_field: attributes.log
+        combine_with: ""
+        is_last_entry: attributes.logtag == "F"
+        output: "merge-multiline-logs"
+        overwrite_with: newest
+        source_identifier: resource["cloudwatch.log.stream"]
+        type: recombine
+      - id: merge-multiline-logs
+        combine_field: attributes.log
+        combine_with: "\n"
+        is_first_entry: attributes.log matches {{ .Values.sumologic.logs.multiline.first_line_regex | quote }}
+        source_identifier: resource["cloudwatch.log.stream"]
+        type: recombine
 receivers:
   awscloudwatch:
     region: {{ .Values.sumologic.logs.collector.otelcloudwatch.region }}
@@ -63,8 +89,11 @@ service:
       receivers:
         - awscloudwatch
       processors:
-        - transform/parse
-        - transform/cloudwatch_logs
+        - transform/set_source_identifier
+        - groupbyattrs/stream
+        - transform/parsejson
+        - logstransform/cloudwatch
+        - transform/metadata
         - batch
       exporters:
         - otlphttp

docs/fargate.md

@@ -1,11 +1,8 @@
 # Fargate
 
-**NOTE: This is the alpha release of EKS Fargate.**
+**Release Note: Generally Available (GA) Release of EKS Fargate.**
 
-The following are some limitations of deploying this helm chart on EKS fargate
-
-- Only supports EKS version 1.24 and above
-- Does not support multiline logs
+Supports EKS version 1.24 and above
 
 The following documentation assumes that you are using eksctl to manage Fargate cluster. Code snippets are using environment variables in
 order to make them as generic and reusable.
@@ -780,7 +777,7 @@ sumologic:
           fluent-bit-cloudwatch:
             ## The log stream prefix, can also be specified as
             ## names: []
-            prefixes: [from-fluent-bit]
+            names: [from-fluent-bit]
 ```
 
 where `my-role` is the name of the role created while setting up [authentication](#authenticate-with-cloudwatch)