Added data compression support for HTTP exporter (#631)

Package.swift

@@ -29,6 +29,7 @@ let package = Package(
         .library(name: "DatadogExporter", type: .static, targets: ["DatadogExporter"]),
         .library(name: "NetworkStatus", type: .static, targets: ["NetworkStatus"]),
         .library(name: "OTelSwiftLog", type: .static, targets: ["OTelSwiftLog"]),
+        .library(name: "DataCompression", type: .static, targets: ["DataCompression"]),
         .executable(name: "simpleExporter", targets: ["SimpleExporter"]),
         .executable(name: "OTLPExporter", targets: ["OTLPExporter"]),
         .executable(name: "OTLPHTTPExporter", targets: ["OTLPHTTPExporter"]),
@@ -110,14 +111,18 @@ let package = Package(
                                "OpenTelemetryProtocolExporterCommon",
                                .product(name: "GRPC", package: "grpc-swift")],
                 path: "Sources/Exporters/OpenTelemetryProtocolGrpc"),
+        .target(name: "DataCompression",
+                dependencies: [],
+                path: "Sources/Exporters/DataCompression"),
         .target(name: "StdoutExporter",
                 dependencies: ["OpenTelemetrySdk"],
                 path: "Sources/Exporters/Stdout"),
         .target(name: "InMemoryExporter",
                 dependencies: ["OpenTelemetrySdk"],
                 path: "Sources/Exporters/InMemory"),
         .target(name: "DatadogExporter",
-                dependencies: ["OpenTelemetrySdk"],
+                dependencies: ["OpenTelemetrySdk",
+                               "DataCompression"],
                 path: "Sources/Exporters/DatadogExporter",
                 exclude: ["NOTICE", "README.md"]),
         .target(name: "PersistenceExporter",
@@ -163,6 +168,7 @@ let package = Package(
         .testTarget(name: "OpenTelemetryProtocolExporterTests",
                     dependencies: ["OpenTelemetryProtocolExporterGrpc",
                                    "OpenTelemetryProtocolExporterHttp",
+                                   "DataCompression",
                                     .product(name: "NIO", package: "swift-nio"),
                                     .product(name: "NIOHTTP1", package: "swift-nio"),
                                     .product(name: "NIOTestUtils", package: "swift-nio")],
@@ -190,7 +196,7 @@ let package = Package(
                 path: "Examples/OTLP Exporter",
                 exclude: ["README.md"]),
         .target(name: "OTLPHTTPExporter",
-                dependencies: ["OpenTelemetryProtocolExporterHttp", "StdoutExporter", "ZipkinExporter", "ResourceExtension", "SignPostIntegration"],
+                dependencies: ["OpenTelemetryProtocolExporterHttp", "StdoutExporter", "ZipkinExporter", "ResourceExtension", "SignPostIntegration", "DataCompression"],
                 path: "Examples/OTLP HTTP Exporter",
                 exclude: ["README.md"]),
         .target(name: "PrometheusSample",

Package@swift-5.6.swift

@@ -30,6 +30,7 @@ let package = Package(
         .library(name: "DatadogExporter", type: .static, targets: ["DatadogExporter"]),
         .library(name: "NetworkStatus", type: .static, targets: ["NetworkStatus"]),
         .library(name: "OTelSwiftLog", type: .static, targets: ["OTelSwiftLog"]),
+        .library(name: "DataCompression", type: .static, targets: ["DataCompression"]),
         .executable(name: "simpleExporter", targets: ["SimpleExporter"]),
         .executable(name: "OTLPExporter", targets: ["OTLPExporter"]),
         .executable(name: "OTLPHTTPExporter", targets: ["OTLPHTTPExporter"]),
@@ -115,14 +116,18 @@ let package = Package(
                                "OpenTelemetryProtocolExporterCommon",
                                .product(name: "GRPC", package: "grpc-swift")],
                 path: "Sources/Exporters/OpenTelemetryProtocolGrpc"),
+        .target(name: "DataCompression",
+                dependencies: [],
+                path: "Sources/Exporters/DataCompression"),
         .target(name: "StdoutExporter",
                 dependencies: ["OpenTelemetrySdk"],
                 path: "Sources/Exporters/Stdout"),
         .target(name: "InMemoryExporter",
                 dependencies: ["OpenTelemetrySdk"],
                 path: "Sources/Exporters/InMemory"),
         .target(name: "DatadogExporter",
-                dependencies: ["OpenTelemetrySdk"],
+                dependencies: ["OpenTelemetrySdk",
+                               "DataCompression"],
                 path: "Sources/Exporters/DatadogExporter",
                 exclude: ["NOTICE", "README.md"]),
         .target(name: "PersistenceExporter",
@@ -173,6 +178,7 @@ let package = Package(
         .testTarget(name: "OpenTelemetryProtocolExporterTests",
                     dependencies: ["OpenTelemetryProtocolExporterGrpc",
                                    "OpenTelemetryProtocolExporterHttp",
+                                   "DataCompression",
                                     .product(name: "NIO", package: "swift-nio"),
                                     .product(name: "NIOHTTP1", package: "swift-nio"),
                                     .product(name: "NIOTestUtils", package: "swift-nio")],
@@ -207,7 +213,7 @@ let package = Package(
         ),
         .executableTarget(
             name: "OTLPHTTPExporter",
-            dependencies: ["OpenTelemetrySdk", "OpenTelemetryProtocolExporterHttp", "StdoutExporter", "ZipkinExporter", "ResourceExtension", "SignPostIntegration"],
+            dependencies: ["OpenTelemetrySdk", "OpenTelemetryProtocolExporterHttp", "StdoutExporter", "ZipkinExporter", "ResourceExtension", "SignPostIntegration", "DataCompression"],
             path: "Examples/OTLP HTTP Exporter",
             exclude: ["README.md"]
         ),

Package@swift-5.9.swift

@@ -23,6 +23,7 @@ let package = Package(
         .library(name: "PersistenceExporter", targets: ["PersistenceExporter"]),
         .library(name: "InMemoryExporter", targets: ["InMemoryExporter"]),
         .library(name: "OTelSwiftLog", targets: ["OTelSwiftLog"]),
+        .library(name: "DataCompression", type: .static, targets: ["DataCompression"]),
         .executable(name: "ConcurrencyContext", targets: ["ConcurrencyContext"]),
         .executable(name: "loggingTracer", targets: ["LoggingTracer"]),
     ],
@@ -70,6 +71,9 @@ let package = Package(
                                "OpenTelemetryProtocolExporterCommon",
                                .product(name: "GRPC", package: "grpc-swift")],
                 path: "Sources/Exporters/OpenTelemetryProtocolGrpc"),
+        .target(name: "DataCompression",
+                dependencies: [],
+                path: "Sources/Exporters/DataCompression"),
         .target(name: "StdoutExporter",
                 dependencies: ["OpenTelemetrySdk"],
                 path: "Sources/Exporters/Stdout"),
@@ -100,6 +104,7 @@ let package = Package(
         .testTarget(name: "OpenTelemetryProtocolExporterTests",
                     dependencies: ["OpenTelemetryProtocolExporterGrpc",
                                    "OpenTelemetryProtocolExporterHttp",
+                                   "DataCompression",
                                    .product(name: "NIO", package: "swift-nio"),
                                    .product(name: "NIOHTTP1", package: "swift-nio"),
                                    .product(name: "NIOTestUtils", package: "swift-nio")],
@@ -240,7 +245,7 @@ extension Package {
                 ),
                 .executableTarget(
                     name: "OTLPHTTPExporter",
-                    dependencies: ["OpenTelemetrySdk", "OpenTelemetryProtocolExporterHttp", "StdoutExporter", "ZipkinExporter", "ResourceExtension", "SignPostIntegration"],
+                    dependencies: ["OpenTelemetrySdk", "OpenTelemetryProtocolExporterHttp", "StdoutExporter", "ZipkinExporter", "ResourceExtension", "SignPostIntegration", "DataCompression"],
                     path: "Examples/OTLP HTTP Exporter",
                     exclude: ["README.md"]
                 ),
@@ -256,7 +261,8 @@ extension Package {
                             dependencies: ["ResourceExtension", "OpenTelemetrySdk"],
                             path: "Tests/InstrumentationTests/SDKResourceExtensionTests"),
                 .target(name: "DatadogExporter",
-                        dependencies: ["OpenTelemetrySdk"],
+                        dependencies: ["OpenTelemetrySdk",
+                                       "DataCompression"],
                         path: "Sources/Exporters/DatadogExporter",
                         exclude: ["NOTICE", "README.md"]),
                 .testTarget(name: "DatadogExporterTests",

Sources/Exporters/DatadogExporter/Utils/DataCompression/DataCompression.swift → Sources/Exporters/DataCompression/DataCompression.swift

@@ -28,9 +28,11 @@
 
 
 import Foundation
+
+#if canImport(Compression)
 import Compression
 
-extension Data
+public extension Data
 {
     /// Compresses the data.
     /// - parameter withAlgorithm: Compression algorithm to use. See the `CompressionAlgorithm` type
@@ -247,7 +249,7 @@ extension Data
 
     /// Calculate the Adler32 checksum of the data.
     /// - returns: Adler32 checksum type. Can still be further advanced.
-    func adler32() -> Adler32
+    internal func adler32() -> Adler32
     {
         var res = Adler32()
         res.advance(withChunk: self)
@@ -256,7 +258,7 @@ extension Data
 
     /// Calculate the Crc32 checksum of the data.
     /// - returns: Crc32 checksum type. Can still be further advanced.
-    func crc32() -> Crc32
+    internal func crc32() -> Crc32
     {
         var res = Crc32()
         res.advance(withChunk: self)
@@ -514,3 +516,4 @@ fileprivate func perform(_ config: Config, source: UnsafePointer<UInt8>, sourceS
         }
     }
 }
+#endif

Sources/Exporters/DatadogExporter/Upload/RequestBuilder.swift

@@ -3,6 +3,9 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+#if canImport(Compression)
+import DataCompression
+#endif
 import Foundation
 
 /// Builds `URLRequest` for sending data to Datadog.

Sources/Exporters/OpenTelemetryProtocolCommon/common/OtlpConfiguration.swift

@@ -5,6 +5,12 @@
 
 import Foundation
 
+public enum CompressionType {
+    case gzip
+    case deflate
+    case none
+}
+
 public struct OtlpConfiguration {
     public static let DefaultTimeoutInterval : TimeInterval = TimeInterval(10)
 
@@ -23,9 +29,15 @@ public struct OtlpConfiguration {
     // let compression
     public let headers : [(String,String)]?
     public let timeout : TimeInterval
+    public let compression: CompressionType
 
-    public init(timeout : TimeInterval  = OtlpConfiguration.DefaultTimeoutInterval, headers: [(String,String)]? = nil) {
+    public init(
+        timeout : TimeInterval  = OtlpConfiguration.DefaultTimeoutInterval,
+        compression: CompressionType = .gzip,
+        headers: [(String,String)]? = nil
+    ) {
         self.headers = headers
         self.timeout = timeout
+        self.compression = compression
     }
 }

Sources/Exporters/OpenTelemetryProtocolHttp/OtlpHttpExporterBase.swift

@@ -3,6 +3,9 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+#if canImport(Compression)
+import DataCompression
+#endif
 import Foundation
 import SwiftProtobuf
 import OpenTelemetryProtocolExporterCommon
@@ -28,20 +31,44 @@ public class OtlpHttpExporterBase {
     }
   }
 
-  public func createRequest(body: Message, endpoint: URL) -> URLRequest {
-    var request = URLRequest(url: endpoint)
-
-    do {
-      request.httpMethod = "POST"
-      request.httpBody = try body.serializedData()
-      request.setValue(Headers.getUserAgentHeader(), forHTTPHeaderField: Constants.HTTP.userAgent)
-      request.setValue("application/x-protobuf", forHTTPHeaderField: "Content-Type")
-    } catch {
-      print("Error serializing body: \(error)")
+    public func createRequest(body: Message, endpoint: URL) -> URLRequest {
+        var request = URLRequest(url: endpoint)
+
+        do {
+            let rawData = try body.serializedData()
+            request.httpMethod = "POST"
+            request.setValue(Headers.getUserAgentHeader(), forHTTPHeaderField: Constants.HTTP.userAgent)
+            request.setValue("application/x-protobuf", forHTTPHeaderField: "Content-Type")
+
+            var compressedData = rawData
+
+#if canImport(Compression)
+            switch config.compression {
+            case .gzip:
+                if let data = rawData.gzip() {
+                    compressedData = data
+                    request.setValue("gzip", forHTTPHeaderField: "Content-Encoding")
+                }
+
+            case .deflate:
+                if let data = rawData.deflate() {
+                    compressedData = data
+                    request.setValue("deflate", forHTTPHeaderField: "Content-Encoding")
+                }
+
+            case .none:
+                break
+            }
+#endif
+            // Apply final data. Could be compressed or raw
+            // but it doesn't matter here
+            request.httpBody = compressedData
+        } catch {
+            print("Error serializing body: \(error)")
+        }
+
+        return request
     }
-
-    return request
-  }
 
   public func shutdown(explicitTimeout: TimeInterval? = nil) {
 

Sources/Exporters/OpenTelemetryProtocolHttp/StableOtlpHTTPExporterBase.swift

@@ -3,6 +3,9 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+#if canImport(Compression)
+import DataCompression
+#endif
 import Foundation
 import OpenTelemetryProtocolExporterCommon
 import SwiftProtobuf
@@ -44,14 +47,38 @@ public class StableOtlpHTTPExporterBase {
         }
 
         do {
+            let rawData = try body.serializedData()
             request.httpMethod = "POST"
-            request.httpBody = try body.serializedData()
             request.setValue(Headers.getUserAgentHeader(), forHTTPHeaderField: Constants.HTTP.userAgent)
             request.setValue("application/x-protobuf", forHTTPHeaderField: "Content-Type")
+
+            var compressedData = rawData
+
+#if canImport(Compression)
+            switch config.compression {
+            case .gzip:
+                if let data = rawData.gzip() {
+                    compressedData = data
+                    request.setValue("gzip", forHTTPHeaderField: "Content-Encoding")
+                }
+
+            case .deflate:
+                if let data = rawData.deflate() {
+                    compressedData = data
+                    request.setValue("deflate", forHTTPHeaderField: "Content-Encoding")
+                }
+
+            case .none:
+                break
+            }
+#endif
+
+            // Apply final data. Could be compressed or raw
+            // but it doesn't matter here
+            request.httpBody = compressedData
         } catch {
             print("Error serializing body: \(error)")
         }
-
         return request
     }