Support for generate_request_id and preserve_external_request_id listener config

envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/Groups.kt

@@ -34,6 +34,8 @@ data class ListenersConfig(
     val egressHost: String,
     val egressPort: Int,
     val useRemoteAddress: Boolean = defaultUseRemoteAddress,
+    val generateRequestId: Boolean = defaultGenerateRequestId,
+    val preserveExternalRequestId: Boolean = defaultPreserveExternalRequestId,
     val accessLogEnabled: Boolean = defaultAccessLogEnabled,
     val enableLuaScript: Boolean = defaultEnableLuaScript,
     val accessLogPath: String = defaultAccessLogPath,
@@ -46,6 +48,8 @@ data class ListenersConfig(
     companion object {
         const val defaultAccessLogPath = "/dev/stdout"
         const val defaultUseRemoteAddress = false
+        const val defaultGenerateRequestId = false
+        const val defaultPreserveExternalRequestId = false
         const val defaultAccessLogEnabled = false
         const val defaultEnableLuaScript = false
         const val defaultAddUpstreamExternalAddressHeader = false

envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/MetadataNodeGroup.kt

@@ -101,6 +101,10 @@ class MetadataNodeGroup(
 
         val useRemoteAddress = metadata.fieldsMap["use_remote_address"]?.boolValue
             ?: ListenersConfig.defaultUseRemoteAddress
+        val generateRequestId = metadata.fieldsMap["generate_request_id"]?.boolValue
+            ?: ListenersConfig.defaultGenerateRequestId
+        val preserveExternalRequestId = metadata.fieldsMap["preserve_external_request_id"]?.boolValue
+            ?: ListenersConfig.defaultPreserveExternalRequestId
         val accessLogEnabled = metadata.fieldsMap["access_log_enabled"]?.boolValue
             ?: ListenersConfig.defaultAccessLogEnabled
         val enableLuaScript = metadata.fieldsMap["enable_lua_script"]?.boolValue
@@ -120,6 +124,8 @@ class MetadataNodeGroup(
             listenersHostPort.egressHost,
             listenersHostPort.egressPort,
             useRemoteAddress,
+            generateRequestId,
+            preserveExternalRequestId,
             accessLogEnabled,
             enableLuaScript,
             accessLogPath,

envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/EnvoyListenersFactory.kt

@@ -218,6 +218,8 @@ class EnvoyListenersFactory(
                 .setStatPrefix("egress_http")
                 .setRds(egressRds(group.communicationMode, group.version))
                 .setHttpProtocolOptions(egressHttp1ProtocolOptions())
+                .setPreserveExternalRequestId(listenersConfig.preserveExternalRequestId)
+                .setGenerateRequestId(boolValue(listenersConfig.generateRequestId))
 
         addHttpFilters(connectionManagerBuilder, egressFilters, group, globalSnapshot)
 
@@ -296,6 +298,8 @@ class EnvoyListenersFactory(
         val connectionManagerBuilder = HttpConnectionManager.newBuilder()
                 .setStatPrefix(statPrefix)
                 .setUseRemoteAddress(boolValue(listenersConfig.useRemoteAddress))
+                .setGenerateRequestId(boolValue(listenersConfig.generateRequestId))
+                .setPreserveExternalRequestId(listenersConfig.preserveExternalRequestId)
                 .setDelayedCloseTimeout(durationInSeconds(0))
                 .setCommonHttpProtocolOptions(httpProtocolOptions)
                 .setCodecType(HttpConnectionManager.CodecType.AUTO)

envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/RequestIdTest.kt

@@ -0,0 +1,119 @@
+package pl.allegro.tech.servicemesh.envoycontrol
+
+import okhttp3.Headers
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.extension.RegisterExtension
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.MethodSource
+import pl.allegro.tech.servicemesh.envoycontrol.assertions.isOk
+import pl.allegro.tech.servicemesh.envoycontrol.assertions.untilAsserted
+import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulExtension
+import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension
+import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlExtension
+import pl.allegro.tech.servicemesh.envoycontrol.config.service.GenericServiceExtension
+import pl.allegro.tech.servicemesh.envoycontrol.config.service.HttpsEchoContainer
+import pl.allegro.tech.servicemesh.envoycontrol.config.service.asHttpsEchoResponse
+
+class RequestIdTest {
+
+    companion object {
+        @JvmStatic
+        fun extraHeadersSource() = listOf(
+            emptyMap(),
+            mapOf("x-forwarded-for" to "123.321.231.111"),
+            mapOf("x-forwarded-for" to "111.111.222.222,123.123.231.231")
+        )
+
+        @JvmField
+        @RegisterExtension
+        val consul = ConsulExtension()
+
+        @JvmField
+        @RegisterExtension
+        val envoyControl = EnvoyControlExtension(consul)
+
+        @JvmField
+        @RegisterExtension
+        val localService = GenericServiceExtension(HttpsEchoContainer())
+
+        @JvmField
+        @RegisterExtension
+        val envoy = EnvoyExtension(envoyControl, localService)
+
+        @JvmField
+        @RegisterExtension
+        val externalService = GenericServiceExtension(HttpsEchoContainer())
+    }
+
+    @BeforeEach
+    fun setup() {
+        consul.server.operations.registerService(externalService, name = "service-1")
+    }
+
+    @ParameterizedTest
+    @MethodSource("extraHeadersSource")
+    fun `should propagate x-request-id on the egress port when it is available in request`(extraHeaders: Map<String, String>) {
+        // given
+        val requestIdHeader = mapOf("x-request-id" to "egress-fake-request-id")
+
+        untilAsserted {
+            // when
+            val response = envoy.egressOperations
+                .callService(service = "service-1", headers = requestIdHeader + extraHeaders)
+                .asHttpsEchoResponse()
+
+            // then
+            assertThat(response).isOk()
+            assertThat(response.requestHeaders).containsEntry("x-request-id", "egress-fake-request-id")
+        }
+    }
+
+    @ParameterizedTest
+    @MethodSource("extraHeadersSource")
+    fun `should generate x-request-id on the egress port when it is missing in request`(extraHeaders: Map<String, String>) {
+        untilAsserted {
+            // when
+            val response = envoy.egressOperations
+                .callService(service = "service-1", headers = extraHeaders)
+                .asHttpsEchoResponse()
+
+            // then
+            assertThat(response).isOk()
+            assertThat(response.requestHeaders).hasEntrySatisfying("x-request-id") { assertThat(it).isNotBlank() }
+        }
+    }
+
+    @ParameterizedTest
+    @MethodSource("extraHeadersSource")
+    fun `should propagate x-request-id on the ingress port when it is available in request`(extraHeaders: Map<String, String>) {
+        // given
+        val requestIdHeader = mapOf("x-request-id" to "ingress-fake-request-id")
+
+        untilAsserted {
+            // when
+            val response = envoy.ingressOperations
+                .callLocalService(endpoint = "/", headers = Headers.of(requestIdHeader + extraHeaders))
+                .asHttpsEchoResponse()
+
+            // then
+            assertThat(response).isOk()
+            assertThat(response.requestHeaders).containsEntry("x-request-id", "ingress-fake-request-id")
+        }
+    }
+
+    @ParameterizedTest
+    @MethodSource("extraHeadersSource")
+    fun `should generate x-request-id on the ingress port when it is missing in request`(extraHeaders: Map<String, String>) {
+        untilAsserted {
+            // when
+            val response = envoy.ingressOperations
+                .callLocalService(endpoint = "/", headers = Headers.of(extraHeaders))
+                .asHttpsEchoResponse()
+
+            // then
+            assertThat(response).isOk()
+            assertThat(response.requestHeaders).hasEntrySatisfying("x-request-id") { assertThat(it).isNotBlank() }
+        }
+    }
+}

envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/assertions/HttpsEchoResponseAssertions.kt

@@ -0,0 +1,20 @@
+package pl.allegro.tech.servicemesh.envoycontrol.assertions
+
+import org.assertj.core.api.Assertions
+import org.assertj.core.api.ObjectAssert
+import pl.allegro.tech.servicemesh.envoycontrol.config.service.HttpsEchoContainer
+import pl.allegro.tech.servicemesh.envoycontrol.config.service.HttpsEchoResponse
+
+fun ObjectAssert<HttpsEchoResponse>.isOk(): ObjectAssert<HttpsEchoResponse> {
+    matches { it.response.isSuccessful }
+    return this
+}
+
+fun ObjectAssert<HttpsEchoResponse>.hasSNI(serverName: String): ObjectAssert<HttpsEchoResponse> = satisfies {
+    val actualServerName = HttpsEchoResponse.objectMapper.readTree(it.body).at("/connection/servername").textValue()
+    Assertions.assertThat(actualServerName).isEqualTo(serverName)
+}
+
+fun ObjectAssert<HttpsEchoResponse>.isFrom(container: HttpsEchoContainer) = satisfies {
+    Assertions.assertThat(container.containerName()).isEqualTo(it.hostname)
+}

envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/service/HttpsEchoContainer.kt

@@ -4,8 +4,6 @@ import com.fasterxml.jackson.databind.DeserializationFeature
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.kotlin.convertValue
 import okhttp3.Response
-import org.assertj.core.api.Assertions.assertThat
-import org.assertj.core.api.ObjectAssert
 import pl.allegro.tech.servicemesh.envoycontrol.config.BaseEnvoyTest
 import pl.allegro.tech.servicemesh.envoycontrol.config.containers.SSLGenericContainer
 
@@ -43,11 +41,4 @@ class HttpsEchoResponse(val response: Response) {
     val hostname by lazy { objectMapper.readTree(body).at("/os/hostname").textValue() }
 }
 
-fun ObjectAssert<HttpsEchoResponse>.hasSNI(serverName: String): ObjectAssert<HttpsEchoResponse> = satisfies {
-    val actualServerName = HttpsEchoResponse.objectMapper.readTree(it.body).at("/connection/servername").textValue()
-    assertThat(actualServerName).isEqualTo(serverName)
-}
-
-fun ObjectAssert<HttpsEchoResponse>.isFrom(container: HttpsEchoContainer) = satisfies {
-    assertThat(container.containerName()).isEqualTo(it.hostname)
-}
+fun Response.asHttpsEchoResponse() = HttpsEchoResponse(this)

envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/permissions/ClientNameTrustedHeaderTest.kt

@@ -5,6 +5,7 @@ import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.RegisterExtension
+import pl.allegro.tech.servicemesh.envoycontrol.assertions.isFrom
 import pl.allegro.tech.servicemesh.envoycontrol.assertions.isOk
 import pl.allegro.tech.servicemesh.envoycontrol.assertions.untilAsserted
 import pl.allegro.tech.servicemesh.envoycontrol.config.Echo1EnvoyAuthConfig
@@ -15,8 +16,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension
 import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlExtension
 import pl.allegro.tech.servicemesh.envoycontrol.config.service.GenericServiceExtension
 import pl.allegro.tech.servicemesh.envoycontrol.config.service.HttpsEchoContainer
-import pl.allegro.tech.servicemesh.envoycontrol.config.service.HttpsEchoResponse
-import pl.allegro.tech.servicemesh.envoycontrol.config.service.isFrom
+import pl.allegro.tech.servicemesh.envoycontrol.config.service.asHttpsEchoResponse
 import java.time.Duration
 
 class ClientNameTrustedHeaderTest {
@@ -116,66 +116,56 @@ class ClientNameTrustedHeaderTest {
         val response = envoy2.ingressOperations.callLocalService(
             "/endpoint",
             Headers.of(mapOf("x-client-name-trusted" to "fake-service"))
-        )
+        ).asHttpsEchoResponse()
 
         // then
         assertThat(response).isOk()
-        HttpsEchoResponse(response).also {
-            assertThat(it).isFrom(service.container())
-            assertThat(it.requestHeaders["x-client-name-trusted"]).isNull()
-        }
+        assertThat(response).isFrom(service.container())
+        assertThat(response.requestHeaders).doesNotContainKey("x-client-name-trusted")
     }
 
     @Test
     fun `should add trusted client identity header to ingress request to local service`() {
         // when
-        val response = envoy2.egressOperations.callService("echo", emptyMap(), "/endpoint")
+        val response = envoy2.egressOperations.callService("echo", emptyMap(), "/endpoint").asHttpsEchoResponse()
 
         // then
         assertThat(response).isOk()
-        HttpsEchoResponse(response).also {
-            assertThat(it).isFrom(service.container())
-            assertThat(it.requestHeaders["x-client-name-trusted"]).isEqualTo("echo2")
-        }
+        assertThat(response).isFrom(service.container())
+        assertThat(response.requestHeaders).containsEntry("x-client-name-trusted", "echo2")
     }
 
     @Test
     fun `should override trusted client identity header in ingress request to local service`() {
         // when
         val headers = mapOf("x-client-name-trusted" to "fake-service")
-        val response = envoy2.egressOperations.callService("echo", headers, "/endpoint")
+        val response = envoy2.egressOperations.callService("echo", headers, "/endpoint").asHttpsEchoResponse()
 
         // then
         assertThat(response).isOk()
-        HttpsEchoResponse(response).also {
-            assertThat(it).isFrom(service.container())
-            assertThat(it.requestHeaders["x-client-name-trusted"]).isEqualTo("echo2")
-        }
+        assertThat(response).isFrom(service.container())
+        assertThat(response.requestHeaders).containsEntry("x-client-name-trusted", "echo2")
     }
 
     @Test
     fun `should set trusted client identity header based on all URIs in certificate SAN field`() {
         // when
-        val response = envoy4MultipleSANs.egressOperations.callService("echo", emptyMap(), "/endpoint")
+        val response = envoy4MultipleSANs.egressOperations.callService("echo", emptyMap(), "/endpoint").asHttpsEchoResponse()
 
         // then
         assertThat(response).isOk()
-        HttpsEchoResponse(response).also {
-            assertThat(it).isFrom(service.container())
-            assertThat(it.requestHeaders["x-client-name-trusted"]).isEqualTo("echo4, echo4-special, echo4-admin")
-        }
+        assertThat(response).isFrom(service.container())
+        assertThat(response.requestHeaders).containsEntry("x-client-name-trusted", "echo4, echo4-special, echo4-admin")
     }
 
     @Test
     fun `should not set trusted client identity header based on URIs in certificate SAN fields having invalid format`() {
         // when
-        val response = envoy5InvalidSANs.egressOperations.callService("echo", emptyMap(), "/endpoint")
+        val response = envoy5InvalidSANs.egressOperations.callService("echo", emptyMap(), "/endpoint").asHttpsEchoResponse()
 
         // then
         assertThat(response).isOk()
-        HttpsEchoResponse(response).also {
-            assertThat(it).isFrom(service.container())
-            assertThat(it.requestHeaders["x-client-name-trusted"]).isNull()
-        }
+        assertThat(response).isFrom(service.container())
+        assertThat(response.requestHeaders).doesNotContainKey("x-client-name-trusted")
     }
 }

envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ssl/EnvoyHttpsDependencyTest.kt

@@ -7,8 +7,9 @@ import org.junit.jupiter.api.Test
 import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlRunnerTestApp
 import pl.allegro.tech.servicemesh.envoycontrol.config.EnvoyControlTestConfiguration
 import pl.allegro.tech.servicemesh.envoycontrol.config.service.HttpsEchoContainer
-import pl.allegro.tech.servicemesh.envoycontrol.config.service.HttpsEchoResponse
-import pl.allegro.tech.servicemesh.envoycontrol.config.service.hasSNI
+import pl.allegro.tech.servicemesh.envoycontrol.assertions.hasSNI
+import pl.allegro.tech.servicemesh.envoycontrol.assertions.isOk
+import pl.allegro.tech.servicemesh.envoycontrol.config.service.asHttpsEchoResponse
 
 class EnvoyCurrentVersionHttpsDependencyTest : EnvoyHttpsDependencyTest() {
     companion object {
@@ -46,13 +47,13 @@ abstract class EnvoyHttpsDependencyTest : EnvoyControlTestConfiguration() {
     fun `should include SNI in request to upstream`() {
         // when
         val response = untilAsserted {
-            val response = callDomain("my.example.com")
+            val response = callDomain("my.example.com").asHttpsEchoResponse()
 
             assertThat(response).isOk()
             response
         }
 
         // then
-        assertThat(HttpsEchoResponse(response)).hasSNI("my.example.com")
+        assertThat(response).hasSNI("my.example.com")
     }
 }

envoy-control-tests/src/main/resources/envoy/config_ads.yaml

@@ -26,6 +26,8 @@ node:
     egress_host: "0.0.0.0"
     egress_port: 5000
     use_remote_address: true
+    generate_request_id: true
+    preserve_external_request_id: true
     access_log_enabled: false
     add_upstream_external_address_header: true
     resources_dir: "/etc/envoy/extra"

envoy-control-tests/src/main/resources/envoy/config_ads_all_dependencies.yaml

@@ -25,6 +25,8 @@ node:
     egress_host: "0.0.0.0"
     egress_port: 5000
     use_remote_address: true
+    generate_request_id: true
+    preserve_external_request_id: true
     access_log_enabled: false
     resources_dir: "/etc/envoy/extra"
     service_name: test-service

envoy-control-tests/src/main/resources/envoy/config_ads_disabled_endpoint_permissions.yaml

@@ -25,6 +25,8 @@ node:
     egress_host: "0.0.0.0"
     egress_port: 5000
     use_remote_address: true
+    generate_request_id: true
+    preserve_external_request_id: true
     access_log_enabled: false
     resources_dir: "/etc/envoy/extra"
     proxy_settings:

envoy-control-tests/src/main/resources/envoy/config_ads_no_dependencies.yaml

@@ -25,6 +25,8 @@ node:
     egress_host: "0.0.0.0"
     egress_port: 5000
     use_remote_address: true
+    generate_request_id: true
+    preserve_external_request_id: true
     access_log_enabled: false
     resources_dir: "/etc/envoy/extra"
 

envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml

@@ -23,6 +23,8 @@ node:
     egress_host: "0.0.0.0"
     egress_port: 5000
     use_remote_address: true
+    generate_request_id: true
+    preserve_external_request_id: true
     access_log_enabled: false
     add_upstream_external_address_header: true
     resources_dir: "/etc/envoy/extra"