Skip to content

Commit

Permalink
Perform charset-aware decoding of request bodies (#3726)
Browse files Browse the repository at this point in the history 
* Split out decoder pooling

* Implemented charset aware request body decoding

* Fix casting in fallback encoding

* Fix java 7 compilation
  • Loading branch information
@JonasKunz
JonasKunz authored Aug 8, 2024
1 parent fbb5d68 commit ddbf896
Show file tree
Hide file tree
Showing 3 changed files with 197 additions and 50 deletions.
73 changes: 53 additions & 20 deletions apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/DslJsonSerializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,13 @@
import co.elastic.apm.agent.impl.transaction.TransactionImpl;
import co.elastic.apm.agent.report.ApmServerClient;
import co.elastic.apm.agent.sdk.internal.collections.LongList;
import co.elastic.apm.agent.sdk.internal.pooling.ObjectHandle;
import co.elastic.apm.agent.sdk.internal.pooling.ObjectPool;
import co.elastic.apm.agent.sdk.internal.pooling.ObjectPooling;
import co.elastic.apm.agent.sdk.internal.util.IOUtils;
import co.elastic.apm.agent.sdk.logging.Logger;
import co.elastic.apm.agent.sdk.logging.LoggerFactory;
import co.elastic.apm.agent.tracer.configuration.WebConfiguration;
import co.elastic.apm.agent.tracer.metadata.PotentiallyMultiValuedMap;
import co.elastic.apm.agent.tracer.metrics.DslJsonUtil;
import co.elastic.apm.agent.tracer.metrics.Labels;
Expand All @@ -79,15 +84,18 @@
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.CoderResult;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

Expand All @@ -104,6 +112,13 @@ public class DslJsonSerializer {
private static final Logger logger = LoggerFactory.getLogger(DslJsonSerializer.class);
private static final List<String> excludedStackFramesPrefixes = Arrays.asList("java.lang.reflect.", "com.sun.", "sun.", "jdk.internal.");

private static final ObjectPool<? extends ObjectHandle<CharBuffer>> REQUEST_BODY_BUFFER_POOL = ObjectPooling.createWithDefaultFactory(new Callable<CharBuffer>() {
@Override
public CharBuffer call() throws Exception {
return CharBuffer.allocate(WebConfiguration.MAX_BODY_CAPTURE_BYTES);
}
});


private final StacktraceConfigurationImpl stacktraceConfiguration;
private final ApmServerClient apmServerClient;
Expand Down Expand Up @@ -1054,22 +1069,7 @@ private void serializeSpanLinks(List<TraceContextImpl> spanLinks) {
}

private void serializeOTel(SpanImpl span) {
serializeOtel(span, Collections.<IdImpl>emptyList(), requestBodyToString(span.getContext().getHttp().getRequestBody()));
}

@Nullable
private CharSequence requestBodyToString(BodyCaptureImpl requestBody) {
//TODO: perform proper, charset aware conversion to string
ByteBuffer buffer = requestBody.getBody();
if (buffer == null || buffer.position() == 0) {
return null;
}
buffer.flip();
StringBuilder result = new StringBuilder();
while (buffer.hasRemaining()) {
result.append((char) buffer.get());
}
return result;
serializeOtel(span, Collections.<IdImpl>emptyList(), span.getContext().getHttp().getRequestBody());
}

private void serializeOTel(TransactionImpl transaction) {
Expand All @@ -1079,11 +1079,12 @@ private void serializeOTel(TransactionImpl transaction) {
}
}

private void serializeOtel(AbstractSpanImpl<?> span, List<IdImpl> profilingStackTraceIds, @Nullable CharSequence httpRequestBody) {
private void serializeOtel(AbstractSpanImpl<?> span, List<IdImpl> profilingStackTraceIds, @Nullable BodyCaptureImpl httpRequestBody) {
OTelSpanKind kind = span.getOtelKind();
Map<String, Object> attributes = span.getOtelAttributes();

boolean hasAttributes = !attributes.isEmpty() || !profilingStackTraceIds.isEmpty() || httpRequestBody != null;
boolean hasRequestBody = httpRequestBody != null && httpRequestBody.getBody() != null;
boolean hasAttributes = !attributes.isEmpty() || !profilingStackTraceIds.isEmpty() || hasRequestBody;
boolean hasKind = kind != null;
if (hasKind || hasAttributes) {
writeFieldName("otel");
Expand Down Expand Up @@ -1133,12 +1134,12 @@ private void serializeOtel(AbstractSpanImpl<?> span, List<IdImpl> profilingStack
}
jw.writeByte(ARRAY_END);
}
if (httpRequestBody != null) {
if (hasRequestBody) {
if (!isFirstAttrib) {
jw.writeByte(COMMA);
}
writeFieldName("http.request.body.content");
jw.writeString(httpRequestBody);
writeRequestBodyAsString(jw, httpRequestBody);
}
jw.writeByte(OBJECT_END);
}
Expand All @@ -1148,6 +1149,38 @@ private void serializeOtel(AbstractSpanImpl<?> span, List<IdImpl> profilingStack
}
}


private void writeRequestBodyAsString(JsonWriter jw, BodyCaptureImpl requestBody) {
try (ObjectHandle<CharBuffer> charBufferHandle = REQUEST_BODY_BUFFER_POOL.createInstance()) {
CharBuffer charBuffer = charBufferHandle.get();
try {
decodeRequestBodyBytes(requestBody, charBuffer);
charBuffer.flip();
jw.writeString(charBuffer);
} finally {
((Buffer) charBuffer).clear();
}
}
}

private void decodeRequestBodyBytes(BodyCaptureImpl requestBody, CharBuffer charBuffer) {
ByteBuffer bodyBytes = requestBody.getBody();
((Buffer) bodyBytes).flip(); //make ready for reading
CharSequence charset = requestBody.getCharset();
if (charset != null) {
CoderResult result = IOUtils.decode(bodyBytes, charBuffer, charset.toString());
if (result != null && !result.isMalformed() && !result.isUnmappable()) {
return;
}
}
//fallback to decoding by simply casting bytes to chars
((Buffer) bodyBytes).position(0);
((Buffer) charBuffer).clear();
while (bodyBytes.hasRemaining()) {
charBuffer.put((char) (((int) bodyBytes.get()) & 0xFF));
}
}

private void serializeNumber(Number n, JsonWriter jw) {
if (n instanceof Integer) {
NumberConverter.serialize(n.intValue(), jw);
Expand Down
51 changes: 44 additions & 7 deletions ...agent-core/src/test/java/co/elastic/apm/agent/report/serialize/DslJsonSerializerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
import co.elastic.apm.agent.report.ApmServerClient;
import co.elastic.apm.agent.sdk.internal.collections.LongList;
import co.elastic.apm.agent.sdk.internal.util.IOUtils;
import co.elastic.apm.agent.tracer.configuration.WebConfiguration;
import com.dslplatform.json.JsonWriter;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
Expand All @@ -74,6 +75,7 @@

import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.CharBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
Expand Down Expand Up @@ -432,18 +434,53 @@ void testSpanHttpContextSerialization() {
}

@Test
void testSpanHttpRequestBodySerialization() {
SpanImpl span = new SpanImpl(tracer);
void testSpanHttpRequestBodySerialization() throws UnsupportedEncodingException {
String noBody = extractRequestBodyJson(createSpanWithRequestBody(null, "utf-8"));
assertThat(noBody).isNull();

BodyCaptureImpl bodyCapture = span.getContext().getHttp().getRequestBody();
bodyCapture.markEligibleForCapturing();
bodyCapture.startCapture("utf-8", 50);
bodyCapture.append("foobar".getBytes(StandardCharsets.UTF_8), 0, 6);
String emptyBody = extractRequestBodyJson(createSpanWithRequestBody(new byte[0], "utf-8"));
assertThat(emptyBody).isEqualTo("");

String invalidCharset = extractRequestBodyJson(createSpanWithRequestBody("testö".getBytes("utf-8"), "bad charset!"));
assertThat(invalidCharset).isEqualTo("testö");

String noCharset = extractRequestBodyJson(createSpanWithRequestBody("testö".getBytes("utf-8"), null));
assertThat(noCharset).isEqualTo("testö");

String utf8 = extractRequestBodyJson(createSpanWithRequestBody("special charßßß!äöü".getBytes("utf-8"), "utf-8"));
assertThat(utf8).isEqualTo("special charßßß!äöü");

String utf16 = extractRequestBodyJson(createSpanWithRequestBody("special charßßß!äöü".getBytes("utf-16"), "utf-16"));
assertThat(utf16).isEqualTo("special charßßß!äöü");

String invalidUtf8Sequence = extractRequestBodyJson(createSpanWithRequestBody(new byte[]{'t', 'e', 's', 't', (byte) 0xC2, (byte) 0xC2}, "utf-8"));
assertThat(invalidUtf8Sequence).isEqualTo("testÂÂ");
}

private String extractRequestBodyJson(SpanImpl span) {
JsonNode spanJson = readJsonString(writer.toJsonString(span));
JsonNode otel = spanJson.get("otel");
if (otel == null) {
return null;
}
JsonNode attribs = otel.get("attributes");
assertThat(attribs.get("http.request.body.content").textValue()).isEqualTo("foobar");
JsonNode bodyContent = attribs.get("http.request.body.content");
if (bodyContent == null) {
return null;
}
return bodyContent.textValue();
}

private SpanImpl createSpanWithRequestBody(@Nullable byte[] bodyBytes, @Nullable String charset) {
SpanImpl span = new SpanImpl(tracer);
BodyCaptureImpl bodyCapture = span.getContext().getHttp().getRequestBody();
bodyCapture.markEligibleForCapturing();
bodyCapture.startCapture(charset, WebConfiguration.MAX_BODY_CAPTURE_BYTES);

if (bodyBytes != null) {
bodyCapture.append(bodyBytes, 0, bodyBytes.length);
}
return span;
}

public static boolean[][] getContentCombinations() {
Expand Down
Loading

0 comments on commit ddbf896

Please sign in to comment.