Skip to content

Commit

Permalink
Reuse JMX Insights in Scraper + Tomcat support (#1485)
Browse files Browse the repository at this point in the history
  • Loading branch information
SylvainJuge authored Oct 16, 2024
1 parent b0aae23 commit dc66266
Show file tree
Hide file tree
Showing 10 changed files with 568 additions and 38 deletions.
15 changes: 14 additions & 1 deletion jmx-scraper/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,32 @@ otelJava.moduleName.set("io.opentelemetry.contrib.jmxscraper")

application.mainClass.set("io.opentelemetry.contrib.jmxscraper.JmxScraper")

repositories {
mavenCentral()
mavenLocal()
// TODO: remove snapshot repository once 2.9.0 is released
maven {
setUrl("https://oss.sonatype.org/content/repositories/snapshots")
}
}

dependencies {
// TODO remove snapshot dependency on upstream once 2.9.0 is released
// api(enforcedPlatform("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha:2.9.0-SNAPSHOT-alpha",))
api(enforcedPlatform("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha:2.9.0-alpha-SNAPSHOT"))

implementation("io.opentelemetry:opentelemetry-api")
implementation("io.opentelemetry:opentelemetry-sdk")
implementation("io.opentelemetry:opentelemetry-sdk-metrics")
implementation("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure")

runtimeOnly("io.opentelemetry:opentelemetry-exporter-otlp")
runtimeOnly("io.opentelemetry:opentelemetry-exporter-logging")

implementation("io.opentelemetry.instrumentation:opentelemetry-jmx-metrics")

testImplementation("org.junit-pioneer:junit-pioneer")
testImplementation("io.opentelemetry:opentelemetry-sdk-testing")
testImplementation("org.awaitility:awaitility")
}

testing {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ public void start() {
// for now only configure through JVM args
List<String> arguments = new ArrayList<>();
arguments.add("java");
arguments.add("-Dotel.metrics.exporter=otlp");
arguments.add("-Dotel.exporter.otlp.endpoint=" + endpoint);

if (!targetSystems.isEmpty()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@

package io.opentelemetry.contrib.jmxscraper.target_systems;

import static io.opentelemetry.contrib.jmxscraper.target_systems.MetricAssertions.assertGauge;
import static io.opentelemetry.contrib.jmxscraper.target_systems.MetricAssertions.assertTypedGauge;
import static io.opentelemetry.contrib.jmxscraper.target_systems.MetricAssertions.assertTypedSum;

import io.opentelemetry.contrib.jmxscraper.JmxScraperContainer;
import io.opentelemetry.contrib.jmxscraper.TestAppContainer;
import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
import java.util.Arrays;
import java.util.List;
import org.testcontainers.containers.GenericContainer;

Expand All @@ -25,7 +29,55 @@ protected JmxScraperContainer customizeScraperContainer(JmxScraperContainer scra
}

@Override
protected void verifyMetrics(List<ExportMetricsServiceRequest> metrics) {
// TODO: Verify gathered metrics
protected void verifyMetrics() {
// those values depend on the JVM GC configured
List<String> gcLabels =
Arrays.asList(
"Code Cache",
"PS Eden Space",
"PS Old Gen",
"Metaspace",
"Compressed Class Space",
"PS Survivor Space");
List<String> gcCollectionLabels = Arrays.asList("PS MarkSweep", "PS Scavenge");

waitAndAssertMetrics(
metric -> assertGauge(metric, "jvm.classes.loaded", "number of loaded classes", "1"),
metric ->
assertTypedSum(
metric,
"jvm.gc.collections.count",
"total number of collections that have occurred",
"1",
gcCollectionLabels),
metric ->
assertTypedSum(
metric,
"jvm.gc.collections.elapsed",
"the approximate accumulated collection elapsed time in milliseconds",
"ms",
gcCollectionLabels),
metric -> assertGauge(metric, "jvm.memory.heap.committed", "current heap usage", "by"),
metric -> assertGauge(metric, "jvm.memory.heap.init", "current heap usage", "by"),
metric -> assertGauge(metric, "jvm.memory.heap.max", "current heap usage", "by"),
metric -> assertGauge(metric, "jvm.memory.heap.used", "current heap usage", "by"),
metric ->
assertGauge(metric, "jvm.memory.nonheap.committed", "current non-heap usage", "by"),
metric -> assertGauge(metric, "jvm.memory.nonheap.init", "current non-heap usage", "by"),
metric -> assertGauge(metric, "jvm.memory.nonheap.max", "current non-heap usage", "by"),
metric -> assertGauge(metric, "jvm.memory.nonheap.used", "current non-heap usage", "by"),
metric ->
assertTypedGauge(
metric, "jvm.memory.pool.committed", "current memory pool usage", "by", gcLabels),
metric ->
assertTypedGauge(
metric, "jvm.memory.pool.init", "current memory pool usage", "by", gcLabels),
metric ->
assertTypedGauge(
metric, "jvm.memory.pool.max", "current memory pool usage", "by", gcLabels),
metric ->
assertTypedGauge(
metric, "jvm.memory.pool.used", "current memory pool usage", "by", gcLabels),
metric -> assertGauge(metric, "jvm.threads.count", "number of threads", "1"));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
/*
* Copyright The OpenTelemetry Authors
* SPDX-License-Identifier: Apache-2.0
*/

package io.opentelemetry.contrib.jmxscraper.target_systems;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.entry;

import io.opentelemetry.proto.common.v1.KeyValue;
import io.opentelemetry.proto.metrics.v1.Metric;
import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.assertj.core.api.MapAssert;

/** Metrics assertions */
class MetricAssertions {

private MetricAssertions() {}

static void assertGauge(Metric metric, String name, String description, String unit) {
assertThat(metric.getName()).isEqualTo(name);
assertThat(metric.getDescription()).isEqualTo(description);
assertThat(metric.getUnit()).isEqualTo(unit);
assertThat(metric.hasGauge()).isTrue();
assertThat(metric.getGauge().getDataPointsList())
.satisfiesExactly(point -> assertThat(point.getAttributesList()).isEmpty());
}

static void assertSum(Metric metric, String name, String description, String unit) {
assertSum(metric, name, description, unit, /* isMonotonic= */ true);
}

static void assertSum(
Metric metric, String name, String description, String unit, boolean isMonotonic) {
assertThat(metric.getName()).isEqualTo(name);
assertThat(metric.getDescription()).isEqualTo(description);
assertThat(metric.getUnit()).isEqualTo(unit);
assertThat(metric.hasSum()).isTrue();
assertThat(metric.getSum().getDataPointsList())
.satisfiesExactly(point -> assertThat(point.getAttributesList()).isEmpty());
assertThat(metric.getSum().getIsMonotonic()).isEqualTo(isMonotonic);
}

static void assertTypedGauge(
Metric metric, String name, String description, String unit, List<String> types) {
assertThat(metric.getName()).isEqualTo(name);
assertThat(metric.getDescription()).isEqualTo(description);
assertThat(metric.getUnit()).isEqualTo(unit);
assertThat(metric.hasGauge()).isTrue();
assertTypedPoints(metric.getGauge().getDataPointsList(), types);
}

static void assertTypedSum(
Metric metric, String name, String description, String unit, List<String> types) {
assertThat(metric.getName()).isEqualTo(name);
assertThat(metric.getDescription()).isEqualTo(description);
assertThat(metric.getUnit()).isEqualTo(unit);
assertThat(metric.hasSum()).isTrue();
assertTypedPoints(metric.getSum().getDataPointsList(), types);
}

@SafeVarargs
static void assertSumWithAttributes(
Metric metric,
String name,
String description,
String unit,
Consumer<MapAssert<String, String>>... attributeGroupAssertions) {
assertThat(metric.getName()).isEqualTo(name);
assertThat(metric.getDescription()).isEqualTo(description);
assertThat(metric.getUnit()).isEqualTo(unit);
assertThat(metric.hasSum()).isTrue();
assertAttributedPoints(metric.getSum().getDataPointsList(), attributeGroupAssertions);
}

@SafeVarargs
static void assertGaugeWithAttributes(
Metric metric,
String name,
String description,
String unit,
Consumer<MapAssert<String, String>>... attributeGroupAssertions) {
assertThat(metric.getName()).isEqualTo(name);
assertThat(metric.getDescription()).isEqualTo(description);
assertThat(metric.getUnit()).isEqualTo(unit);
assertThat(metric.hasGauge()).isTrue();
assertAttributedPoints(metric.getGauge().getDataPointsList(), attributeGroupAssertions);
}

@SuppressWarnings("unchecked")
private static void assertTypedPoints(List<NumberDataPoint> points, List<String> types) {
Consumer<MapAssert<String, String>>[] assertions =
types.stream()
.map(
type ->
(Consumer<MapAssert<String, String>>)
attrs -> attrs.containsOnly(entry("name", type)))
.toArray(Consumer[]::new);

assertAttributedPoints(points, assertions);
}

@SuppressWarnings("unchecked")
private static void assertAttributedPoints(
List<NumberDataPoint> points,
Consumer<MapAssert<String, String>>... attributeGroupAssertions) {
Consumer<Map<String, String>>[] assertions =
Arrays.stream(attributeGroupAssertions)
.map(assertion -> (Consumer<Map<String, String>>) m -> assertion.accept(assertThat(m)))
.toArray(Consumer[]::new);
assertThat(points)
.extracting(
numberDataPoint ->
numberDataPoint.getAttributesList().stream()
.collect(
Collectors.toMap(
KeyValue::getKey, keyValue -> keyValue.getValue().getStringValue())))
.satisfiesExactlyInAnyOrder(assertions);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@

package io.opentelemetry.contrib.jmxscraper.target_systems;

import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;

import com.linecorp.armeria.server.ServerBuilder;
import com.linecorp.armeria.server.grpc.GrpcService;
import com.linecorp.armeria.testing.junit5.server.ServerExtension;
Expand All @@ -13,11 +16,18 @@
import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse;
import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc;
import io.opentelemetry.proto.metrics.v1.Metric;
import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
Expand All @@ -30,6 +40,7 @@
import org.testcontainers.containers.output.Slf4jLogConsumer;

public abstract class TargetSystemIntegrationTest {
private static final Logger logger = LoggerFactory.getLogger(TargetSystemIntegrationTest.class);
private static final Logger targetSystemLogger = LoggerFactory.getLogger("TargetSystemContainer");
private static final Logger jmxScraperLogger = LoggerFactory.getLogger("JmxScraperContainer");
private static final String TARGET_SYSTEM_NETWORK_ALIAS = "targetsystem";
Expand Down Expand Up @@ -105,10 +116,45 @@ void endToEndTest() {
scraper = customizeScraperContainer(scraper);
scraper.start();

verifyMetrics(otlpServer.getMetrics());
verifyMetrics();
}

protected void waitAndAssertMetrics(Iterable<Consumer<Metric>> assertions) {
await()
.atMost(Duration.ofSeconds(30))
.untilAsserted(
() -> {
List<ExportMetricsServiceRequest> receivedMetrics = otlpServer.getMetrics();
assertThat(receivedMetrics).describedAs("no metric received").isNotEmpty();

List<Metric> metrics =
receivedMetrics.stream()
.map(ExportMetricsServiceRequest::getResourceMetricsList)
.flatMap(rm -> rm.stream().map(ResourceMetrics::getScopeMetricsList))
.flatMap(Collection::stream)
.filter(
// TODO: disabling batch span exporter might help remove unwanted metrics
sm -> sm.getScope().getName().equals("io.opentelemetry.jmx"))
.flatMap(sm -> sm.getMetricsList().stream())
.collect(Collectors.toList());

assertThat(metrics)
.describedAs("metrics reported but none from JMX scraper")
.isNotEmpty();

for (Consumer<Metric> assertion : assertions) {
assertThat(metrics).anySatisfy(assertion);
}
});
}

protected abstract void verifyMetrics(List<ExportMetricsServiceRequest> metrics);
@SafeVarargs
@SuppressWarnings("varargs")
protected final void waitAndAssertMetrics(Consumer<Metric>... assertions) {
waitAndAssertMetrics(Arrays.asList(assertions));
}

protected abstract void verifyMetrics();

protected JmxScraperContainer customizeScraperContainer(JmxScraperContainer scraper) {
return scraper;
Expand Down Expand Up @@ -137,6 +183,10 @@ protected void configure(ServerBuilder sb) {
public void export(
ExportMetricsServiceRequest request,
StreamObserver<ExportMetricsServiceResponse> responseObserver) {

// verbose but helpful to diagnose what is received
logger.debug("receiving metrics {}", request);

metricRequests.add(request);
responseObserver.onNext(ExportMetricsServiceResponse.getDefaultInstance());
responseObserver.onCompleted();
Expand Down
Loading

0 comments on commit dc66266

Please sign in to comment.