AMLII-1148 - Adding Unit test to check GC metrics emitted by each type of Java garbage collector

pom.xml

@@ -200,7 +200,13 @@
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers</artifactId>
-            <version>1.18.0</version>
+            <version>1.19.3</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-library</artifactId>
+            <version>1.3</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

src/main/java/org/datadog/jmxfetch/Instance.java

@@ -16,7 +16,6 @@
 import java.io.InputStream;
 import java.lang.management.ManagementFactory;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;

src/test/java/org/datadog/jmxfetch/TestCommon.java

@@ -1,9 +1,5 @@
 package org.datadog.jmxfetch;
 
-import static org.hamcrest.CoreMatchers.hasItem;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
@@ -19,24 +15,24 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import javax.management.InstanceAlreadyExistsException;
 import javax.management.InstanceNotFoundException;
 import javax.management.MBeanRegistrationException;
 import javax.management.MBeanServer;
 import javax.management.MalformedObjectNameException;
 import javax.management.NotCompliantMBeanException;
 import javax.management.ObjectName;
+
+import org.junit.After;
+import org.junit.BeforeClass;
+
 import org.datadog.jmxfetch.reporter.ConsoleReporter;
 import org.datadog.jmxfetch.reporter.Reporter;
 import org.datadog.jmxfetch.util.CustomLogger;
+import org.datadog.jmxfetch.util.MetricsAssert;
 import org.datadog.jmxfetch.util.LogLevel;
-import org.junit.After;
-import org.junit.BeforeClass;
 
 final class ConfigUtil {
     public static Path writeConfigYamlToTemp(String content, String yamlName) throws IOException {
@@ -261,49 +257,7 @@ public void assertMetric(
             List<String> additionalTags,
             int countTags,
             String metricType) {
-        List<String> tags = new ArrayList<String>(commonTags);
-        tags.addAll(additionalTags);
-
-        for (Map<String, Object> m : metrics) {
-            String mName = (String) (m.get("name"));
-            Double mValue = (Double) (m.get("value"));
-            Set<String> mTags = new HashSet<String>(Arrays.asList((String[]) (m.get("tags"))));
-
-            if (mName.equals(name)) {
-
-                if (!value.equals(-1)) {
-                    assertEquals((Double) value.doubleValue(), mValue);
-                } else if (!lowerBound.equals(-1) || !upperBound.equals(-1)) {
-                    assertTrue(mValue > (Double) lowerBound.doubleValue());
-                    assertTrue(mValue < (Double) upperBound.doubleValue());
-                }
-
-                if (countTags != -1) {
-                    assertEquals(countTags, mTags.size());
-                }
-                for (String t : tags) {
-                    assertThat(mTags, hasItem(t));
-                }
-
-                if (metricType != null) {
-                    assertEquals(metricType, m.get("type"));
-                }
-                // Brand the metric
-                m.put("tested", true);
-
-                return;
-            }
-        }
-        fail(
-                "Metric assertion failed (name: "
-                        + name
-                        + ", value: "
-                        + value
-                        + ", tags: "
-                        + tags
-                        + ", #tags: "
-                        + countTags
-                        + ").");
+        MetricsAssert.assertMetric(name, value, lowerBound, upperBound, commonTags, additionalTags, countTags, metricType, this.metrics);
     }
 
     public void assertMetric(

src/test/java/org/datadog/jmxfetch/TestGCMetrics.java

@@ -0,0 +1,208 @@
+package org.datadog.jmxfetch;
+
+import static org.datadog.jmxfetch.util.MetricsAssert.assertDomainPresent;
+import static org.datadog.jmxfetch.util.server.app.org.datadog.jmxfetch.util.server.JDKImage.JDK_11;
+import static org.datadog.jmxfetch.util.server.app.org.datadog.jmxfetch.util.server.JDKImage.JDK_17;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.management.MBeanServerConnection;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+import javax.management.remote.JMXServiceURL;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.junit.Test;
+
+import org.datadog.jmxfetch.reporter.ConsoleReporter;
+import org.datadog.jmxfetch.util.MetricsAssert;
+import org.datadog.jmxfetch.util.server.MisbehavingJMXServer;
+import org.datadog.jmxfetch.util.server.SimpleAppContainer;
+
+@Slf4j
+public class TestGCMetrics extends TestCommon {
+
+    @Test
+    public void testJMXDirectBasic() throws Exception {
+        try (final SimpleAppContainer container = new SimpleAppContainer()) {
+            container.start();
+            final String ipAddress = container.getIp();
+            final String remoteJmxServiceUrl = String.format(
+                "service:jmx:rmi:///jndi/rmi://%s:%s/jmxrmi", ipAddress, container.getRMIPort());
+            final JMXServiceURL jmxUrl = new JMXServiceURL(remoteJmxServiceUrl);
+            final JMXConnector conn = JMXConnectorFactory.connect(jmxUrl);
+            final MBeanServerConnection mBeanServerConnection = conn.getMBeanServerConnection();
+            assertDomainPresent("java.lang", mBeanServerConnection);
+        }
+    }
+
+    @Test
+    public void testDefaultOldGC() throws IOException {
+        try (final MisbehavingJMXServer server = new MisbehavingJMXServer.Builder().build()) {
+            final List<Map<String, Object>> actualMetrics = startAndGetMetrics(server, false);
+            List<String> gcGenerations = Arrays.asList(
+                "G1 Old Generation",
+                "G1 Young Generation");
+            assertGCMetric(actualMetrics, "jvm.gc.cms.count", gcGenerations);
+            assertGCMetric(actualMetrics, "jvm.gc.parnew.time", gcGenerations);
+        }
+    }
+
+    @Test
+    public void testDefaultNewGCMetricsUseParallelGC() throws IOException {
+        try (final MisbehavingJMXServer server = new MisbehavingJMXServer.Builder().withJDKImage(
+            JDK_11).appendJavaOpts("-XX:+UseParallelGC").build()) {
+            final List<Map<String, Object>> actualMetrics = startAndGetMetrics(server, true);
+            assertThat(actualMetrics, hasSize(13));
+            assertGCMetric(actualMetrics,
+                "jvm.gc.minor_collection_count", "PS Scavenge", "counter");
+            assertGCMetric(actualMetrics,
+                "jvm.gc.minor_collection_time", "PS Scavenge", "counter");
+            assertGCMetric(actualMetrics,
+                "jvm.gc.major_collection_count", "PS MarkSweep", "counter");
+            assertGCMetric(actualMetrics,
+                "jvm.gc.major_collection_time", "PS MarkSweep", "counter");
+        }
+    }
+
+    @Test
+    public void testDefaultNewGCMetricsUseConcMarkSweepGC() throws IOException {
+        try (final MisbehavingJMXServer server = new MisbehavingJMXServer.Builder().withJDKImage(
+            JDK_11).appendJavaOpts("-XX:+UseConcMarkSweepGC").build()) {
+            final List<Map<String, Object>> actualMetrics = startAndGetMetrics(server, true);
+            assertThat(actualMetrics, hasSize(13));
+            assertGCMetric(actualMetrics,
+                "jvm.gc.minor_collection_count", "ParNew", "counter");
+            assertGCMetric(actualMetrics,
+                "jvm.gc.minor_collection_time", "ParNew", "counter");
+            assertGCMetric(actualMetrics,
+                "jvm.gc.major_collection_count", "ConcurrentMarkSweep", "counter");
+            assertGCMetric(actualMetrics,
+                "jvm.gc.major_collection_time", "ConcurrentMarkSweep", "counter");
+        }
+    }
+
+    @Test
+    public void testDefaultNewGCMetricsUseG1GC() throws IOException {
+        try (final MisbehavingJMXServer server = new MisbehavingJMXServer.Builder().withJDKImage(
+            JDK_17).appendJavaOpts("-XX:+UseG1GC").build()) {
+            final List<Map<String, Object>> actualMetrics = startAndGetMetrics(server, true);
+            assertThat(actualMetrics, hasSize(13));
+            assertGCMetric(actualMetrics,
+                "jvm.gc.minor_collection_count", "G1 Young Generation", "counter");
+            assertGCMetric(actualMetrics,
+                "jvm.gc.minor_collection_time", "G1 Young Generation", "counter");
+            assertGCMetric(actualMetrics,
+                "jvm.gc.major_collection_count", "G1 Old Generation", "counter");
+            assertGCMetric(actualMetrics,
+                "jvm.gc.major_collection_time", "G1 Old Generation", "counter");
+        }
+    }
+
+    @Test
+    public void testDefaultNewGCMetricsUseZGC() throws IOException {
+        try (final MisbehavingJMXServer server = new MisbehavingJMXServer.Builder().withJDKImage(
+            JDK_17).appendJavaOpts("-XX:+UseZGC").build()) {
+            final List<Map<String, Object>> actualMetrics = startAndGetMetrics(server, true);
+            assertThat(actualMetrics, hasSize(13));
+            assertGCMetric(actualMetrics,
+                "jvm.gc.zgc_pauses_collection_count", "ZGC Pauses", "counter");
+            assertGCMetric(actualMetrics,
+                "jvm.gc.zgc_pauses_collection_time", "ZGC Pauses", "counter");
+            assertGCMetric(actualMetrics,
+                "jvm.gc.zgc_cycles_collection_count", "ZGC Cycles", "counter");
+            assertGCMetric(actualMetrics,
+                "jvm.gc.zgc_cycles_collection_time", "ZGC Cycles", "counter");
+        }
+    }
+
+    private List<Map<String, Object>> startAndGetMetrics(final MisbehavingJMXServer server,
+        final boolean newGCMetrics) throws IOException {
+        server.start();
+        this.initApplicationWithYamlLines(
+            "init_config:",
+            "  is_jmx: true",
+            "  new_gc_metrics: " + newGCMetrics,
+            "",
+            "instances:",
+            "    -   name: jmxint_container",
+            "        host: " + server.getIp(),
+            "        collect_default_jvm_metrics: true",
+            "        max_returned_metrics: 300000",
+            "        port: " + server.getRMIPort());
+        // Run one iteration first
+        this.app.doIteration();
+        // And then pull get the metrics or else reporter does not have correct number of metrics
+        ((ConsoleReporter) appConfig.getReporter()).getMetrics();
+
+        // Actual iteration we care about
+        this.app.doIteration();
+        return ((ConsoleReporter) appConfig.getReporter()).getMetrics();
+    }
+
+    private static void assertGCMetric(final List<Map<String, Object>> actualMetrics,
+        final String expectedMetric,
+        final String gcGeneration,
+        final String metricType) {
+        MetricsAssert.assertMetric(
+            expectedMetric,
+            -1,
+            -1,
+            10.0,
+            Collections.singletonList(String.format("name:%s", gcGeneration)),
+            Arrays.asList(
+                "instance:jmxint_container",
+                "jmx_domain:java.lang",
+                "type:GarbageCollector"),
+            5,
+            metricType,
+            actualMetrics);
+    }
+
+    private static void assertGCMetric(final List<Map<String, Object>> actualMetrics,
+        final String expectedMetric,
+        final List<String> gcGenerations) {
+        final List<Map<String, Object>> filteredMetrics = new ArrayList<>();
+        for (Map<String, Object> actualMetric : actualMetrics) {
+            final String name = (String) actualMetric.get("name");
+            if(expectedMetric.equals(name)) {
+                filteredMetrics.add(actualMetric);
+            }
+        }
+        assertThat(filteredMetrics, hasSize(gcGenerations.size()));
+        for (final String name : gcGenerations) {
+            log.debug("Asserting for metric '{}'", name);
+            boolean found = false;
+            for (Map<String, Object> filteredMetric : filteredMetrics) {
+                final Set<String> mTags = new HashSet<>(
+                    Arrays.asList((String[]) (filteredMetric.get("tags"))));
+
+                if(mTags.contains(String.format("name:%s", name))) {
+                    assertThat(mTags, not(empty()));
+                    assertThat(mTags, hasSize(5));
+                    log.debug("mTags '{}' has size: {}\n{}", name, mTags.size(), mTags);
+                    assertThat(mTags, hasItems(
+                        "instance:jmxint_container",
+                        "jmx_domain:java.lang",
+                        "type:GarbageCollector",
+                        String.format("name:%s", name)));
+                    found = true;
+                }
+            }
+            assertThat(String.format("Did not find metric '%s'", name), found, is(true));
+        }
+    }
+}

src/test/java/org/datadog/jmxfetch/TestReconnectContainer.java

@@ -1,6 +1,10 @@
 package org.datadog.jmxfetch;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import static org.datadog.jmxfetch.util.MetricsAssert.assertDomainPresent;
+import static org.datadog.jmxfetch.util.MetricsAssert.isDomainPresent;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -12,7 +16,6 @@
 import javax.management.remote.JMXConnectorFactory;
 import javax.management.remote.JMXServiceURL;
 
-import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestRule;
@@ -39,21 +42,6 @@ public class TestReconnectContainer extends TestCommon {
     private JMXServerSupervisorClient supervisorClient;
     private static Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(log);
 
-    private static boolean isDomainPresent(String domain, MBeanServerConnection mbs) {
-        boolean found = false;
-        try {
-            String[] domains = mbs.getDomains();
-            for (int i = 0; i < domains.length; i++) {
-                if (domains[i].equals(domain)) {
-                    found = true;
-                }
-            }
-        } catch (IOException e) {
-            found = false;
-        }
-        return found;
-    }
-
     private static ImageFromDockerfile img = new ImageFromDockerfile()
         .withFileFromPath(".", Paths.get("./tools/misbehaving-jmx-server/"));
 
@@ -101,7 +89,7 @@ public void testJMXDirectBasic() throws Exception {
         JMXConnector conn = JMXConnectorFactory.connect(jmxUrl);
         MBeanServerConnection mBeanServerConnection = conn.getMBeanServerConnection();
 
-        assertEquals(true, isDomainPresent("Bohnanza", mBeanServerConnection));
+        assertDomainPresent("Bohnanza", mBeanServerConnection);
     }
 
     @Test
@@ -117,15 +105,15 @@ public void testJMXDirectReconnect() throws Exception {
         JMXConnector conn = JMXConnectorFactory.connect(jmxUrl);
         MBeanServerConnection mBeanServerConnection = conn.getMBeanServerConnection();
 
-        assertEquals(true, isDomainPresent("Bohnanza", mBeanServerConnection));
+        assertDomainPresent("Bohnanza", mBeanServerConnection);
 
         this.controlClient.jmxCutNetwork();
 
-        assertEquals(false, isDomainPresent("Bohnanza", mBeanServerConnection));
+        assertFalse(isDomainPresent("Bohnanza", mBeanServerConnection));
 
         this.controlClient.jmxRestoreNetwork();
 
-        assertEquals(true, isDomainPresent("Bohnanza", mBeanServerConnection));
+        assertDomainPresent("Bohnanza", mBeanServerConnection);
     }
 
     @Test