Merge branch 'master' into brpc_config

.github/workflows/build8.yml → .github/workflows/build-agent-jdk8.yml

@@ -1,7 +1,7 @@
 # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time
 # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
 
-name: JDK8 Build
+name: Agent Build on JDK8
 
 on:
   push:
@@ -39,10 +39,7 @@ jobs:
         run: mvn clean install -T 1C -ntp --activate-profiles shaded -Dmaven.javadoc.skip=true -DskipTests
 
       - name: Compilation
-        run: mvn clean install -T 1C -ntp --activate-profiles jooq,server,agent,component -Dmaven.javadoc.skip=true -DskipTests
-
-      - name: Code Check
-        run: mvn --ntp -T 1C animal-sniffer:check checkstyle:checkstyle com.github.spotbugs:spotbugs-maven-plugin:check
+        run: mvn clean install -T 1C -ntp --activate-profiles agent,component -Dmaven.javadoc.skip=true -DskipTests
 
       #
       # Follow steps are for release, we only enable them for changes on the master branch

.github/workflows/build17.yml → .github/workflows/build-all-jdk17.yml

@@ -29,5 +29,9 @@ jobs:
         with:
           java-version: '17'
           distribution: 'adopt'
+
       - name: Compilation
         run: mvn clean install -T 1C -ntp --activate-profiles shaded,jooq,server,component,agent -Dmaven.javadoc.skip=true -DskipTests
+
+      - name: Code Check
+        run: mvn --ntp -T 1C animal-sniffer:check checkstyle:checkstyle com.github.spotbugs:spotbugs-maven-plugin:check

LICENSE

@@ -213,17 +213,17 @@ Apache License version 2.0
 ================================
 
   This product contains schema definition related code adapted from Apache Druid
-    * component/component-commons/src/main/java/org/bithon/component/commons/utils/HumanReadableBytes.java
-    * server/collector/src/main/java/org/bithon/server/metric/TimestampSpec.java
-    * server/collector/src/main/java/org/bithon/server/metric/parser/DateTimes.java
-    * server/collector/src/main/java/org/bithon/server/metric/parser/ParserUtils.java
-    * server/collector/src/main/java/org/bithon/server/metric/parser/TimestampParser.java
+    * component/component-commons/src/main/java/org/bithon/component/commons/utils/HumanReadableNumber.java
+    * server/server-commons/src/main/java/org/bithon/server/commons/time/DateTimes.java
     * hooks/pre-push.sh
     * setup-hooks.sh
 
  This product contains a util method from Apache commons-math
     * component/component-commons/src/main/java/org/bithon/component/commons/utils/NumberUtils.java
 
+ This product contains a class from Spring Framework (https://github.com/spring-projects/spring-framework)
+    * server/server-starter/src/main/java/org/bithon/server/starter/config/UrlHandlerFilter.java
+
 qos.ch
 ================================
 
@@ -247,3 +247,9 @@ qos.ch
     * server/server-commons/src/main/java/org/bithon/server/commons/autocomplete/LexerFactory.java
     * server/server-commons/src/main/java/org/bithon/server/commons/autocomplete/ParserFactory.java
     * server/server-commons/src/main/java/org/bithon/server/commons/autocomplete/TransitionWrapper.java
+
+MIT License
+================================
+
+  This product contains UUIDv7 related code adapted from https://github.com/f4b6a3/uuid-creator
+    * component/component-commons/src/main/java/org/bithon/component/commons/uuid/UUIDv7Generator.java

README.md

@@ -52,14 +52,13 @@ git submodule update --init
 
 ## 2. choose a right JDK
 
-JDK 1.8 is recommended because the agent is compatible with higher JRE that is used to run your Java applications.
-
+Since the project is built upon SpringBoot 3.0, a JDK 17 or higher is required to build this project.
 If you have multiple JDKs on your machine, use `export JAVA_HOME={YOUR_JDK_HOME}` command to set correct JDK.
 
 For example
 
 ```bash
-export JAVA_HOME=/Library/Java/JavaVirtualMachines/openjdk-8.jdk/Contents/Home
+export JAVA_HOME=/Library/Java/JavaVirtualMachines/openjdk-17.jdk/Contents/Home
 ```
 
 ## 3. build the project
@@ -129,7 +128,9 @@ Make sure to re-build the project after changing the configuration file above.
 
 # JDKs Compatibility
 
-The following matrix lists the JDKs that have been tested on macOS. And in theory, this matrix works both for Windows and Linux.
+Even the project is built by JDK 17 and above, the agent is compatible with JDK 1.8+.
+The following matrix lists the JDKs that are compatible with the agent on macOS. 
+And in theory, this matrix works both for Windows and Linux.
 
 | JDK           | Supported | 
 |---------------|-----------|
@@ -162,10 +163,10 @@ If the target application runs under JDK 11 and above, the following arguments s
 | JDK - HTTP Client      | 1.8         |             | [✓](doc/metrics/http-outgoing/README.md) | ✓ |
 | Alibaba Druid          | 1.0.28      |             | ✓                                        |         |
 | Apache Druid           | 0.16        | 24.0        |                                                | ✓ |
-| Apache Kafka           | 0.10        |             | ✓                                        | ✓ |
+| Apache Kafka(1)        | 0.10        |             | ✓                                        | ✓ |
 | Apache OZone           | 1.3.0       |             |                                                | ✓ |
 | Eclipse Glassfish      | 2.34        |             |                                                | ✓ |
-| GRPC                   | 1.6.0       |             | ✓                                        |         |
+| GRPC                   | 1.57.0      |             | ✓                                        |         |
 | Google Guice           | 4.1.0       |             |                                                | ✓ |
 | HTTP Client - Apache   | 4.5.2       | 5.x         | [✓](doc/metrics/http-outgoing/README.md) | ✓ |
 | HTTP Client - Jetty    | 9.4.6       |             | [✓](doc/metrics/http-outgoing/README.md) | ✓ |
@@ -176,7 +177,7 @@ If the target application runs under JDK 11 and above, the following arguments s
 | MySQL                  | 5.x         | 8.x         | ✓                                        |         |
 | Quartz                 | 2.x         |             | ✓                                        | ✓ |
 | Redis - Jedis          | 2.9         | 5.x         | ✓                                        | ✓ |
-| Redis - Lettuce(1)     | 5.1.2       | 6.x         | ✓                                        | ✓ |
+| Redis - Lettuce(2)     | 5.1.2       | 6.x         | ✓                                        | ✓ |
 | Redis - Redisson       | 3.19.0      |             | ✓                                        | ✓ |
 | Spring Boot            | 1.5         | 3.0+        |                                                | ✓ |
 | Spring Bean            | 4.3.12      |             |                                                | ✓ |
@@ -191,8 +192,9 @@ If the target application runs under JDK 11 and above, the following arguments s
 | xxl-job                | 2.3.0       |             |                                                | ✓ |
 
 ## Restrictions
-1. For Lettuce, the tracing support is only available when it's used with Spring Data Redis API.
-2. For Redisson, metrics and tracing only work when connections are successfully established between the client and redis servers.
+1. For Apache Kafka clients 3.7, the consumer metrics only works when the `group.protocol` is configured as `classic` which is the default configuration of the consumer client. 
+2. For Lettuce, the tracing support is only available when it's used with Spring Data Redis API.
+
 
 
 

agent/agent-core/src/main/java/org/bithon/agent/configuration/source/PropertySource.java

@@ -107,20 +107,28 @@ public static PropertySource from(PropertySourceType source,
     /**
      * Merge two configuration nodes into one recursively
      */
-    public static JsonNode merge(JsonNode to, JsonNode from, boolean isReplace) {
-        if (from == null) {
-            return to;
+    public static JsonNode merge(JsonNode target, JsonNode source, boolean isReplace) {
+        if (source == null) {
+            return target;
         }
 
-        Iterator<String> names = from.fieldNames();
+        if (target instanceof ArrayNode && source instanceof ArrayNode) {
+            if (isReplace) {
+                ((ArrayNode) target).removeAll();
+            }
+            ((ArrayNode) target).addAll((ArrayNode) source);
+            return target;
+        }
+
+        Iterator<String> names = source.fieldNames();
         while (names.hasNext()) {
 
             String fieldName = names.next();
-            JsonNode targetNode = to.get(fieldName);
-            JsonNode sourceNode = from.get(fieldName);
+            JsonNode targetNode = target.get(fieldName);
+            JsonNode sourceNode = source.get(fieldName);
 
             if (targetNode == null) {
-                ((ObjectNode) to).set(fieldName, sourceNode);
+                ((ObjectNode) target).set(fieldName, sourceNode);
                 continue;
             }
 
@@ -144,15 +152,15 @@ public static JsonNode merge(JsonNode to, JsonNode from, boolean isReplace) {
                     }
                 } else {
                     // use the source node to replace the 'to' node
-                    ((ObjectNode) to).set(fieldName, sourceNode);
+                    ((ObjectNode) target).set(fieldName, sourceNode);
                 }
             } else {
                 // use the source node to replace the 'to' node
-                ((ObjectNode) to).set(fieldName, sourceNode);
+                ((ObjectNode) target).set(fieldName, sourceNode);
             }
         }
 
-        return to;
+        return target;
     }
 
     public PropertySourceType getType() {

agent/agent-core/src/test/java/org/bithon/agent/configuration/TestConfigurationManager.java

@@ -28,6 +28,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -85,8 +86,8 @@ public void test_DynamicConfiguration() throws IOException {
         manager.addPropertySource(PropertySource.from(PropertySourceType.INTERNAL,
                                                       "1",
                                                       "test.a=1\n" +
-                                                          "test.b=7\n" +
-                                                          "test.percentage=8%"));
+                                                      "test.b=7\n" +
+                                                      "test.percentage=8%"));
 
         TestConfig testConfig = manager.getConfig(TestConfig.class);
         Assert.assertEquals(1, testConfig.getA());
@@ -99,7 +100,7 @@ public void test_DynamicConfiguration() throws IOException {
         manager.addPropertySource(PropertySource.from(PropertySourceType.INTERNAL,
                                                       "2",
                                                       "test.a=2\ntest.b=8\ntest.percentage=500%"
-                                                     ));
+        ));
         Assert.assertEquals(2, testConfig.getA());
         Assert.assertEquals(8, testConfig.getB());
         Assert.assertEquals(5, testConfig.getPercentage().intValue());
@@ -163,7 +164,7 @@ public void test_CommandLineArgs() {
 
                                                        // Also set the external configuration
                                                        "-Dbithon.configuration.location=" + externalConfigLocation
-                                                      ));
+                             ));
 
             ConfigurationManager manager = ConfigurationManager.create(defaultConfigLocation);
 
@@ -188,7 +189,7 @@ public void test_Environment() {
 
                                                        // Also set the external configuration
                                                        "-Dbithon.configuration.location=" + externalConfigLocation
-                                                      ));
+                             ));
 
             configurationMock.when(Helper::getEnvironmentVariables)
                              .thenReturn(ImmutableMap.of("bithon_t", "t1",
@@ -235,7 +236,7 @@ public void test_PropFromDifferentSource() {
 
                                                        // Also set the external configuration
                                                        "-Dbithon.configuration.location=" + externalConfigLocation
-                                                      ));
+                             ));
 
             configurationMock.when(Helper::getEnvironmentVariables)
                              .thenReturn(ImmutableMap.of("bithon_t", "t1",
@@ -385,7 +386,7 @@ public void test_BindToSimpleTypes() {
                                                        // Override the in file configuration
                                                        "-Dbithon.test.b=true",
                                                        "-Dbithon.test.percentage=8%"
-                                                      ));
+                             ));
 
             ConfigurationManager manager = ConfigurationManager.create(defaultConfigLocation);
 
@@ -408,7 +409,7 @@ public void test_BindToArray() {
                                                        "-Dbithon.test.p[0]=8%",
                                                        "-Dbithon.test.p[2]=9%",
                                                        "-Dbithon.test.p[3]=10%"
-                                                      ));
+                             ));
 
             ConfigurationManager manager = ConfigurationManager.create(defaultConfigLocation);
 
@@ -419,4 +420,32 @@ public void test_BindToArray() {
                                      manager.getConfig("test.p", HumanReadablePercentage[].class, true));
         }
     }
+
+    @ConfigurationProperties(path = "test.arrayList")
+    public static class StringListConfig extends ArrayList<String> {
+    }
+
+    @Test
+    public void test_BindToArray_ReplaceDefault() {
+        // If the property is not given, the default value is the one in the external configuration file
+        {
+            ConfigurationManager manager = ConfigurationManager.create(externalConfigLocation);
+            StringListConfig config = manager.getConfig(StringListConfig.class);
+            Assert.assertEquals(Collections.singletonList("from file a"), config);
+        }
+
+        // When the property is given, the default one is overridden
+        try (MockedStatic<Helper> configurationMock = Mockito.mockStatic(Helper.class)) {
+            configurationMock.when(Helper::getCommandLineInputArgs)
+                             .thenReturn(Arrays.asList("-Xms512M",
+                                                       // Array configuration
+                                                       "-Dbithon.test.arrayList[0]=1",
+                                                       "-Dbithon.test.arrayList[1]=2"
+                             ));
+
+            ConfigurationManager manager = ConfigurationManager.create(externalConfigLocation);
+            StringListConfig config = manager.getConfig(StringListConfig.class);
+            Assert.assertEquals(Arrays.asList("1", "2"), config);
+        }
+    }
 }

agent/agent-core/src/test/resources/conf/external.yml

@@ -2,3 +2,5 @@
 # This file here is for test cases
 test:
   prop: "from external file"
+  arrayList:
+    - "from file a"

agent/agent-dispatcher-brpc/src/main/java/org/bithon/agent/dispatcher/brpc/BrpcAgentController.java

@@ -62,7 +62,7 @@ public BrpcAgentController(AgentControllerConfig config) {
 
         AppInstance appInstance = AppInstance.getInstance();
         brpcClient = BrpcClientBuilder.builder()
-                                      .applicationName(appInstance.getQualifiedAppName())
+                                      .applicationName(appInstance.getQualifiedName())
                                       .clientId("ctrl")
                                       .server(new RoundRobinEndPointProvider(endpoints))
                                       .ioThreads(2)
@@ -75,13 +75,13 @@ public BrpcAgentController(AgentControllerConfig config) {
 
         if (appInstance.getPort() > 0) {
             // Set the default the appId
-            brpcClient.setHeader(Headers.HEADER_APP_ID, appInstance.getHostAndPort());
+            brpcClient.setHeader(Headers.HEADER_APP_ID, appInstance.getInstanceName());
         }
 
         // Update appId once the port is configured,
         // so that the management API in the server side can find this agent by appId correctly
         appInstance.addListener((port) -> {
-            brpcClient.setHeader(Headers.HEADER_APP_ID, AppInstance.getInstance().getHostAndPort());
+            brpcClient.setHeader(Headers.HEADER_APP_ID, AppInstance.getInstance().getInstanceName());
 
             if (refreshListener != null) {
                 try {
@@ -106,11 +106,9 @@ public Map<String, String> getAgentConfiguration(String appName, String env, lon
 
         AppInstance appInstance = AppInstance.getInstance();
         BrpcMessageHeader header = BrpcMessageHeader.newBuilder()
-                                                    .setAppName(appInstance.getAppName())
+                                                    .setAppName(appInstance.getName())
                                                     .setEnv(appInstance.getEnv())
-                                                    .setInstanceName(appInstance.getHostAndPort())
-                                                    .setHostIp(appInstance.getHostIp())
-                                                    .setPort(appInstance.getPort())
+                                                    .setInstanceName(appInstance.getInstanceName())
                                                     .setAppType(ApplicationType.JAVA)
                                                     .build();
         try {

agent/agent-dispatcher-brpc/src/main/java/org/bithon/agent/dispatcher/brpc/BrpcEventMessageChannel.java

@@ -68,19 +68,15 @@ public BrpcEventMessageChannel(DispatcherConfig dispatcherConfig) {
 
         AppInstance appInstance = AppInstance.getInstance();
         this.header = BrpcMessageHeader.newBuilder()
-                                       .setAppName(appInstance.getQualifiedAppName())
+                                       .setAppName(appInstance.getQualifiedName())
                                        .setEnv(appInstance.getEnv())
-                                       .setInstanceName(appInstance.getHostAndPort())
-                                       .setHostIp(appInstance.getHostIp())
-                                       .setPort(appInstance.getPort())
+                                       .setInstanceName(appInstance.getInstanceName())
                                        .setAppType(ApplicationType.JAVA)
                                        .build();
         appInstance.addListener(port -> this.header = BrpcMessageHeader.newBuilder()
-                                                                       .setAppName(appInstance.getQualifiedAppName())
+                                                                       .setAppName(appInstance.getQualifiedName())
                                                                        .setEnv(appInstance.getEnv())
-                                                                       .setInstanceName(appInstance.getHostAndPort())
-                                                                       .setHostIp(appInstance.getHostIp())
-                                                                       .setPort(appInstance.getPort())
+                                                                       .setInstanceName(appInstance.getInstanceName())
                                                                        .setAppType(ApplicationType.JAVA)
                                                                        .build());
     }