chore: split java connector tests (#19382)

risingwavelabs · Nov 14, 2024 · 4d3f633 · 4d3f633
1 parent df07ffb
commit 4d3f633
Show file tree

Hide file tree

Showing 37 changed files with 202 additions and 83 deletions.
diff --git a/java/connector-node/assembly/assembly.xml b/java/connector-node/assembly/assembly.xml
@@ -34,6 +34,7 @@
             <includes>
                 <!-- The connector service -->
                 <include>*:risingwave-connector-service</include>
+                <include>*:tracing</include>
 
                 <!-- Source connectors -->
                 <include>*:risingwave-source-cdc</include>

diff --git a/java/connector-node/assembly/pom.xml b/java/connector-node/assembly/pom.xml
@@ -54,6 +54,10 @@
             <groupId>com.risingwave</groupId>
             <artifactId>s3-common</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.risingwave</groupId>
+            <artifactId>tracing</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/...a/com/risingwave/connector/TestUtils.java → ...a/com/risingwave/connector/TestUtils.java b/...a/com/risingwave/connector/TestUtils.java → ...a/com/risingwave/connector/TestUtils.java
diff --git a/java/connector-node/risingwave-connector-service/pom.xml b/java/connector-node/risingwave-connector-service/pom.xml
@@ -26,10 +26,6 @@
             <groupId>com.risingwave</groupId>
             <artifactId>proto</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.risingwave</groupId>
-            <artifactId>tracing</artifactId>
-        </dependency>
         <dependency>
             <groupId>com.risingwave</groupId>
             <artifactId>java-binding</artifactId>
@@ -42,12 +38,18 @@
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
         </dependency>
-
-
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-text</artifactId>
@@ -64,6 +66,8 @@
             <groupId>io.prometheus</groupId>
             <artifactId>simpleclient_httpserver</artifactId>
         </dependency>
+
+        <!-- test dependencies -->
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

diff --git a/...or-node/risingwave-connector-service/src/main/java/com/risingwave/connector/FileSink.java b/...or-node/risingwave-connector-service/src/main/java/com/risingwave/connector/FileSink.java
@@ -29,6 +29,8 @@
 import java.util.UUID;
 import java.util.stream.IntStream;
 
+// `FileSink` only serves for testing purpose. It is NOT the file sink in RisingWave docs.
+// TODO: consider rename or remove it to avoid confusion
 public class FileSink extends SinkWriterBase {
     private final FileWriter sinkWriter;
 

diff --git a/...singwave/connector/sink/FileSinkTest.java → ...singwave/connector/sink/FileSinkTest.java b/...singwave/connector/sink/FileSinkTest.java → ...singwave/connector/sink/FileSinkTest.java
diff --git a/...onnector/sink/SinkStreamObserverTest.java → ...onnector/sink/SinkStreamObserverTest.java b/...onnector/sink/SinkStreamObserverTest.java → ...onnector/sink/SinkStreamObserverTest.java
diff --git a/...test/src/test/resources/log4j2.properties → ...vice/src/test/resources/log4j2.properties b/...test/src/test/resources/log4j2.properties → ...vice/src/test/resources/log4j2.properties
diff --git a/...or-node/risingwave-connector-test/pom.xml → ...de/risingwave-sink-deltalake-test/pom.xml b/...or-node/risingwave-connector-test/pom.xml → ...de/risingwave-sink-deltalake-test/pom.xml
@@ -9,9 +9,15 @@
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
-    <name>risingwave-connector-test</name>
+    <name>risingwave-sink-deltalake-test</name>
     <packaging>jar</packaging>
-    <artifactId>risingwave-connector-test</artifactId>
+    <artifactId>risingwave-sink-deltalake-test</artifactId>
+    <description>
+        This module contains the unit test of Delta Lake Sink. Usually we just put the unit test alongside
+        the main code, but Delta Lake's test relies on Apache Spark, which contains numerous dependencies and
+        can easily conflict with ours. By splitting it as a dedicated module, these Spark's dependencies
+        (especially the stale ones) only affect the test itself.
+    </description>
 
     <properties>
         <iceberg.version>1.0.0</iceberg.version>
@@ -28,11 +34,6 @@
             <artifactId>risingwave-sink-deltalake</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>com.risingwave</groupId>
-            <artifactId>risingwave-sink-iceberg</artifactId>
-            <scope>test</scope>
-        </dependency>
         <dependency>
             <groupId>com.risingwave</groupId>
             <artifactId>s3-common</artifactId>
@@ -53,14 +54,22 @@
         <dependency>
             <groupId>io.grpc</groupId>
             <artifactId>grpc-protobuf</artifactId>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-api</artifactId>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j2-impl</artifactId>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -101,30 +110,8 @@
             <version>3.24.2</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>com.zaxxer</groupId>
-            <artifactId>HikariCP</artifactId>
-            <version>5.0.1</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.testcontainers</groupId>
-            <artifactId>testcontainers</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.testcontainers</groupId>
-            <artifactId>mysql</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.testcontainers</groupId>
-            <artifactId>postgresql</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.testcontainers</groupId>
-            <artifactId>elasticsearch</artifactId>
-            <version>${testcontainers.version}</version>
-            <scope>test</scope>
-        </dependency>
+
+        <!-- Force using jackson-databind 2.13.5, as required by spark-3.2_2.12  -->
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
@@ -138,46 +125,5 @@
             <scope>test</scope>
         </dependency>
 
-        <dependency>
-            <groupId>com.risingwave</groupId>
-            <artifactId>risingwave-connector-service</artifactId>
-            <scope>test</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.risingwave</groupId>
-                    <artifactId>tracing</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>com.risingwave</groupId>
-            <artifactId>risingwave-sink-jdbc</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.risingwave</groupId>
-            <artifactId>risingwave-sink-es-7</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.risingwave</groupId>
-            <artifactId>risingwave-sink-cassandra</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.risingwave</groupId>
-            <artifactId>risingwave-sink-mock-flink-runtime</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.risingwave</groupId>
-            <artifactId>risingwave-sink-mock-flink-common</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.risingwave</groupId>
-            <artifactId>risingwave-sink-mock-flink-http-sink</artifactId>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 </project>
diff --git a/...ink/deltalake/DeltaLakeLocalSinkTest.java → ...ink/deltalake/DeltaLakeLocalSinkTest.java b/...ink/deltalake/DeltaLakeLocalSinkTest.java → ...ink/deltalake/DeltaLakeLocalSinkTest.java
diff --git a/...k/deltalake/DeltaLakeSinkFactoryTest.java → ...k/deltalake/DeltaLakeSinkFactoryTest.java b/...k/deltalake/DeltaLakeSinkFactoryTest.java → ...k/deltalake/DeltaLakeSinkFactoryTest.java
diff --git a/java/connector-node/risingwave-sink-deltalake-test/src/test/resources/log4j2.properties b/java/connector-node/risingwave-sink-deltalake-test/src/test/resources/log4j2.properties
@@ -0,0 +1,10 @@
+rootLogger.level=ERROR
+# declare the appender to use
+appenders=console
+# appender properties
+appender.console.type=Console
+appender.console.name=stdout
+appender.console.layout.type=PatternLayout
+appender.console.layout.pattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%t] %c{2}:%L - %m%n
+rootLogger.appenderRefs=console
+rootLogger.appenderRef.console.ref=stdout
diff --git a/java/connector-node/risingwave-sink-es-7/pom.xml b/java/connector-node/risingwave-sink-es-7/pom.xml
@@ -63,6 +63,30 @@
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpclient</artifactId>
         </dependency>
+
+        <!-- test dependencies -->
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>elasticsearch</artifactId>
+            <version>${testcontainers.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j2-impl</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
 </project>
diff --git a/...nector/sink/elasticsearch/EsSinkTest.java → ...nector/sink/elasticsearch/EsSinkTest.java b/...nector/sink/elasticsearch/EsSinkTest.java → ...nector/sink/elasticsearch/EsSinkTest.java
diff --git a/java/connector-node/risingwave-sink-es-7/src/test/resources/log4j2.properties b/java/connector-node/risingwave-sink-es-7/src/test/resources/log4j2.properties
@@ -0,0 +1,10 @@
+rootLogger.level=ERROR
+# declare the appender to use
+appenders=console
+# appender properties
+appender.console.type=Console
+appender.console.name=stdout
+appender.console.layout.type=PatternLayout
+appender.console.layout.pattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%t] %c{2}:%L - %m%n
+rootLogger.appenderRefs=console
+rootLogger.appenderRef.console.ref=stdout
diff --git a/java/connector-node/risingwave-sink-jdbc/pom.xml b/java/connector-node/risingwave-sink-jdbc/pom.xml
@@ -42,5 +42,43 @@
             <artifactId>mysql-connector-j</artifactId>
         </dependency>
 
+
+        <!-- test dependencies -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j2-impl</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>mysql</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>postgresql</artifactId>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 </project>
diff --git a/...ave/connector/sink/jdbc/JDBCSinkTest.java → ...ave/connector/sink/jdbc/JDBCSinkTest.java b/...ave/connector/sink/jdbc/JDBCSinkTest.java → ...ave/connector/sink/jdbc/JDBCSinkTest.java
diff --git a/java/connector-node/risingwave-sink-jdbc/src/test/resources/log4j2.properties b/java/connector-node/risingwave-sink-jdbc/src/test/resources/log4j2.properties
@@ -0,0 +1,10 @@
+rootLogger.level=ERROR
+# declare the appender to use
+appenders=console
+# appender properties
+appender.console.type=Console
+appender.console.name=stdout
+appender.console.layout.type=PatternLayout
+appender.console.layout.pattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%t] %c{2}:%L - %m%n
+rootLogger.appenderRefs=console
+rootLogger.appenderRef.console.ref=stdout
diff --git a/java/connector-node/risingwave-sink-mock-flink/risingwave-sink-mock-flink-http-sink/pom.xml b/java/connector-node/risingwave-sink-mock-flink/risingwave-sink-mock-flink-http-sink/pom.xml
@@ -33,6 +33,28 @@
             <artifactId>risingwave-sink-mock-flink-runtime</artifactId>
             <scope>provided</scope>
         </dependency>
+
+        <!-- test dependencies -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j2-impl</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>
diff --git a/...ingwave/connector/sink/http/HttpTest.java → ...ingwave/connector/sink/http/HttpTest.java b/...ingwave/connector/sink/http/HttpTest.java → ...ingwave/connector/sink/http/HttpTest.java
diff --git a/...sink-mock-flink/risingwave-sink-mock-flink-http-sink/src/test/resources/log4j2.properties b/...sink-mock-flink/risingwave-sink-mock-flink-http-sink/src/test/resources/log4j2.properties
@@ -0,0 +1,10 @@
+rootLogger.level=ERROR
+# declare the appender to use
+appenders=console
+# appender properties
+appender.console.type=Console
+appender.console.name=stdout
+appender.console.layout.type=PatternLayout
+appender.console.layout.pattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%t] %c{2}:%L - %m%n
+rootLogger.appenderRefs=console
+rootLogger.appenderRef.console.ref=stdout
diff --git a/java/connector-node/risingwave-sink-mock-flink/risingwave-sink-mock-flink-runtime/pom.xml b/java/connector-node/risingwave-sink-mock-flink/risingwave-sink-mock-flink-runtime/pom.xml
@@ -57,6 +57,28 @@
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-connector-base</artifactId>
         </dependency>
+
+        <!-- test dependencies -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j2-impl</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>
diff --git a/...e/connector/sink/flink/FlinkMockTest.java → ...e/connector/sink/flink/FlinkMockTest.java b/...e/connector/sink/flink/FlinkMockTest.java → ...e/connector/sink/flink/FlinkMockTest.java
diff --git a/...e-sink-mock-flink/risingwave-sink-mock-flink-runtime/src/test/resources/log4j2.properties b/...e-sink-mock-flink/risingwave-sink-mock-flink-runtime/src/test/resources/log4j2.properties
@@ -0,0 +1,10 @@
+rootLogger.level=ERROR
+# declare the appender to use
+appenders=console
+# appender properties
+appender.console.type=Console
+appender.console.name=stdout
+appender.console.layout.type=PatternLayout
+appender.console.layout.pattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%t] %c{2}:%L - %m%n
+rootLogger.appenderRefs=console
+rootLogger.appenderRef.console.ref=stdout