Merge pull request #100 from exasol/fix-udf-logging

exasol · Aug 14, 2024 · b2fdf33 · b2fdf33
2 parents fbb4faa + 12cc55c
commit b2fdf33
Show file tree

Hide file tree

Showing 15 changed files with 256 additions and 203 deletions.
diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml
diff --git a/.project-keeper.yml b/.project-keeper.yml
@@ -12,9 +12,6 @@ build:
   runnerOs: ubuntu-20.04
   freeDiskSpace: true
   exasolDbVersions:
-    - "8.24.0"
-    - "7.1.25"
+    - "8.26.0" # "8.29.1"
+    - "7.1.29"
 excludes:
-  # Custom extension artifact
-  - "W-PK-CORE-153: Project-keeper version 4.1.0 is outdated. Please update project-keeper to latest version 4.2.0."
-  - "E-PK-CORE-18: Outdated content: '.github/workflows/release_droid_upload_github_release_assets.yml'"
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,25 +1,25 @@
 {
-  "editor.formatOnSave": true,
-  "editor.codeActionsOnSave": {
-    "source.organizeImports": "explicit",
-    "source.generate.finalModifiers": "explicit",
-    "source.fixAll": "explicit"
-  },
-  "java.codeGeneration.useBlocks": true,
-  "java.saveActions.organizeImports": true,
-  "java.sources.organizeImports.starThreshold": 3,
-  "java.sources.organizeImports.staticStarThreshold": 3,
-  "java.test.config": {
-    "vmArgs": [
-      "-Djava.util.logging.config.file=src/test/resources/logging.properties"
-    ]
-  },
-  "java.configuration.updateBuildConfiguration": "automatic",
-  "files.watcherExclude": {
-    "**/target": true
-  },
-  "sonarlint.connectedMode.project": {
-    "connectionId": "exasol",
-    "projectKey": "com.exasol:kafka-connector-extension"
-  }
+    "editor.formatOnSave": true,
+    "editor.codeActionsOnSave": {
+        "source.organizeImports": "explicit",
+        "source.generate.finalModifiers": "explicit",
+        "source.fixAll": "explicit"
+    },
+    "java.codeGeneration.useBlocks": true,
+    "java.saveActions.organizeImports": true,
+    "java.sources.organizeImports.starThreshold": 3,
+    "java.sources.organizeImports.staticStarThreshold": 3,
+    "java.test.config": {
+        "vmArgs": [
+            "-Djava.util.logging.config.file=src/test/resources/logging.properties"
+        ]
+    },
+    "java.configuration.updateBuildConfiguration": "automatic",
+    "files.watcherExclude": {
+        "**/target": true
+    },
+    "sonarlint.connectedMode.project": {
+        "connectionId": "exasol",
+        "projectKey": "com.exasol:kafka-connector-extension"
+    }
 }
diff --git a/dependencies.md b/dependencies.md
diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md
diff --git a/doc/changes/changes_1.7.7.md b/doc/changes/changes_1.7.7.md
@@ -0,0 +1,30 @@
+# Kafka Connector Extension 1.7.7, released 2024-??-??
+
+Code name: Fix logging
+
+## Summary
+
+This release fixes logging of the UDF by adding required libraries. The log level is `WARN` by default and can be changed by rebuilding the adapter JAR. See the [Exasol documentation](https://docs.exasol.com/db/latest/database_concepts/udf_scripts/debug_udf_script_output.htm) for how to configure logging of UDFs.
+
+## Features
+
+* ISSUE_NUMBER: description
+
+## Dependency Updates
+
+### Exasol Kafka Connector Extension
+
+#### Compile Dependency Updates
+
+* Added `ch.qos.logback:logback-classic:1.5.6`
+* Added `org.slf4j:slf4j-api:2.0.16`
+
+#### Test Dependency Updates
+
+* Removed `ch.qos.logback:logback-classic:1.5.3`
+* Removed `ch.qos.logback:logback-core:1.5.3`
+* Updated `com.exasol:exasol-testcontainers:7.0.1` to `7.1.1`
+
+#### Plugin Dependency Updates
+
+* Updated `org.itsallcode:openfasttrace-maven-plugin:1.8.0` to `2.0.0`
diff --git a/doc/development/developer_guide.md b/doc/development/developer_guide.md
@@ -12,3 +12,9 @@ Please read the common [developer guide for the Scala projects][dev-guide].
 
 [dev-guide]: https://github.com/exasol/import-export-udf-common-scala/blob/master/doc/development/developer_guide.md
 [import-export-udf]: https://docs.exasol.com/loading_data/user_defined_import_export_using_udfs.htm
+
+## Configure Logging
+
+This project uses Logback as it is compatible to SLF4J already used by the third-party dependencies. The log level is set to `WARN` by default to avoid performance overhead. For debugging you can increase the log level in [logback.xml](../../src/main/resources/logback.xml) and rebuild the adapter JAR.
+
+See the [Exasol documentation](https://docs.exasol.com/db/latest/database_concepts/udf_scripts/debug_udf_script_output.htm) for how to enable log output for UDFs.
diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md
@@ -61,7 +61,7 @@ checksum provided together with the jar file.
 To check the SHA256 sum of the downloaded jar, run the command:
 
 ```sh
-sha256sum exasol-kafka-connector-extension-1.7.6.jar
+sha256sum exasol-kafka-connector-extension-1.7.7.jar
 ```
 
 ### Building From Source
@@ -84,7 +84,7 @@ sbt assembly
 ```
 
 The packaged jar file should be located at
-`target/scala-2.12/exasol-kafka-connector-extension-1.7.6.jar`.
+`target/scala-2.12/exasol-kafka-connector-extension-1.7.7.jar`.
 
 ### Create an Exasol BucketFS Bucket
 
@@ -106,7 +106,7 @@ jar, please make sure the BucketFS ports are open.
 Upload the jar file using the `curl` command:
 
 ```bash
-curl -X PUT -T exasol-kafka-connector-extension-1.7.6.jar \
+curl -X PUT -T exasol-kafka-connector-extension-1.7.7.jar \
   http://w:<WRITE_PASSWORD>@<EXASOL_DATANODE>:2580/<BUCKET_NAME>/
 ```
 
@@ -135,12 +135,12 @@ OPEN SCHEMA KAFKA_EXTENSION;
 
 CREATE OR REPLACE JAVA SET SCRIPT KAFKA_CONSUMER(...) EMITS (...) AS
   %scriptclass com.exasol.cloudetl.kafka.KafkaConsumerQueryGenerator;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-kafka-connector-extension-1.7.6.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-kafka-connector-extension-1.7.7.jar;
 /
 
 CREATE OR REPLACE JAVA SET SCRIPT KAFKA_IMPORT(...) EMITS (...) AS
   %scriptclass com.exasol.cloudetl.kafka.KafkaTopicDataImporter;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-kafka-connector-extension-1.7.6.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-kafka-connector-extension-1.7.7.jar;
 /
 
 CREATE OR REPLACE JAVA SET SCRIPT KAFKA_METADATA(
@@ -150,7 +150,7 @@ CREATE OR REPLACE JAVA SET SCRIPT KAFKA_METADATA(
 )
 EMITS (partition_index DECIMAL(18, 0), max_offset DECIMAL(36,0)) AS
   %scriptclass com.exasol.cloudetl.kafka.KafkaTopicMetadataReader;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-kafka-connector-extension-1.7.6.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-kafka-connector-extension-1.7.7.jar;
 /
 ```
 

diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom
diff --git a/pom.xml b/pom.xml
@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.exasol</groupId>
     <artifactId>kafka-connector-extension</artifactId>
-    <version>1.7.6</version>
+    <version>1.7.7</version>
     <name>Exasol Kafka Connector Extension</name>
     <description>Exasol Kafka Extension for accessing Apache Kafka</description>
     <url>https://github.com/exasol/kafka-connector-extension/</url>
@@ -111,6 +111,17 @@
             <artifactId>guava</artifactId>
             <version>33.1.0-jre</version>
         </dependency>
+        <!-- Upgrade slf4j-api to allow using the latest logback version -->
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>2.0.16</version>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>1.5.6</version>
+        </dependency>
         <!-- Test Dependencies -->
         <dependency>
             <groupId>org.scalatest</groupId>
@@ -133,7 +144,7 @@
         <dependency>
             <groupId>com.exasol</groupId>
             <artifactId>exasol-testcontainers</artifactId>
-            <version>7.0.1</version>
+            <version>7.1.1</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -212,20 +223,6 @@
             <version>9.4.54.v20240208</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <!-- Upgrade transitive dependency of org.apache.zookeeper:zookeeper to fix CVE-2023-6378 -->
-            <groupId>ch.qos.logback</groupId>
-            <artifactId>logback-core</artifactId>
-            <version>1.5.3</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <!-- Upgrade transitive dependency of org.apache.zookeeper:zookeeper to fix CVE-2023-6378 -->
-            <groupId>ch.qos.logback</groupId>
-            <artifactId>logback-classic</artifactId>
-            <version>1.5.3</version>
-            <scope>test</scope>
-        </dependency>
         <dependency>
             <groupId>io.confluent</groupId>
             <artifactId>kafka-streams-avro-serde</artifactId>
@@ -463,23 +460,6 @@
                     </excludeVulnerabilityIds>
                 </configuration>
             </plugin>
-            <plugin>
-                <groupId>org.basepom.maven</groupId>
-                <artifactId>duplicate-finder-maven-plugin</artifactId>
-                <configuration>
-                    <skip>${duplicate-finder.skip}</skip>
-                    <printEqualFiles>false</printEqualFiles>
-                    <failBuildInCaseOfEqualContentConflict>false</failBuildInCaseOfEqualContentConflict>
-                    <failBuildInCaseOfConflict>false</failBuildInCaseOfConflict>
-                    <ignoredDependencies>
-                        <dependency>
-                            <groupId>com.exasol</groupId>
-                            <artifactId>import-export-udf-common-scala_${scala.compat.version}</artifactId>
-                            <version>1.1.1</version>
-                        </dependency>
-                    </ignoredDependencies>
-                </configuration>
-            </plugin>
             <plugin>
                 <groupId>com.exasol</groupId>
                 <artifactId>project-keeper-maven-plugin</artifactId>
@@ -495,7 +475,7 @@
             <plugin>
                 <groupId>org.itsallcode</groupId>
                 <artifactId>openfasttrace-maven-plugin</artifactId>
-                <version>1.8.0</version>
+                <version>2.0.0</version>
                 <executions>
                     <execution>
                         <id>trace-requirements</id>
@@ -641,7 +621,7 @@
     <parent>
         <artifactId>kafka-connector-extension-generated-parent</artifactId>
         <groupId>com.exasol</groupId>
-        <version>1.7.6</version>
+        <version>1.7.7</version>
         <relativePath>pk_generated_parent.pom</relativePath>
     </parent>
 </project>
diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Commented out to fix OFT tracing, see https://github.com/itsallcode/openfasttrace/issues/429 -->
+<!-- <!DOCTYPE configuration> -->
+
+<configuration>
+  <import class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"/>
+  <import class="ch.qos.logback.core.ConsoleAppender"/>
+
+  <appender name="STDOUT" class="ConsoleAppender">
+    <encoder class="PatternLayoutEncoder">
+      <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n</pattern>
+    </encoder>
+  </appender>
+
+  <root level="warn">
+    <appender-ref ref="STDOUT"/>
+  </root>
+</configuration>
diff --git a/src/test/java/com/exasol/cloudetl/kafka/ExtensionIT.java b/src/test/java/com/exasol/cloudetl/kafka/ExtensionIT.java
@@ -47,7 +47,7 @@ class ExtensionIT extends AbstractScriptExtensionIT {
     static void setup() throws FileNotFoundException, BucketAccessException, TimeoutException, SQLException,
             ExecutionException, InterruptedException {
         if (System.getProperty("com.exasol.dockerdb.image") == null) {
-            System.setProperty("com.exasol.dockerdb.image", "8.23.1");
+            System.setProperty("com.exasol.dockerdb.image", "8.29.1");
         }
         exasolTestSetup = new ExasolTestSetupFactory(Paths.get("no-cloud-setup")).getTestSetup();
         ExasolVersionCheck.assumeExasolVersion8(exasolTestSetup);

diff --git a/src/test/java/com/exasol/cloudetl/kafka/IntegrationTestConstants.java b/src/test/java/com/exasol/cloudetl/kafka/IntegrationTestConstants.java
@@ -6,7 +6,6 @@ public class IntegrationTestConstants {
     public static final String PROJECT_VERSION = MavenProjectVersionGetter.getCurrentProjectVersion();
     public static String JAR_FILE_NAME = "exasol-kafka-connector-extension-" + PROJECT_VERSION + ".jar";
     public static String TEST_SCHEMA_NAME = "kafka_schema";
-    public static String DEFAULT_EXASOL_DOCKER_IMAGE = "8.24.0";
     public static String LOCALSTACK_DOCKER_IMAGE = "localstack/localstack:2.2";
     public static String DOCKER_IP_ADDRESS = "172.17.0.1";
 }
diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Commented out to fix OFT tracing, see https://github.com/itsallcode/openfasttrace/issues/429 -->
+<!-- <!DOCTYPE configuration> -->
+
+<configuration>
+  <import class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"/>
+  <import class="ch.qos.logback.core.ConsoleAppender"/>
+
+  <appender name="STDOUT" class="ConsoleAppender">
+    <encoder class="PatternLayoutEncoder">
+      <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n</pattern>
+    </encoder>
+  </appender>
+
+  <root level="info">
+    <appender-ref ref="STDOUT"/>
+  </root>
+
+  <logger name="com.exasol" additivity="false" level="trace" />
+  <logger name="org.testcontainers" additivity="false" level="debug" />
+</configuration>
diff --git a/src/test/scala/com/exasol/cloudetl/kafka/ittests/docker/BaseDockerIntegrationTest.scala b/src/test/scala/com/exasol/cloudetl/kafka/ittests/docker/BaseDockerIntegrationTest.scala
@@ -15,7 +15,7 @@ import org.scalatest.funsuite.AnyFunSuite
 
 trait BaseDockerIntegrationTest extends AnyFunSuite with BeforeAndAfterAll {
   private[this] val JAR_NAME_PATTERN = "exasol-kafka-connector-extension-"
-  private[this] val DEFAULT_EXASOL_DOCKER_IMAGE = "8.24.0"
+  private[this] val DEFAULT_EXASOL_DOCKER_IMAGE = "8.29.1"
 
   val network = DockerNamedNetwork("kafka-it-tests", true)
   val exasolContainer = {
@@ -120,7 +120,7 @@ trait BaseDockerIntegrationTest extends AnyFunSuite with BeforeAndAfterAll {
   }
 
   private[this] def getExasolDockerImageVersion(): String = {
-    val dockerVersion = System.getenv("EXASOL_DOCKER_VERSION")
+    val dockerVersion = System.getenv("EXASOL_DB_VERSION")
     if (dockerVersion == null) {
       DEFAULT_EXASOL_DOCKER_IMAGE
     } else {