From a831fdc78ef839bafcd39a6c15484c206ec214c6 Mon Sep 17 00:00:00 2001 From: Max Lapan Date: Tue, 20 Feb 2024 13:18:28 +0100 Subject: [PATCH] Custom krb5.conf file location (#87) * Configuration option * Check for file exstence and set the property * Fix optional dereference * Remove empty line * Unit test * Trigger the exception * Reformating * Unit test for case when file exists * New parameter documentation * Prepare the release * Fix artefacts ref * Update doc/changes/changes_1.7.3.md Co-authored-by: Christoph Kuhnke * Update doc/user_guide/user_guide.md Co-authored-by: Christoph Kuhnke * Docs updates * Update commons-compress to fix CVE-2024-25710 and CVE-2024-26308 * PK update --------- Co-authored-by: Christoph Kuhnke --- dependencies.md | 244 +++++++++--------- doc/changes/changelog.md | 1 + doc/changes/changes_1.7.3.md | 22 ++ doc/user_guide/user_guide.md | 18 +- error_code_config.yml | 2 +- pk_generated_parent.pom | 2 +- pom.xml | 11 +- .../kafka/KafkaConsumerProperties.scala | 34 ++- src/test/resources/test_krb5.conf | 0 .../kafka/KafkaConsumerPropertiesTest.scala | 24 +- 10 files changed, 224 insertions(+), 134 deletions(-) create mode 100644 doc/changes/changes_1.7.3.md create mode 100644 src/test/resources/test_krb5.conf diff --git a/dependencies.md b/dependencies.md index 5884959..9dc4085 100644 --- a/dependencies.md +++ b/dependencies.md @@ -3,67 +3,68 @@ ## Compile Dependencies -| Dependency | License | -| ------------------------------------------ | ------------------------------------- | -| [Scala Library][0] | [Apache-2.0][1] | -| [Import Export UDF Common Scala][2] | [MIT License][3] | -| [error-reporting-java][4] | [MIT License][5] | -| [kafka-avro-serializer][6] | [Apache License 2.0][7] | -| [scala-collection-compat][8] | [Apache-2.0][1] | -| [Guava: Google Core Libraries for Java][9] | [Apache License, Version 2.0][10] | -| [Apache Kafka][11] | [The Apache License, Version 2.0][10] | -| [snappy-java][12] | [Apache-2.0][13] | +| Dependency | License | +| ------------------------------------------- | ------------------------------------- | +| [Scala Library][0] | [Apache-2.0][1] | +| [Import Export UDF Common Scala][2] | [MIT License][3] | +| [error-reporting-java][4] | [MIT License][5] | +| [Apache Commons Compress][6] | [Apache-2.0][7] | +| [kafka-avro-serializer][8] | [Apache License 2.0][9] | +| [scala-collection-compat][10] | [Apache-2.0][1] | +| [Guava: Google Core Libraries for Java][11] | [Apache License, Version 2.0][12] | +| [Apache Kafka][13] | [The Apache License, Version 2.0][12] | +| [snappy-java][14] | [Apache-2.0][15] | ## Test Dependencies | Dependency | License | | ------------------------------------------ | ----------------------------------------------------------------------------- | -| [scalatest][14] | [the Apache License, ASL Version 2.0][15] | -| [scalatestplus-mockito][16] | [Apache-2.0][15] | -| [mockito-core][17] | [MIT][18] | -| [Test containers for Exasol on Docker][19] | [MIT License][20] | -| [Test Database Builder for Java][21] | [MIT License][22] | -| [Matcher for SQL Result Sets][23] | [MIT License][24] | -| [Extension integration tests library][25] | [MIT License][26] | -| [embedded-kafka-schema-registry][27] | [MIT][18] | -| [JSON in Java][28] | [Public Domain][29] | -| [Apache ZooKeeper - Server][30] | [Apache License, Version 2.0][31] | -| [Logback Core Module][32] | [Eclipse Public License - v 1.0][33]; [GNU Lesser General Public License][34] | -| [Logback Classic Module][35] | [Eclipse Public License - v 1.0][33]; [GNU Lesser General Public License][34] | -| [kafka-streams-avro-serde][36] | [Apache License 2.0][7] | -| [avro4s-core][37] | [MIT][18] | -| [Apache Avro][38] | [Apache-2.0][31] | -| [Testcontainers :: Kafka][39] | [MIT][40] | +| [scalatest][16] | [the Apache License, ASL Version 2.0][17] | +| [scalatestplus-mockito][18] | [Apache-2.0][17] | +| [mockito-core][19] | [MIT][20] | +| [Test containers for Exasol on Docker][21] | [MIT License][22] | +| [Test Database Builder for Java][23] | [MIT License][24] | +| [Matcher for SQL Result Sets][25] | [MIT License][26] | +| [Extension integration tests library][27] | [MIT License][28] | +| [embedded-kafka-schema-registry][29] | [MIT][20] | +| [JSON in Java][30] | [Public Domain][31] | +| [Apache ZooKeeper - Server][32] | [Apache License, Version 2.0][7] | +| [Logback Core Module][33] | [Eclipse Public License - v 1.0][34]; [GNU Lesser General Public License][35] | +| [Logback Classic Module][36] | [Eclipse Public License - v 1.0][34]; [GNU Lesser General Public License][35] | +| [kafka-streams-avro-serde][37] | [Apache License 2.0][9] | +| [avro4s-core][38] | [MIT][20] | +| [Apache Avro][39] | [Apache-2.0][7] | +| [Testcontainers :: Kafka][40] | [MIT][41] | ## Plugin Dependencies -| Dependency | License | -| ------------------------------------------------------- | ---------------------------------------------- | -| [SonarQube Scanner for Maven][41] | [GNU LGPL 3][42] | -| [Apache Maven Toolchains Plugin][43] | [Apache License, Version 2.0][31] | -| [Apache Maven Compiler Plugin][44] | [Apache-2.0][31] | -| [Apache Maven Enforcer Plugin][45] | [Apache-2.0][31] | -| [Maven Flatten Plugin][46] | [Apache Software Licenese][31] | -| [org.sonatype.ossindex.maven:ossindex-maven-plugin][47] | [ASL2][10] | -| [scala-maven-plugin][48] | [Public domain (Unlicense)][49] | -| [ScalaTest Maven Plugin][50] | [the Apache License, ASL Version 2.0][15] | -| [Apache Maven Javadoc Plugin][51] | [Apache-2.0][31] | -| [Maven Surefire Plugin][52] | [Apache-2.0][31] | -| [Versions Maven Plugin][53] | [Apache License, Version 2.0][31] | -| [duplicate-finder-maven-plugin Maven Mojo][54] | [Apache License 2.0][7] | -| [Apache Maven Assembly Plugin][55] | [Apache-2.0][31] | -| [Apache Maven JAR Plugin][56] | [Apache License, Version 2.0][31] | -| [Artifact reference checker and unifier][57] | [MIT License][58] | -| [Maven Failsafe Plugin][59] | [Apache-2.0][31] | -| [JaCoCo :: Maven Plugin][60] | [Eclipse Public License 2.0][61] | -| [error-code-crawler-maven-plugin][62] | [MIT License][63] | -| [Reproducible Build Maven Plugin][64] | [Apache 2.0][10] | -| [Project Keeper Maven plugin][65] | [The MIT License][66] | -| [OpenFastTrace Maven Plugin][67] | [GNU General Public License v3.0][68] | -| [Scalastyle Maven Plugin][69] | [Apache 2.0][7] | -| [spotless-maven-plugin][70] | [The Apache Software License, Version 2.0][31] | -| [scalafix-maven-plugin][71] | [BSD-3-Clause][72] | -| [Exec Maven Plugin][73] | [Apache License 2][31] | +| Dependency | License | +| ------------------------------------------------------- | --------------------------------------------- | +| [SonarQube Scanner for Maven][42] | [GNU LGPL 3][43] | +| [Apache Maven Toolchains Plugin][44] | [Apache License, Version 2.0][7] | +| [Apache Maven Compiler Plugin][45] | [Apache-2.0][7] | +| [Apache Maven Enforcer Plugin][46] | [Apache-2.0][7] | +| [Maven Flatten Plugin][47] | [Apache Software Licenese][7] | +| [org.sonatype.ossindex.maven:ossindex-maven-plugin][48] | [ASL2][12] | +| [scala-maven-plugin][49] | [Public domain (Unlicense)][50] | +| [ScalaTest Maven Plugin][51] | [the Apache License, ASL Version 2.0][17] | +| [Apache Maven Javadoc Plugin][52] | [Apache-2.0][7] | +| [Maven Surefire Plugin][53] | [Apache-2.0][7] | +| [Versions Maven Plugin][54] | [Apache License, Version 2.0][7] | +| [duplicate-finder-maven-plugin Maven Mojo][55] | [Apache License 2.0][9] | +| [Apache Maven Assembly Plugin][56] | [Apache-2.0][7] | +| [Apache Maven JAR Plugin][57] | [Apache License, Version 2.0][7] | +| [Artifact reference checker and unifier][58] | [MIT License][59] | +| [Maven Failsafe Plugin][60] | [Apache-2.0][7] | +| [JaCoCo :: Maven Plugin][61] | [Eclipse Public License 2.0][62] | +| [error-code-crawler-maven-plugin][63] | [MIT License][64] | +| [Reproducible Build Maven Plugin][65] | [Apache 2.0][12] | +| [Project Keeper Maven plugin][66] | [The MIT License][67] | +| [OpenFastTrace Maven Plugin][68] | [GNU General Public License v3.0][69] | +| [Scalastyle Maven Plugin][70] | [Apache 2.0][9] | +| [spotless-maven-plugin][71] | [The Apache Software License, Version 2.0][7] | +| [scalafix-maven-plugin][72] | [BSD-3-Clause][73] | +| [Exec Maven Plugin][74] | [Apache License 2][7] | [0]: https://www.scala-lang.org/ [1]: https://www.apache.org/licenses/LICENSE-2.0 @@ -71,71 +72,72 @@ [3]: https://github.com/exasol/import-export-udf-common-scala/blob/main/LICENSE [4]: https://github.com/exasol/error-reporting-java/ [5]: https://github.com/exasol/error-reporting-java/blob/main/LICENSE -[6]: http://confluent.io/kafka-avro-serializer -[7]: http://www.apache.org/licenses/LICENSE-2.0.html -[8]: http://www.scala-lang.org/ -[9]: https://github.com/google/guava -[10]: http://www.apache.org/licenses/LICENSE-2.0.txt -[11]: https://kafka.apache.org -[12]: https://github.com/xerial/snappy-java -[13]: https://www.apache.org/licenses/LICENSE-2.0.html -[14]: http://www.scalatest.org -[15]: http://www.apache.org/licenses/LICENSE-2.0 -[16]: https://github.com/scalatest/scalatestplus-mockito -[17]: https://github.com/mockito/mockito -[18]: https://opensource.org/licenses/MIT -[19]: https://github.com/exasol/exasol-testcontainers/ -[20]: https://github.com/exasol/exasol-testcontainers/blob/main/LICENSE -[21]: https://github.com/exasol/test-db-builder-java/ -[22]: https://github.com/exasol/test-db-builder-java/blob/main/LICENSE -[23]: https://github.com/exasol/hamcrest-resultset-matcher/ -[24]: https://github.com/exasol/hamcrest-resultset-matcher/blob/main/LICENSE -[25]: https://github.com/exasol/extension-manager/ -[26]: https://github.com/exasol/extension-manager/blob/main/LICENSE -[27]: https://github.com/embeddedkafka/embedded-kafka-schema-registry -[28]: https://github.com/douglascrockford/JSON-java -[29]: https://github.com/stleary/JSON-java/blob/master/LICENSE -[30]: http://zookeeper.apache.org/zookeeper -[31]: https://www.apache.org/licenses/LICENSE-2.0.txt -[32]: http://logback.qos.ch/logback-core -[33]: http://www.eclipse.org/legal/epl-v10.html -[34]: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html -[35]: http://logback.qos.ch/logback-classic -[36]: http://confluent.io/kafka-streams-avro-serde -[37]: https://github.com/sksamuel/avro4s -[38]: https://avro.apache.org -[39]: https://java.testcontainers.org -[40]: http://opensource.org/licenses/MIT -[41]: http://sonarsource.github.io/sonar-scanner-maven/ -[42]: http://www.gnu.org/licenses/lgpl.txt -[43]: https://maven.apache.org/plugins/maven-toolchains-plugin/ -[44]: https://maven.apache.org/plugins/maven-compiler-plugin/ -[45]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ -[46]: https://www.mojohaus.org/flatten-maven-plugin/ -[47]: https://sonatype.github.io/ossindex-maven/maven-plugin/ -[48]: http://github.com/davidB/scala-maven-plugin -[49]: http://unlicense.org/ -[50]: https://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin -[51]: https://maven.apache.org/plugins/maven-javadoc-plugin/ -[52]: https://maven.apache.org/surefire/maven-surefire-plugin/ -[53]: https://www.mojohaus.org/versions/versions-maven-plugin/ -[54]: https://basepom.github.io/duplicate-finder-maven-plugin -[55]: https://maven.apache.org/plugins/maven-assembly-plugin/ -[56]: https://maven.apache.org/plugins/maven-jar-plugin/ -[57]: https://github.com/exasol/artifact-reference-checker-maven-plugin/ -[58]: https://github.com/exasol/artifact-reference-checker-maven-plugin/blob/main/LICENSE -[59]: https://maven.apache.org/surefire/maven-failsafe-plugin/ -[60]: https://www.jacoco.org/jacoco/trunk/doc/maven.html -[61]: https://www.eclipse.org/legal/epl-2.0/ -[62]: https://github.com/exasol/error-code-crawler-maven-plugin/ -[63]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE -[64]: http://zlika.github.io/reproducible-build-maven-plugin -[65]: https://github.com/exasol/project-keeper/ -[66]: https://github.com/exasol/project-keeper/blob/main/LICENSE -[67]: https://github.com/itsallcode/openfasttrace-maven-plugin -[68]: https://www.gnu.org/licenses/gpl-3.0.html -[69]: http://www.scalastyle.org -[70]: https://github.com/diffplug/spotless -[71]: https://github.com/evis/scalafix-maven-plugin -[72]: https://opensource.org/licenses/BSD-3-Clause -[73]: https://www.mojohaus.org/exec-maven-plugin +[6]: https://commons.apache.org/proper/commons-compress/ +[7]: https://www.apache.org/licenses/LICENSE-2.0.txt +[8]: http://confluent.io/kafka-avro-serializer +[9]: http://www.apache.org/licenses/LICENSE-2.0.html +[10]: http://www.scala-lang.org/ +[11]: https://github.com/google/guava +[12]: http://www.apache.org/licenses/LICENSE-2.0.txt +[13]: https://kafka.apache.org +[14]: https://github.com/xerial/snappy-java +[15]: https://www.apache.org/licenses/LICENSE-2.0.html +[16]: http://www.scalatest.org +[17]: http://www.apache.org/licenses/LICENSE-2.0 +[18]: https://github.com/scalatest/scalatestplus-mockito +[19]: https://github.com/mockito/mockito +[20]: https://opensource.org/licenses/MIT +[21]: https://github.com/exasol/exasol-testcontainers/ +[22]: https://github.com/exasol/exasol-testcontainers/blob/main/LICENSE +[23]: https://github.com/exasol/test-db-builder-java/ +[24]: https://github.com/exasol/test-db-builder-java/blob/main/LICENSE +[25]: https://github.com/exasol/hamcrest-resultset-matcher/ +[26]: https://github.com/exasol/hamcrest-resultset-matcher/blob/main/LICENSE +[27]: https://github.com/exasol/extension-manager/ +[28]: https://github.com/exasol/extension-manager/blob/main/LICENSE +[29]: https://github.com/embeddedkafka/embedded-kafka-schema-registry +[30]: https://github.com/douglascrockford/JSON-java +[31]: https://github.com/stleary/JSON-java/blob/master/LICENSE +[32]: http://zookeeper.apache.org/zookeeper +[33]: http://logback.qos.ch/logback-core +[34]: http://www.eclipse.org/legal/epl-v10.html +[35]: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html +[36]: http://logback.qos.ch/logback-classic +[37]: http://confluent.io/kafka-streams-avro-serde +[38]: https://github.com/sksamuel/avro4s +[39]: https://avro.apache.org +[40]: https://java.testcontainers.org +[41]: http://opensource.org/licenses/MIT +[42]: http://sonarsource.github.io/sonar-scanner-maven/ +[43]: http://www.gnu.org/licenses/lgpl.txt +[44]: https://maven.apache.org/plugins/maven-toolchains-plugin/ +[45]: https://maven.apache.org/plugins/maven-compiler-plugin/ +[46]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ +[47]: https://www.mojohaus.org/flatten-maven-plugin/ +[48]: https://sonatype.github.io/ossindex-maven/maven-plugin/ +[49]: http://github.com/davidB/scala-maven-plugin +[50]: http://unlicense.org/ +[51]: https://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin +[52]: https://maven.apache.org/plugins/maven-javadoc-plugin/ +[53]: https://maven.apache.org/surefire/maven-surefire-plugin/ +[54]: https://www.mojohaus.org/versions/versions-maven-plugin/ +[55]: https://basepom.github.io/duplicate-finder-maven-plugin +[56]: https://maven.apache.org/plugins/maven-assembly-plugin/ +[57]: https://maven.apache.org/plugins/maven-jar-plugin/ +[58]: https://github.com/exasol/artifact-reference-checker-maven-plugin/ +[59]: https://github.com/exasol/artifact-reference-checker-maven-plugin/blob/main/LICENSE +[60]: https://maven.apache.org/surefire/maven-failsafe-plugin/ +[61]: https://www.jacoco.org/jacoco/trunk/doc/maven.html +[62]: https://www.eclipse.org/legal/epl-2.0/ +[63]: https://github.com/exasol/error-code-crawler-maven-plugin/ +[64]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE +[65]: http://zlika.github.io/reproducible-build-maven-plugin +[66]: https://github.com/exasol/project-keeper/ +[67]: https://github.com/exasol/project-keeper/blob/main/LICENSE +[68]: https://github.com/itsallcode/openfasttrace-maven-plugin +[69]: https://www.gnu.org/licenses/gpl-3.0.html +[70]: http://www.scalastyle.org +[71]: https://github.com/diffplug/spotless +[72]: https://github.com/evis/scalafix-maven-plugin +[73]: https://opensource.org/licenses/BSD-3-Clause +[74]: https://www.mojohaus.org/exec-maven-plugin diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index f2a1448..f027c51 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [1.7.3](changes_1.7.3.md) * [1.7.2](changes_1.7.2.md) * [1.7.1](changes_1.7.1.md) * [1.7.0](changes_1.7.0.md) diff --git a/doc/changes/changes_1.7.3.md b/doc/changes/changes_1.7.3.md new file mode 100644 index 0000000..b0c9942 --- /dev/null +++ b/doc/changes/changes_1.7.3.md @@ -0,0 +1,22 @@ +# Exasol Kafka Connector Extension 1.7.3, released 2024-02-20 + +Code name: Custom `krb5.conf` files support. + +## Summary + +Implemented support for custom `krb5.conf` files. +Updated transient dependency to fix CVE-2024-25710 and CVE-2024-26308. + +## Features + +* #86: Add support for custom krb5.conf + +## Dependency Updates + +### Compile Dependency Updates + +* Added `org.apache.commons:commons-compress:1.26.0` + +### Plugin Dependency Updates + +* Updated `com.exasol:project-keeper-maven-plugin:3.0.0` to `3.0.1` diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index ca35eec..2b8d90e 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -61,7 +61,7 @@ checksum provided together with the jar file. To check the SHA256 sum of the downloaded jar, run the command: ```sh -sha256sum exasol-kafka-connector-extension-1.7.2.jar +sha256sum exasol-kafka-connector-extension-1.7.3.jar ``` ### Building From Source @@ -84,7 +84,7 @@ sbt assembly ``` The packaged jar file should be located at -`target/scala-2.12/exasol-kafka-connector-extension-1.7.2.jar`. +`target/scala-2.12/exasol-kafka-connector-extension-1.7.3.jar`. ### Create an Exasol BucketFS Bucket @@ -106,7 +106,7 @@ jar, please make sure the BucketFS ports are open. Upload the jar file using the `curl` command: ```bash -curl -X PUT -T exasol-kafka-connector-extension-1.7.2.jar \ +curl -X PUT -T exasol-kafka-connector-extension-1.7.3.jar \ http://w:@:2580// ``` @@ -135,12 +135,12 @@ OPEN SCHEMA KAFKA_EXTENSION; CREATE OR REPLACE JAVA SET SCRIPT KAFKA_CONSUMER(...) EMITS (...) AS %scriptclass com.exasol.cloudetl.kafka.KafkaConsumerQueryGenerator; - %jar /buckets/bfsdefault//exasol-kafka-connector-extension-1.7.2.jar; + %jar /buckets/bfsdefault//exasol-kafka-connector-extension-1.7.3.jar; / CREATE OR REPLACE JAVA SET SCRIPT KAFKA_IMPORT(...) EMITS (...) AS %scriptclass com.exasol.cloudetl.kafka.KafkaTopicDataImporter; - %jar /buckets/bfsdefault//exasol-kafka-connector-extension-1.7.2.jar; + %jar /buckets/bfsdefault//exasol-kafka-connector-extension-1.7.3.jar; / CREATE OR REPLACE JAVA SET SCRIPT KAFKA_METADATA( @@ -150,7 +150,7 @@ CREATE OR REPLACE JAVA SET SCRIPT KAFKA_METADATA( ) EMITS (partition_index DECIMAL(18, 0), max_offset DECIMAL(36,0)) AS %scriptclass com.exasol.cloudetl.kafka.KafkaTopicMetadataReader; - %jar /buckets/bfsdefault//exasol-kafka-connector-extension-1.7.2.jar; + %jar /buckets/bfsdefault//exasol-kafka-connector-extension-1.7.3.jar; / ``` @@ -489,6 +489,8 @@ keyTab="/buckets/bfsdefault/bucket1/kafka.keytab" principal="principal@DOMAIN.COM"; ``` +In some complex setups, you might need to provide a custom ``krb5.conf`` file. Thes could be done by uploading it to the BucketFS and providing the path in ``SASL_KRB5CONF_LOCATION`` parameter, similar to ``SASL_JAAS_LOCATION``. + ## Importing Data From Azure Event Hubs To import data from [Azure Event Hubs][azure-event-hubs], we are going to create @@ -690,6 +692,10 @@ not in import statement itself. more complex configuration of SASL authentication. It should refer to the file stored inside a bucket in Exasol BucketFS. +* ``SASL_KRB5CONF_LOCATION`` - It is the location of the custom ``krb5.conf`` file. + It should refer to the file stored inside a bucket in Exasol BucketFS. In default + configuration, the path starts with ``/buckets/bfsdefault//`` + [gh-releases]: https://github.com/exasol/kafka-connector-extension/releases [schema-registry]: https://docs.confluent.io/current/schema-registry/index.html [kafka-security]: https://kafka.apache.org/documentation/#security diff --git a/error_code_config.yml b/error_code_config.yml index fd29f70..2b5b23b 100644 --- a/error_code_config.yml +++ b/error_code_config.yml @@ -2,4 +2,4 @@ error-tags: KCE: packages: - com.exasol.cloudetl - highest-index: 27 + highest-index: 28 diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom index 621c02c..80b010d 100644 --- a/pk_generated_parent.pom +++ b/pk_generated_parent.pom @@ -3,7 +3,7 @@ 4.0.0 com.exasol kafka-connector-extension-generated-parent - 1.7.2 + 1.7.3 pom UTF-8 diff --git a/pom.xml b/pom.xml index c622a64..b907898 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.exasol kafka-connector-extension - 1.7.2 + 1.7.3 Exasol Kafka Connector Extension Exasol Kafka Extension for accessing Apache Kafka https://github.com/exasol/kafka-connector-extension/ @@ -56,6 +56,11 @@ error-reporting-java 1.0.1 + + org.apache.commons + commons-compress + 1.26.0 + io.confluent kafka-avro-serializer @@ -439,7 +444,7 @@ com.exasol project-keeper-maven-plugin - 3.0.0 + 3.0.1 @@ -597,7 +602,7 @@ kafka-connector-extension-generated-parent com.exasol - 1.7.2 + 1.7.3 pk_generated_parent.pom diff --git a/src/main/scala/com/exasol/cloudetl/kafka/KafkaConsumerProperties.scala b/src/main/scala/com/exasol/cloudetl/kafka/KafkaConsumerProperties.scala index 2f71ab0..5a364a0 100644 --- a/src/main/scala/com/exasol/cloudetl/kafka/KafkaConsumerProperties.scala +++ b/src/main/scala/com/exasol/cloudetl/kafka/KafkaConsumerProperties.scala @@ -299,6 +299,28 @@ class KafkaConsumerProperties(private val properties: Map[String, String]) exten } } + /** + * Gets custom krb5.conf file location. Checks that user provided path exists. + * @return file path or empty string + */ + final def getSASLKrb5ConfLocation(): String = { + val krb5confLocation = get(SASL_KRB5CONF_LOCATION.userPropertyName) + if (krb5confLocation.isEmpty) { + "" + } else { + if (!Files.isRegularFile(Paths.get(krb5confLocation.get))) { + throw new KafkaConnectorException( + ExaError + .messageBuilder("E-KCE-28") + .message("Unable to find the custom krb5.conf file at {{JAAS_LOCATION}}", krb5confLocation) + .mitigation(BUCKETFS_CHECK_MITIGATION) + .toString() + ) + } + krb5confLocation.get + } + } + /** Returns the Kafka consumer properties as Java map. */ final def getProperties(): java.util.Map[String, AnyRef] = { val props = MMap.empty[String, String] @@ -328,6 +350,7 @@ class KafkaConsumerProperties(private val properties: Map[String, String]) exten if (isSASLEnabled()) { props.put(SASL_MECHANISM.kafkaPropertyName, getSASLMechanism()) props.put(SASL_JAAS_CONFIG.kafkaPropertyName, getSASLJaasConfig()) + props.put(SASL_KRB5CONF_LOCATION.kafkaPropertyName, getSASLKrb5ConfLocation()) addOptionalParametersForSASL(props) } props.toMap.asInstanceOf[Map[String, AnyRef]].asJava @@ -374,7 +397,6 @@ class KafkaConsumerProperties(private val properties: Map[String, String]) exten .toString() ) } - } /** @@ -757,6 +779,16 @@ object KafkaConsumerProperties { "" ) + /** + * SASL krb5.conf file location. It is can be used when [[SECURITY_PROTOCOL]] is set to {@code SASL_PLAINTEXT} + * or {@code SASL_SSL} to provide location of custom krb5.conf. + */ + private[kafka] final val SASL_KRB5CONF_LOCATION: Config[String] = Config[String]( + "SASL_KRB5CONF_LOCATION", + "java.security.krb5.conf", + "" + ) + /** * SASL password. It is used when [[SASL_MECHANISM]] is set to {@code PLAIN}, {@code Digest-*} or {@code SCRAM-*}. */ diff --git a/src/test/resources/test_krb5.conf b/src/test/resources/test_krb5.conf new file mode 100644 index 0000000..e69de29 diff --git a/src/test/scala/com/exasol/cloudetl/kafka/KafkaConsumerPropertiesTest.scala b/src/test/scala/com/exasol/cloudetl/kafka/KafkaConsumerPropertiesTest.scala index 8abede0..eba92ae 100644 --- a/src/test/scala/com/exasol/cloudetl/kafka/KafkaConsumerPropertiesTest.scala +++ b/src/test/scala/com/exasol/cloudetl/kafka/KafkaConsumerPropertiesTest.scala @@ -473,6 +473,9 @@ class KafkaConsumerPropertiesTest extends AnyFunSuite with BeforeAndAfterEach wi private[this] val DUMMY_SASL_JAAS_FILE = Paths.get(getClass.getResource("/kafka_client_jaas.conf").toURI).toAbsolutePath + private[this] val DUMMY_KRB5CONF_FILE = + Paths.get(getClass.getResource("/test_krb5.conf").toURI).toAbsolutePath + test("apply returns a SSL enabled consumer properties") { val properties = getSecurityEnabledConsumerProperties("SSL", Option(DUMMY_KEYSTORE_FILE), Option(DUMMY_TRUSTSTORE_FILE)) @@ -553,10 +556,28 @@ class KafkaConsumerPropertiesTest extends AnyFunSuite with BeforeAndAfterEach wi assert(properties.getProperties().get(SSL_TRUSTSTORE_PASSWORD.kafkaPropertyName) === "tspw") } + test("error is throws when non-existent krb5.conf file passed") { + val properties = + getSecurityEnabledConsumerProperties("SASL_SSL", krb5confFile = Option(Paths.get("krb5_non_existing"))) + val thrown = intercept[KafkaConnectorException] { + properties.getProperties() + } + val message = thrown.getMessage() + assert(message.contains("Unable to find the custom krb5.conf file")) + assert(message.contains("Please make sure it is successfully uploaded to BucketFS bucket")) + } + + test("property is set when existing krb5.conf file passed") { + val properties = getSecurityEnabledConsumerProperties("SASL_SSL", krb5confFile = Option(DUMMY_KRB5CONF_FILE)) + val props = properties.getProperties() + assert(props.get(SASL_KRB5CONF_LOCATION.kafkaPropertyName) === s"$DUMMY_KRB5CONF_FILE") + } + private[this] def getSecurityEnabledConsumerProperties( securityProtocol: String, keystoreFile: Option[Path] = None, - truststoreFile: Option[Path] = None + truststoreFile: Option[Path] = None, + krb5confFile: Option[Path] = None ): KafkaConsumerProperties = { val properties = Map( "BOOTSTRAP_SERVERS" -> "kafka01", @@ -571,6 +592,7 @@ class KafkaConsumerPropertiesTest extends AnyFunSuite with BeforeAndAfterEach wi val stringBuilder = new StringBuilder() keystoreFile.foreach(file => stringBuilder.append("SSL_KEYSTORE_LOCATION=").append(file).append(";")) truststoreFile.foreach(file => stringBuilder.append("SSL_TRUSTSTORE_LOCATION=").append(file).append(";")) + krb5confFile.foreach(file => stringBuilder.append("SASL_KRB5CONF_LOCATION=").append(file).append(";")) if (securityProtocol === "SSL") { addSimpleSSLParameters(stringBuilder) } else if (securityProtocol === "SASL_SSL") {