Merge branch 'main' of https://github.com/bakdata/kpops-examples into…

… test/argocd

.github/workflows/build-and-publish.yaml

@@ -0,0 +1,68 @@
+name: Build and Publish
+
+on:
+  push:
+    tags: ["**"]
+    branches: ["**"]
+
+env:
+  java-version: 21
+  gradle-version: 8.4
+  working-directory: "./word-count/code"
+
+jobs:
+  build-and-test:
+    name: Build, Test and Assess code quality
+    uses: bakdata/ci-templates/.github/workflows/[email protected]
+    with:
+      java-version: 21
+      gradle-version: 8.4
+      gradle-cache: true
+      gradle-cache-read-only: false
+      gradle-refresh-dependencies: false
+      working-directory: "./word-count/code"
+    secrets:
+      signing-key-id: ${{ secrets.SONATYPE_SIGNING_KEY_ID }}
+      signing-password: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
+      signing-secret-key-ring: ${{ secrets.SONATYPE_SIGNING_SECRET_KEY_RING }}
+      sonar-token: ${{ secrets.SONARCLOUD_TOKEN }}
+      sonar-organization: ${{ secrets.SONARCLOUD_ORGANIZATION }}
+
+  build-jib:
+    name: Build images
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: ${{ env.working-directory }}
+    needs: build-and-test
+
+    steps:
+      - name: Login into docker
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push sentence producer image
+        uses: bakdata/ci-templates/actions/[email protected]
+        with:
+          full-image-name: bakdata/kpops-demo-sentence-producer
+          class: com.bakdata.kpops.examples.SentenceProducer
+          java-version: ${{ env.java-version }}
+          gradle-version: ${{ env.gradle-version }}
+          gradle-cache: true
+          gradle-cache-read-only: true
+          gradle-refresh-dependencies: false
+          working-directory: ${{ env.working-directory }}
+
+      - name: Build and push word count image
+        uses: bakdata/ci-templates/actions/[email protected]
+        with:
+          full-image-name: bakdata/kpops-demo-word-count-app
+          class: com.bakdata.kpops.examples.WordCountApplication
+          java-version: ${{ env.java-version }}
+          gradle-version: ${{ env.gradle-version }}
+          gradle-cache: true
+          gradle-cache-read-only: true
+          gradle-refresh-dependencies: false
+          working-directory: ${{ env.working-directory }}

.github/workflows/release.yaml

@@ -0,0 +1,27 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      release-type:
+        description: "The scope of the release (major, minor or patch)."
+        type: choice
+        required: true
+        default: patch
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  java-gradle-release:
+    name: Java Gradle Release
+    uses: bakdata/ci-templates/.github/workflows/[email protected]
+    with:
+      java-version: 21
+      release-type: "${{ inputs.release-type }}"
+      working-directory: "./word-count/code"
+    secrets:
+      github-email: "${{ secrets.GH_EMAIL }}"
+      github-username: "${{ secrets.GH_USERNAME }}"
+      github-token: ${{ secrets.GH_TOKEN }}

defaults.yaml → atm-fraud/defaults.yaml

@@ -1,32 +1,29 @@
 kubernetes-app:
   namespace: ${NAMESPACE}
 
-streams-bootstrap:
-  app:
+streams-bootstrap-v2:
+  values:
     imageTag: "1.0.0"
     labels:
       pipeline: ${pipeline.name}
-    replicaCount: 1
-
-kafka-app:
-  app:
     streams:
       brokers: ${config.kafka_brokers}
       schemaRegistryUrl: ${config.schema_registry.url}
       optimizeLeaveGroupBehavior: false
     prometheus:
       jmx:
         enabled: false
-  debug: true
 
-producer-app:
+    replicaCount: 1
+
+producer-app-v2:
   to:
     topics:
       ${output_topic_name}:
         partitions_count: 3
 
-streams-app:
-  app:
+streams-app-v2:
+  values:
     commandLine:
       PRODUCTIVE: false
   to:

atm-fraud/pipeline.yaml

@@ -1,50 +1,50 @@
-- type: producer-app
+- type: producer-app-v2
   name: account-producer
-  app:
+  values:
     image: ${DOCKER_REGISTRY}/atm-demo-accountproducer
     schedule: 0 12 * * *
     suspend: true
 
-- type: producer-app
+- type: producer-app-v2
   name: transaction-avro-producer
-  app:
+  values:
     image: ${DOCKER_REGISTRY}/atm-demo-transactionavroproducer
     commandLine:
       REAL_TX: 19
       ITERATION: 20
     schedule: 0 12 * * *
     suspend: true
 
-- type: streams-app
+- type: streams-app-v2
   name: transaction-joiner
-  app:
+  values:
     image: ${DOCKER_REGISTRY}/atm-demo-transactionjoiner
     annotations:
       consumerGroup: atm-transactionjoiner-atm-fraud-joinedtransactions-topic
 
-- type: streams-app
+- type: streams-app-v2
   name: fraud-detector
-  app:
+  values:
     image: ${DOCKER_REGISTRY}/atm-demo-frauddetector
     annotations:
       consumerGroup: atm-frauddetector-atm-fraud-possiblefraudtransactions-topic
 
-- type: streams-app
+- type: streams-app-v2
   name: account-linker
   from:
     components:
       fraud-detector:
         type: input
       account-producer:
-        role: accounts
-  app:
+        label: accounts
+  values:
     image: ${DOCKER_REGISTRY}/atm-demo-accountlinker
     annotations:
       consumerGroup: atm-accountlinker-atm-fraud-output-topic
 
 - type: kafka-sink-connector
   name: postgresql-connector
-  app:
+  config:
     connector.class: io.confluent.connect.jdbc.JdbcSinkConnector
     tasks.max: 1
     connection.url: jdbc:postgresql://postgresql-dev.${NAMESPACE}.svc.cluster.local:5432/app_db

requirements.txt

@@ -1 +1 @@
-kpops~=3.0
+kpops~=8.1.4

word-count/README.md

@@ -12,5 +12,5 @@ gradle jib -Djib.to.image=bakdata/kpops-demo-sentence-producer -Djib.container.m
 
 
 # deploy streams-app
-gradle jib -Djib.to.image=bakdata/kpops-demo-word-count-app-Djib.container.mainClass=com.bakdata.kpops.examples.WordCountApplication
+gradle jib -Djib.to.image=bakdata/kpops-demo-word-count-app -Djib.container.mainClass=com.bakdata.kpops.examples.WordCountApplication
 ```

word-count/code/build.gradle.kts

@@ -1,9 +1,13 @@
 description = "Word count pipeline with Kafka Streams"
+
 plugins {
     java
     idea
-    id("io.freefair.lombok") version "6.6.1"
-    id("com.google.cloud.tools.jib") version "3.3.1"
+    id("com.bakdata.release") version "1.4.1"
+    id("com.bakdata.sonar") version "1.4.1"
+    id("com.bakdata.sonatype") version "1.4.1"
+    id("io.freefair.lombok") version "8.11"
+    id("com.google.cloud.tools.jib") version "3.4.4"
 }
 
 group = "com.bakdata.kpops.examples"
@@ -14,9 +18,19 @@ repositories {
 }
 
 
-configure<JavaPluginExtension> {
-    sourceCompatibility = JavaVersion.VERSION_17
-    targetCompatibility = JavaVersion.VERSION_17
+java {
+    toolchain {
+        languageVersion = JavaLanguageVersion.of(21)
+    }
+}
+
+configure<com.bakdata.gradle.SonatypeSettings> {
+    developers {
+        developer {
+            name.set("Ramin Gharib")
+            id.set("raminqaf")
+        }
+    }
 }
 
 tasks {
@@ -32,28 +46,34 @@ tasks {
 }
 
 dependencies {
-    implementation(group = "com.bakdata.kafka", name = "streams-bootstrap", version = "2.8.0")
-    implementation(group = "org.apache.logging.log4j", name = "log4j-slf4j-impl", version = "2.19.0")
-    implementation(group = "com.google.guava", name = "guava", version = "31.1-jre")
+    val streamsBootstrapVersion = "3.1.0"
+    implementation(group = "com.bakdata.kafka", name = "streams-bootstrap-cli", version = streamsBootstrapVersion)
+    val log4jVersion = "2.24.2"
+    implementation(group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version = log4jVersion)
+    val guavaVersion = "33.3.1-jre"
+    implementation(group = "com.google.guava", name = "guava", version = guavaVersion )
 
-    val junitVersion: String by project
+    val junitVersion = "5.11.3"
     testImplementation(group = "org.junit.jupiter", name = "junit-jupiter-api", version = junitVersion)
     testRuntimeOnly(group = "org.junit.jupiter", name = "junit-jupiter-engine", version = junitVersion)
-    testImplementation(group = "org.assertj", name = "assertj-core", version = "3.24.2")
-    val fluentKafkaVersion = "2.8.1"
+    val assertJVersion = "3.26.3"
+    testImplementation(group = "org.assertj", name = "assertj-core", version = assertJVersion)
+
+    testImplementation(group = "com.bakdata.kafka", name = "streams-bootstrap-test", version = streamsBootstrapVersion)
+    val fluentKafkaVersion = "2.14.0"
     testImplementation(
         group = "com.bakdata.fluent-kafka-streams-tests",
         name = "fluent-kafka-streams-tests-junit5",
         version = fluentKafkaVersion
     )
-    val kafkaVersion: String by project
-    testImplementation(group = "net.mguenther.kafka", name = "kafka-junit", version = kafkaVersion) {
+    val kafkaJunitVersion = "3.6.0"
+    testImplementation(group = "net.mguenther.kafka", name = "kafka-junit", version = kafkaJunitVersion) {
         exclude(group = "org.slf4j", module = "slf4j-log4j12")
     }
 }
 
 jib {
     from {
-        image = "eclipse-temurin:17.0.6_10-jre"
+        image = "eclipse-temurin:21.0.5_11-jre"
     }
 }

word-count/code/gradle.properties

@@ -1,5 +1,3 @@
 version=1.0.0-SNAPSHOT
 org.gradle.caching=true
 org.gradle.parallel=true
-junitVersion=5.9.2
-kafkaVersion=3.3.0

word-count/code/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
 networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

word-count/code/gradlew

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# SPDX-License-Identifier: Apache-2.0
+#
 
 ##############################################################################
 #
@@ -55,7 +57,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -83,10 +85,9 @@ done
 # This is normally unused
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
+' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -133,26 +134,29 @@ location of your Java installation."
     fi
 else
     JAVACMD=java
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
+    fi
 fi
 
 # Increase the maximum file descriptors if we can.
 if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
         # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045 
+        # shellcheck disable=SC2039,SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
         # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045 
+        # shellcheck disable=SC2039,SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -197,11 +201,15 @@ if "$cygwin" || "$msys" ; then
     done
 fi
 
-# Collect all arguments for the java command;
-#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
-#     shell script including quotes and variable substitutions, so put them in
-#     double quotes to make sure that they get re-expanded; and
-#   * put everything else in single quotes, so that it's not re-expanded.
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \