From f56ae2577d47c728906a87556a2e647ba4b72914 Mon Sep 17 00:00:00 2001 From: Aleksandar Osenov Date: Thu, 16 Nov 2023 17:05:02 +0200 Subject: [PATCH 1/4] Removing zookeeper. Added kafka drop --- rdbms-image/docker-compose.yml | 69 ++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/rdbms-image/docker-compose.yml b/rdbms-image/docker-compose.yml index 7c05955..9f6865c 100644 --- a/rdbms-image/docker-compose.yml +++ b/rdbms-image/docker-compose.yml @@ -1,30 +1,27 @@ version: "3.9" services: - zookeeper: - image: docker.io/bitnami/zookeeper:3.8 - ports: - - "2181:2181" - restart: unless-stopped - environment: - - ALLOW_ANONYMOUS_LOGIN=yes - profiles: - - infra - - all kafka: image: devicehive/devicehive-kafka:${DH_TAG:-3.5.0} ports: - "9092:9092" - links: - - "zookeeper" + container_name: kafka restart: unless-stopped environment: - - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT - - KAFKA_CFG_LISTENERS=CLIENT://:9092 - - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka:9092 - - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT - - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_ENABLE_KRAFT=yes - ALLOW_PLAINTEXT_LISTENER=yes + - KAFKA_CFG_NODE_ID=1001 + - KAFKA_KRAFT_CLUSTER_ID=OTMwNzFhYTY1ODNiNGE5OT + + - KAFKA_CFG_PROCESS_ROLES=broker,controller + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1001@kafka:29092 + + - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://localhost:9092,INTERNAL://kafka:19092 + - KAFKA_CFG_LISTENERS=CLIENT://:9092,CONTROLLER://kafka:29092,INTERNAL://:19092 + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL + healthcheck: test: /bin/nc -z localhost 9092 interval: 2s @@ -62,10 +59,11 @@ services: restart: unless-stopped environment: - FRANZ.KAFKA_HOSTS: ${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} - PROXY.WEB_SOCKET_SERVER_HOST: 'wsproxy' - PROXY.ENABLE_PLUGIN_MANAGER: 'false' PROXY.APP_LOG_LEVEL: 'debug' + ENVSEPARATOR : _ + FRANZ_KAFKA_HOSTS: ${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + PROXY_WEB_SOCKET_SERVER_HOST: 'wsproxy' + PROXY_ENABLE_PLUGIN_MANAGER: 'false' profiles: - infra - all @@ -96,14 +94,11 @@ services: links: - "postgres" - "kafka" - - "zookeeper" - "dh_frontend" - "dh_auth" - "wsproxy" restart: unless-stopped environment: - DH_ZK_ADDRESS: ${DH_ZK_ADDRESS:-zookeeper} - DH_ZK_PORT: ${DH_ZK_PORT:-2181} DH_KAFKA_BOOTSTRAP_SERVERS: ${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} DH_POSTGRES_ADDRESS: ${DH_POSTGRES_ADDRESS:-postgres} DH_POSTGRES_PORT: ${DH_POSTGRES_PORT:-5432} @@ -118,6 +113,7 @@ services: DH_AUTH_URL: http://dh_auth:8090/auth/rest REDIS_MASTER_HOST: ${REDIS_MASTER_HOST:-redis_master} REDIS_MASTER_PORT: ${REDIS_MASTER_PORT:-6379} + REDIS_MASTER_PASSWORD: ${REDIS_MASTER_PASSWORD} JWT_SECRET: ${JWT_SECRET} SPRING_PROFILES_ACTIVE: ${DH_PLUGIN_SPRING_PROFILES_ACTIVE:-ws-kafka-proxy} profiles: @@ -131,14 +127,11 @@ services: links: - "postgres" - "kafka" - - "zookeeper" - "dh_backend" - "dh_auth" - "wsproxy" restart: unless-stopped environment: - DH_ZK_ADDRESS: ${DH_ZK_ADDRESS:-zookeeper} - DH_ZK_PORT: ${DH_ZK_PORT:-2181} DH_KAFKA_BOOTSTRAP_SERVERS: ${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} DH_POSTGRES_ADDRESS: ${DH_POSTGRES_ADDRESS:-postgres} DH_POSTGRES_PORT: ${DH_POSTGRES_PORT:-5432} @@ -152,6 +145,9 @@ services: DH_AUTH_URL: ${DH_AUTH_URL:-http://dh_auth:8090/auth/rest} REDIS_MASTER_HOST: ${REDIS_MASTER_HOST:-redis_master} REDIS_MASTER_PORT: ${REDIS_MASTER_PORT:-6379} + REDIS_MASTER_PASSWORD: ${REDIS_MASTER_PASSWORD} + DH_BACKEND_ADDRESS: ${DH_BACKEND_ADDRESS:-dh_backend} + DH_BACKEND_PORT: ${DH_BACKEND_PORT:-8000} JWT_SECRET: ${JWT_SECRET} SPRING_PROFILES_ACTIVE: ${DH_FE_SPRING_PROFILES_ACTIVE:-ws-kafka-proxy-frontend} profiles: @@ -166,12 +162,9 @@ services: - "kafka" - "postgres" - "wsproxy" - - "zookeeper" - "dh_backend" restart: unless-stopped environment: - DH_ZK_ADDRESS: ${DH_ZK_ADDRESS:-zookeeper} - DH_ZK_PORT: ${DH_ZK_PORT:-2181} DH_KAFKA_BOOTSTRAP_SERVERS: ${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} DH_POSTGRES_ADDRESS: ${DH_POSTGRES_ADDRESS:-postgres} DH_POSTGRES_PORT: ${DH_POSTGRES_PORT:-5432} @@ -182,6 +175,7 @@ services: ROOT_LOG_LEVEL: ${ROOT_LOG_LEVEL:-WARN} REDIS_MASTER_HOST: ${REDIS_MASTER_HOST:-redis_master} REDIS_MASTER_PORT: ${REDIS_MASTER_PORT:-6379} + REDIS_MASTER_PASSWORD: ${REDIS_MASTER_PASSWORD} JWT_SECRET: ${JWT_SECRET} DH_WS_PROXY: wsproxy:3000 SPRING_PROFILES_ACTIVE: ${DH_AUTH_SPRING_PROFILES_ACTIVE:-ws-kafka-proxy} @@ -222,13 +216,10 @@ services: links: - "kafka" - "postgres" - - "zookeeper" - "wsproxy" - "redis_master" restart: unless-stopped environment: - DH_ZK_ADDRESS: ${DH_ZK_ADDRESS:-zookeeper} - DH_ZK_PORT: ${DH_ZK_PORT:-2181} DH_KAFKA_BOOTSTRAP_SERVERS: ${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} DH_POSTGRES_ADDRESS: ${DH_POSTGRES_ADDRESS:-postgres} DH_POSTGRES_PORT: ${DH_POSTGRES_PORT:-5432} @@ -241,6 +232,7 @@ services: DH_WS_PROXY: wsproxy:3000 REDIS_MASTER_HOST: ${REDIS_MASTER_HOST:-redis_master} REDIS_MASTER_PORT: ${REDIS_MASTER_PORT:-6379} + REDIS_MASTER_PASSWORD: ${REDIS_MASTER_PASSWORD} ROOT_LOG_LEVEL: ${ROOT_LOG_LEVEL:-INFO} SPRING_PROFILES_ACTIVE: ${DH_BE_SPRING_PROFILES_ACTIVE:-ws-kafka-proxy-backend} profiles: @@ -260,6 +252,19 @@ services: - dev - all + kafdrop: + image: 'obsidiandynamics/kafdrop:4.0.1' + restart: "no" + ports: + - "9000:9000" + + environment: + - KAFKA_BROKERCONNECT=${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + - JVM_OPTS=-Xms32M -Xmx64M + - SERVER_SERVLET_CONTEXTPATH=/ + + container_name: kafdrop + volumes: devicehive-db: devicehive-redis: From f42b57e08bb15da66d1a5fb8be7fe034b992ccf4 Mon Sep 17 00:00:00 2001 From: Aleksandar Osenov Date: Tue, 21 Nov 2023 13:08:03 +0200 Subject: [PATCH 2/4] Bring back zookeper --- rdbms-image/docker-compose.yml | 43 +++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/rdbms-image/docker-compose.yml b/rdbms-image/docker-compose.yml index 9f6865c..9aa6560 100644 --- a/rdbms-image/docker-compose.yml +++ b/rdbms-image/docker-compose.yml @@ -1,27 +1,30 @@ version: "3.9" services: + zookeeper: + image: docker.io/bitnami/zookeeper:3.8 + ports: + - "2181:2181" + restart: unless-stopped + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + profiles: + - infra + - all kafka: image: devicehive/devicehive-kafka:${DH_TAG:-3.5.0} ports: - "9092:9092" - container_name: kafka + links: + - "zookeeper" restart: unless-stopped environment: - - KAFKA_ENABLE_KRAFT=yes + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT + - KAFKA_CFG_LISTENERS=CLIENT://:9092 + - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka:9092 + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT + - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 - ALLOW_PLAINTEXT_LISTENER=yes - - KAFKA_CFG_NODE_ID=1001 - - KAFKA_KRAFT_CLUSTER_ID=OTMwNzFhYTY1ODNiNGE5OT - - - KAFKA_CFG_PROCESS_ROLES=broker,controller - - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER - - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT - - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1001@kafka:29092 - - - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://localhost:9092,INTERNAL://kafka:19092 - - KAFKA_CFG_LISTENERS=CLIENT://:9092,CONTROLLER://kafka:29092,INTERNAL://:19092 - - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL - healthcheck: test: /bin/nc -z localhost 9092 interval: 2s @@ -94,11 +97,14 @@ services: links: - "postgres" - "kafka" + - "zookeeper" - "dh_frontend" - "dh_auth" - "wsproxy" restart: unless-stopped environment: + DH_ZK_ADDRESS: ${DH_ZK_ADDRESS:-zookeeper} + DH_ZK_PORT: ${DH_ZK_PORT:-2181} DH_KAFKA_BOOTSTRAP_SERVERS: ${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} DH_POSTGRES_ADDRESS: ${DH_POSTGRES_ADDRESS:-postgres} DH_POSTGRES_PORT: ${DH_POSTGRES_PORT:-5432} @@ -127,11 +133,14 @@ services: links: - "postgres" - "kafka" + - "zookeeper" - "dh_backend" - "dh_auth" - "wsproxy" restart: unless-stopped environment: + DH_ZK_ADDRESS: ${DH_ZK_ADDRESS:-zookeeper} + DH_ZK_PORT: ${DH_ZK_PORT:-2181} DH_KAFKA_BOOTSTRAP_SERVERS: ${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} DH_POSTGRES_ADDRESS: ${DH_POSTGRES_ADDRESS:-postgres} DH_POSTGRES_PORT: ${DH_POSTGRES_PORT:-5432} @@ -162,9 +171,12 @@ services: - "kafka" - "postgres" - "wsproxy" + - "zookeeper" - "dh_backend" restart: unless-stopped environment: + DH_ZK_ADDRESS: ${DH_ZK_ADDRESS:-zookeeper} + DH_ZK_PORT: ${DH_ZK_PORT:-2181} DH_KAFKA_BOOTSTRAP_SERVERS: ${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} DH_POSTGRES_ADDRESS: ${DH_POSTGRES_ADDRESS:-postgres} DH_POSTGRES_PORT: ${DH_POSTGRES_PORT:-5432} @@ -216,10 +228,13 @@ services: links: - "kafka" - "postgres" + - "zookeeper" - "wsproxy" - "redis_master" restart: unless-stopped environment: + DH_ZK_ADDRESS: ${DH_ZK_ADDRESS:-zookeeper} + DH_ZK_PORT: ${DH_ZK_PORT:-2181} DH_KAFKA_BOOTSTRAP_SERVERS: ${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} DH_POSTGRES_ADDRESS: ${DH_POSTGRES_ADDRESS:-postgres} DH_POSTGRES_PORT: ${DH_POSTGRES_PORT:-5432} From 305af75bd294a0eaa2019156f7c9c8d84c9ebc7d Mon Sep 17 00:00:00 2001 From: Aleksandar Osenov Date: Wed, 29 Nov 2023 12:46:44 +0200 Subject: [PATCH 3/4] Small docker compose for local run kafka config change --- rdbms-image/docker-compose.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rdbms-image/docker-compose.yml b/rdbms-image/docker-compose.yml index 9aa6560..224a8fe 100644 --- a/rdbms-image/docker-compose.yml +++ b/rdbms-image/docker-compose.yml @@ -19,10 +19,11 @@ services: - "zookeeper" restart: unless-stopped environment: - - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT - - KAFKA_CFG_LISTENERS=CLIENT://:9092 - - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka:9092 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT + - KAFKA_CFG_LISTENERS=INTERNAL://:29092,CLIENT://:9092 + - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka:29092,CLIENT://kafka:9092 - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT + - KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 - ALLOW_PLAINTEXT_LISTENER=yes healthcheck: From 0777104903bc8741751b289efc69d8ff17c00336 Mon Sep 17 00:00:00 2001 From: Aleksandar Osenov Date: Mon, 11 Dec 2023 13:52:57 +0200 Subject: [PATCH 4/4] Added second kafka, second zookeeper --- rdbms-image/docker-compose.yml | 47 +++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/rdbms-image/docker-compose.yml b/rdbms-image/docker-compose.yml index 224a8fe..ce28028 100644 --- a/rdbms-image/docker-compose.yml +++ b/rdbms-image/docker-compose.yml @@ -10,6 +10,17 @@ services: profiles: - infra - all + zookeeper-2: + image: docker.io/bitnami/zookeeper:3.8 + ports: + - "2182:2182" + restart: unless-stopped + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + profiles: + - infra + - all + - local kafka: image: devicehive/devicehive-kafka:${DH_TAG:-3.5.0} @@ -17,14 +28,14 @@ services: - "9092:9092" links: - "zookeeper" + - "zookeeper-2" restart: unless-stopped environment: - - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT - - KAFKA_CFG_LISTENERS=INTERNAL://:29092,CLIENT://:9092 - - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka:29092,CLIENT://kafka:9092 - - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT - - KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL - - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181,zookeeper-2:2182 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT + - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka:9092 + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL + - KAFKA_CFG_LISTENERS=INTERNAL://:9092 - ALLOW_PLAINTEXT_LISTENER=yes healthcheck: test: /bin/nc -z localhost 9092 @@ -34,6 +45,29 @@ services: profiles: - infra - all + kafka-2: + image: devicehive-kafka:local + ports: + - "9093:9093" + links: + - "zookeeper" + - "zookeeper-2" + restart: unless-stopped + environment: + - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181,zookeeper-2:2182 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT + - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-2:9092 + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL + - KAFKA_CFG_LISTENERS=INTERNAL://:9092 + - ALLOW_PLAINTEXT_LISTENER=yes + healthcheck: + test: /bin/nc -z localhost 9093 + interval: 2s + timeout: 2s + retries: 15 + profiles: + - infra + - all postgres: image: postgres:14.4 @@ -56,6 +90,7 @@ services: - "3000:3000" links: - "kafka" + - "kafka-2" depends_on: kafka: