From e867510e817fbd88705175fa2dcace99fe207e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82abek?= Date: Tue, 31 Dec 2024 14:17:07 +0100 Subject: [PATCH 1/2] fix flink properties --- examples/dev/flink/flink-properties.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/dev/flink/flink-properties.yml b/examples/dev/flink/flink-properties.yml index 12f45da9518..c64ec68ca3e 100644 --- a/examples/dev/flink/flink-properties.yml +++ b/examples/dev/flink/flink-properties.yml @@ -1,6 +1,9 @@ taskmanager.numberOfTaskSlots: 8 +# Nu requires a little bit more metaspace than Flink default allocate based on process size +taskmanager.memory.process.size: 1500m +taskmanager.memory.jvm-metaspace.size: 400m -state.backend: filesystem +state.backend.type: filesystem state.checkpoints.dir: file:///opt/flink/data/checkpoints state.savepoints.dir: file:///opt/flink/data/savepoints @@ -21,8 +24,8 @@ metrics.reporter.influxdb_reporter.port: 8087 metrics.reporter.influxdb_reporter.db: nussknacker_metrics metrics.reporter.influxdb_reporter.scope.variables.excludes: tm_id;job_id;task_id;task_attempt_id;operator_id;task_attempt_num;task_name metrics.scope.jm: local..jobmanagerGlobal -metrics.scope.jm.job: local..jobmanagerJob. +metrics.scope.jm-job: local..jobmanagerJob. metrics.scope.tm: local..taskmanagerGlobal. -metrics.scope.tm.job: local..taskmanagerJob.. +metrics.scope.tm-job: local..taskmanagerJob.. metrics.scope.task: local..taskmanagerTask.... metrics.scope.operator: local..taskmanagerTask.... From af4b6915c095f92e9cdbf9bfa48a257eeb56e749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82abek?= Date: Tue, 31 Dec 2024 14:56:20 +0100 Subject: [PATCH 2/2] add separate kafka listener for external connections --- examples/dev/local-testing.docker-compose.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/dev/local-testing.docker-compose.yml b/examples/dev/local-testing.docker-compose.yml index 3e4365a0778..7b9782044fe 100644 --- a/examples/dev/local-testing.docker-compose.yml +++ b/examples/dev/local-testing.docker-compose.yml @@ -94,12 +94,13 @@ services: restart: unless-stopped hostname: nu-kafka ports: - - 3032:9092 + - 3032:3032 environment: KAFKA_CFG_NODE_ID: 0 KAFKA_CFG_PROCESS_ROLES: "controller,broker" - KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093" - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:3032" + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT" + KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092,EXTERNAL://localhost:3032" KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "0@kafka:9093" KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER" healthcheck: