ci: remove unnessary kafka & zookeeper

ci/docker-compose.yml

@@ -238,44 +238,20 @@ services:
 # # protobuf/avro schema registry. Should be removed after the support.
 # # Related tracking issue:
 # # https://github.com/redpanda-data/redpanda/issues/1878
-  zookeeper:
-    container_name: zookeeper
-    image: confluentinc/cp-zookeeper:latest
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
-    ports:
-      - "2181:2181"
-
   schemaregistry:
     container_name: schemaregistry
     hostname: schemaregistry
     image: confluentinc/cp-schema-registry:latest
     depends_on:
-      - kafka
+      - message_queue
     ports:
       - "8082:8082"
     environment:
       SCHEMA_REGISTRY_HOST_NAME: schemaregistry
-      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
       SCHEMA_REGISTRY_LISTENERS: http://schemaregistry:8082
-      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9093,PLAINTEXT_INTERNAL://localhost:29093
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: message_queue:29092
       SCHEMA_REGISTRY_DEBUG: 'true'
 
-  kafka:
-    container_name: kafka
-    image: confluentinc/cp-kafka:latest
-    ports:
-      - "29093:29093"
-    depends_on:
-      - zookeeper
-    environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,PLAINTEXT_INTERNAL://localhost:29093
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-
   pulsar-server:
     container_name: pulsar-server
     image: apachepulsar/pulsar:latest

ci/workflows/pull-request.yml

@@ -178,6 +178,7 @@ steps:
           run: source-test-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
+          upload-container-logs: always
       - ./ci/plugins/upload-failure-logs
     timeout_in_minutes: 18
     retry: *auto-retry

e2e_test/source/basic/nosim_kafka.slt

@@ -54,7 +54,7 @@ statement ok
 CREATE TABLE kafka_json_schema_plain with (
     connector = 'kafka',
     kafka.topic = 'kafka_json_schema',
-    kafka.brokers = 'kafka:9093',
+    kafka.brokers = 'message_queue:29092',
     kafka.scan.startup.mode = 'earliest'
 ) FORMAT PLAIN ENCODE JSON (schema.registry = 'http://schemaregistry:8082');
 
@@ -64,7 +64,7 @@ INCLUDE KEY AS rw_key
 with (
     connector = 'kafka',
     kafka.topic = 'kafka_upsert_json_schema',
-    kafka.brokers = 'kafka:9093',
+    kafka.brokers = 'message_queue:29092',
     kafka.scan.startup.mode = 'earliest'
 ) FORMAT UPSERT ENCODE JSON (schema.registry = 'http://schemaregistry:8082');
 

scripts/source/prepare_ci_kafka.sh

@@ -58,7 +58,7 @@ for filename in $kafka_data_files; do
     elif [[ "$topic" = *avro_json ]]; then
         python3 source/schema_registry_producer.py "message_queue:29092" "http://message_queue:8081" "$filename" "topic" "avro"
     elif [[ "$topic" = *json_schema ]]; then
-        python3 source/schema_registry_producer.py "kafka:9093" "http://schemaregistry:8082" "$filename" "topic" "json"
+        python3 source/schema_registry_producer.py "message_queue:29092" "http://schemaregistry:8082" "$filename" "topic" "json"
     else
         cat "$filename" | kcat -P -K ^  -b message_queue:29092 -t "$topic"
     fi