docker-compose.yml

---
version: '3'
services:

  # For better performance (and fewer headaches) comment out any services you aren't using.
  # For services that do run, also consider commenting out
  # - persistent storage
  # - external port mapping
  # as these are frequent sources of contention and random errors.


  ###
  # Base Kafka services
  ###

  # Apache zookeeper does what now?
  kafka-zookeeper:
    image: confluentinc/cp-zookeeper:6.1.1
    hostname: kafka-zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  # The kafka broker
  kafka-broker:
    image: confluentinc/cp-server:6.1.1
    hostname: kafka-broker
    depends_on:
      - kafka-zookeeper
    ports:
      - "19092:19092"
      - "9092:9092"
      - "9101:9101"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'kafka-zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,LOCAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker:29092,PLAINTEXT_HOST://kafka-broker:9092, LOCAL://localhost:19092
      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_JMX_PORT: 9101
      KAFKA_JMX_HOSTNAME: localhost
      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka-broker:29092
      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      CONFLUENT_METRICS_ENABLE: 'true'
      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'

  # Registry for kafka message schemas
  kafka-schema-registry:
    image: confluentinc/cp-schema-registry:6.1.1
    hostname: kafka-schema-registry
    depends_on:
      - kafka-broker
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka-broker:29092'
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081


 

  ###
  # Kafka UIs (landoop)
  ###

  # RESTful proxy for kafka
  kafka-rest-proxy:
    image: confluentinc/cp-kafka-rest:6.1.1
    hostname: kafka-rest-proxy
    depends_on:
      - kafka-broker
      - kafka-schema-registry
    ports:
      - 8082:8082
    environment:
      KAFKA_REST_HOST_NAME: kafka-rest-proxy
      KAFKA_REST_BOOTSTRAP_SERVERS: 'kafka-broker:29092'
      KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
      KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://kafka-schema-registry:8081'

  # viewable at localhost:8093
  landoop-topics-ui:
    image: landoop/kafka-topics-ui
    ports:
      - 8093:8000
    depends_on:
      - kafka-rest-proxy
    environment:
      KAFKA_REST_PROXY_URL: http://kafka-rest-proxy:8082
      PROXY: 'true'
      
  # viewable at localhost:8094
  landoop-schema-ui:
    image: landoop/schema-registry-ui
    ports:
      - 8094:8000
    depends_on:
      - kafka-rest-proxy
      - kafka-schema-registry
    environment:
      SCHEMAREGISTRY_URL: "http://kafka-schema-registry:8081"
      PROXY: 'true'

  ###
  # Example producers
  ###

  # Example producer for strainmeter bottle files.  contains 1 Day, Hour, and Min file, which it reads, parses, and produces to the kafka topic gtsm_etl
  strain-producer:
    image: unavdocker/strain_bottle_producer_prototype:latest
    hostname: strain-producer
    depends_on:
      - kafka-broker
      - kafka-schema-registry
    environment:
      BOOTSTRAP_SERVERS: "kafka-broker:9092"
      SCHEMA_REGISTRY_URL: "http://kafka-schema-registry:8081"
  

  ###
  # Apache Flink Web stack
  ###

  # a flink container that is used to open the flink sql prompt
  flink-sql-client:
    image: flink:1.14.2-scala_2.12-java8
    hostname: flink-sql-client
    #   to make a shell available.
    #   we are not starting any service
    command: /bin/bash
    stdin_open: true # docker run -i
    tty: true        # docker run -t
    depends_on:
      - kafka-broker
      - flink-jobmanager
    environment:
      FLINK_JOBMANAGER_HOST: flink-jobmanager
      FLINK_JOBMANAGER_REST_PORT: 8083
      ZOOKEEPER_CONNECT: kafka-zookeeper
      KAFKA_BOOTSTRAP: kafka-broker
      ES_HOST: elasticsearch
    volumes:
      - ./flink/update_flink_install.sh:/opt/flink/update_flink_install.sh

  flink-jobmanager:
    image: flink:1.14.2-scala_2.12-java8
    hostname: flink-jobmanager
    ports:
      - "8083:8083"
    command: jobmanager
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: flink-jobmanager
        # added to bind to a different port
        jobmanager.web.port: 8083

  flink-taskmanager:
    image: flink:1.14.2-scala_2.12-java8
    hostname: flink-taskmanager
    depends_on:
      - flink-jobmanager
    command: taskmanager
    volumes:
      - ${PWD}/flink/flink-taskmanager-log4j-console.properties:/opt/flink/conf/log4j-console.properties
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: flink-jobmanager
        taskmanager.numberOfTaskSlots: 10