docker-compose.yaml

version: "3.4"
x-airflow-common: &airflow-common
  build: ./airflow
  environment: &airflow-common-env
    AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth"
    AIRFLOW__CORE__EXECUTOR: CeleryExecutor
    AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
    AIRFLOW__CORE__FERNET_KEY: ""
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "true"
    AIRFLOW__CORE__LOAD_EXAMPLES: "true"
    AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow
    AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
    MLFLOW_S3_ENDPOINT_URL: http://minio:9000
    AWS_ACCESS_KEY_ID: minioadmin
    AWS_SECRET_ACCESS_KEY: minioadmin
  volumes:
    - ./airflow/dags:/opt/airflow/dags
    - ./airflow/logs:/opt/airflow/logs
    - ./airflow/plugins:/opt/airflow/plugins
  user: "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-50000}"
  depends_on:
    - redis
    - postgres

services:
  nginx:
    container_name: nginx
    build: ./nginx
    depends_on:
      - python-model-service1
      - python-model-service2
    ports:
      - "80:80"
      - "443:443"
    # volumes:
    #     - ./nginx.conf:/etc/nginx/nginx.conf
    restart: always

  python-model-service1:
    container_name: python-service-1
    depends_on:
      - kafka
    build:
      context: ./python-model-service-1
      dockerfile: Dockerfile
    ports:
      - "8000:8000"
    volumes:
      - ./python-model-service-1:/python-model-service-1
    restart: always

  python-model-service2:
    container_name: python-service-2
    depends_on:
      - kafka
    build:
      context: ./python-model-service-2
      dockerfile: Dockerfile
    ports:
      - "8001:8001"
    volumes:
      - ./python-model-service-2:/python-model-service-2
    restart: always

  bridge-server:
    container_name: bridge_server
    build: ./bridge-server
    ports:
      - "8002:8002"
    volumes:
      - ./bridge-server/app.py:/app/app.py
      - ./bridge-server/dag_mapping.py:/app/dag_mapping.py
    restart: always

  postgres:
    container_name: postgres
    build: ./postgres
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    volumes:
      - postgres-db-volume:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "airflow"]
      interval: 5s
      retries: 5
    restart: always

  prometheus:
    container_name: prometheus
    build: ./prometheus
    depends_on:
      - python-model-service1
      - python-model-service2
      - kafka
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
    restart: always

  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      KAFKA_JMX_PORT: 39999

  kafka:
    image: confluentinc/cp-kafka:latest
    container_name: kafka
    restart: always
    depends_on:
      - zookeeper
    ports:
      - "9092:9092"
      - "7071:7071"
      - "49999:49999"
    # https://rmoff.net/2018/08/02/kafka-listeners-explained/
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_JMX_PORT: 49999
      KAFKA_OPTS: "-javaagent:/usr/local/bin/jmx_prometheus_javaagent-0.9.jar=7071:/etc/jmx-exporter/kafka-2_0_0.yml"
    volumes:
      - ./kafka/jmx-exporter/kafka-2_0_0.yml:/etc/jmx-exporter/kafka-2_0_0.yml
      - ./kafka/jmx-exporter/jmx_prometheus_javaagent-0.9.jar:/usr/local/bin/jmx_prometheus_javaagent-0.9.jar

  kafka-workers:
    container_name: kafka-workers
    depends_on:
      - kafka
    build: ./workers
    restart: always

  # kafka-manager:
  #     image: kafkamanager/kafka-manager
  #     restart: always
  #     depends_on:
  #         - zookeeper
  #         - kafka
  #     ports:
  #         - "9000:9000"
  #     environment:
  #         ZK_HOSTS: "zookeeper:2181"
  #     volumes:
  #         - ./nginx.conf:/etc/nginx/nginx.conf

  mlflow:
    container_name: mlflow-server
    depends_on:
      - minio
    build: ./mlflow
    environment:
      MLFLOW_S3_ENDPOINT_URL: http://minio:9000
      AWS_ACCESS_KEY_ID: minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
    ports:
      - "5000:5000"
    command: mlflow server --backend-store-uri sqlite:///mlflow.db --default-artifact-root s3://mlflow --host 0.0.0.0 -p 5000
    restart: always

  minio:
    container_name: minio
    image: minio/minio
    ports:
      - "9000:9000"
      - "9001:9001"
    environment:
      AWS_ACCESS_KEY_ID: minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
    volumes:
      - data:/data
    command: server /data --console-address ":9001"
    restart: always

  influxdb:
    container_name: influxdb
    build: ./influxdb
    ports:
      - 8086:8086
      - 8083:8083
      - 2003:2003
    env_file:
      - ./influxdb/influxdb.env
    restart: always

  chronograf:
    container_name: chronograf
    image: chronograf:1.8.4-alpine
    depends_on:
      - influxdb
    user: "0"
    ports:
      - 8888:8888
    env_file:
      - ./chronograf/chronograf.env
    restart: always

  grafana:
    container_name: grafana
    image: grafana/grafana:6.3.6
    depends_on:
      - influxdb
      - prometheus
    user: "0"
    ports:
      - 3000:3000
    env_file:
      - ./grafana/grafana.env
    volumes:
      - ./grafana/provisioning/datasources/:/etc/grafana/provisioning/datasources/
      - ./grafana/provisioning/dashboards/:/etc/grafana/provisioning/dashboards/
      - ./grafana/provisioning/notifiers/:/etc/grafana/provisioning/notifiers/
    restart: always

  redis:
    container_name: redis
    image: redis:latest
    ports:
      - 6379:6379
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 30s
      retries: 50
    restart: always

  locust:
    container_name: locust
    build: ./locust
    ports:
      - "8089:8089"
    restart: always

  airflow-webserver:
    <<: *airflow-common
    command: webserver
    ports:
      - 8080:8080
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 10s
      timeout: 10s
      retries: 5
    restart: always

  airflow-scheduler:
    <<: *airflow-common
    depends_on:
      - airflow-webserver
    command: scheduler
    restart: always

  airflow-worker:
    <<: *airflow-common
    depends_on:
      - airflow-scheduler
    command: celery worker
    restart: always

  airflow-init:
    <<: *airflow-common
    environment:
      <<: *airflow-common-env
      _AIRFLOW_DB_UPGRADE: "true"
      _AIRFLOW_WWW_USER_CREATE: "true"
      _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
      _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
    command: version

  flower:
    <<: *airflow-common
    ports:
      - 5555:5555
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
      interval: 10s
      timeout: 10s
      retries: 5
    command: celery flower
    restart: always

volumes:
  postgres-db-volume:
  data: