docker-compose.yml

version: "3.8"
services:
  spark-master:
    image: spark-master:2.4.7
    ports:
      - "9090:8080"
      - "7077:7077"
    volumes:
      - ./apps:/opt/apps
      - ./data:/opt/data
    environment:
      - "SPARK_LOCAL_IP=spark-master"
    env_file:
      - ./env/master.env
    deploy:
      mode: global
  spark-worker:
    image: spark-worker:2.4.7
    depends_on:
      - spark-master
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - SPARK_WORKER_CORES=1
      - SPARK_WORKER_MEMORY=1G
      - SPARK_DRIVER_MEMORY=128m
      - SPARK_EXECUTOR_MEMORY=256m
    env_file:
      - ./env/worker.env
    volumes:
      - ./apps:/opt/apps
      - ./data:/opt/data
    deploy:
      mode: replicated
      replicas: 6
  spark-submit:
    image: spark-submit:2.4.7
    depends_on:
      - spark-master
      - spark-worker
    env_file:
      - ./env/submit.env
    volumes:
      - ./apps:/opt/apps
      - ./data:/opt/data
    deploy:
      mode: replicated
      replicas: 0