docker-compose.yaml

version: '3'

# This docker-compose is for developer convenience, not for running in production.

services:

  namenode:
    image: apache/hadoop:3.3.6
    hostname: namenode
    command: ["hdfs", "namenode"]
    ports:
      - 9870:9870
    environment:
      - ENSURE_NAMENODE_DIR=/tmp/hadoop-root/dfs/name
      - CORE-SITE.XML_fs.default.name=hdfs://namenode
      - CORE-SITE.XML_fs.defaultFS=hdfs://namenode
      - HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020
      - HDFS-SITE.XML_dfs.permissions.enabled=false
      - HDFS-SITE.XML_dfs.replication=1

  datanode:
    image: apache/hadoop:3.3.6
    command: ["hdfs", "datanode"]
    environment:
      - CORE-SITE.XML_fs.default.name=hdfs://namenode
      - CORE-SITE.XML_fs.defaultFS=hdfs://namenode
      - HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020
      - HDFS-SITE.XML_dfs.permissions.enabled=false
      - HDFS-SITE.XML_dfs.replication=1

  yarn-resourcemanager:
    build:
      context: .
      dockerfile: Dockerfile
    container_name: yarn-resourcemanager
    ports:
      - 8088:8088  # web ui
    environment:
      - YARN_MODE=resourcemanager
      # S3 config
      - MINIO_URL=http://minio:9002
      - MINIO_ACCESS_KEY=minio
      - MINIO_SECRET_KEY=minio123
      # HDFS config
      - CORE-SITE.XML_fs.default.name=hdfs://namenode
      - CORE-SITE.XML_fs.defaultFS=hdfs://namenode
      - HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020
      - HDFS-SITE.XML_dfs.permissions.enabled=false
      - HDFS-SITE.XML_dfs.replication=1

  yarn-nodemanager:
    build:
      context: .
      dockerfile: Dockerfile
    container_name: yarn-nodemanager
    ports:
      - 8042:8042  # web ui
    environment:
      - YARN_MODE=nodemanager
      - YARN_RESOURCEMANAGER_HOSTNAME=yarn-resourcemanager
      # S3 config
      - MINIO_URL=http://minio:9002
      - MINIO_ACCESS_KEY=minio
      - MINIO_SECRET_KEY=minio123
      # HDFS config
      - CORE-SITE.XML_fs.default.name=hdfs://namenode
      - CORE-SITE.XML_fs.defaultFS=hdfs://namenode
      - HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020
      - HDFS-SITE.XML_dfs.permissions.enabled=false
      - HDFS-SITE.XML_dfs.replication=1

  minio:
    image: minio/minio
    container_name: spark-minio
    ports:
      - "9002:9002"
      # MinIO Console is available at http://localhost:9003
      - "9003:9003"
    environment:
      MINIO_ROOT_USER: minio
      MINIO_ROOT_PASSWORD: minio123
    healthcheck:
      # reference: https://github.com/rodrigobdz/docker-compose-healthchecks?tab=readme-ov-file#minio-release2023-11-01t18-37-25z-and-older
      test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9002' || exit 1
      interval: 1s
      timeout: 10s
      retries: 5
    # Note there is no bucket by default
    command: server --address 0.0.0.0:9002 --console-address 0.0.0.0:9003 /data

  minio-create-bucket:
    image: minio/mc
    depends_on:
      minio:
        condition: service_healthy
    entrypoint: >
      bash -c "
      mc alias set minio http://minio:9002 minio minio123 &&
      if ! mc ls minio/yarn 2>/dev/null; then
        mc mb minio/yarn && echo 'Bucket yarn created'
      else
        echo 'bucket yarn already exists'
      fi
      "

  spark-container:
    image: bitnami/spark:3.5.1
    container_name: spark-container
    depends_on: 
      - yarn-nodemanager
    environment:
      - YARN_RESOURCE_MANAGER_URL=http://yarn-resourcemanager:8032
      - MINIO_URL=http://minio:9002
      - MINIO_ACCESS_KEY=minio
      - MINIO_SECRET_KEY=minio123
      - YARN_CONF_DIR=/opt/yarn/conf
    volumes:
      - ./conf/hadoop:/opt/yarn/conf