docker-compose.spark.yaml

version: '3.4'
services:
  # portainer:
  #   container_name: portainer
  #   image: portainer/portainer
  #   restart: "always"
  #   volumes:
  #     - /var/run/docker.sock:/var/run/docker.sock
  #   ports:
  #     - 9000:9000
  spark-master:
    image: bde2020/spark-master:2.4.5-hadoop2.7
    #network_mode: host
    container_name: spark-master
    ports:
      - "7077:7077"
      - "4040:4040"
    env_file:
      - .env
    environment:
      - INIT_DAEMON_STEP=setup_spark

  spark-worker-1:
    image: bde2020/spark-worker:2.4.5-hadoop2.7
    container_name: spark-worker-1
    #network_mode: host
    environment:
      - SPARK_WORKER_WEBUI_PORT=8781
    ports:
      - "8781:8781"
    depends_on:
      - spark-master
    env_file:
      - .env
  spark-worker-2:
    image: bde2020/spark-worker:2.4.5-hadoop2.7
    container_name: spark-worker-2
    #network_mode: host
    environment:
      - SPARK_WORKER_WEBUI_PORT=8782
    ports:
      - "8782:8782"
    depends_on:
      - spark-master
    env_file:
      - .env
      
  jupyter:
    image: jupyter/pyspark-notebook
    container_name: jupyter
    depends_on:
      - spark-master
    env_file:
      - .env
    ports:
      - "8989:8888"
      - "4041:4040"
    volumes:
      - ./:/home/jovyan/