compose.yaml

version: '3'
networks:
    goodonions:
        driver: bridge
        ipam:
            driver: default
            config:
                - subnet: "10.0.0.0/24"

services:  


  spark_master:
    image: 'bitnami/spark:latest'
    container_name: spark_master
    hostname: spark_master
    user: root
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no    
      - SPARK_LOCAL_HOSTNAME=10.0.0.2 
    ports:
      - '7077:7077'
      - '4040:4040'
      - '8080:8080'
    volumes:
      - './spark/master/spark_data:/bitnami'
    env_file:
      - .env
    networks:
      goodonions:
        ipv4_address: 10.0.0.2

  spark-worker:
    image: 'bitnami/spark:latest'
    container_name: spark_worker
    hostname: spark_worker
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=10.0.0.2:7077
      - SPARK_WORKER_MEMORY=2G
      - SPARK_WORKER_CORES=1
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
      - SPARK_LOCAL_HOSTNAME=10.0.0.3
      - SPARK_WORKER_OPTS=-Dspark.worker.cleanup.interval=60 -Dspark.worker.cleanup.appDataTtl=120 -Dspark.worker.cleanup.enabled=true

    ports:
      - '8081:8081'
    depends_on:
      - spark_master
    env_file:
      - .env
    networks:
      goodonions:
        ipv4_address: 10.0.0.3

  zookeeper:
    image: 'bitnami/zookeeper:latest'
    container_name: zookeeper
    hostname: zookeeper
    ports:
      - '2181:2181'
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes

    networks:
      goodonions:
        ipv4_address: 10.0.0.4

  kafka:
    image: 'bitnami/kafka:latest'
    container_name: kafka
    restart: always
    hostname: kafka
    ports:
      - '9093:9093'
    environment: 
    - KAFKA_BROKER_ID=1
    - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
    - KAFKA_LISTENERS=INTERNAL://:9094,EXTERNAL://:9093
    - KAFKA_ADVERTISED_LISTENERS=INTERNAL://kafka:9094,EXTERNAL://localhost:9093  #
    - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT        #INTERNAL PLAIN COSI ZOOKEEPER NON CAGA IL CAZZO 
    - KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
    - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
    - KAFKA_CFG_CONFLUENT_SUPPORT_METRICS_ENABLE=false
    - KAFKA_CFG_DELETE_TOPIC_ENABLE=true
    - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
    depends_on:
      - zookeeper
    #volumes:
      #- ./kafka/data:/bitnami/kafka
    networks:
      goodonions:
        ipv4_address: 10.0.0.5

  cassandra:
      image: 'bitnami/cassandra:latest'
      container_name: cassandra
      hostname: cassandra
      ports:
        - '9042:9042'      
      volumes:
        - ./cassandra/cass1:/bitnami/cassandra
        - ./cassandra/schema:/schema
      networks:
        goodonions:
          ipv4_address: 10.0.0.6

  redis:
    image: 'redis'
    container_name: redis
    hostname: redis
    ports:
      - '6379:6379'
    # deploy:
    #   resources:
    #     limits:
    #       memory: 1G
    networks:
        goodonions:
          ipv4_address: 10.0.0.7