docker-compose.yaml

version: '3.7'
services: 
    cassandra:
        build: 
            context: cassandra
        hostname: cassandra
        image: dotingestion2:cassandra
        container_name: cassandra
        environment: 
            - CASSANDRA_BROADCAST_ADDRESS=cassandra
            - CASSANDRA_KEYSPACE=dota_ks
            - CASSANDRA_MATCH_TABLE=matches
        ports: 
            - "9042:9042"

    zookeeper:
        hostname: zookeeper
        image: confluentinc/cp-zookeeper:6.1.1
        container_name: zookeeper
        ports:
            - "2181:2181"
        environment:
            ZOOKEEPER_CLIENT_PORT: 2181
            ZOOKEEPER_SERVER_ID: "1"

    kafkaserver:
        hostname: kafkaserver
        image: confluentinc/cp-kafka:6.1.1
        container_name: kafkaserver
        depends_on:
          - zookeeper
        ports:
          - "9092:9092"
        environment:
          KAFKA_BROKER_ID: 0
          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafkaserver:9092
          KAFKA_MESSAGE_MAX_BYTES: 1677722
          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
          KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
          KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
          KAFKA_DEFAULT_REPLICATION_FACTOR: 1
          KAFKA_CONFLUENT_METADATA_TOPIC_REPLICATION_FACTOR: 1

    connect-cassandra:
        build: 
            context: connect-cassandra
        image: dotingestion2:kafka-connect-cassandra
        container_name: connect-cassandra
        depends_on:
          - kafkaserver
          - cassandra

    connect-elasticsearch:
        build: 
            context: connect-elastic
        image: dotingestion2:kafka-connect-elasticsearch
        container_name: connect-elasticsearch
        depends_on:
          - kafkaserver
          - elasticsearch

    kafkastream:
        build: 
            context: stream
        image: dotingestion2:kafkastream
        container_name: kafkastream
        depends_on: 
            - kafkaserver
            - kafkaproducer

    kafkaproducer:
        build: 
            context: ingestion
        image: dotingestion2:ingestion
        container_name: kafkaproducer
        environment:
            API_ENDPOINT: http://api.steampowered.com/IDOTA2Match_570/GetMatchHistoryBySequenceNum/V001/?key={}&start_at_match_seq_num={}
            TOPIC: dota_raw
            MATCH_SEQ_NUM: steam
            HISTORY_ENDPOINT: https://api.steampowered.com/IDOTA2Match_570/GetMatchHistory/V001/?key={}&matches_requested=1
            INTERVAL: "40"
        depends_on: 
            - kafkaserver

    webui:
        image: provectuslabs/kafka-ui:latest
        container_name: kafkaWebUI
        environment:
            KAFKA_CLUSTERS_0_NAME: my_cluster
            KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181
            KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafkaServer:9092
        ports: 
            - 8080:8080
        depends_on: 
            - kafkaserver

    spark:
        build: 
            context: spark
        image: dotingestion2:spark
        container_name: spark
        depends_on: 
            - kafkaserver
            - elasticsearch
            - connect-elasticsearch

    elasticsearch:
        hostname: elasticsearch
        image: docker.elastic.co/elasticsearch/elasticsearch:7.12.1
        container_name: elasticsearch
        ports:
            - "9200:9200"
        environment:
            - node.name=elasticsearch
            - cluster.name=elasticsearch-docker-cluster
            - discovery.seed_hosts=elasticsearch
            - cluster.initial_master_nodes=elasticsearch
            - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
        depends_on: 
            - kafkaserver

    kibana:
        image: docker.elastic.co/kibana/kibana:7.12.1
        container_name: kibana
        ports:
            - "5601:5601"
        depends_on: 
            - kafkaserver

    api:
        build: 
            context: api
        image: dotingestion2:api
        container_name: api
        ports:
            - "3000:3000"
        environment: 
            PORT: "3000"
        depends_on: 
            - kafkaserver