docker-compose.yaml

version: '5'

services:
  spark-master: 
    build: spark
    container_name: spark-master
    hostname: spark-master
    ports:
      - "8080:8080"
    environment:
      - SPARK_MASTER_HOST=spark-master
    volumes:
      - spark-binaries:/opt/bitnami/spark

  spark-worker1:
    build: spark
    container_name: spark-worker1
    hostname: spark-worker1
    ports:
      - 8082:8082
    links:
      - spark-master
    depends_on:
      - spark-master
    environment:
      SPARK_MODE: worker
      SPARK_WORKER_WEBUI_PORT: 8082
      SPARK_WORKER_PORT: 8083

  zeppelin:
    build: 
      context: zeppelin
      dockerfile: Dockerfile        # Use this line if want to build from scratch
      # dockerfile: Dockerfile_hub  # Use this line if you want to use the pre-built image from Docker Hub
    container_name: zeppelin
    hostname: zeppelin.local # You can change it into "zeppelin.local" and add into /etc/hosts
    ports:
      - "9999:8080"
    links:
      - spark-master
    depends_on:
      - spark-master
    environment:
      SPARK_HOME: /opt/bitnami/spark
      ZEPPELIN_LOG_DIR: /opt/zeppelin/logs
      ZEPPELIN_NOTEBOOK_DIR: /opt/zeppelin/notebook
    volumes:
      - spark-binaries:/opt/bitnami/spark
      - ./notebook:/opt/zeppelin/notebook       # notebook folder in main project : path in docker
      - "./data:/data" # put any data like CSV, Parquet, etc. in this folder and access it via /data/... in Zeppelin
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9999"]
      interval: 10s
      timeout: 5s
      retries: 3
    deploy:
      resources:
        reservations:
          devices:
          - driver: nvidia
            device_ids: ['0']
            capabilities: [gpu]
volumes:
  spark-binaries: