logitravel/spark-docker repository overview

docker build -t logitravel/spark-docker .

docker network create spark

docker run -e MASTER_HOSTNAME="master" \
           -e MASTER="spark://master:7077" \
           -e SPARK_PUBLIC_DNS="localhost" \
           --network=spark \
           -h master \
           --name master \
           -p "8080:8080" \
           -ti logitravel/spark-docker start-master

docker run -e MASTER="spark://master:7077" \
           -e SPARK_CONF_DIR="/conf" \
           -e SPARK_WORKER_CORES="2" \
           -e SPARK_WORKER_MEMORY="512m" \
           -e SPARK_WORKER_PORT="8881" \
           -e SPARK_WORKER_WEBUI_PORT="8081" \
           -e SPARK_PUBLIC_DNS="localhost" \
           --network=spark \
           --link master:master \
           -ti logitravel/spark-docker start-worker

docker run -e MASTER="spark://master:7077" \
           -e MAIN_CLASS="HelloWorld" \
           -e DRIVER_MEMORY="1g" \
           -e EXECUTOR_MEMORY="512m" \
           -e EXECUTOR_CORES="2" \
           -e TOTAL_EXECUTOR_CORES="2" \
           -e DEPENDENCIES="org.apache.spark:spark-streaming_2.11:2.1.0" \
           -e JOB_NAME="app" \
           -e JAR="https://github.com/logitravel/spark-hello-world/releases/download/1.0/spark-hello-world_2.11-1.0.jar" \
           --network=spark \
           --link master:master \
           -p "4040:4040" \
           -ti logitravel/spark-docker start-driver

docker swarm init

docker network create -d overlay spark --attachable

docker stack deploy -c docker-compose.yml spark

docker service scale spark_worker=100