bumory1987/sparks repository overview

⁠step 1 -> 2 -> 3 -> 4 -> 5

#1 docker network
docker network create --driver bridge spark

#2 storage
docker run -it --privileged --network spark -d --name storage bumory1987/storage:spstorage

#3 spark_worker
docker run -d --network spark --cpus="2" --memory="4g" --name master --privileged -p 8083:8083 -p 7077:7077 -e "SPARK_MASTER_HOST=master" -e "STORAGE=storage" bumory1987/sparks:masterv2

#4 spark_worker
docker run -d --cpus="1" --memory="3g" --network spark --name worker_0 --privileged -p 8090:8090 -e "SPARK_MASTER_HOST=master" -e "SPARK_MASTER_PORT=7077" -e "SPARK_MASTER_WEBUI_PORT=8083" -e "SPARK_WORKER_WEBUI_PORT=8090" -e "STORAGE=storage" bumory1987/sparks:workerv2

docker run -d --cpus="1" --memory="3g" --network spark --name worker_1 --privileged -p 8091:8090 -e "SPARK_MASTER_HOST=master" -e "SPARK_MASTER_PORT=7077" -e "SPARK_MASTER_WEBUI_PORT=8083" -e "SPARK_WORKER_WEBUI_PORT=8090" -e "STORAGE=storage" bumory1987/sparks:workerv2

docker run -d --cpus="1" --memory="3g" --network spark --name worker_2 --privileged -p 8092:8090 -e "SPARK_MASTER_HOST=master" -e "SPARK_MASTER_PORT=7077" -e "SPARK_MASTER_WEBUI_PORT=8083" -e "SPARK_WORKER_WEBUI_PORT=8090" -e "STORAGE=storage" bumory1987/sparks:workerv2

#5 ui-> password:1234 (after deploying wait 2 min until jupyterlab setting finish)
docker run -d --network spark --privileged --name controller -p 8889:8889 -e "STORAGE=storage" bumory1987/sparks:ui

#6 ip address
spark ui -> localhost:8889
spark master -> localhost:8083