chming1016/spark repository overview

docker pull chming1016/spark

docker network create spark

#!/bin/bash
N=${1:-4}
echo "start spark-master..."
docker run -itd \
                --net spark \
                -p 8080:8080 \
                --name spark-master \
                --hostname spark-master \
                chming1016/spark /bin/bash &> /dev/null

i=1
while [ $i -lt $N ]
do
        echo "start spark-worker$i..."
        docker run -itd \
                        --net spark \
                        --name spark-worker$i \
                        --hostname spark-worker$i \
                        chming1016/spark /bin/bash &> /dev/null
        i=$(( $i + 1 ))
done

sleep 1
echo "start ssh service..."
docker exec -it spark-master sh -c 'service ssh start' > /dev/null
docker exec -it spark-worker1 sh -c 'service ssh start' > /dev/null
docker exec -it spark-worker2 sh -c 'service ssh start' > /dev/null
docker exec -it spark-worker3 sh -c 'service ssh start' > /dev/null

echo "start Spark Cluster..."
docker exec -it spark-master sh -c '/usr/local/spark/sbin/start-all.sh' > /dev/null

docker exec -it spark-master bash

pyspark --master spark://spark-master:7077 --executor-memory 1g --executor-cores 1