adobetp/docker-hadoop-client repository overview

$SPARK_HOME/bin/spark-submit --class org.apache.spark.examples.SparkPi \
    --master local[4] \
    --num-executors 1 \
    --driver-memory 512m \
    --executor-memory 512m \
    --executor-cores 1 \
    $SPARK_HOME/examples/jars/spark-examples*.jar 10

hadoop fs -copyFromLocal ${SPARK_HOME}/README.md /tmp/data

${SPARK_HOME}/bin/spark-shell --master yarn-client --driver-memory 512m --executor-memory 512m

val file = sc.textFile("/tmp/data")
val counts = file.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey(_ + _)
counts.saveAsTextFile("/tmp/wordcount")

scala> counts.count()

hadoop fs -ls /tmp/wordcount

hadoop fs -cat /tmp/wordcount/part-00000