datalinked/spark repository overview

cd $SPARK_HOME \
docker build -t spark:latest -f kubernetes/dockerfiles/spark/Dockerfile .

 FROM datalinked/spark:2.4.3

 ENV SPARK_HOME /opt/spark
 ENV PATH $PATH:$SPARK_HOME/bin

 ENV SPARK_SUBMIT_ARGS spark-submit \
                    --master 'local[2]' \
                     --driver-memory 1G \
                     --class org.apache.spark.examples.SparkPi \
		             --name spark-pi \
		             /opt/spark/examples/jars/spark-examples_2.11-2.4.3.jar

 CMD ["/bin/bash", "-c", "$SPARK_SUBMIT_ARGS"] 

docker build -t spark-pi .

docker run spark-pi

21/01/25 10:27:34 INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1161
21/01/25 10:27:34 INFO DAGScheduler: Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at SparkPi.scala:34) (first 15 tasks are for partitions Vector(0, 1))
21/01/25 10:27:34 INFO TaskSchedulerImpl: Adding task set 0.0 with 2 tasks
21/01/25 10:27:34 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost, executor driver, partition 0, PROCESS_LOCAL, 7866 bytes)
21/01/25 10:27:34 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, localhost, executor driver, partition 1, PROCESS_LOCAL, 7866 bytes)
21/01/25 10:27:34 INFO Executor: Running task 1.0 in stage 0.0 (TID 1)
21/01/25 10:27:34 INFO Executor: Running task 0.0 in stage 0.0 (TID 0)
21/01/25 10:27:34 INFO Executor: Fetching spark://f359169e37e7:39951/jars/spark-examples_2.11-2.4.3.jar with timestamp 1611570452260
21/01/25 10:27:34 INFO TransportClientFactory: Successfully created connection to f359169e37e7/172.17.0.3:39951 after 87 ms (0 ms spent in bootstraps)
21/01/25 10:27:34 INFO Utils: Fetching spark://f359169e37e7:39951/jars/spark-examples_2.11-2.4.3.jar to /tmp/spark-857fdf84-762d-48fa-a611-8780c8719d35/userFiles-e291d606-e828-4e20-a76a-fbe57159bf1d/fetchFileTemp6173381185321356138.tmp
21/01/25 10:27:34 INFO Executor: Adding file:/tmp/spark-857fdf84-762d-48fa-a611-8780c8719d35/userFiles-e291d606-e828-4e20-a76a-fbe57159bf1d/spark-examples_2.11-2.4.3.jar to class loader
21/01/25 10:27:35 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 824 bytes result sent to driver
21/01/25 10:27:35 INFO Executor: Finished task 1.0 in stage 0.0 (TID 1). 824 bytes result sent to driver
21/01/25 10:27:35 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 544 ms on localhost (executor driver) (1/2)
21/01/25 10:27:35 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 613 ms on localhost (executor driver) (2/2)
21/01/25 10:27:35 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool
21/01/25 10:27:35 INFO DAGScheduler: ResultStage 0 (reduce at SparkPi.scala:38) finished in 1.155 s
21/01/25 10:27:35 INFO DAGScheduler: Job 0 finished: reduce at SparkPi.scala:38, took 1.479874 s
Pi is roughly 3.139235696178481
21/01/25 10:27:35 INFO SparkUI: Stopped Spark web UI at http://f359169e37e7:4040
21/01/25 10:27:35 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
21/01/25 10:27:35 INFO MemoryStore: MemoryStore cleared
21/01/25 10:27:35 INFO BlockManager: BlockManager stopped
21/01/25 10:27:35 INFO BlockManagerMaster: BlockManagerMaster stopped
21/01/25 10:27:35 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
21/01/25 10:27:35 INFO SparkContext: Successfully stopped SparkContext
21/01/25 10:27:35 INFO ShutdownHookManager: Shutdown hook called
21/01/25 10:27:35 INFO ShutdownHookManager: Deleting directory /tmp/spark-7f77f3d8-59c0-4337-9e14-f2db06ab206f
21/01/25 10:27:35 INFO ShutdownHookManager: Deleting directory /tmp/spark-857fdf84-762d-48fa-a611-8780c8719d35