fluxdatareactor/spark repository overview

# Default system properties included when running spark-submit.
# This is useful for setting default environmental settings.

spark.driver.port 7001
spark.fileserver.port 7002
spark.broadcast.port 7003
spark.replClassServer.port 7004
spark.blockManager.port 7005

spark.broadcast.factory=org.apache.spark.broadcast.HttpBroadcastFactory
spark.port.maxRetries 4

# Default system properties included when running spark-submit.
# This is useful for setting default environmental settings.

#spark.driver.port 7101
spark.fileserver.port 7012
spark.broadcast.port 7013
spark.replClassServer.port 7014
spark.blockManager.port 7015

spark.broadcast.factory=org.apache.spark.broadcast.HttpBroadcastFactory
spark.port.maxRetries 4

version: "2.0"
services:

  master:
    container_name: master
    image: fluxdatareactor/spark:1.0.0
    hostname: master
    command: bin/spark-class org.apache.spark.deploy.master.Master -h master
    environment:
      SPARK_MASTER: spark://master:7077
      SPARK_CONF_DIR: /conf
      PYSPARK_PYTHON: python3
    expose:
      - 7001
      - 7002
      - 7003
      - 7004
      - 7005
      - 7006
      - 7077
      - 6066
    ports:
      - 4040:4040
      - 6066:6066
      - 7077:7077
      - 8080:8080
    volumes:
      - ./conf/master:/conf
      - ./data:/tmp/data

  worker:
    image: fluxdatareactor/spark:1.0.0
    hostname: worker
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077
    environment:
      SPARK_CONF_DIR: /conf
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 1g
      SPARK_WORKER_PORT: 8881
      SPARK_WORKER_WEBUI_PORT: 8081
      PYSPARK_PYTHON: python3
    depends_on:
      - master
    expose:
      - 7012
      - 7013
      - 7014
      - 7015
      - 7016
      - 8881
    ports:
      - 8081
    volumes:
      - ./conf/worker:/conf
      - ./data:/tmp/data

antlr-runtime-3.4.jar
datanucleus-api-jdo-3.2.6.jar
datanucleus-core-3.2.10.jar
datanucleus-rdbms-3.2.9.jar
derby-10.10.2.0.jar
hive-exec-1.2.1.spark.jar (1.1.0)
hive-metastore-1.2.1.spark.jar (1.1.0)
jdo-api-3.0.1.jar
libfb303-0.9.3.jar
libthrift-0.9.3.jar
scala-compiler-2.10.4.jar (1.1.0) spark 1.6
scala-parser-combinators_2.11-1.0.1.jar
scala-reflect-2.10.4.jar 
scala-xml_2.11-1.0.1.jar
scalap-2.11.0.jar
snappy-0.2.jar
spark-hive_2.10-1.1.0.jar (1.1.0)