cadic/pyspark repository overview

FROM apache/spark:3.5.0 as base
USER root
ENV TERM xterm-256color
RUN apt update ; apt install rsync sudo -y
ENV PATH=$PATH:/usr/local/bin:/opt/spark/bin:/opt/spark/sbin \
    JAVA_HOME=/opt/java/openjdk \
    SPARK_HOME=/opt/spark \
    SPARK_MASTER_PORT=7077 \
    SPARK_MASTER_WEBUI_PORT=8080 \
    SPARK_LOG_DIR=/opt/spark/logs \
    SPARK_MASTER_LOG=/opt/spark/logs/spark-master.out \
    SPARK_WORKER_LOG=/opt/spark/logs/spark-worker.out \
    SPARK_WORKER_WEBUI_PORT=8080 \
    SPARK_WORKER_PORT=7000 \
    MASTER_HOST="0.0.0.0" \
    SPARK_MASTER='spark://$MASTER_HOST:$SPARK_MASTER_PORT' \
    WORKLOAD="master" \
    SPARK_VERSION="3.5.0"\
    SPARK_NO_DAEMONIZE=true

# Foo
RUN adduser --disabled-password --gecos '' foo
RUN adduser foo sudo
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
RUN sudo chown -R foo:foo /home/foo

# Install deps
RUN pip install jupyterlab pandas PyArrow grpcio google-api-python-client grpcio-status pyspark==$SPARK_VERSION

# Change workdir
WORKDIR /workspace
RUN chown -R foo /workspace

FROM base as sandbox
COPY docker/static/spark /spark
RUN chown -R foo /opt
USER foo
ENTRYPOINT [ "/spark" ]

version: "3.9"

x-default: &default
  image: cadic/pyspark
  restart: always
  build:
    context: .
    dockerfile: docker/Dockerfile
  volumes:
    - ./:/workspace

services:
  master:
    <<: *default
    container_name: master
    environment:
      - WORKLOAD=master
    hostname: master
    ports:
      - 8080:8080
      - 8081:8081
      - 8888:8888
      - 15002:15002
      - 4040:4040

  worker:
    <<: *default
    environment:
      - WORKLOAD=worker
      - MASTER_HOST=master
    depends_on:
      - master
    deploy:
      mode: replicated
      replicas: 16
      resources:
        limits:
          cpus: "2"

docker compose build
docker compose up -d
docker exec -it master bash