turingears/spark3-k8s repository overview

FROM gcr.io/spark-operator/spark:v3.0.0
USER root

RUN rm -rf $SPARK_HOME/jars/guava-14.0.1.jar
RUN mkdir -p /opt/hadoop/conf

ADD https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-latest-hadoop2.jar $SPARK_HOME/jars
ADD https://repo1.maven.org/maven2/com/google/guava/guava/23.0/guava-23.0.jar $SPARK_HOME/jars

COPY core-site.xml /opt/hadoop/conf

ENV HADOOP_CONF_DIR "/opt/hadoop/conf"
ENV HADOOP_OPTS "$HADOOP_OPTS -Dgs.project.id=$GS_PROJECT_ID"

apiVersion: "sparkoperator.k8s.io/v1beta2"
kind: SparkApplication
metadata:
  name: spark-leo666
  namespace: spark-operator
spec:
  type: Scala
  mode: cluster
  image: turingears/spark3-k8s:v2
  imagePullPolicy: Always
  mainClass: cl.turingears.fund.App
  mainApplicationFile: "gs://jobs-5a7132df-4905-4ac0-9c3b-63e51746ef71/spark3/jar/encrypt_spark-0.0.1-SNAPSHOT-shaded.jar"
  arguments:
    - "ewoiaW5wdXRQYXRoIjogImdzOi8vY29zYTEvdGVzdC9pbjIiLAoib3V0cHV0UGF0aCI6ICJnczovL2Nvc2EyL3Rlc3Rfc3BhcmszL2JpZ291dHB1dDIiLAoicGFzc3dvcmQiOiAiZWplbXBsbyIsCiJzYWx0IjogImVqZW1wbG8yIiwKImNvbHVtbnNUb0VuY3J5cHQiOiAiSURfU0tVIgp9"
  sparkVersion: "3.0.0"
  restartPolicy:
    type: Never
  volumes:
    - name: "test-volume"
      hostPath:
        path: "/tmp"
        type: Directory
  driver:
    cores: 2
    coreLimit: "2400m"
    memory: "2048m"
    labels:
      version: 3.0.0
    serviceAccount: spark
    volumeMounts:
      - name: "test-volume"
        mountPath: "/tmp"
  
  executor:
    cores: 2
    instances: 14
    memory: "4096m"
    labels:
      version: 3.0.0
    volumeMounts:
      - name: "test-volume"
        mountPath: "/tmp"