platpus/aws-glue-python repository overview

FROM centos:centos8

ENV MAVEN=https://aws-glue-etl-artifacts.s3.amazonaws.com/glue-common/apache-maven-3.6.0-bin.tar.gz
ENV SPARK=https://aws-glue-etl-artifacts.s3.amazonaws.com/glue-1.0/spark-2.4.3-bin-hadoop2.8.tgz
ENV GLUE=https://github.com/awslabs/aws-glue-libs.git

RUN mkdir -p opt
RUN yum install -y python3 java-1.8.0-openjdk java-1.8.0-openjdk-devel tar git wget zip
WORKDIR /opt

RUN curl -O $MAVEN
RUN curl -O $SPARK
RUN git clone -b glue-1.0 --single-branch $GLUE

RUN tar zxfv apache-maven-3.6.0-bin.tar.gz
# Use -o option to extract spark archive: --no-same-owner = Extract files as root
# Avoid issues later e.g. "Container cannot be mapped to a host ID"
RUN tar zxfvo spark-2.4.3-bin-hadoop2.8.tgz
RUN rm spark-2.4.3-bin-hadoop2.8.tgz
RUN rm apache-maven-3.6.0-bin.tar.gz
RUN mv $(rpm -q -l java-1.8.0-openjdk-devel | grep "/bin$" | rev | cut -d"/" -f2- |rev) /usr/lib/jvm/jdk
ENV SPARK_HOME /opt/spark-2.4.3-bin-spark-2.4.3-bin-hadoop2.8
ENV MAVEN_HOME /opt/apache-maven-3.6.0
ENV JAVA_HOME /usr/lib/jvm/jdk
ENV GLUE_HOME /opt/aws-glue-libs
ENV PATH $PATH:$MAVEN_HOME/bin:$SPARK_HOME/bin:$JAVA_HOME/bin:$GLUE_HOME/bin

# Wacky workaround to get past issue with p4j error (credit @svajiraya - https://github.com/awslabs/aws-glue-libs/issues/25)
RUN rm -rf /opt/aws-glue-libs/jars/netty*
# Known issue needs removing some of the libraries https://github.com/awslabs/aws-glue-libs/issues/25.
RUN sed -i '/mvn -f/a rm /opt/aws-glue-libs/jarsv1/javax.servlet-3.*' /opt/aws-glue-libs/bin/glue-setup.sh

# Download & cache maven dependencies in docker image
RUN sh /opt/aws-glue-libs/bin/glue-setup.sh

RUN yum clean all
RUN rm -rf /var/cache/yum
CMD ["bash"]