vincent1995/iceberg-flink repository overview

docker-compose up

docker exec -it hiveserver2 beeline -u 'jdbc:hive2://localhost:10000/'

docker exec -it hive_jobmanager_1 bash
./bin/sql-client.sh

version: '3.9'
services:
  postgres:
    image: postgres
    restart: unless-stopped
    container_name: postgres
    hostname: postgres
    environment:
      POSTGRES_DB: 'metastore_db'
      POSTGRES_USER: 'hive'
      POSTGRES_PASSWORD: 'password'
    ports:
      - '5432:5432'
    volumes:
      - ./hive-db:/var/lib/postgresql

  metastore:
    image: apache/hive:4.0.0-alpha-2
    depends_on:
      - postgres
    restart: unless-stopped
    container_name: metastore
    hostname: metastore
    environment:
      DB_DRIVER: postgres
      SERVICE_NAME: 'metastore'
      SERVICE_OPTS: '-Xmx1G -Djavax.jdo.option.ConnectionDriverName=org.postgresql.Driver
                     -Djavax.jdo.option.ConnectionURL=jdbc:postgresql://postgres:5432/metastore_db
                     -Djavax.jdo.option.ConnectionUserName=hive
                     -Djavax.jdo.option.ConnectionPassword=password'
    ports:
        - '9083:9083'
    volumes:
        - ./warehouse:/opt/hive/data/warehouse
 
  hiveserver2:
    image: apache/hive:4.0.0-alpha-2
    depends_on:
      - metastore
    restart: unless-stopped
    container_name: hiveserver2
    environment:
      HIVE_SERVER2_THRIFT_PORT: 10000
      SERVICE_OPTS: '-Xmx1G -Dhive.metastore.uris=thrift://metastore:9083'
      IS_RESUME: 'true'
      SERVICE_NAME: 'hiveserver2'
    ports:
      - '10000:10000'
      - '10002:10002'
    volumes:
      - ./warehouse:/opt/hive/data/warehouse

  jobmanager:
    image: vincent1995/iceberg-flink:1.16.2_1.4.1_1
    ports:
      - "8081:8081"
    command: jobmanager
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager       
    volumes :
      - ./warehouse:/opt/hive/data/warehouse

  taskmanager:
    image: vincent1995/iceberg-flink:1.16.2_1.4.1_1
    depends_on:
      - jobmanager
    command: taskmanager
    scale: 1
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        taskmanager.numberOfTaskSlots: 2
    volumes :
      - ./warehouse:/opt/hive/data/warehouse
    
                
  sql-client:
    image: vincent1995/iceberg-flink:1.16.2_1.4.1_1
    command: bin/sql-client.sh
    depends_on:
      - jobmanager
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        rest.address: jobmanager
    volumes :
      - ./warehouse:/opt/hive/data/warehouse  

###############################################################################
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################

FROM eclipse-temurin:8-jre-jammy

# Install dependencies
RUN set -ex; \
  apt-get update; \
  apt-get -y install gpg libsnappy1v5 gettext-base libjemalloc-dev; \
  rm -rf /var/lib/apt/lists/*

# Grab gosu for easy step-down from root
ENV GOSU_VERSION 1.11
RUN set -ex; \
  wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)"; \
  wget -nv -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc"; \
  export GNUPGHOME="$(mktemp -d)"; \
  for server in ha.pool.sks-keyservers.net $(shuf -e \
                          hkp://p80.pool.sks-keyservers.net:80 \
                          keyserver.ubuntu.com \
                          hkp://keyserver.ubuntu.com:80 \
                          pgp.mit.edu) ; do \
      gpg --batch --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || : ; \
  done && \
  gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \
  gpgconf --kill all; \
  rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \
  chmod +x /usr/local/bin/gosu; \
  gosu nobody true

# Configure Hadoop
# ENV APACHE_HADOOP_URL=https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz
ENV HADOOP_HOME=/opt/hadoop

WORKDIR $HADOOP_HOME
COPY hadoop.tar.gz ./
RUN set -ex; \
  # wget -nv -O hadoop.tar.gz "$APACHE_HADOOP_URL"; \
  tar -xf hadoop.tar.gz --strip-components=1; \
  rm hadoop.tar.gz;

ENV HADOOP_CLASSPATH=/opt/hadoop/etc/hadoop:/opt/hadoop/share/hadoop/common/lib/*:/opt/hadoop/share/hadoop/common/*:/opt/hadoop/share/hadoop/hdfs:/opt/hadoop/share/hadoop/hdfs/lib/*:/opt/hadoop/share/hadoop/hdfs/*:/opt/hadoop/share/hadoop/yarn/lib/*:/opt/hadoop/share/hadoop/yarn/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/contrib/capacity-scheduler/*.jar
ENV HADOOP_CONF_DIR=/opt/hadoop/etc/hadoop


# Configure Flink version
# ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.16.2/flink-1.16.2-bin-scala_2.12.tgz \
    # FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.16.2/flink-1.16.2-bin-scala_2.12.tgz.asc \
    # GPG_KEY=8D56AE6E7082699A4870750EA4E8C4C05EE6861F \
    # CHECK_GPG=true 
    # ICEBERG_JAR=https://repo.maven.apache.org/maven2/org/apache/iceberg/iceberg-flink-runtime-1.16/1.4.1/iceberg-flink-runtime-1.16-1.4.1.jar \
    # ICEBERG_HIVE_JAR=https://repo1.maven.org/maven2/org/apache/flink/flink-sql-connector-hive-2.3.9_2.12/1.16.2/flink-sql-connector-hive-2.3.9_2.12-1.16.2.jar 

# Prepare environment
ENV FLINK_HOME=/opt/flink
ENV PATH=$FLINK_HOME/bin:$PATH
RUN groupadd --system --gid=9999 flink && \
    useradd --system --home-dir $FLINK_HOME --uid=9999 --gid=flink flink
WORKDIR $FLINK_HOME

COPY flink.tgz ./
COPY iceberg-flink-runtime-1.16-1.4.1.jar ./
COPY flink-sql-connector-hive-2.3.9_2.12-1.16.2.jar ./
# Install Flink
RUN set -ex; \
  # wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
  # \
  # if [ "$CHECK_GPG" = "true" ]; then \
  #   wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
  #   export GNUPGHOME="$(mktemp -d)"; \
  #   for server in ha.pool.sks-keyservers.net $(shuf -e \
  #                           hkp://p80.pool.sks-keyservers.net:80 \
  #                           keyserver.ubuntu.com \
  #                           hkp://keyserver.ubuntu.com:80 \
  #                           pgp.mit.edu) ; do \
  #       gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
  #   done && \
  #   gpg --batch --verify flink.tgz.asc flink.tgz; \
  #   gpgconf --kill all; \
  #   rm -rf "$GNUPGHOME" flink.tgz.asc; \
  # fi; \
  \
  tar -xf flink.tgz --strip-components=1; \
  rm flink.tgz; \
  \
  chown -R flink:flink .; \
  \
  # wget -nv -O iceberg-flink-runtime-1.16-1.4.1.jar "$ICEBERG_JAR"; \
  mv iceberg-flink-runtime-1.16-1.4.1.jar $FLINK_HOME/lib; \
  # wget -nv -O flink-sql-connector-hive-2.3.9_2.12-1.16.2.jar "$ICEBERG_HIVE_JAR"; \
  mv flink-sql-connector-hive-2.3.9_2.12-1.16.2.jar "$FLINK_HOME/lib"; \
  # wget -nv -O hadoop-common-2.8.5.jar "$HADOOP_COMMON_JAR"; \
  # mv hadoop-common-2.8.5.jar "$FLINK_HOME/lib"; \
  \
  # Replace default REST/RPC endpoint bind address to use the container's network interface \
  sed -i 's/rest.address: localhost/rest.address: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \
  sed -i 's/rest.bind-address: localhost/rest.bind-address: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \
  sed -i 's/jobmanager.bind-host: localhost/jobmanager.bind-host: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \
  sed -i 's/taskmanager.bind-host: localhost/taskmanager.bind-host: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \
  sed -i '/taskmanager.host: localhost/d' $FLINK_HOME/conf/flink-conf.yaml;


# Configure container
COPY docker-entrypoint.sh /
RUN chmod +x /docker-entrypoint.sh; \
    chown flink:flink /docker-entrypoint.sh 
ENTRYPOINT ["/docker-entrypoint.sh"]
EXPOSE 6123 8081
CMD ["help"]