run a hadoop cluster in docker containers.
300
基于Hadoop2.9.2版本，使用docker搭建高可用集群，高可用包含hdfs namenode HA，yarn HA
Hadoop高可用集群依赖于Zookeeper，此镜像中不包含zookeeper，因此需要单独提供zookeeper服务。 要运行hadoop-ha集群，有两种方式可选：直接从docker hub拉取相关镜像并启动容器，或者自行构建。
docker pull okdokey/zookeeper:3.4.14
docker pull okdokey/hadoop:2.9.2-HA
命令：
docker-compose -f docker-compose.yml up #初次启动容器集群
docker-compose -f docker-compose.yml start #启动容器服务集群
docker-compose -f docker-compose.yml stop #关闭容器服务集群
编写docker-compoose.yml文件，内容参考如下：
version: "2"
services:
hadoop1:
image: hadoop:2.9.2-HA
container_name: hadoop-nn1-rm1
hostname: hadoop-nn1-rm1
depends_on:
- zk1
- zk2
- zk3
networks:
hadoop:
aliases:
- master
- nn1
- rm1
environment:
ZK_SERVERS: zk1:2181,zk2:2181,zk3:2181
HADOOP_WORKER_NAMES: hadoop1 hadoop2 hadoop3
HDFS_JOURNAL_NODES: hadoop1:8485,hadoop2:8485,hadoop3:8485
YARN_RM_AM_MAX_ATTEMPTS: 4
YARN_APP_MIN_MEMORY: 512
YARN_APP_MAX_MEMORY: 4096
YARN_NM_MEMROY: 4096
YARN_NM_VMEM_PMEM_RATIO: 5
YARN_NM_CPU_VCORES: 2
ports:
- "8020:8020"
- "8030:8030"
- "8031:8031"
- "8032:8032"
- "8033:8033"
- "8042:8042"
- "8044:8044"
- "8045:8045"
- "8046:8046"
- "8047:8047"
- "8048:8048"
- "8049:8049"
- "8088:8088"
- "8089:8089"
- "8090:8090"
- "8091:8091"
- "8188:8188"
- "8190:8190"
- "8480:8480"
- "8481:8481"
- "8485:8485"
- "8788:8788"
- "10200:10200"
- "50010:50010"
- "50020:50020"
- "50070:50070"
- "50075:50075"
- "10020:10020"
- "19888:19888"
- "19890:19890"
- "10033:10033"
command: [ "-m", "-d" ]
hadoop2:
image: hadoop:2.9.2-HA
container_name: hadoop-nn2-rm2
hostname: hadoop-nn2-rm2
depends_on:
- zk1
- zk2
- zk3
networks:
hadoop:
aliases:
- standby
- nn2
- rm2
environment:
ZK_SERVERS: zk1:2181,zk2:2181,zk3:2181
HADOOP_WORKER_NAMES: hadoop1 hadoop2 hadoop3
HDFS_JOURNAL_NODES: hadoop1:8485,hadoop2:8485,hadoop3:8485
YARN_RM_AM_MAX_ATTEMPTS: 4
YARN_APP_MIN_MEMORY: 512
YARN_APP_MAX_MEMORY: 4096
YARN_NM_MEMROY: 4096
YARN_NM_VMEM_PMEM_RATIO: 5
YARN_NM_CPU_VCORES: 2
ports:
- "9088:8088"
- "50071:50070"
command: [ "-d" ]
hadoop3:
image: hadoop:2.9.2-HA
container_name: hadoop-slave1
hostname: hadoop-slave1
depends_on:
- zk1
- zk2
- zk3
networks:
hadoop:
aliases:
- slave1
environment:
ZK_SERVERS: zk1:2181,zk2:2181,zk3:2181
HADOOP_WORKER_NAMES: hadoop1 hadoop2 hadoop3
HDFS_JOURNAL_NODES: hadoop1:8485,hadoop2:8485,hadoop3:8485
YARN_RM_AM_MAX_ATTEMPTS: 4
YARN_APP_MIN_MEMORY: 512
YARN_APP_MAX_MEMORY: 4096
YARN_NM_MEMROY: 4096
YARN_NM_VMEM_PMEM_RATIO: 5
YARN_NM_CPU_VCORES: 2
command: [ "-d" ]
zk1:
image: okdokey/zookeeper:3.4.14
hostname: zk1
container_name: zk1
networks:
hadoop:
aliases:
- zk1
ports:
- 2181:2181
environment:
ZK_MY_ID: 1
ZK_SERVERS: server.1=0.0.0.0:2888:3888 server.2=zk2:2888:3888 server.3=zk3:2888:3888
zk2:
image: okdokey/zookeeper:3.4.14
hostname: zk2
container_name: zk2
networks:
hadoop:
aliases:
- zk2
ports:
- 2182:2181
environment:
ZK_MY_ID: 2
ZK_SERVERS: server.1=zk1:2888:3888 server.2=0.0.0.0:2888:3888 server.3=zk3:2888:3888
zk3:
image: okdokey/zookeeper:3.4.14
hostname: zk3
container_name: zk3
networks:
hadoop:
aliases:
- zk3
ports:
- 2183:2181
environment:
ZK_MY_ID: 3
ZK_SERVERS: server.1=zk1:2888:3888 server.2=zk2:2888:3888 server.3=0.0.0.0:2888:3888
networks:
hadoop:
你可以选择直接从Dockerfile构建，构建目录在docker-build，命令如下：
docker build -t hadoop:2.9.2-HA docker-build/
然后使用docker-compose启动集群即可。
正常情况这样hadoop高可用集群就启动成功了，访问localhost:8088可以查看yarn集群状态，localhost:50070可以查看dfs状态
Content type
Image
Digest
Size
883.1 MB
Last updated
over 7 years ago
docker pull okdokey/hadoop:2.9.2-HA