tanlull/hadoop_spark_lab repository overview

⁠Run with:

docker run -d -p 0.0.0.0:2222:22 -p 0.0.0.0:50070:50070 --name hadoop tanlull/hadoop_spark_lab

⁠SSH

port: 2222 user: user01 password: password

⁠Lab information :

########## Setup Apache Hadoop 2.7.2 ##########

⁠extract zip and move folder

cd tar -xvf hadoop-2.7.2.tar.gz sudo mv ./hadoop-2.7.2 /usr/local/hadoop

⁠set startup path

sudo nano ~/.bashrc export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ export PATH=$PATH:$JAVA_HOME/bin export HADOOP_HOME=/usr/local/hadoop export PATH=$PATH:$HADOOP_HOME/bin export PATH=$PATH:$HADOOP_HOME/sbin

⁠create hadoop log path

source ~/.bashrc sudo mkdir /var/log/hadoop sudo chown -R user01:user01 /var/log/hadoop cd /usr/local/hadoop/etc/hadoop vi hadoop-env.sh export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64

⁠config yarn log path

nano yarn-env.sh YARN_LOG_DIR=/var/log/hadoop

⁠lookup self IP

ifconfig [Suppost that we have IP = 172.17.0.3 ]

⁠config core-site.xml

vi core-site.xml fs.defaultFS hdfs://172.17.0.3:9000

⁠create namenode and datanode path

sudo mkdir -p /var/hadoop_data/namenode sudo mkdir -p /var/hadoop_data/datanode sudo chown user01:user01 -R /var/hadoop_data

⁠config hdfs-site.xml

vi hdfs-site.xml dfs.replication 1 dfs.namenode.name.dir file:/var/hadoop_data/namenode dfs.datanode.data.dir file:/var/hadoop_data/datanode

⁠config yarn-site.xml

vi yarn-site.xml yarn.resourcemanager.hostname 172.17.0.3 yarn.resourcemanager.scheduler.address 172.17.0.3:8030 yarn.resourcemanager.resource-tracker.address 172.17.0.3:8031 yarn.resourcemanager.address 172.17.0.3:8032 yarn.resourcemanager.admin.address 172.17.0.3:8033 yarn.resourcemanager.webapp.address 172.17.0.3:8088 yarn.nodemanager.aux-services mapreduce_shuffle yarn.nodemanager.aux-services.mapreduce.shuffle.class org.apache.hadoop.mapred.ShuffleHandler

⁠config mapred-site.xml

cp mapred-site.xml.template mapred-site.xml vi mapred-site.xml mapreduce.framework.name yarn

⁠format namenode

hdfs namenode -format

⁠lookup namenode

cd /var/hadoop_data/namenode/ ll cd current/ ls

⁠start dfs

cd /usr/local/hadoop/etc/hadoop start-dfs.sh [answer yes] jps

⁠start yarn

start-yarn.sh jps

netstat -a

[docker run -d -p 0.0.0.0:2222:22 -p 0.0.0.0:50070:50070 --name hadoop tanlull/hadoop_yarn_run]

⁠open browser on ubuntu

URL: http://192.168.99.100:50070/⁠

########## Apache Hadoop HDFS ##########

⁠back to hadoop directory

cd /usr/local/hadoop/etc/hadoop/

⁠create input and ouput folder on dfs

hdfs dfs -mkdir /inputs hdfs dfs -mkdir /outputs

⁠copy file to dfs

hdfs dfs -copyFromLocal ~/sales_data_jan2009.csv /inputs/ hdfs dfs -ls /inputs

⁠read file content on dfs

hdfs dfs -cat /inputs/sales_data_jan2009.csv

⁠remove file on dfs

hdfs dfs -rm /inputs/sales_data_jan2009.csv

⁠copy file to dfs (again)

hdfs dfs -copyFromLocal ~/sales_data_jan2009.csv /inputs/sales_data_jan2009.csv hdfs dfs -ls /inputs/

⁠(optional) review linux file system

cd /var/hadoop_data/datanode/ ls cd current/ cd BP-2114074188-127.0.1.1-1474558767749/ ls cd current/ ls cd finalized/ ls cd subdir0/ ls cd subdir0/ ls