fix broken links, formatting correction, hide datanode port on master

Author: mjaglan

Dockerfile

@@ -18,10 +18,10 @@ RUN ssh-keygen -t rsa -f $HOME/.ssh/id_rsa -P "" \
     && cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys
 
 # download & extract & move hadoop & clean up
-RUN wget -O /hadoop.tar.gz -q http://apache.osuosl.org/hadoop/common/hadoop-2.7.2/hadoop-2.7.2.tar.gz \
 # TODO: write a way of untarring file to "/usr/local/hadoop" directly
+RUN wget -O /hadoop.tar.gz -q https://iu.box.com/shared/static/u9wy21nev5hxznhuhu0v6dzmcqhkhaz7.gz \
 	&& tar xfz hadoop.tar.gz \
-	&& mv /hadoop-2.7.2 /usr/local/hadoop \
+	&& mv /hadoop-2.7.3 /usr/local/hadoop \
 	&& rm /hadoop.tar.gz
 
 # hadoop environment variables
@@ -35,7 +35,7 @@ RUN mkdir -p $HADOOP_HOME/hdfs/namenode \
 # setup configs - [standalone, pseudo-distributed mode, fully distributed mode]
 # NOTE: Directly using COPY/ ADD will NOT work if you are NOT using absolute paths inside the docker image.
 # Temporary files: http://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s18.html
-COPY config/* /tmp/
+COPY config/ /tmp/
 RUN mv /tmp/ssh_config $HOME/.ssh/config \
     && mv /tmp/hadoop-env.sh $HADOOP_HOME/etc/hadoop/hadoop-env.sh \
     && mv /tmp/core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml \

README.md

@@ -19,9 +19,12 @@ Follow the instructions from [Post-installation steps for Linux](https://docs.do
 	```
 - Run the following script
 	```
-	. ./restart-all.sh
+	# Here, N = number of slave nodes to create (default value is 3).
+	. ./restart-all.sh   N
+
 	```
 
+
 ## After Starting Hadoop System
 
 The [hadoop-services.sh](scripts/hadoop-services.sh) is running following commands after starting Hadoop Multi-Node Cluster -
@@ -59,15 +62,15 @@ The [hadoop-services.sh](scripts/hadoop-services.sh) is running following comman
 
 - (Optional) Hadoop MRBENCH Test
 
-# Web UI
 
-- NameNode can be accessed on host machine browser at [http://HOST-OS-IP:8088/](http://0.0.0.0:8088/)
+## Web UI
 
-- Resource Manager can be accessed on host machine browser at [http://HOST-OS-IP:50070/](http://0.0.0.0:50070/)
+- NameNode can be accessed on browser at [http://CONTAINER-IP:8088/](http://0.0.0.0:8088/)
 
-- Secondary NameNode can be accessed on host machine browser at [http://HOST-OS-IP:50090/](http://0.0.0.0:50090/)
+- Resource Manager can be accessed on browser at [http://CONTAINER-IP:50070/](http://0.0.0.0:50070/)
+
+- Secondary can be accessed on browser at [http://CONTAINER-IP:50090/](http://0.0.0.0:50090/)
 
-- DataNode can be accessed on host machine browser at [http://HOST-OS-IP:50075/](http://0.0.0.0:50075/)
 
 ## Tools
 ```
@@ -77,11 +80,12 @@ Eclipse IDE for Java EE Developers Oxygen (4.7.0)
 Eclipse Docker Tooling 3.1.0
 ```
 
-# Configuration References
-- [Apache Hadoop 2.7.2 docs](https://hadoop.apache.org/docs/r2.7.2/)
-- [core-default.xml](https://hadoop.apache.org/docs/r2.7.2/hadoop-project-dist/hadoop-common/core-default.xml)
-- [hdfs-default.xml](https://hadoop.apache.org/docs/r2.7.2/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml)
-- [mapred-default.xml](https://hadoop.apache.org/docs/r2.7.2/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml)
-- [yarn-default.xml](https://hadoop.apache.org/docs/r2.7.2/hadoop-yarn/hadoop-yarn-common/yarn-default.xml)
-- [DeprecatedProperties.html](https://hadoop.apache.org/docs/r2.7.2/hadoop-project-dist/hadoop-common/DeprecatedProperties.html)
+
+## Configuration References
+- [Apache Hadoop 2.7.3 docs](https://hadoop.apache.org/docs/r2.7.3/)
+- [core-default.xml](https://hadoop.apache.org/docs/r2.7.3/hadoop-project-dist/hadoop-common/core-default.xml)
+- [hdfs-default.xml](https://hadoop.apache.org/docs/r2.7.3/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml)
+- [mapred-default.xml](https://hadoop.apache.org/docs/r2.7.3/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml)
+- [yarn-default.xml](https://hadoop.apache.org/docs/r2.7.3/hadoop-yarn/hadoop-yarn-common/yarn-default.xml)
+- [DeprecatedProperties.html](https://hadoop.apache.org/docs/r2.7.3/hadoop-project-dist/hadoop-common/DeprecatedProperties.html)
 

restart-all.sh

@@ -10,8 +10,7 @@ docker stop $(docker ps -a -q)
 docker rm $(docker ps -a -q)
 
 # if desired, clean up images
-#docker rmi $(docker images -q)
-#docker rmi  "$IMG_NAME"
+docker rmi $(docker images -q)
 
 # total number of slave nodes
 N=${1:-3}
@@ -51,13 +50,11 @@ Daemon                   Default Port  Configuration Parameter
 -----------------------  ------------ ----------------------------------
 ResourceManager          8088         yarn.resourcemanager.webapp.address
 Namenode                 50070        dfs.http.address
-Datanodes                50075        dfs.datanode.http.address
 Secondarynamenode        50090        dfs.secondary.http.address
 '
 HADOOP_MASTER="$HOST_PREFIX"-master
 docker run --name $HADOOP_MASTER -h $HADOOP_MASTER --net=$NETWORK_NAME \
-		-p  8088:8088  -p 50070:50070 \
-		-p 50075:50075 -p 50090:50090 \
+		-p  8088:8088  -p 50070:50070 -p 50090:50090 \
 		-itd "$IMG_NAME"
 
 # see active docker containers