ERP 17.08 - Build, Configure, Install and Test Hadoop, Spark, Hive, Hbase in containerized environment

git clone --depth 1 https://git.linaro.org/leg/bigdata/bigtop-trunk.git/ -b erp17.08 
cd bigtop-trunk

docker images

docker run -v `pwd`:/ws bigtop/slaves:erp17.08-deb-8-aarch64 bash -l -c 'cd /ws ; ./gradlew bigtop-groovy-deb bigtop-utils-deb bigtop-jsvc-deb bigtop-tomcat-deb'

docker run -v `pwd`:/ws bigtop/slaves:erp17.08-deb-8-aarch64 bash -l -c 'cd /ws ; ./gradlew hadoop-deb hbase-deb spark-deb hive-deb'

docker run -v `pwd`:/ws bigtop/slaves:erp17.08-deb-8-aarch64 bash -l -c 'cd /ws ; ./gradlew deb'

docker run -v `pwd`:/ws bigtop/slaves:erp17.08-deb-8-aarch64 bash -l -c 'cd /ws ; ./gradlew flume-deb mahout-deb pig-deb sqoop-deb zookeeper-deb'

docker run -v `pwd`:/ws bigtop/slaves:erp17.08-deb-8-aarch64 bash -l -c 'cd /ws ; ./gradlew apt'

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

docker:
        memory_limit: "4g"
        image:  "bigtop/puppet:debian-8-aarch64"

repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.2.0/debian/8-aarch64/aarch64"
distro: debian
components: [hdfs, yarn, mapreduce, hive]
enable_local_repo: true
smoke_test_components: [hive]
jdk: "openjdk-8-jdk"

cd provisioner/docker
./docker-hadoop.sh -C <smoke_test_cfg_yaml> -c <node_count> -s -d

./docker-hadoop.sh -E

usage: docker-hadoop.sh [-C file ] args
       -C file                                   Use alternate file for config.yaml
  commands:
       -c NUM_INSTANCES, --create NUM_INSTANCES  Create a Docker based Bigtop Hadoop cluster
       -d, --destroy                             Destroy the cluster
       -e, --exec INSTANCE_NO|INSTANCE_NAME      Execute command on a specific instance. Instance can be specified by name or number.
                                                 For example: docker-hadoop.sh --exec 1 bash
                                                              docker-hadoop.sh --exec docker_bigtop_1 bash
       -E, --env-check                           Check whether required tools has been installed
       -l, --list                                List out container status for the cluster
       -p, --provision                           Deploy configuration changes
       -s, --smoke-tests                         Run Bigtop smoke tests
       -h, --help

$ vim provisioner/docker/config_debian-8-aarch64.yaml

Do the below changes

components: [hdfs, yarn, mapreduce, hive, spark]
enable_local_repo: true
smoke_test_components: [spark]
jdk: "openjdk-8-jdk"

save and quit.

$ vim provisioner/utils/smoke-tests.sh

Add the below lines

export HADOOP_HOME=/usr/lib/hadoop
export HADOOP_PREFIX=$HADOOP_HOME
export HADOOP_OPTS="-Djava.library.path=$HADOOP_PREFIX/lib/native"
export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec
export HADOOP_CONF_DIR=/etc/hadoop/conf
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce
export HADOOP_HDFS_HOME=/usr/lib/hadoop-hdfs
export YARN_HOME=/usr/lib/hadoop-yarn
export HADOOP_YARN_HOME=/usr/lib/hadoop-yarn/
export HADOOP_USER_NAME=hdfs
export CLASSPATH=$CLASSPATH:.
export CLASSPATH=$CLASSPATH:$HADOOP_HOME/hadoop-common-2.7.2.jar:$HADOOP_HOME/client/hadoop-hdfs-2.7.2.jar:$HADOOP_HOME/hadoop-auth-2.7.2.jar:/usr/lib/hadoop-mapreduce/*:/usr/lib/hive/lib/*:/usr/lib/hadoop/lib/*:
export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::")
export PATH=/usr/lib/hadoop/libexec:/etc/hadoop/conf:$HADOOP_HOME/bin/:$PATH
export SPARK_HOME=/usr/lib/spark
export PATH=$HADOOP_HOME\bin:$PATH
export SPARK_DIST_CLASSPATH=$HADOOP_HOME\bin\hadoop:$CLASSPATH:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/lib/*:/usr/lib/hadoop-mapreduce/*:.
export CLASSPATH=$CLASSPATH:/usr/lib/hadoop/lib/*:.

export SPARK_HOME=/usr/lib/spark
export SPARK_MASTER_IP=<IP Address of your host>
export SPARK_MASTER_PORT=7077

save and quit.

Run the command smoke test commands

$ ./docker-hadoop.sh -C config_debian-8-aarch64.yaml -c 3 -s -d

If the smoke test is still failing.  It might be because of you have't set the "SPARK_MASTER_IP" properly.  Another way to do is that remove the $masterMode and add local[*] as below

$ vim bigtop-tests/smoke-tests/spark/TestSpark.groovy

make the below change

final String SPARK_SHELL = SPARK_HOME + "/bin/spark-shell --master local[*]"

save and quit. Now run the spark smoke-tests again.