Fix Hadoop Installation aarch64

build-and-run-tests-hadoop-aarch64.sh

set -ex
[ "$EUID" -eq 0 ] && exit "this script must be run with non-root user!"
sudo apt-get update -y

DEBIAN_FRONTEND=noninteractive sudo apt-get install -y build-essential autoconf automake libtool cmake zlib1g-dev pkg-config libssl-dev libsasl2-dev libsnappy-dev bats

# install openjdk-8-jdk
sudo apt-get install openjdk-8-jdk -y
export JAVA_HOME=`dirname $(dirname $(update-alternatives --list javac |grep java-8))`

# install maven-3.6.2
[ -d "/opt/apache-maven-3.6.2/" ] || wget -O - "https://www.apache.org/dist/maven/maven-3/3.6.2/binaries/apache-maven-3.6.2-bin.tar.gz" | tar xz -C /opt/
export PATH=/opt/apache-maven-3.6.2/bin:$PATH

javac -version
mvn --version

# Need to keep the localhost in the 1st line
grep "127.0.0.1 $(hostname)" /etc/hosts || sudo sed  -i '2i\127.0.0.1 '$(hostname)'' /etc/hosts
# fix tests in org.apache.hadoop.yarn.server.resourcemanager.recovery.TestFSRMStateStore
grep "127.0.0.1 $(hostname)." /etc/hosts || sudo sed  -i '3i\127.0.0.1 '$(hostname).'' /etc/hosts

# Install Docker
type docker || curl -sSL https://get.docker.com/ | sh -

if [ ! -d "/opt/protobuf-3.7.1" ]; then
    wget -O - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz | tar zx -C ~/
    pushd ~/protobuf-3.7.1
    ./configure --prefix=/opt/protobuf-3.7.1
    sudo make install
    popd
    sudo rm -fr ~/protobuf-3.7.1
fi
export PROTOBUF_HOME=/opt/protobuf-3.7.1
export PATH="${PATH}:/opt/protobuf-3.7.1/bin"

# phantomjs 2.1.1 require libicu55 which only in ubuntu xenial
sudo echo "deb http://ports.ubuntu.com/ubuntu-ports xenial main" >> /etc/apt/sources.list
sudo apt-get update
sudo apt-get install -y libicu55

git clone https://github.com/google/snappy ~/snappy
pushd ~/snappy
mkdir build
cd build/ && cmake ../ && make
sudo make install
popd

if ! type phantomjs; then
    readonly phant_dir="$(mktemp -d --tmpdir phantomjs.XXXXXX)"
    wget -O - "https://github.com/liusheng/phantomjs/releases/download/2.1.1/phantomjs-2.1.1-linux-aarch64.tar.bz2" | sudo tar xj -C "$phant_dir"
    sudo cp "$phant_dir"/phantomjs-2.1.1-linux-aarch64/bin/phantomjs /usr/bin/
fi

# Install protoc-gen-grpc-java for ARM platform
if [ ! -f ~/.m2/repository/io/grpc/protoc-gen-grpc-java/1.15.1/protoc-gen-grpc-java-1.15.1-linux-aarch_64.exe ]; then
    mkdir -p ~/.m2/repository/io/grpc/protoc-gen-grpc-java/1.15.1/
    pushd ~/.m2/repository/io/grpc/protoc-gen-grpc-java/1.15.1/
    wget http://home.apache.org/~aajisaka/repository/io/grpc/protoc-gen-grpc-java/1.15.1/protoc-gen-grpc-java-1.15.1-linux-aarch_64.exe
    wget http://home.apache.org/~aajisaka/repository/io/grpc/protoc-gen-grpc-java/1.15.1/protoc-gen-grpc-java-1.15.1.pom
    wget http://home.apache.org/~aajisaka/repository/io/grpc/protoc-gen-grpc-java/1.15.1/protoc-gen-grpc-java-1.15.1.pom.sha1
    popd
fi

# NOTE: the tests of TestAuxServices need to create some file and directories which must not have group
# and other permissions, and the files' parents direcotries(must not writable by group or other).
# so we need change all the permissions of directories and the "umask" the umask will effect the new
# created files and directories' permissions e.g. the 077 means the new created permissions: 777 - 077 = 700 (files: 666 - 077 = 600)
sudo chmod go-w ~/ -R
[[ "$(umask)" =~ "022" ]] || echo "umask 022" >> ~/.profile
. ~/.profile
umask

mkdir -p ~/src/
[ -d ~/src/hadoop ] || git clone http://github.com/apache/hadoop ~/src/hadoop
cd ~/src/hadoop

# Install manually compiled netty-all package
wget -O netty-all-4.1.27-linux-aarch64.jar https://git.io/Je8K3
mvn install:install-file -DgroupId=io.netty -Dfile=netty-all-4.1.27-linux-aarch64.jar -DartifactId=netty-all -Dversion=4.1.27.Final -Dpackaging=jar
rm netty-all-4.1.27-linux-aarch64.jar

# install hacked wildfly-openssl-1.0.7.Final to support aarch64 see: https://github.com/wildfly-security/wildfly-openssl/issues/72
curl -O -L https://github.com/liusheng/package/files/3904116/wildfly-openssl-1.0.7.Final.jar.tar.gz
tar zxf wildfly-openssl-1.0.7.Final.jar.tar.gz
mvn install:install-file -DgroupId=org.wildfly.openssl -Dfile=wildfly-openssl-1.0.7.Final.jar -DartifactId=wildfly-openssl -Dversion=1.0.7.Final -Dpackaging=jar
rm wildfly-openssl-1.0.7.Final.jar*

mkdir -p ~/hadoop-logs/
# Install hadoop in Maven local repo -Pdist,native,aarch64
mvn clean install -e -B -Pdist,native -Dtar -DskipTests -Dmaven.javadoc.skip 2>&1 | tee ~/hadoop-logs/hadoop_build.log

# Compile hadoop
sudo cp -r hadoop-dist/target/hadoop-3.3.0-SNAPSHOT /opt/
export PATH=/opt/hadoop-3.3.0-SNAPSHOT/bin:$PATH
hadoop version
# hadoop checknative -a

pushd hadoop-yarn-project/
mvn test -B -e -fn | tee ~/hadoop-logs/hadoop_yarn_test.log
popd

Dockerfile

FROM ubuntu:bionic

SHELL ["/bin/bash", "-o", "pipefail", "-c"]

######
# Install common dependencies from packages. Versions here are either
# sufficient or irrelevant.
#
# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
# Ubuntu Java.  See Java section below!
######
RUN apt-get -q update \
    && apt-get -q install -y --no-install-recommends \
        build-essential \
        autoconf \
        automake \
        libtool \
        cmake \
        zlib1g-dev \
        pkg-config \
        libssl-dev \
        libssl1.0.0 \
        libsasl2-dev \
        libsnappy-dev \
        curl \
        sudo \
        git \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*

#######
# OpenJDK 8
#######
RUN apt-get -q update \
    && apt-get -q install -y --no-install-recommends openjdk-8-jdk \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*

ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64

RUN apt-get -q update \
    &&	apt-get -q install -y maven \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*


######
# Install Google Protobuf 3.7.1 (2.6.0 ships with Xenial)
######
RUN mkdir -p /opt/protobuf-src \
    && curl -L -s -S \
      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
      -o /opt/protobuf.tar.gz \
    && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
    && cd /opt/protobuf-src \
    && ./configure --prefix=/opt/protobuf \
    && make install \
    && cd /root \
    && rm -rf /opt/protobuf-src
ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin"

# phantomjs 2.1.1 require libicu55 which only in ubuntu xenial
RUN echo "deb http://ports.ubuntu.com/ubuntu-ports xenial main" >> /etc/apt/sources.list
RUN apt-get -q update \
    &&	apt-get -q install -y libicu55 \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*
RUN curl -L -s -S \
      https://github.com/liusheng/phantomjs/releases/download/2.1.1/phantomjs-2.1.1-linux-aarch64.tar.bz2 \
      -o /tmp/phantomjs-2.1.1-linux-aarch64.tar.bz2 \
    && tar xjf /tmp/phantomjs-2.1.1-linux-aarch64.tar.bz2 -C /tmp/ \
    && cp /tmp/phantomjs-2.1.1-linux-aarch64/bin/phantomjs /usr/bin/

RUN git clone https://github.com/google/snappy /opt/snappy \
    && cd /opt/snappy \
    && mkdir build \
    && cd build/ && cmake ../ && make \
    && make install

RUN useradd -m -d /home/hadoop -s /bin/bash hadoop && echo hadoop:hadoop | chpasswd && adduser hadoop sudo
RUN echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers

USER hadoop
WORKDIR /home/hadoop

RUN curl -O -L https://github.com/liusheng/package/files/3904116/wildfly-openssl-1.0.7.Final.jar.tar.gz \
    && tar zxf wildfly-openssl-1.0.7.Final.jar.tar.gz \
    && mvn install:install-file -DgroupId=org.wildfly.openssl -Dfile=wildfly-openssl-1.0.7.Final.jar -DartifactId=wildfly-openssl -Dversion=1.0.7.Final -Dpackaging=jar \
    && rm wildfly-openssl-1.0.7.Final.jar*

RUN curl -L -s -S https://git.io/Je8K3 -o netty-all-4.1.27-linux-aarch64.jar \
    && mvn install:install-file -DgroupId=io.netty -Dfile=netty-all-4.1.27-linux-aarch64.jar -DartifactId=netty-all -Dversion=4.1.27.Final -Dpackaging=jar \
    && rm netty-all-4.1.27-linux-aarch64.jar

RUN sudo echo "umask 022" >> ~/.profile

# Install protoc-gen-grpc-java for ARM platform
RUN mkdir -p ~/.m2/repository/io/grpc/protoc-gen-grpc-java/1.15.1/ \
    && cd ~/.m2/repository/io/grpc/protoc-gen-grpc-java/1.15.1/ \
    && curl -L -O -s -S http://home.apache.org/~aajisaka/repository/io/grpc/protoc-gen-grpc-java/1.15.1/protoc-gen-grpc-java-1.15.1-linux-aarch_64.exe \
    && curl -L -O -s -S http://home.apache.org/~aajisaka/repository/io/grpc/protoc-gen-grpc-java/1.15.1/protoc-gen-grpc-java-1.15.1.pom \
    && curl -L -O -s -S http://home.apache.org/~aajisaka/repository/io/grpc/protoc-gen-grpc-java/1.15.1/protoc-gen-grpc-java-1.15.1.pom.sha1

RUN mkdir -p ~/hadoop-logs/

# RUN git clone https://github.com/apache/hadoop \
#     && cd hadoop \
#     && mvn clean install -e -B -Pdist,native -Dtar -DskipTests -Dmaven.javadoc.skip 2>&1 | tee ~/hadoop-logs/hadoop_build.log \
# ENV PATH "${PATH}:~/hadoop/hadoop-dist/target/hadoop-3.3.0-SNAPSHOT/bin"


CMD git clone https://github.com/apache/hadoop \
    && cd hadoop \
    && mvn clean install -e -B -Pdist,native -Dtar -DskipTests -Dmaven.javadoc.skip 2>&1 | tee ~/hadoop-logs/hadoop_build.log \
    && export PATH=~/hadoop/hadoop-dist/target/hadoop-3.3.0-SNAPSHOT/bin:$PATH \
    && cd hadoop-yarn-project/ \
    && mvn test -B -e -fn | tee ~/hadoop-logs/hadoop_yarn_test.log

hadoop_install_aarch64.sh

# Install patched protoc-2.5.0
apt-get install -y build-essential autoconf automake libtool cmake zlib1g-dev pkg-config libssl-dev libsasl2-dev
curl -Lo /opt/protobuf-2.5.0.tar.gz https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
tar zxf /opt/protobuf-2.5.0.tar.gz -C /opt/
wget -O /opt/protobuf-2.5.0/protobuf-2.5.0-arm64.patch https://www.dropbox.com/s/713wql5cw9dfxhx/protobuf-2.5.0-arm64.patch?dl=0
cd /opt/protobuf-2.5.0/ && patch -p1 < ./protobuf-2.5.0-arm64.patch
./autogen.sh
./configure
make install
ldconfig
protoc --version

# Install protoc-gen-grpc-java for ARM platform
mkdir -p ~/.m2/repository/io/grpc/protoc-gen-grpc-java/1.15.1/
pushd ~/.m2/repository/io/grpc/protoc-gen-grpc-java/1.15.1/
wget http://home.apache.org/~aajisaka/repository/io/grpc/protoc-gen-grpc-java/1.15.1/protoc-gen-grpc-java-1.15.1-linux-aarch_64.exe
wget http://home.apache.org/~aajisaka/repository/io/grpc/protoc-gen-grpc-java/1.15.1/protoc-gen-grpc-java-1.15.1.pom
wget http://home.apache.org/~aajisaka/repository/io/grpc/protoc-gen-grpc-java/1.15.1/protoc-gen-grpc-java-1.15.1.pom.sha1
popd