Easy installation and test of xgboost scripts for hadoop/hive/hivemall on ubuntu16.04/arm64

1-host_prepare_qemu.sh

#!/bin/sh

# http://www.mztn.org/dragon/arm64_01.html

mkdir -p qemu-arm64
cd qemu-arm64
wget http://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-arm64-uefi1.img

sudo qemu-img resize xenial-server-cloudimg-arm64-uefi1.img +10G

sudo modprobe nbd max_part=63
sudo qemu-nbd -c /dev/nbd0 xenial-server-cloudimg-arm64-uefi1.img
mkdir mnt
sudo mount /dev/nbd0p1 mnt
 
sudo cp mnt/boot/vmlinuz-4.4.0-89-generic .
sudo cp mnt/boot/initrd.img-4.4.0-89-generic .
sudo chown ito: *-generic

sudo umount mnt
sudo qemu-nbd -d /dev/nbd0

qemu-system-aarch64 -m 2048 -cpu cortex-a57 -nographic -machine virt \
 -kernel vmlinuz-4.4.0-89-generic \
 -append 'root=/dev/vda1 rw rootwait mem=2048M console=ttyS0 \
  console=ttyAMA0,38400n8 init=/usr/lib/cloud-init/uncloud-init \
  ds=nocloud ubuntu-pass=upass' \
 -drive if=none,id=image,file=xenial-server-cloudimg-arm64-uefi1.img \
 -initrd initrd.img-4.4.0-89-generic \
 -device virtio-blk-device,drive=image \
 -netdev user,id=user0 \
 -device virtio-net-device,netdev=user0 \
 -redir tcp:10022::22

2-vm_init.sh

#!/bin/sh

# ubuntu/upass

sudo su -
wget https://gist.github.com/amaya382/3206f8a9fc469083a0a0904e1eea1a0f/raw/a5d49ba69900bdda2be90915eeede92482f0f243/3-vm_install_hadoop_hive_hivemall.bash
chmod +x 3-vm_install_hadoop_hive_hivemall.bash
./3-vm_install_hadoop_hive_hivemall.bash

# scp -P 10022 /path/to/libxgboost4j.so ubuntu@localhost:~/
# mkdir -p /opt/hivemall/xgboost/src/main/resources/lib/linux-arm64
# mv /home/ubuntu/libxgboost4j.so /opt/hivemall/xgboost/src/main/resources/lib/linux-arm64

# Using ports: 8088 19888 50070
export JAVA_HOME='/usr/lib/jvm/java-8-openjdk-arm64' && export HADOOP_HOME=/usr/local/hadoop && export HIVE_HOME=/usr/local/hive && export PATH=${HADOOP_HOME}/bin:${HIVE_HOME}/bin:${PATH} && export HADOOP_OPTS='-Dsystem:java.io.tmpdir=/tmp -Dsystem:user.name=root -Dderby.stream.error.file=/root/derby.log -Djava.library.path=${HADOOP_HOME}/lib' && export HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_HOME}/lib/native && /root/bin/init.sh && /root/bin/prepare_iris.sh && hive

3-vm_install_hadoop_hive_hivemall.bash

#!/bin/bash

cd /root/

PREBUILD=true
HADOOP_VERSION=2.7.3
HIVE_VERSION=2.1.1
HIVEMALL_REPOSITORY='https://github.com/amaya382/incubator-hivemall.git'
HIVEMALL_BRANCH='cross-compiling'
BASE_URL='https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename='

export JAVA_HOME='/usr/lib/jvm/java-8-openjdk-arm64'
export HADOOP_HOME='/usr/local/hadoop'
export HIVE_HOME='/usr/local/hive'
export HIVEMALL_PATH='/opt/hivemall'
export HADOOP_OPTS='-Dsystem:java.io.tmpdir=/tmp -Dsystem:user.name=root -Dderby.stream.error.file=/root/derby.log'
export PATH="${HADOOP_HOME}/bin:${HIVE_HOME}/bin:${PATH}"

set -eux && \
    apt update && \
    apt install -y --no-install-recommends openjdk-7-jdk git openssh-server maven g++ make && \
    \
    git clone -b ${HIVEMALL_BRANCH} ${HIVEMALL_REPOSITORY} ${HIVEMALL_PATH} && \
    \
    wget ${BASE_URL}hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz -O - \
      | tar xz && \
    mv -f hadoop-${HADOOP_VERSION} ${HADOOP_HOME} && \
    sed -i -e 's!${JAVA_HOME}!'"${JAVA_HOME}!" ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh && \
    [ ! -e ~/.ssh/id_rsa ] && ssh-keygen -q -P '' -f ~/.ssh/id_rsa || : && \
    cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && \
    [ ! -e ~/.ssh/config ] && echo 'host *\n  StrictHostKeyChecking no' > ~/.ssh/config || : &&
    cp -f ${HIVEMALL_PATH}/resources/docker/etc/hadoop/*.xml ${HADOOP_HOME}/etc/hadoop && \
    hdfs namenode -format && \
    \
    wget ${BASE_URL}hive/hive-${HIVE_VERSION}/apache-hive-${HIVE_VERSION}-bin.tar.gz -O - \
      | tar xz && \
    mv -f apache-hive-${HIVE_VERSION}-bin ${HIVE_HOME} && \
    cat ${HIVE_HOME}/conf/hive-default.xml.template \
      | sed -e 's!databaseName=metastore_db!databaseName=/root/metastore_db!' \
      > ${HIVE_HOME}/conf/hive-site.xml && \
    \
    cd ${HIVEMALL_PATH} && \
    HIVEMALL_VERSION=`cat VERSION` && \
    \
    (if ${PREBUILD}; then \
      cd ${HIVEMALL_PATH} && bin/build.sh; \
    fi) && \
    \
    mkdir -p /root/bin /root/hivemall && \
    find ${HIVEMALL_PATH}/resources/docker/home/bin -mindepth 1 -maxdepth 1 \
      -exec sh -c 'f={} && ln -s $f /root/bin/${f##*/}' \; && \
    ln -s ${HIVEMALL_PATH}/resources/docker/home/.hiverc /root && \
    ln -s ${HIVEMALL_PATH}/resources/ddl/define-all.hive /root/hivemall/define-all.hive && \
    ln -s ${HIVEMALL_PATH}/target/hivemall-core-${HIVEMALL_VERSION}-with-dependencies.jar \
      /root/hivemall/hivemall-core-with-dependencies.jar

4-hive_trial_operation.hql

add jar /opt/hivemall/target/hivemall-xgboost-0.60-0.4.2-rc.2-with-dependencies.jar;
source /opt/hivemall/resources/ddl/define-additional.hive;

set hivevar:f0_min=4.3;
set hivevar:f0_max=7.9;
set hivevar:f1_min=2.0;
set hivevar:f1_max=4.4;
set hivevar:f2_min=1.0;
set hivevar:f2_max=6.9;
set hivevar:f3_min=0.1;
set hivevar:f3_max=2.5;

use iris;
create or replace view iris_scaled
as
select
  rowid, 
  label,
  add_bias(array(
     concat("1:", rescale(features[0],${hivevar:f0_min},${hivevar:f0_max})), 
     concat("2:", rescale(features[1],${hivevar:f1_min},${hivevar:f1_max})), 
     concat("3:", rescale(features[2],${hivevar:f2_min},${hivevar:f2_max})), 
     concat("4:", rescale(features[3],${hivevar:f3_min},${hivevar:f3_max}))
  )) as features
from 
  iris_raw;

-- select * from iris_scaled limit 3;
-- 1       Iris-setosa     ["1:0.22222215","2:0.625","3:0.0677966","4:0.041666664","0:1.0"]
-- 2       Iris-setosa     ["1:0.16666664","2:0.41666666","3:0.0677966","4:0.041666664","0:1.0"]
-- 3       Iris-setosa     ["1:0.11111101","2:0.5","3:0.05084745","4:0.041666664","0:1.0"]

select train_xgboost_classifier(features, case when label = 'Iris-setosa' then 1.0 else 0.0 end) from iris_scaled;