Setup Custom Python/TensorFlow env with Miniconda3 at KISTI Nurion HPC (KNL/SKL)

setup_custom_tensorflow_at_nurion.sh

#!/bin/bash

# 0. import required modules
module add gcc/7.2.0 openmpi/3.1.0 htop/2.2.0

# 1. install Miniconda3 (Miniconda3-4.5.4/Python 3.6.5)
# refer: https://repo.continuum.io/miniconda/
# add miniconda PATH in ~/.bashrc
wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh
chmod +x Miniconda3-4.5.4-Linux-x86_64.sh
./Miniconda3-4.5.4-Linux-x86_64.sh

# 2. setup basic python packages (for tf) via conda install
# we use numpy v1.16.3
# we use tensorflow r1.12.2 / pre-install public tf binary first for requirements installation
# refer proj4: https://anaconda.org/conda-forge/proj4/files
conda install -c conda-forge numpy==1.16.3 matplotlib scipy opencv cython pandas geopy geos proj4==4.9.3 scikit-learn scikit-image netcdf4 h5py tensorflow==1.12.0 numexpr sympy
pip install cartopy
pip uninstall tensorflow

# 3. setup bazel for custom tf build
# we use bazel v0.19.1 for tf r1.12.2 build
# refer bazel: https://github.com/bazelbuild/bazel/releases/tag/0.19.1
# source bazel configuration in ~/.bashrc: source ${HOME}/.bazel/bin/bazel-complete.bash
#
# default home directory quota of Nurion is poor -> must initialize bazel temp dir on /tmp,
# by export TEST_TMPDIR=/tmp/bazeltmp (refer: https://docs.bazel.build/versions/master/output_directories.html)
export TEST_TMPDIR=/tmp/bazeltmp
# wget https://github.com/bazelbuild/bazel/releases/download/0.19.1/bazel-0.19.1-installer-linux-x86_64.sh
wget https://github.com/bazelbuild/bazel/releases/download/0.19.2/bazel-0.19.2-installer-linux-x86_64.sh
chmod +x bazel-0.19.2-installer-linux-x86_64.sh
./bazel-0.19.2-installer-linux-x86_64.sh --user

# 4. get & build tensorflow source (r1.12.2) with Intel MKL
# refer: https://github.com/tensorflow/tensorflow/releases/tag/v1.12.2
# YOU MUST BUILD tensorflow on computing node (either SKL or KNL) by qsub interactive mode (due to the dirty bash on login node)
# call-knl: qsub -I -l select=1:ncpus=68 -q normal -A etc
# call-skl: qsub -I -l select=1:ncpus=40 -q norm_skl -A etc
# from now on, we're going to work on $HOME/tfbuild
# DO NOT FORGET export TEST_TMPDIR=/tmp/bazeltmp, otherwise you MUST experience a quota limitation (total num. of files)
# refer common CPU instructions for KNL/SKL while configure: -msse4.1 -msse4.2 -mavx -mavx2 -mfma -mavx512f
mkdir ~/tfbuild && cd ~/tfbuild
wget https://github.com/tensorflow/tensorflow/archive/v1.12.3.tar.gz -O tensorflow-v1.12.3.tgz
tar -xf tensorflow-v1.12.3.tgz
cd tensorflow-1.12.3
./configure
bazel build --copt="-DEIGEN_USE_VML" --config=opt --config=mkl //tensorflow/tools/pip_package:build_pip_package --verbose_failures
bazel-bin/tensorflow/tools/pip_package/build_pip_package ..
pip install ../tensorflow-1.12.3-cp36-cp36m-linux_x86_64.whl

# 5. test tensorflow session initialization over Intel MKL
python -c "import tensorflow as tf;sess = tf.Session();print(sess);"