How to (properly) install HOOMD-blue into a virtual environment on Summit

hoomd_on_summit.txt

# Jens Glaser <[email protected]> 05/21/2020

# The goal is to avoid installing *any* precompiled binaries that are not
# supported by OLCF.  This is very important, as these usually come with their
# own versions of libstdc++, which pollute your environment and make it hard to
# track down ABI conflicts in your builds later on.

# we will therefore *avoid* conda, and use pip virtual environments

# for packages that support source compilation through pip, we will use that (pip install --no-binary <pkg> <pkg>)

# make sure you have the following modules loaded
$ module list

  1) hsi/5.0.2.p5   3) lsf-tools/2.0           5) DefApps        7) cuda/10.1.243   9) cmake/3.15.2
  2) xalt/1.2.0     4) darshan-runtime/3.1.7   6) python/3.7.0   8) gcc/7.4.0      10) spectrum-mpi/10.3.1.2-20200121

module save my_favorite_modules
echo "module restore my_favorite_modules" >> ~/.bashrc

# gcc 7.4.0 is the latest gcc version supported by CUDA on Summit
# we use gcc, not clang (clang is not fully supported on summit yet, plus it uses the stdc++ headers installed on the
# system for compatbility, which happen to be gcc 4.8.5's, which we don't want)
echo "export CC=\$OLCF_GCC_ROOT/bin/gcc" >> ~/.bashrc # can't rely on the right gcc being in the PATH
echo "export CXX=\$OLCF_GCC_ROOT/bin/g++" >> ~/.bashrc
source ~/.bashrc

# create and activate a virtualenv somewhere in /ccs/proj. Replace <project_id> with your OLCF project id (no caps)
echo "export PROJ_ID=<project_id>" >> ~/.bashrc
source ~/.bashrc
mkdir -p /ccs/proj/$PROJ_ID/$USER/envs
python -m venv /ccs/proj/$PROJ_ID/$USER/envs/hoomd
echo ". /ccs/proj/\${PROJ_ID}/$USER/envs/hoomd/bin/activate" >> ~/.bashrc
echo "export LD_LIBRARY_PATH=\${VIRTUAL_ENV}/lib:\${LD_LIBRARY_PATH}" >> ~/.bashrc
source ~/.bashrc

# numpy
pip install --no-binary numpy numpy

# scipy
pip install --no-binary cython cython
mkdir -p /tmp/$USER
cd /tmp/$USER
git clone https://github.com/scipy/scipy.git
cd scipy
module load netlib-lapack
export LAPACK=$OLCF_NETLIB_LAPACK_ROOT/lib64/liblapack.so
export BLAS=$OLCF_NETLIB_LAPACK_ROOT/lib64/libcblas.so
python setup.py install

# eigen
mkdir -p /tmp/$USER
cd /tmp/$USER
git clone https://gitlab.com/libeigen/eigen.git
cd eigen
mkdir -p build
cd build
export CMAKE_PREFIX_PATH=$VIRTUAL_ENV
cmake -D CMAKE_INSTALL_PREFIX=$VIRTUAL_ENV ..
make install

# pybind11
# (the pypi version is currently missing CMake config files)
mkdir -p /tmp/$USER
cd /tmp/$USER
pip install --no-binary pytest pytest
git clone https://github.com/pybind/pybind11.git
cd pybind11
mkdir -p build
cd build
export CMAKE_PREFIX_PATH=$VIRTUAL_ENV
cmake -DPYBIND11_TEST=OFF -D CMAKE_INSTALL_PREFIX=$VIRTUAL_ENV ..
make install

# cereal
mkdir -p /tmp/$USER
cd /tmp/$USER
git clone https://github.com/USCiLab/cereal.git
cd cereal
mkdir -p build
cd build
export CMAKE_PREFIX_PATH=$VIRTUAL_ENV
cmake -DSKIP_PORTABILITY_TEST=ON -D CMAKE_INSTALL_PREFIX=$VIRTUAL_ENV ..
make -j 8 install

#
# for added speed of compilation, we install ninja
# this is optional, if ninja is not available just don't use it as a makefile tool (-Gninja) with cmake later on
pip install scikit-build
mkdir -p /tmp/$USER
cd /tmp/$USER
git clone https://github.com/ninja-build/ninja
cd ninja
./configure.py --bootstrap
cp ninja $VIRTUAL_ENV/bin

# *If* we want to use the jit feature in HOOMD, we also need LLVM and clang (in addition to gcc).
# Otherwise, skip this step (and set -DBUILD_JIT=OFF later).

# Unfortunately the llvm module on summit is mutually exclusive with the gcc module, so we have
# to build llvm from source (yay!).

# Warning: due to cgroups limitations on Summit, we cannot compile too many
# projects in /tmp in one session. It may be a good idea to logout from
# **all** sessions on that particular login node *before* this step, and login
# again. Pro Tip: you can ssh to a specific login node directly, e.g.,
# login1.summit.olcf.ornl.gov

# install llvm/clang
mkdir -p /tmp/$USER
cd /tmp/$USER
git clone https://github.com/llvm/llvm-project
cd llvm-project
git checkout release/10.x
mkdir -p build
cd build
cmake -D CMAKE_INSTALL_PREFIX=$VIRTUAL_ENV -DLLVM_ENABLE_PROJECTS=clang -GNinja \
    -DBUILD_SHARED_LIBS=ON \ # necessary for HOOMD's jit module
    -DCMAKE_BUILD_TYPE=Release \ # important, the default is Debug
    -DLLVM_TARGETS_TO_BUILD="PowerPC" \ # cuts compilation time
#    -DCMAKE_EXE_LINKER_FLAGS="-Wl,--hash-size=23" \ # these linker options are an attempt at reducing the memory footprint
#    -DCMAKE_SHARED_LINKER_FLAGS="-Wl,--hash-size=23" \
    ../llvm
ninja -j 8 install # may have to adjust

# Build HOOMD-blue
cd /ccs/proj/$PROJ_ID/$USER
git clone https://github.com/glotzerlab/hoomd-blue.git
cd hoomd-blue
git checkout next # check out your desired branch
git submodule update --init
export HOOMD_SOURCE_DIR=`pwd`/hoomd-blue
mkdir -p /tmp/$USER/hoomd-build
cd /tmp/$USER/hoomd-build
CMAKE_PREFIX_PATH=$VIRTUAL_ENV cmake -D ENABLE_GPU=ON -D BUILD_JIT=ON -D ENABLE_TBB=OFF -D ENABLE_MPI=ON \
    -DENABLE_MPI_CUDA=ON -D BUILD_MD=OFF -D BUILD_METAL=OFF -D BUILD_MPCD=OFF -DBUILD_DEM=OFF \
    $HOOMD_SOURCE_DIR
make -j 4 install