Building HPX on SuperMIC

README.md

Building HPX on SuperMIC

Why this script exists?

I couldn't build HPX on SuperMIC and was getting errors. @sithhell wasn't convinced so I wrote this script for him to be able to reproduce my problem.

Explanation:

SuperMIC's Boost module being loaded was the issue. It was adding itself to CPATH and that was forcing g++ to use the headers of the module in place of our Boost.

Solution:

Not using Boost modules or any solution that adds Boost to CPATH and confuses the compiler

Running

To run this script inside an interactive job, first obtain a session:

qsub -I -V ${DISPLAY:+-X} -l nodes=1:ppn=20 -l walltime=00:30:00

and then in the obtained session:

bash <(curl -sL https://gist.githubusercontent.com/parsa/1cdee2bc2f05d9d75df71c581c2620ac/raw/build_hpx.sh)

build_boost.sh

#!/bin/bash
#
# Copyright (c) 2009-2012 Bryce Adelstein-Lelbach
# Copyright (c) 2016 Parsa Amini
#
# Distributed under the Boost Software License, Version 1.0. (See accompanying
# file BOOST_LICENSE_1_0.rst or copy at http://www.boost.org/LICENSE_1_0.txt)

# Short Url for this file: git.io/vXPXb

usage()
{
    echo "Usage: $0 -d directory -v version [args]"
    echo
    echo "This script downloads and builds the Boost C++ Libraries."
    echo
    echo "Options:"
    echo "  -d    Directory where Boost should be built."
    echo "  -v    Version of Boost to build (format: X.YY.Z)"
    echo "  -n    Don't download Boost (expects tarball named boost_X.YY.Z.tar.bz2 in the -d directory) [default: 1.55.0]"
    echo "  -x    Libraries to exclude (format: exclude0,exclude1...) [default: mpi,graph_parallel,python]"
    echo "  -t    Number of threads to use while building [default: number of processors]"
    echo "  -c    Compiler [default: automatically detected]"
}

DIRECTORY=

# Dot version, e.g. X.YY.Z
DOT_VERSION=1.55.0

# Underscore version, e.g. X_YY_Z
US_VERSION=1_55_0

DOWNLOAD=1

# HPX does not need these, and they have external dependencies, so skip them.
EXCLUDES=mpi,graph_parallel,python

# Physical, not logical cores.
THREADS=`grep -c ^processor /proc/cpuinfo`

COMPILER=

CAT_PIPE=cat
if [[ -f "$(which pv)" ]]; then
    CAT_PIPE=pv
fi

###############################################################################
# Argument parsing
while getopts "hnt:d:v:c:x:" OPTION; do case $OPTION in
    h)
        usage
        exit 0
        ;;
    n)
        DOWNLOAD=0
        ;;
    d)
        # Try to make the directories.
        mkdir -p $OPTARG/release > /dev/null 2>&1
        mkdir -p $OPTARG/debug > /dev/null 2>&1

        if [[ -d $OPTARG/release && -w $OPTARG/release ]] && \
           [[ -d $OPTARG/debug   && -w $OPTARG/debug   ]];
        then
            DIRECTORY=$OPTARG
        else
            echo "ERROR: -d argument was invalid"; echo
            usage
            exit 1
        fi
        ;;
    v)
        if [[ $OPTARG =~ ^[0-9][.][0-9][0-9][.][0-9]$ ]]; then
            DOT_VERSION=$OPTARG
            US_VERSION=${OPTARG//./_}
        else
            echo "ERROR: -v argument was invalid"; echo
            usage
            exit 1
        fi
        ;;
    x)
        EXCLUDES=$OPTARG
        ;;
    t)
        if [[ $OPTARG =~ ^[0-9]+$ ]]; then
            THREADS=$OPTARG
        else
            echo "ERROR: -t argument was invalid"; echo
            usage
            exit 1
        fi
        ;;
    c)
        COMPILER=$OPTARG
        ;;
    ?)
        usage
        exit 1
        ;;
esac; done

if ! [[ $DIRECTORY ]]; then
    echo "ERROR: no directory specified"; echo
    usage
    exit 1
fi

if ! [[ $DOT_VERSION && $US_VERSION ]]; then
    echo "ERROR: no version specified"; echo
    usage
    exit 1
fi

if [[ $EXCLUDES ]]; then
    EXCLUDES="--without-libraries=$EXCLUDES"
fi

if [[ $COMPILER ]]; then
    COMPILER="--with-toolset=$COMPILER"
fi

###############################################################################
DIRECTORY=$(cd $DIRECTORY 2>&1 >/dev/null; pwd)
ORIGINAL_DIRECTORY=$PWD

BJAM=$DIRECTORY/source/b2

error()
{
    cd $ORIGINAL_DIRECTORY
    exit 1
}

cd $DIRECTORY

if [[ $DOWNLOAD == "1" ]]; then
    curl -LO downloads.sourceforge.net/sourceforge/boost/boost/$DOT_VERSION/boost_$US_VERSION.tar.bz2
    if ! [[ $? == "0" ]]; then echo "ERROR: Unable to download Boost"; error; fi
fi

$CAT_PIPE boost_$US_VERSION.tar.bz2 | tar --no-same-owner -xjf -
if ! [[ $? == "0" ]]; then echo "ERROR: Unable to unpack `pwd`/boost_$US_VERSION.tar.bz2"; error; fi

mv boost_$US_VERSION source

cd $DIRECTORY/source

# Boostrap the Boost build system, Boost.Build.
$DIRECTORY/source/bootstrap.sh $EXCLUDES $COMPILER

$BJAM --stagedir=$DIRECTORY/debug/stage variant=debug -j${THREADS}
if ! [[ $? == "0" ]]; then echo "ERROR: Debug build of Boost failed"; error; fi

$BJAM --stagedir=$DIRECTORY/release/stage variant=release -j${THREADS}
if ! [[ $? == "0" ]]; then echo "ERROR: Release build of Boost failed"; error; fi

# Build the Boost.Wave preprocessor.
cd $DIRECTORY/source/tools/wave/build
$BJAM dist-bin -j${THREADS} variant=release

# Build the Quickbook documentation framework.
cd $DIRECTORY/source/tools/quickbook
$BJAM dist-bin -j${THREADS} variant=release

# Copy over the BoostBook DTD and XML code to the staging directory.
cd $DIRECTORY/source/tools
$BJAM dist-share-boostbook

# Build the auto_index indexing tool.
cd $DIRECTORY/source/tools/auto_index/build
$BJAM i -j${THREADS} variant=release

# These links are necessary to ensure that the stage directories are usable
# Boost source trees.
create_links()
{
    ln -fs $DIRECTORY/source/bjam bjam
    ln -fs $DIRECTORY/source/boost boost
    ln -fs $DIRECTORY/source/boost-build.jam boost-build.jam
    ln -fs $DIRECTORY/source/boostcpp.jam boostcpp.jam
    ln -fs $DIRECTORY/source/boost.css boost.css
    ln -fs $DIRECTORY/source/boost.png boost.png
    ln -fs $DIRECTORY/source/dist dist
    ln -fs $DIRECTORY/source/doc doc
    ln -fs $DIRECTORY/source/index.htm index.htm
    ln -fs $DIRECTORY/source/index.html index.html
    ln -fs $DIRECTORY/source/Jamroot Jamroot
    ln -fs $DIRECTORY/source/libs libs
    ln -fs $DIRECTORY/source/LICENSE_1_0.txt LICENSE_1_0.txt
    ln -fs $DIRECTORY/source/project-config.jam project-config.jam
    ln -fs $DIRECTORY/source/rst.css rst.css
    ln -fs $DIRECTORY/source/tools tools
}

cd $DIRECTORY/debug
create_links

cd $DIRECTORY/release
create_links

echo
echo "Successfully built Boost ${DOT_VERSION}"
echo
echo "Debug root:"
echo "  BOOST_ROOT=$DIRECTORY/debug"
echo
echo "Release root:"
echo "  BOOST_ROOT=$DIRECTORY/release"

build_hpx.sh

#!/usr/bin/env bash
#
# Copyright (c) 2016 Parsa Amini
#
# Distributed under the Boost Software License, Version 1.0. (See accompanying
# file BOOST_LICENSE_1_0.rst or copy at http://www.boost.org/LICENSE_1_0.txt)

# Short Url for this file: git.io/vXPXR

# Description {{{ #
#  Problem:
#    HPX builds were failing.
#
#  Explanation:
#    SuperMIC's Boost module being loaded was the issue. It was adding
#    itself to CPATH and that was forcing g++ to use the headers of the
#    module in place of our Boost.
#
#  Solution:
#    Not using Boost modules or any solution that adds Boost to CPATH and
#    confuses the compiler
#
# To run this script inside an interactive job:
#   qsub -I -l nodes=1:ppn=20 -l walltime=00:30:00
# and then in the obtained session:
#   bash <(curl -s https://gist.githubusercontent.com/parsa/1cdee2bc2f05d9d75df71c581c2620ac/raw/build_hpx.sh)
# }}} Description #

# Parameters {{{ #
# For debugging
set -x

# Working directory for this program
export WORK="/worka/work/$USER"
# Bryce's Boost building script
export BOOST_BUILDER_URL="https://gist.githubusercontent.com/parsa/1cdee2bc2f05d9d75df71c581c2620ac/raw/build_boost.sh"
# Where Boost will be available at after installation
export BOOST_PATH="$WORK/hbf/boost_1_61_0_gcc_4_9/debug"
# HPX's HTTPS Git URL
export HPX_GIT_URL="https://github.com/stellar-group/hpx.git"
# }}} Parameters #

# Resolve Relative to Absolute Paths {{{ #
function resolve_path() {
    echo $(cd ${1} 2>&1 >/dev/null && pwd)
}
# }}} Resolve Relative to Absolute Paths #

# Prepare Environment {{{ #
function prep_env() {
    # Death to all loaded modules
    module purge
    # Load the bare minimum we need
    module load cmake/2.8.12/INTEL-14.0.2 \
                gcc/4.9.0 \
                hwloc/1.10.0/INTEL-14.0.2 \
                mvapich2/2.0/INTEL-14.0.2

    #export PATH="/usr/local/compilers/gcc/4.9.0/bin:/usr/local/compilers/Intel/parallel_studio_xe_2015/impi/5.0.1.035/intel64/bin:/usr/lib64/qt-3.3/bin:/usr/local/packages/hwloc/1.10.0/INTEL-14.0.2/bin:/usr/local/packages/license/allinea/4.2.1/bin:/usr/local/packages/cmake/2.8.12/INTEL-14.0.2/bin:/usr/local/packages/python/2.7.7-anaconda/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/usr/local/bin:/usr/local/compilers/pgi/linux86-64/6.1/bin:/usr/local/compilers/Intel/parallel_studio_xe_2015/composer_xe_2015.0.090/bin/intel64:/opt/ibutils/bin:/opt/dell/srvadmin/bin:"
    #export LIBRARY_PATH="/usr/local/packages/mvapich2/2.0/INTEL-14.0.2/lib:/usr/local/compilers/gcc/4.9.0/lib64:/usr/local/compilers/gcc/4.9.0/lib:/usr/local/packages/hwloc/1.10.0/INTEL-14.0.2//lib:/usr/local/packages/boost/1.55.0/INTEL-14.0.2-python-2.7.7-anaconda/lib:/usr/local/packages/python/2.7.7-anaconda/lib:/usr/local/compilers/Intel/parallel_studio_xe_2015/composer_xe_2015.0.090/compiler/lib/intel64:/usr/local/compilers/Intel/parallel_studio_xe_2015/composer_xe_2015.0.090/mkl/lib/intel64"
}
# }}} Prepare Environment #

# Get Ninja {{{ #
function setup_ninja() {
    if [[ ! -f "$(which ninja)" ]]; then
        # Get Ninja
        curl -sLo ninja.zip "https://github.com/ninja-build/ninja/releases/download/v1.7.1/ninja-linux.zip"
        unzip ninja.zip
        
        # Add ninja executable to path
        [[ :"${PATH}": == *:"${PWD}":* ]] && export PATH="${PWD}:${PATH}"
    fi
}
# }}} Get Ninja #

# Boost via Bryce {{{ #
function setup_boost() {
    # Boost via Bryce
    curl -sLo "build_boost.sh" "$BOOST_BUILDER_URL"
    chmod +x build_boost.sh
    ./build_boost.sh -c "gcc cxxflags=-std=c++11" -v 1.61.0 -t 22 -d boost_1_61_0_gcc_4_9
    return $?
}
# }}} Boost via Bryce #

# Get HPX {{{ #
function setup_hpx() {
    # No need to clone the whole repository
    git clone --depth=1 "$HPX_GIT_URL" repo

    mkdir build
    pushd build

    local cmake_args=$(cat <<-EOT
        -DCMAKE_BUILD_TYPE=Debug
        -DCMAKE_CXX_FLAGS=-std=c++11
        -DCMAKE_C_COMPILER=gcc
        -DCMAKE_CXX_COMPILER=g++
        -DBOOST_ROOT=$BOOST_PATH
        -DBOOST_INCLUDEDIR=$BOOST_PATH/source
        -DBOOST_LIBRARYDIR=$BOOST_PATH/stage/lib
        -DBoost_NO_SYSTEM_PATHS=True
        -DCMAKE_EXPORT_COMPILE_COMMANDS=True
        -DHPX_WITH_MALLOC=custom
        -DHPX_WITH_PARCELPORT_MPI=True
        -DHPX_WITH_PARCELPORT_TCP=False
        -DHPX_WITH_PARCELPORT_IBVERBS=False
        -DHPX_WITH_EXAMPLES=True
        -GNinja
EOT
    )

    cmake $cmake_args $(resolve_path ../repo)
    ninja core
}
# }}} Get HPX #

# Prepare Working Directory {{{ #
function prep_workdir() {
    # Working directory
    pushd $WORK
    mkdir hbf
    pushd hbf
}
# }}} Prepare Working Directory #

prep_env
prep_workdir
setup_ninja
setup_boost || exit 1
setup_hpx