Build script for universal utm python bindings for Linux

utm-manylinux.sh

#
# Build script for distribution independent binary Python wheels
# for utm library on Linux.
#
# Install Docker
#
#    sudo apt install docker
#    sudo usermod -a -G docker $USER  # logout required
#
# Pull manylinux image
#
#    docker pull quay.io/pypa/manylinux1_x86_64
#
# Run manylinux image
#
#    docker run -i -t -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 /bin/bash
#
# Build bindings and binary wheels for Python versions 3.5, 3.6, 3.7 and 3.8
#
#    ># cd /io
#    ># bash utm-manylinux.sh
#

UTM_VERSION="0.7.3"
UTM_MODULES="tm-table tm-grammar tm-eventsetup"
UTM_GITBASE="https://gitlab.cern.ch/cms-l1t-utm/utm.git"
UTM_MODULES_GITBASE="https://github.com/cms-l1-globaltrigger"

YUM_DEPENDENCIES="autoconf automake make glibc-devel kernel-headers gcc gcc-c++ zlib-devel pcre-devel libffi-devel"

PYTHON35_VERSION="3.5.7"
PYTHON36_VERSION="3.6.8"
PYTHON37_VERSION="3.7.4"
PYTHON38_VERSION="3.8.0"
PYTHON_VERSIONS="$PYTHON35_VERSION $PYTHON36_VERSION $PYTHON37_VERSION $PYTHON38_VERSION"
PERL_VERSION="5.30.0"
SSL_VERSION="1.1.1c"
XERCES_VERSION="3.2.2"
SWIG_VERSION="4.0.1"
BOOST_VERSION="1.71.0"

LOCKS_DIR=/tmp
BUILD_DIR=build

set -e

# Print colored info message
function _info {
    echo -e "\033[33;1m:: $@\033[0m"
}

# Downloads source tarball if not present
function bootstrap {
    fname=$(basename -- "$1")
    if [ ! -f "$fname" ]
    then
        curl -L $1 --output $fname
    fi
}

mkdir -p $LOCKS_DIR
mkdir -p $BUILD_DIR
cd $BUILD_DIR

_info "installing dependencies..."
yum install -y $YUM_DEPENDENCIES

_info "installing perl $PERL_VERSION..."
if [ ! -f $LOCKS_DIR/PERL.LOCK ]
then

    # Perl
    bootstrap https://www.cpan.org/src/5.0/perl-$PERL_VERSION.tar.gz
    if [ ! -d perl-$PERL_VERSION ]
    then
        tar xzf perl-$PERL_VERSION.tar.gz
    fi
    cd perl-$PERL_VERSION
    if [ ! -f BUILD.LOCK ]
    then
        ./Configure -des
        make
        touch BUILD.LOCK
    fi
    make install
    cd ..

    touch $LOCKS_DIR/PERL.LOCK

fi

_info "installing openssl $SSL_VERSION..."
if [ ! -f $LOCKS_DIR/OPENSSL.LOCK ]
then

    # openssl
    bootstrap https://www.openssl.org/source/openssl-$SSL_VERSION.tar.gz
    if [ ! -d openssl-$SSL_VERSION ]
    then
        tar xzf openssl-$SSL_VERSION.tar.gz
    fi
    cd openssl-$SSL_VERSION
    if [ ! -f BUILD.LOCK ]
    then
        ./config
        make
        touch BUILD.LOCK
    fi
    make install
    cd ..

    touch $LOCKS_DIR/OPENSSL.LOCK

fi

_info "creating openssl links for Python3..."
cd /usr/local/ssl
if [ ! -e include ]
then
    ln -s /usr/local/include include
fi
if [ ! -e lib ]
then
    ln -s /usr/local/lib64 lib
fi
cd -

_info "installing boost $BOOST_VERSION..."
if [ ! -f $LOCKS_DIR/BOOST.LOCK ]
then

    # build boost
    BOOST_VERSION_STR=${BOOST_VERSION//./_}
    bootstrap https://dl.bintray.com/boostorg/release/$BOOST_VERSION/source/boost_$BOOST_VERSION_STR.tar.gz
    if [ ! -d boost_$BOOST_VERSION_STR ]
    then
        tar xzf boost_$BOOST_VERSION_STR.tar.gz
    fi
    cd boost_$BOOST_VERSION_STR
    if [ ! -f BUILD.LOCK ]
    then
        ./bootstrap.sh --without-libraries=python
        ./b2 --with-filesystem --with-system
        touch BUILD.LOCK
    fi
    ./b2 install
    cd ..

    touch $LOCKS_DIR/BOOST.LOCK
fi

_info "installing xerces-c $XERCES_VERSION..."
if [ ! -f $LOCKS_DIR/XERCES.LOCK ]
then

    # build xerces
    XERCES_VERSION_STR=${XERCES_VERSION//./_}
    bootstrap https://github.com/apache/xerces-c/archive/Xerces-C_$XERCES_VERSION_STR.tar.gz
    if [ ! -d xerces-c-Xerces-C_$XERCES_VERSION_STR ]
    then
        tar xzf Xerces-C_$XERCES_VERSION_STR.tar.gz
    fi
    cd xerces-c-Xerces-C_$XERCES_VERSION_STR
    if [ ! -f BUILD.LOCK ]
    then
        ./reconf
        ./configure
        make
        touch BUILD.LOCK
    fi
    make install
    cd ..

    touch $LOCKS_DIR/XERCES.LOCK
fi

_info "installing SWIG $SWIG_VERSION..."
if [ ! -f $LOCKS_DIR/SWIG.LOCK ]
then

    # swig
    bootstrap https://github.com/swig/swig/archive/rel-$SWIG_VERSION.tar.gz
    if [ ! -d swig-rel-$SWIG_VERSION ]
    then
        tar xzf rel-$SWIG_VERSION.tar.gz
    fi
    cd swig-rel-$SWIG_VERSION
    if [ ! -f BUILD.LOCK ]
    then
        ./autogen.sh
        ./configure
        make
        touch BUILD.LOCK
    fi
    make install
    cd ..

    touch $LOCKS_DIR/SWIG.LOCK
fi

# Python
for version in $PYTHON_VERSIONS
do

    _info "installing Python $version..."
    if [ ! -f $LOCKS_DIR/PYTHON$version.LOCK ]
    then
        bootstrap https://www.python.org/ftp/python/$version/Python-$version.tgz
        if [ ! -d Python-$version ]
        then
            tar xzf Python-$version.tgz
        fi
        cd Python-$version
        if [ ! -f BUILD.LOCK ]
        then
            ./configure
            make
            touch BUILD.LOCK
        fi
        make altinstall
        cd ..
        touch $LOCKS_DIR/PYTHON$version.LOCK
    fi

done

export BOOST_BASE=/io/boost_$BOOST_VERSION_STR
export XERCES_C_BASE=/io/xerces-c-Xerces-C_$XERCES_VERSION
export XERCES_LIB_DIR=$XERCES_C_BASE/lib

_info "building utm $UTM_VERSION..."
if [ ! -f $LOCKS_DIR/UTM.LOCK ]
then
    # utm
    rm -rf utm
    git clone $UTM_GITBASE utm
    cd utm
    git checkout utm_$UTM_VERSION
    # HACK: build utm with SWIG support!
    make all CPPFLAGS="-DNDEBUG -DSWIG"
    cd ..
    touch $LOCKS_DIR/UTM.LOCK
fi

for version in $PYTHON_VERSIONS
do
    version="${version%.*}"

    _info "building wheels for python $version..."

    rm -rf bindings_$version
    mkdir bindings_$version
    cd bindings_$version

    python$version -m venv env_$version
    . env_$version/bin/activate
    pip install --upgrade pip
    pip install wheel==0.31.1 # HACK broken auditwheel
    pip install auditwheel
    . ../utm/setup.sh # source utm location for this env

    for module in $UTM_MODULES
    do
        _info "cloning $module $UTM_VERSION..."
        git clone $UTM_MODULES_GITBASE/$module.git $module
        cd $module
        git checkout $UTM_VERSION
        _info "building $module..."
        python setup.py bdist_wheel
        _info "patching $module wheel..."
        auditwheel repair dist/*linux_x86_64.whl
        cd ..
    done

    deactivate

    cd ..

done

_info "list of generated Python wheels:"
for filename in $(find $BUILD_DIR -name 'tm-*manylinux*.whl')
do
    echo "$filename"
done

_info "done."