Install sysadmin environment for EuPathDB BRC workflow software

workflow-software-sysadmin-init.sh

#!/bin/sh
set -e

# Usage: workflow-software-sysadmin-init.sh [/path/to/workflow-software]
# were [/path/to/workflow-software] is where to install the sysadmin
# directory. Defaults to $HOME.
#
# This script nitializes and bootstraps the `${wfs_dir}/sysadmin`
# directory with required infrastructure. The end result is an
# environment where the workpuppet script can be run to maintain
# software installations.
#
# More info at https://wiki.apidb.org/index.php/PreparingClusters
#
# It has special provisions when run via a Vagrant VM for
# testing/development. Such as with
# [email protected]:EuPathDB/vagrant-workflow-software.git

############################################################################
# Functions
############################################################################

function int_handler() {
  echo "Interrupted."
  kill $PPID
  exit 1
}

trap 'int_handler' INT

############################################################################
# Desired versions
############################################################################

puppet_cluster_branch=master
ruby_ver=2.1.7 # puppet 3 is not well supported under ruby 2.2; https://tickets.puppetlabs.com/browse/PUP-3796
openssl_ver=1.0.1p
rubygems_ver=2.4.8
git_ver=2.5.0

############################################################################
# Global variables
############################################################################

# $wfs_dir is the parent directory for workflow-software. It will
# contain subdirectories bin, etc, lib, software, sysadmin, tmp, var.
# This script only sets up the sysadmin directory. Example values
# include: holly:/eupath/workflow-software
# consign:/project/eupathdblab/workflow-software
# sapelo:/usr/local/lab/eupathdb/workflow-software
wfs_dir="${1-$HOME}"
export admin_path="${wfs_dir}/sysadmin"

# status file to record installed versions and signal a complete
# installation.
is_installed="${admin_path}/installed"

############################################################################
# Confirmation prompt before continuing
############################################################################

[[ -f "$is_installed" ]] && {
  echo "Already installed. Remove $is_installed to reinstall.";
  echo "Even better, remove all of ${admin_path} to ensure a clean install."
  exit 0;
}

if tty -s; then
    echo -e "I will install into \e[1m${admin_path}\e[0m."
    echo -n "Shall I continue?[y/n] "
    read ans
    case "$(echo -n $ans | tr '[:upper:]' '[:lower:]')" in
    y|yes)
      ;;
    *) 
      exit 1
      ;;
    esac
fi

############################################################################
# Install core software
############################################################################

mkdir -p "${admin_path}"/{software,src}

## Clean environment #######################################################

export PATH=/usr/bin:/bin
unset RUBYOPT RUBYLIB GEM_HOME GEM_PATH

## Download sources ########################################################

echo "downloading source code ..."
cd "${admin_path}/src/"
if ! type git >/dev/null 2>&1; then
  echo "  - git-${git_ver}"
  curl -s -O "https://software.apidb.org/source/git-${git_ver}.tar.gz"
fi
echo "  - ruby-${ruby_ver}"
curl -s -O "https://software.apidb.org/source/ruby-${ruby_ver}.tar.gz"
echo "  - openssl-${openssl_ver}"
curl -s -O "https://software.apidb.org/source/openssl-${openssl_ver}.tar.gz"
echo "  - rubygems-${rubygems_ver}"
curl -s -O "https://software.apidb.org/source/rubygems-${rubygems_ver}.tgz"

## unpack ##################################################################

for f in *gz; do
  file "$f" | grep -q 'gzip compressed' || { echo "$PWD/$f is not valid"; exit 1; }
  tar zxf "$f";
done;

## Git (if needed) #########################################################

if ! type git >/dev/null 2>&1; then
  echo "No system Git found. Installing Git for ourselves."
  cd "git-${git_ver}"
  ./configure --prefix="${admin_path}/software/git-${git_ver}"
  make
  make install
  export PATH="${admin_path}/software/git-${git_ver}/bin:$PATH"
fi

## openssl #################################################################

cd "${admin_path}/src/openssl-${openssl_ver}"

./config --prefix="${admin_path}/software/openssl-${openssl_ver}" shared
make install

## Ruby ####################################################################

cd "${admin_path}/src/ruby-${ruby_ver}"
./configure \
  --prefix="${admin_path}/software/ruby-${ruby_ver}" \
  --with-openssl-dir="${admin_path}/software/openssl-${openssl_ver}" \
  --disable-option-checking \
  --disable-install-doc
make
make install

## RubyGems ################################################################

# Add our Ruby version to PATH
export PATH="${admin_path}/software/ruby-${ruby_ver}/bin:$PATH"

cd ${admin_path}/src/rubygems-${rubygems_ver}

ruby setup.rb --prefix="${admin_path}/software/rubygems-${rubygems_ver}"  --no-document

## bundler #################################################################

# get the new gem executable from RubyGems in PATH
export PATH="${admin_path}/software/rubygems-${rubygems_ver}/bin:$PATH"

export GEM_HOME="${admin_path}/software/rubygems-${rubygems_ver}/gems"
export RUBYLIB="${admin_path}/software/rubygems-${rubygems_ver}/lib"

# rubygems.org is using new CA so need to add it to older RubyGem versions
curl -s https://secure.globalsign.net/cacert/Root-R1.crt | \
  openssl x509 -outform PEM -inform DEF \
  -out ${RUBYLIB}/rubygems/ssl_certs/GlobabSignCA-custom_installed_by_ebrc.pem

gem install bundler --no-document

############################################################################
# Populate sysadmin bin directory
############################################################################

echo "Populate sysadmin bin directory with symlinks"

if [ -d "$admin_path/bin" ]; then
  rm -rf "$admin_path/bin"
fi

mkdir "$admin_path/bin"
cd "${admin_path}/bin"

# maxdepth of 3 will find
# software/rubygems-1.8.15/bin
# and
# software/rubygems-1.8.15/gems/bin
# but not e.g.
# software/rubygems-1.8.15/gems/gems/bundler-1.0.21/bin/ .
# This avoids duplicates.
find ../software -maxdepth 4 -path '*/bin/*' -type f | while read binfile; do
  ln -fs "${binfile}"
done

############################################################################
# Create sysadmin bashrc. There are two important things. One is to add
# the bin dir you created in to your $PATH. The second is to configure
# rubygems as described at
# http://docs.rubygems.org/read/chapter/15#page101 and
# http://docs.rubygems.org/read/chapter/3#page70 .
############################################################################

echo "Create admin bashrc."
cat > ${admin_path}/bashrc <<EOF
umask 0022
export admin_path="${admin_path}"
export PATH="\${admin_path}/bin:/usr/bin:/bin"
export GEM_HOME="\${admin_path}/software/rubygems-${rubygems_ver}/gems"
export GEM_PATH=''
export RUBYLIB="\${admin_path}/software/rubygems-${rubygems_ver}/lib"
export RUBYOPT=rubygems
EOF

############################################################################
# Checkout Puppet manifests
############################################################################

echo "Checkout Puppet manifests."
if [[ "$USER" == "vagrant" ]]; then
  # On Vagrant VMs, checkout puppet code on host volume to aid editing,
  # committing on host and to retain changes across vagrant destroy.
  git_wd=/vagrant/scratch/puppet-cluster
  ln -nsf "$git_wd" "${admin_path}/puppet"
else
  git_wd="${admin_path}/puppet"
fi
if [[ ! -d "${git_wd}" ]]; then
  git clone -b "${puppet_cluster_branch}" ssh://[email protected]:2112/puppet-cluster.git "${git_wd}"
fi

############################################################################
# Populate bin directory with puppet executables
############################################################################

# Manually copy get_latest_package.rb in to $PATH . This script is used by a generate()
# function that is executed at manifest compile time - that is,
# before Puppet installs any files - so it needs to be pre-installed.
# Puppet will install updates to this file so we cp rather than ln.
cp "${admin_path}/puppet/modules/software/files/get_latest_package.rb" "${admin_path}/bin/"

# Manually copy the puppet wrapper script.
# Puppet will install updates to this file so we cp rather than ln.
cp "${admin_path}/puppet/modules/software/files/workpuppet" "${admin_path}/bin/"

############################################################################
# special case for Vagrant development environments
############################################################################

if [[ "$USER" == "vagrant" ]]; then
  # The yum repo contents will be downloaded to the guest volume shared
  # with Vagrant host so it persists across vagrant destroy. This is
  # just to save download time when re-provisioning a VM. 
  # This is Vagrant-specific. On a real cluster this directory is
  # created in situ by the workpuppet script.
  mkdir -p /vagrant/scratch/yum-workflow
  ln -nfs "/vagrant/scratch/yum-workflow" "${admin_path}/yum-workflow"
fi

############################################################################
# Install python-hashlib if RHEL 5
#
# RedHat 5 Only: There's an additional requirement for python hashlib on
# zcluster (and other RedHat 5 with yum 3.2.22 and without
# python-hashlib-20081119-4.el5 installed). This is needed so RedHat yum
# 3.2.22 can handle the sha256 checksums that modern createrepo commands
# generate (Pulp runs createrepo and there is no way to pass the -s
# option to the createrepo command to override the sha256 default). The
# python-devel package is required (or Python headers supplied somehow).
# Also set PYTHONPATH in bashrc.
############################################################################

operatingsystemmajrelease="$(
  source "${admin_path}/bashrc"
  cd "${admin_path}/puppet"
  bundle exec facter operatingsystemmajrelease
)"

if [[ "${operatingsystemmajrelease}" == "5" ]]; then
  echo install python-hashlib
  cd "${admin_path}/src"
  curl -LO http://pypi.python.org/packages/source/h/hashlib/hashlib-20081119.zip
  if [[ -d "hashlib-20081119" ]]; then
    rm -rf "hashlib-20081119"
  fi
  unzip hashlib-20081119.zip
  cd hashlib-20081119
  if [[ -d "${admin_path}/software/hashlib-20081119" ]]; then
    rm -rf "${admin_path}/software/hashlib-20081119"
  fi
  python setup.py install --prefix=${admin_path}/software/hashlib-20081119
  echo 'export PYTHONPATH=${admin_path}/software/hashlib-20081119/lib64/python2.4/site-packages/' >> "${admin_path}/bashrc"
fi

############################################################################
# Document installed versions.
# The file also serves as a flag that installation is complete.
############################################################################

echo "setting $is_installed"
date > "$is_installed"
echo "ruby $ruby_ver" >> "$is_installed"
echo "openssl $openssl_ver" >> "$is_installed"
echo "rubygems $rubygems_ver" >> "$is_installed"
(
  source "${admin_path}/bashrc"
  cd "${admin_path}/puppet"
  echo -n 'puppet ' >> "$is_installed"
  bundle exec puppet --version  >> "$is_installed"
  echo -n 'facter ' >> "$is_installed"
  bundle exec facter --version >> "$is_installed"
)