Simplified version of a script for providing a big bunch of things with Spack

simple-spack-install.sh

#!/bin/sh
set -e

# having an app tree set up messes up the build.  Clear the field.
if [[ $(type -t module) == "function" ]]; then
    module purge
fi

cat << "EOYAML" > etc/spack/packages.yaml
packages:
  all:
    compiler: [[email protected], [email protected], [email protected]]
  ack:
    version: [2.18]
  ant:
    version: [1.9.9]
  aspell:
    version: [0.60.6.1]
  aspell6-de:
    version: [6-de-20030222-1]
  aspell6-en:
    version: [2017.01.22-0]
  aspell6-es:
    version: [1.11-2]
  bamtools:
    version: [2.4.1]
  bamutil:
    version: [1.0.13]
  bats:
    version: [0.4.0]
  bcftools:
    version: [1.6]
  bcl2fastq2:
    version: [2.18.0.12]
  bedtools2:
    version: [2.26.0]
  blast-plus:
    version: [2.6.0]
    variants: ~python~perl
  blat:
    version: [3.5]
  boost:
    version: [1.63.0]
  bowtie2:
    version: [2.3.0]
  busco:
    version: [3.0.1]
  bwa:
    version: [0.7.15]
  cairo:
    variants: +X
  canu:
    version: [1.5]
  cask:
    version: [0.8.1]
  cctools:
    version: [6.1.1]
  clustalo:
    version: [1.2.4]
  clustalw:
    version: [2.1]
  cmake:
    version: [3.7.2]
  curl:
    version: [7.56.0]
  daligner:
    version: [1.0]
  direnv:
    version: [2.11.3]
  emacs:
    version: [25.3]
    variants: +X
  falcon:
    version: ['2017-05-30']
  fastqc:
    version: [0.11.4]
  fastx-toolkit:
    version: [0.0.14]
  flash:
    version: [1.2.11]
  freebayes:
    version: [1.1.0]
  gcc:
    version: [5.4.0]
  git:
    version: [2.14.1]
  git-lfs:
    version: [2.3.0]
  gnuplot:
    version: [5.0.5]
  go:
    version: [1.9.2]
    # tests have two false positives, user_namespaces and threads/exceptions
    variants: ~test
  graphviz:
    version: [2.38.0]
    variants: +pangocairo+libgd
  hdf5:
    version: [1.10.0-patch1]
  htop:
    version: [2.0.2]
  httpie:
    version: [0.9.9]
  jdk:
    version: [8u92-b14]
  jq:
    version: [1.5]
  lftp:
    version: [4.8.1]
  lmod:
    version: [7.7]
  maven:
    version: [3.5.0]
  miniconda2:
    version: [4.3.14]
  miniconda3:
    version: [4.3.14]
  mosh:
    version: [1.3.2]
  mono:
    version: [4.8.0.524]
    variants: +patch-folder-path
  multitail:
    version: [6.4.2]
  mummer:
    version: [3.23]
  muscle:
    version: [3.8.1551]
  ncftp:
    version: [3.2.6]
  nextflow:
    version: [0.23.3]
  ngmlr:
    version: [0.2.5]
  octave:
    version: [4.2.1]
  pango:
    variants: +X
  parallel:
    version: [20170322]
  pcre:
    # Work around lack of backtracking in concretization.
    # Check out explanation here: https://github.com/LLNL/spack/issues/5465
    variants: +jit
  perl:
    version: [5.24.1]
  picard:
    version: [2.9.2]
  pigz:
    version: [2.3.4]
  prank:
    version: [150803]
  primer3:
    version: [2.3.7]
  python:
    version: [2.7.13, 3.6.0]
  r:
    version: [3.3.2]
    variants: +X
  samtools:
    version: [1.6]
  seqtk:
    version: [1.2]
  snakemake:
    version: [3.11.2]
  sniffles:
    version: [1.0.5]
  sqlite:
    version: [3.18.0]
  staden-io-lib:
    version: [1.14.8]
  star:
    version: [2.5.3a]
  strelka:
    version: [2.8.2]
  subread:
    version: [1.5.2]
  swig:
    version: [3.0.12]
  texlive:
    variants: scheme=basic
  the-silver-searcher:
    version: [2.1.0]
  tmux:
    version: [2.5]
  tophat:
    version: [2.1.1]
  tree:
    version: [1.7.0]
  trimmomatic:
    version: [0.36]
  unison:
    version: [2.48.4]
  valgrind:
    version: [3.12.0]
    variants: ~mpi
  vardictjava:
    version: [1.5.1]
  vcftools:
    version: [0.1.14]
  vim:
    version: [8.0.0454]
    variants: +big
  wget:
    version: [1.17]
  zsh:
    version: [5.3.1]
EOYAML

cat << "EOYAML" > etc/spack/modules.yaml
modules:
  enable::
    - lmod
  lmod:
    core_compilers:
      - '[email protected]'
      - '[email protected]'
    hash_length: 0
    whitelist:
      - gcc
    blacklist:
      - '%[email protected]'
      - '%[email protected]'
    verbose_autoload: false
    all:
      suffixes:
        '+X': x-windows
        # must make sense vis a vis the pythons being installed below
        '^[email protected]': py2.7.9
        '^[email protected]': py3.5.1
        '^[email protected]': py3.6.0
      filter:
        environment_blacklist: ['CPATH', 'LIBRARY_PATH', 'LD_LIBRARY_PATH']
      environment:
        set:
          '${PACKAGE}_ROOT': '${PREFIX}'
    ^python:
      autoload:  'direct'
      filter:
        environment_whitelist: ['LD_LIBRARY_PATH']
EOYAML

[email protected]

if ! spack location -i ${our_compiler} > /dev/null 2>&1; then
    # Compiler

    ## HEADS UP: This is a bit fragile.  Make sure that you don't have
    ## a ~/.spack/.../compilers.yaml file that defines a 5.4.0,
    ## otherwise you're likely to build "our_compiler" with that one
    ## and lmod won't generate a set of Core modules.  Alt, we could
    ## grab the version from the system compiler and specify it here
    ## explicitly.  More work than it's worth for now.
    spack install ${our_compiler}

    # This might not do what you want if there's already
    # a 5.4.0 on the system....
    # Now all of the things will be built with %[email protected]
    spack compiler add --scope site $(spack location -i ${our_compiler})
fi

# R things
spack install r
spack install r-packrat
spack activate r-packrat
spack install [email protected]
spack install r-packrat ^[email protected]
spack activate r-packrat ^[email protected]
spack install [email protected]
spack install r-packrat ^[email protected]
spack activate r-packrat ^[email protected]

# Python things
# Must install our "main" python2 and python3 packages before the
# others to work around name collisions in the modulefiles for the few
# extensions we install (our main versions win by "first come first
# served").
# HEADS UP: Be careful to keep versions in sync with the suffixes in
# modules.yaml above.
spack install python            # default from packages.yaml
spack install py-virtualenv
spack install py-pip
spack activate py-virtualenv
spack activate py-pip

spack install [email protected]
spack install py-virtualenv  ^[email protected]
spack install py-pip         ^[email protected]
spack activate py-virtualenv ^[email protected]
spack activate py-pip        ^[email protected]

spack install [email protected]
spack install py-virtualenv  ^[email protected]
spack install py-pip         ^[email protected]
spack activate py-virtualenv ^[email protected]
spack activate py-pip        ^[email protected]

spack install [email protected]
spack install py-virtualenv  ^[email protected]
spack install py-pip         ^[email protected]
spack activate py-virtualenv ^[email protected]
spack activate py-pip        ^[email protected]

# Other Python packages that we want to install.
# These will pick a python from packages.yaml defaults
spack install py-flake8

# another batch of things that need to be installed
# and activated
spack install aspell
spack install aspell6-de
spack activate aspell6-de
spack install aspell6-en
spack activate aspell6-en
spack install aspell6-es
spack activate aspell6-es

spack install ack
spack install ant
spack install [email protected]
spack install bamtools
spack install bamutil
spack install bats
spack install bcftools
spack install [email protected]
spack install bcl2fastq2
spack install [email protected]
spack install [email protected]
spack install bedtools2
spack install blast-plus
spack install [email protected]
spack install blat
spack install boost
spack install bowtie2
spack install busco
spack install bwa
spack install [email protected]
spack install canu
spack install cask
spack install cctools
spack install clustalo
spack install clustalw
spack install cmake
spack install curl
spack install daligner
spack install direnv
spack install emacs
spack install [email protected]
spack install falcon
spack install fastqc
spack install fastx-toolkit
spack install flash
spack install freebayes
spack install git
spack install [email protected]
spack install git-lfs
spack install gnuplot
spack install go
spack install graphviz
spack install hdf5
spack install htop
spack install httpie
spack install jdk
spack install jdk@8u73-b02
spack install jdk@7u80-b0
spack install jq
spack install lftp
spack install lmod
spack install maven
spack install [email protected]
spack install miniconda2
spack install miniconda3
spack install mosh
spack install -j 1 mono
spack install multitail
spack install mummer
spack install muscle
spack install ncftp
spack install nextflow
spack install [email protected]
spack install [email protected]
spack install ngmlr
spack install octave
spack install parallel
spack install perl
spack install [email protected]
spack install [email protected]
spack install [email protected]
spack install picard
spack install [email protected]
spack install pigz
spack install prank
spack install primer3
spack install samtools
spack install [email protected]
spack install seqtk
spack install snakemake
spack install sniffles
spack install sqlite
spack install staden-io-lib
spack install star
spack install [email protected]
spack install swig
spack install strelka
spack install subread
# spack install --no-checksum texlive # live, no reliable checksum
spack install the-silver-searcher
spack install tmux
spack install tophat
spack install tree
spack install trimmomatic
spack install [email protected]
spack install unison
spack install valgrind
spack install vardictjava
spack install vcftools
spack install vim
spack install wget
spack install zsh

# spack module refresh --module-type lmod --delete-tree -y