Install list of haskell packages locally. Create hyperlinked documentation with index. Download package sources and create tags

install-packages.sh

#! /usr/bin/env bash
#
# File: install-packages.sh
#
# Created: Saturday, 29 October 2016
#

# treat undefined variable substitutions as errors
set -u
# propagate errors from all parts of pipes
set -o pipefail

set -e

# Inputs

root=$(cd "$(dirname "$0")" && pwd)
ghc_version="8.4.4"

pkg_db_dir="$root/local/pkg-db.d"
docs_dir="$root/docs"

html_dir="html"
# Dir with with children like ‘base-4.9.0.0/Data-Functor.html’.
docs_html_dir="$docs_dir/$html_dir"
mkdir -p "$docs_html_dir"

# Dir with package db, libraries, shared files
local="$root/local"

ghc_docs_root="$(dirname "$(which ghc)")/../share/doc/ghc-${ghc_version}/html/libraries/"

package_list="$root/package-list-unversioned"
package_download_dir="$root/all-packages"

# Packages for which no source on Hackage exists
special_global_packages="rts|ghc"

if [[ ! -d "$ghc_docs_root" ]]; then
    echo "The ghc documentation directory does not exist: $ghc_docs_root" >&2
    exit 1
fi
if [[ ! -f "$package_list" ]]; then
    echo "The package list file does not exist: $package_list" >&2
    exit 1
fi

# User input

action="${1:-all}"

if ! [[ "$action" = "all" || "$action" = "install" || "$action" = "download" || "$action" = "update-tags" || "$action" = "generate-haddock-docs" ]]; then
    echo "Invalid action: $action." >&2
    echo "Valid actions: install, download, update-tags, generate-haddock-docs, all" >&2
    echo "  install - create local package db and install packages from ‘$package_list’ into it, using the latest LTS stackage snapshot" >&2
    echo "  download - get sources of all packages installed into local package db"
    echo "  update-tags - regenerate tags file"
    echo "  generate-haddock-docs - create offline documentation for installed packages"
fi

# Utils

function execVerbose {
   echo "${@}"
   "${@}"
}

# Actions

if [[ ! -d "$local" ]]; then
    echo "Creating directory $local"
    mkdir -p "$local"
fi

# Create package db
if [[ ! -d "$pkg_db_dir" ]]; then
    echo "Creating package db at $pkg_db_dir"
    ghc-pkg init "$pkg_db_dir"
fi


# Copy builtin docs
if [[ "$action" = "install" || "$action" = "generate-haddock-docs" || "$action" = "all" ]]; then
    echo "Populating '$docs_html_dir' with builtin documentatios"
    for pkg in $(cd "$ghc_docs_root" && find . -maxdepth 1 -type d | sed -re "/^\.\/(rts)-[0-9.]*$/d"); do
        if [[ "$pkg" != "." ]]; then
            if [[ -d "$docs_html_dir/$pkg" ]]; then
                echo "Skipping $pkg - documentation already present"
            else
                echo "Copying documentation for $pkg"
                cp -r "$ghc_docs_root/$pkg" "$docs_html_dir"
            fi
        fi
    done
fi

# Install packages
if [[ "$action" = "install" || "$action" = "all" ]]; then
    echo "Installing packages"
    global_packages_re=""
    for pkg in $(ghc-pkg list --global --simple-output); do
        unversioned="${pkg%-*}"
        if [[ -n "$global_packages_re" ]]; then
            global_packages_re="$global_packages_re|$unversioned"
        else
            global_packages_re="$unversioned"
        fi
    done

    echo "global_packages_re = ${global_packages_re}"
    packages_to_install=$(cat "$package_list" | awk '!/^ *--.*$/' | grep -v -E "^($global_packages_re)$" | xargs echo)

    if [[ ! -f cabal.config ]]; then
        echo "Updating cabal.config"
        wget https://www.stackage.org/lts/cabal.config
    fi

    echo "Installing following packages"
    for pkg in ${packages_to_install}; do
        echo "- $pkg"
    done

    for prog in alex happy c2hs; do
      if ! which "$prog"; then
          echo "Cannot find program '$prog' within PATH" >&2
          exit 1
      fi
    done

        # --haddock-hoogle \
    execVerbose \
        cabal --no-require-sandbox v1-install \
        --prefix "$local" \
        --docdir="$docs_html_dir/\$pkgid" \
        --htmldir="$docs_html_dir/\$pkgid" \
        --package-db="$pkg_db_dir" \
        --disable-split-objs \
        --enable-optimization=0 \
        --enable-shared \
        --disable-library-profiling \
        --disable-profiling \
        --disable-tests \
        --disable-benchmarks \
        --enable-documentation \
        --haddock-hoogle \
        --haddock-html \
        --haddock-hyperlink-source \
        "--haddock-html-location=../\$pkgid" \
        --haddock-internal \
        --haddock-hyperlink-source \
        -f new-base \
        --allow-newer=Cabal \
        --allow-newer=HUnit \
        --allow-newer=QuickCheck \
        --allow-newer=SVGFonts \
        --allow-newer=base \
        --allow-newer=bifunctors \
        --allow-newer=conduit \
        --allow-newer=diagrams-cairo \
        --allow-newer=diagrams-lib \
        --allow-newer=distributed-process \
        --allow-newer=haskell-src-exts \
        --allow-newer=fgl \
        --allow-newer=free \
        --allow-newer=lens \
        --allow-newer=optparse-applicative \
        --allow-newer=pipes \
        --allow-newer=pqueue \
        --allow-newer=process \
        --allow-newer=syb \
        --allow-newer=template-haskell \
        --allow-newer=time \
        -j5 \
        $packages_to_install
        # --allow-newer=haskell-src-exts \
        # --build-log=build.log \
fi

        # --allow-newer=process \
        # --allow-newer=diagrams-cairo \
        # --allow-newer=diagrams-lib \
        # --allow-newer=distributed-process-extras \
        # --allow-newer=distributed-process \
        # --allow-newer=distributed-process-client-server \
        # --allow-newer=binary \
        # --allow-newer=HUnit \
        # --allow-newer=time \
        # --allow-newer=pipes \

function get_packages {
    ghc-pkg "${@}" list --simple-output | sed -e "s/ /\n/g" | sed -re "/^(${special_global_packages})-[0-9.]*$/d"
}

if [[ "$action" = "download" || "$action" = "all" ]]; then
    echo "Downloading package sources for indexing into $package_download_dir"

    mkdir -p "$package_download_dir"

    for pkg in $(get_packages --global --package-db "$pkg_db_dir"); do
        declare -a fs
        fs=( $(find "$package_download_dir" -maxdepth 1 -type d -name "${pkg}*") )

        if [[ "${#fs[@]}" = 0 && ! -d "$pkg" && "$pkg" != z-* ]]; then
            if [[ "$pkg" == "ghc-boot-8.4.4" ]]; then
                pkg="ghc-boot-8.4.3"
            fi
            if [[ "$pkg" == "ghc-boot-th-8.4.4" ]]; then
                pkg="ghc-boot-th-8.4.3"
            fi
            if [[ "$pkg" == "ghci-8.4.4" ]]; then
                pkg="ghci-8.4.3"
            fi
            echo "Downloading $pkg"
            pushd "$package_download_dir" >/dev/null
            cabal get "$pkg"
            popd >/dev/null
        else
            echo "Skipping $pkg"
        fi
    done
fi

if [[ "$action" = "update-tags" || "$action" = "all" ]]; then
    echo "Updating tags"
    time fast-tags -o "$root/tags" -v -R "$package_download_dir" --nomerge
fi

if [[ "$action" = "generate-haddock-docs" || "$action" = "all" ]]; then
    mkdir -p "$docs_dir"

    pushd "$docs_dir" >/dev/null
        haddock_args=""
        for interface_file in $(find "$docs_dir" -type f -name '*.haddock'); do
            interface_file_rel="$(realpath --relative-to "$docs_dir" "$interface_file")"
            echo "interface_file = ${interface_file_rel}"
            html="$(dirname "$interface_file_rel")"
            flag="--read-interface=${html},${interface_file_rel}"
            #flag="--read-interface=file://${html},${interface_file_rel}"
            # flag="--read-interface=${interface_file_rel}"
            if [[ -z "$haddock_args" ]]; then
                haddock_args="$flag"
            else
                haddock_args="$haddock_args $flag"
            fi
        done

        echo "Running haddock"

        # haddock=haddock
        # haddock=/home/sergey/projects/haskell/projects/thirdparty/haddock/dist/build/haddock/haddock
        #haddock=/home/sergey/projects/haskell/projects/thirdparty/haddock/dist-newstyle/build/x86_64-linux/ghc-8.4.3/haddock-2.20.0/x/haddock/build/haddock/haddock
        haddock=/home/sergey/projects/haskell/projects/thirdparty/haddock/.stack-work/install/x86_64-linux-tinfo6/lts-12.5/8.4.3/bin/haddock
        $haddock \
            --verbosity=3 \
            --pretty-html \
            --gen-contents \
            --gen-index \
            --odir="$docs_dir" \
            --title="Standalone Haskell Documentation" \
            --hyperlinked-source \
            $haddock_args
    popd >/dev/null
fi

package-list-unversioned

abstract-deque
abstract-par
ad
adjunctions
aeson
aeson-compat
aeson-pretty
ansi-terminal
appar
array
arrows
--asn1-encoding
--asn1-parse
--asn1-types
async
attoparsec
auto-update
base
base16-bytestring
base64-bytestring
base-compat
base-orphans
base-prelude
bert
bifunctors
binary
binary-tagged
bitarray
blaze-builder
blaze-html
blaze-markup
blaze-textual
bmp
byteable
byteorder
bytestring
bytestring-builder
Cabal
cabal-doctest
cairo
call-stack
case-insensitive
cassava
cereal
charset
Chart
Chart-cairo
-- Chart-diagrams
clock
cmark
cmdargs
colour
comonad
-- compdata
conduit
conduit-combinators
conduit-extra
-- conduit-find
-- conduit-merge
-- conduit-network-stream
conduit-parse
connection
constraints
containers
contravariant
cookie
cpphs
criterion
cryptohash
cryptohash-sha256
cryptonite
cryptonite-conduit
css-text
curl
data-accessor
data-default
data-default-class
data-default-instances-containers
data-default-instances-dlist
data-default-instances-old-locale
data-reify
deepseq
-- derive
diagrams
diagrams-cairo
diagrams-canvas
diagrams-contrib
diagrams-core
diagrams-graphviz
diagrams-gtk
diagrams-html5
diagrams-lib
diagrams-postscript
diagrams-svg
-- diagrams-wx
Diff
digest
direct-sqlite
directory
dirstream
distributed-process
distributed-process-async
distributed-process-client-server
distributed-process-extras
-- distributed-process-registry
--distributed-process-simplelocalnet
-- distributed-process-task
--distributed-process-tests
distributed-static
distributive
dlist
doctemplates
doctest
easy-file
ed25519
either
enclosed-exceptions
erf
errors
exceptions
executable-path
extensible-exceptions
extra
fail
fast-logger
fclabels
fgl
file-embed
filelock
filemanip
filepath
fingertree
fixed
fmlist
foundation
free
fsnotify
gauge
gd
generic-deriving
generics-sop
geniplate-mirror
ghc-boot-th
ghc-exactprint
ghc-paths
ghc-prim
ghc-syb-utils
gio
gitrev
glade
glib
Glob
gloss
gloss-rendering
GLURaw
GLUT
graphviz
--gtk2hs-buildtools
gtk3
gi-gtk-declarative
hackage-security
haddock-library
hakyll
half
hashable
hashtables
haskeline
haskell-gi
haskell-lexer
haskell-src-exts
-- hastache
hex
hinotify
-- hit
hlint
hoopl
hostname
hourglass
hpack
hpc
hs-bibutils
hscolour
hslogger
--hslua
hspec
hspec-core
hspec-discover
hspec-expectations
hspec-smallcheck
HTTP
htoml-megaparsec
http2
http-api-data
http-client
http-client-tls
http-conduit
http-date
http-types
HUnit
hxt
hxt-charproperties
hxt-regex-xmlschema
hxt-unicode
ieee754
IfElse
integer-gmp
integer-logarithms
iproute
JuicyPixels
kan-extensions
language-c
language-haskell-extract
language-javascript
-- language-python
lazysmallcheck
lens
lifted-async
lifted-base
linear
logict
lrucache
--markdown
math-functions
megaparsec
memory
MemoTrie
microlens
microlens-ghc
microlens-mtl
microlens-platform
microlens-th
mime-types
mmark
mmorph
monad-control
monad-journal
monad-logger
monad-loops
monad-memo
monad-par
monad-par-extras
MonadRandom
monad-unlift
mono-traversable
mtl
multistate
mwc-random
nats
neat-interpolation
network
network-transport
network-transport-composed
network-transport-inmemory
network-transport-tcp
network-transport-tests
network-uri
newtype
non-negative
ObjectName
old-locale
old-time
Only
open-browser
OpenGL
OpenGLRaw
operational
optparse-applicative
pandoc
pandoc-citeproc
pandoc-types
pango
parallel
parsec
parsers
parser-combinators
path
path-io
path-pieces
patience
pem
persistent
persistent-sqlite
persistent-template
pid1
pipes
pipes-aeson
pipes-attoparsec
pipes-binary
pipes-bytestring
pipes-concurrency
pipes-extras
pipes-network
pipes-parse
pipes-safe
--pipes-text
pipes-zlib
pointed
pointedlist
polyparse
prelude-extras
pretty
pretty-show
prettyprinter
primitive
process
profunctors
project-template
psqueues
QuickCheck
quickcheck-instances
quickcheck-io
random
random-fu
random-source
rank1dynamic
reactive-banana
-- reactive-banana-wx
recursion-schemes
reducers
reflection
regex-applicative
regex-applicative-text
regex-base
--regex-pcre-builtin
regex-posix
regex-tdfa
regex-tdfa-text
repa
resource-pool
resourcet
retry
rfc5051
rts
safe
safe-exceptions
scientific
sdl2
-- sdl2-compositor
-- sdl2-gfx
sdl2-image
-- sdl2-mixer
sdl2-ttf
semigroupoids
semigroups
setenv
set-extra
SHA
shake
shelly
silently
simple-reflect
simple-sendfile
singletons
skylighting
smallcheck
socks
split
sqlite-simple
StateVar
statistics
stm
stm-chans
stm-conduit
store
store-core
-- streaming
streaming-commons
stringsearch
syb
system-fileio
system-filepath
tagged
tagsoup
tar
tar-conduit
tasty
tasty-golden
tasty-hspec
tasty-hunit
tasty-quickcheck
tasty-smallcheck
tasty-th
template-haskell
temporary
--terminfo
--test-framework
--test-framework-hunit
--test-framework-quickcheck2
--test-framework-th
texmath
text
text-binary
text-metrics
tf-random
th-expand-syns
th-lift
th-lift-instances
th-orphans
th-reify-many
th-utilities
time
time-locale-compat
tls
transformers
transformers-base
transformers-compat
tree-view
trifecta
unbounded-delays
--unexceptionalio
unicode-transforms
uniplate
unix
unix-compat
unix-time
unsafe
unordered-containers
uri-bytestring
utf8-string
uuid-types
vault
vector
vector-algorithms
vector-binary-instances
vector-th-unbox
void
wai
wai-app-static
wai-extra
wai-logger
warp
word8
-- wx
-- wxc
-- wxcore
-- wxdirect
-- x509
-- x509-store
-- x509-system
-- x509-validation
xhtml
xml
xml-conduit
xml-types
yaml
Yampa
zip-archive
zlib
zlib-bindings
tracetree
-- orgmode-parse
safe-exceptions-checked
sexp-grammar
invertible-grammar