Set up matlaber environment

create-matlaber-env.sh

#!/bin/bash

set -xe

export PYTHON_VERSION=3.9
export CUDA_VERSION=11.3
export JUPYTER_PORT="$(id -u)"

##
## Make SSL certs
##

## CA first
CA_NAME=wwbrannonCA

CERTS_DIR="$HOME/docs/certs"
mkdir -p "$CERTS_DIR" && cd "$CERTS_DIR"

if [[ -e ca.key ]] && [[ -e ca.crt ]]; then
    echo "ca.pem and ca-key.pem exist; skipping"
else
    openssl genrsa -out ca-key.pem 2048
    openssl req -x509 -new -nodes -key ca-key.pem -sha256 -days 365 \
                -out ca.pem -subj "/C=US/CN=$CA_NAME"
fi

## Then main cert
rm -f cert.csr cert.key ca.srl cert.ext cert.pem cert-key.pem

cat << EOF > cert.ext
authorityKeyIdentifier = keyid,issuer
basicConstraints = CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
subjectAltName = @alt_names

[alt_names]
DNS.1 = matlaber1.media.mit.edu
DNS.2 = matlaber2.media.mit.edu
DNS.3 = matlaber3.media.mit.edu
DNS.4 = matlaber4.media.mit.edu
DNS.5 = matlaber5.media.mit.edu
DNS.6 = matlaber6.media.mit.edu
DNS.7 = matlaber7.media.mit.edu
DNS.8 = matlaber8.media.mit.edu
DNS.9 = matlaber9.media.mit.edu
DNS.10 = matlaber10.media.mit.edu
DNS.11 = matlaber11.media.mit.edu
DNS.12 = matlaber12.media.mit.edu
DNS.13 = matlaberp1.media.mit.edu
DNS.14 = matlaberp2.media.mit.edu
DNS.15 = matlaberp3.media.mit.edu
DNS.16 = matlaberp4.media.mit.edu
DNS.17 = matlaberp5.media.mit.edu
DNS.18 = matlaberp6.media.mit.edu
EOF

openssl genrsa -out cert-key.pem 2048
openssl req -new -key cert-key.pem -out cert.csr -subj "/C=US/CN=matlaber.media.mit.edu"
openssl x509 -req -in cert.csr -CA ca.pem \
             -CAkey ca-key.pem -CAcreateserial -out cert.pem \
             -days 365 -sha256 -extfile cert.ext

rm -f cert.csr cert.ext ca.srl

cd

#
# Install jupyterlab config
#

mkdir -p "$HOME/.jupyter/"
cat << EOF > "$HOME/.jupyter/jupyter_lab_config.py"
c.ServerApp.certfile = '$CERTS_DIR/cert.pem'
c.ServerApp.keyfile = '$CERTS_DIR/cert-key.pem'

c.ServerApp.ip = '*'
c.ServerApp.port = $JUPYTER_PORT

c.ServerApp.root_dir = 'notebooks'

c.ServerApp.open_browser = False

# use 'from notebook.auth import passwd; passwd()' to generate the hash
# that goes in the c.ServerApp.password field
c.ServerApp.password = ''
c.ServerApp.password_required = True
EOF

#
# Install all the software
# This is much faster with mamba than conda
#

rm -rf "$HOME/miniconda3"

wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
bash Miniconda3-latest-Linux-x86_64.sh -b -p "$HOME/miniconda3"
rm -f Miniconda3-latest-Linux-x86_64.sh

"$HOME/miniconda3/bin/conda" update -y -n base conda
"$HOME/miniconda3/bin/conda" update -y -n base --all
"$HOME/miniconda3/bin/conda" install -y -n base -c defaults -c conda-forge \
    mamba

"$HOME/miniconda3/bin/mamba" create -y -n jupyter -c defaults \
    python=$PYTHON_VERSION notebook jupyter jupyterlab nbconvert nbformat \
    ipywidgets widgetsnbextension
"$HOME/miniconda3/bin/mamba" install -y -n jupyter -c defaults -c conda-forge \
    jupyterlab-git

"$HOME/miniconda3/bin/mamba" create -y -n dl -c defaults \
    python=$PYTHON_VERSION ipykernel ipywidgets
"$HOME/miniconda3/envs/dl/bin/python3" -m ipykernel install --user --name=DL

"$HOME/miniconda3/bin/mamba" install -y -n dl -c pytorch -c defaults \
    pytorch cudatoolkit=$CUDA_VERSION torchtext torchvision torchaudio captum
"$HOME/miniconda3/bin/mamba" install -y -n dl -c defaults -c pyg pyg

"$HOME/miniconda3/bin/mamba" install -y -n dl -c defaults -c conda-forge \
    jinja2 tqdm pyyaml requests unidecode python-louvain sqlalchemy \
    pytest omegaconf mypy pylint \
    \
    numpy pandas numexpr pandasql patsy \
    scipy scikit-learn statsmodels scikit-image networkx \
    graphviz pydot matplotlib seaborn \
    \
    nltk gensim \
    spacy spacy-transformers \
    spacy-model-en_core_web_sm spacy-model-en_core_web_md \
    spacy-model-en_core_web_lg spacy-model-en_core_web_trf \
    \
    transformers datasets sentencepiece emoji \
    tensorboard skorch pytorch-model-summary \
    pytorch-lightning jsonargparse[signatures] docstring_parser \
    dask dask-cuda distributed fairscale \
    bertopic top2vec sentence-transformers \
    bert_score comet faiss sacrebleu seqeval  # huggingface metrics deps

# not on conda
"$HOME/miniconda3/envs/dl/bin/python3" -m pip install deepspeed