Anaconda simple builder

anaconda_build.sh

#!/usr/bin/env bash
set -uxeo pipefail

# https://docs.conda.io/en/latest/miniconda.html#linux-installers

# warning, creates ~/.conda dir, you can delete it manually

###############################################
# setup variables
###############################################
export CONDA_HOME=${TEMP_DIR:-$(mktemp -d)}
export TARGET_DIR=${TARGET_DIR:-/tmp/anaconda}

NAME=${NAME:-ANACONDA}
VERSION=${VERSION:-$(date +'%Y-%m-%d_%H%M%S')}
DEST=${DEST:-/opt/cloudera/parcels}
PARCEL_HOME="${DEST}/${NAME}-${VERSION}"
nameLower="$(echo "$NAME" | awk '{print tolower($0)}')"
nameUpper="$(echo "$NAME" | awk '{print toupper($0)}')"

###############################################
# download and expand miniconda
###############################################
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
    --output /tmp/miniconda.sh
chmod +x /tmp/miniconda.sh
/tmp/miniconda.sh -u -p "$CONDA_HOME" -b

###############################################
# manual activate conda
###############################################
export CONDA_PREFIX="$CONDA_HOME"
export CONDA_EXE="$CONDA_HOME/bin/conda"
export PATH="$CONDA_HOME/bin:$CONDA_HOME/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
export CONDA_PYTHON_EXE="$CONDA_HOME/bin/python"


###############################################
# install all dependecies
###############################################
conda update -y -n base -c defaults conda
conda install -y -c conda-forge conda-pack
conda install -y                                            \
    anaconda                                                \
    cx_oracle keras                                         \
    numpy                                                   \
    pandas                                                  \
    geopandas                                               \
    scipy                                                   \
    scikit-learn                                            \
    matplotlib                                              \
    seaborn                                                 \
    plotly                                                  \
    py-xgboost                                              \
    lightgbm                                                \
    tensorflow                                              \
    pandas-profiling                                        \
    joblib                                                  \
    graphviz                                                \
    descartes                                               \
    fastparquet                                             \
    xarray                                                  \
    flask
conda install -y -c anaconda requests-kerberos

###############################################
# prepare parcel
###############################################

mkdir -p "$CONDA_HOME/meta"

cat > "$CONDA_HOME/meta/env.sh" <<EOF
#!/bin/bash
export ANACONDA_HOME=$PARCEL_HOME
EOF

cat >> "$CONDA_HOME/meta/env.sh" <<'EOF'
export ANACONDA_CDH_PYTHON="${ANACONDA_HOME}/bin/python"
export ANACONDA_PYTHONUSERBASE="${ANACONDA_HOME}"
export ANACONDA_PYTHONPATH="${ANACONDA_HOME}/lib/python3.7/site-packages"
export ANACONDA_PATH=$ANACONDA_HOME/bin:$ANACONDA_HOME/condabin
export ANACONDA_PYSPARK_PYTHON="${ANACONDA_CDH_PYTHON}"
export ANACONDA_PYSPARK_DRIVER_PYTHON="${ANACONDA_CDH_PYTHON}"
EOF
chmod +x "$CONDA_HOME/meta/env.sh"

cat > "$CONDA_HOME/meta/parcel.json" <<EOF
{
  "schema_version": 1,
  "name": "${nameUpper}",
  "version": "${VERSION}",
  "setActiveSymlink": true,
  "provides": [
    "${nameLower}",
    "anaconda-plugin"
  ],
  "depends": "CDH",
  "replaces": "${nameUpper}",
  "conflicts": "",
  "scripts": {
    "defines": "env.sh"
  },
  "groups": [],
  "users": {},
  "packages": [],
  "components": []
}
EOF

###############################################
# generate parcel file
###############################################

conda pack --output "${TARGET_DIR}/${nameUpper}-${VERSION}-el7.parcel"          \
--arcroot "${nameUpper}-${VERSION}"                                             \
--format tar.gz                                                                 \
--force                                                                         \
--dest-prefix "${PARCEL_HOME}"

sha1sum "${TARGET_DIR}/${nameUpper}-${VERSION}-el7.parcel"                      \
| cut -f1 -d" "                                                                 \
| tee "${TARGET_DIR}/${nameUpper}-${VERSION}-el7.parcel.sha"                    \
> "${TARGET_DIR}/${nameUpper}-${VERSION}-el7.parcel.sha1"

cat > "${TARGET_DIR}/manifest.json" <<EOF
{
    "parcels": [
        {
            "replaces": "",
            "hash": "$(cat "${TARGET_DIR}/${nameUpper}-${VERSION}-el7.parcel.sha")",
            "depends": "CDH",
            "components": [],
            "conflicts": "",
            "parcelName": "${nameUpper}-${VERSION}-el7.parcel"
        }
    ],
    "lastUpdated": $(date +%s)000
}
EOF

chmod 0644 "${TARGET_DIR}/${nameUpper}-${VERSION}"*

###############################################
# clean up all
###############################################
rm -rf "$CONDA_HOME"

echo "Parcel prepared in $TARGET_DIR"
ls -lh "$TARGET_DIR"