sample deploy

README

sample script to deploy

logging.cfg

[loggers]
keys=root, luigiStatus, taskStatus, waglStatus, rasterio

[handlers]
keys=consoleHandler, fileHandler, taskfileHandler, waglfileHandler

[formatters]
keys=simpleFormatter, JSONL

[logger_root]
level=DEBUG
handlers=consoleHandler

[logger_luigiStatus]
level=INFO
handlers=fileHandler
qualname=luigi-interface
propagate=0

[logger_taskStatus]
level=INFO
handlers=taskfileHandler
qualname=task
propagate=0

[logger_waglStatus]
level=INFO
handlers=waglfileHandler
qualname=status
propagate=0

[logger_rasterio]
level=WARNING
handlers=consoleHandler
qualname=rasterio
propagate=0

[handler_consoleHandler]
class=StreamHandler
level=DEBUG
formatter=simpleFormatter
args=(sys.stderr,)

[handler_fileHandler]
class=FileHandler
level=INFO
formatter=simpleFormatter
args=('luigi-interface.log',)

[handler_taskfileHandler]
class=FileHandler
level=INFO
formatter=JSONL
args=('task-log.jsonl',)

[handler_waglfileHandler]
class=FileHandler
level=INFO
formatter=JSONL
args=('status-log.jsonl',)

[formatter_simpleFormatter]
format=%(asctime)s: %(levelname)s: %(message)s

[formatter_JSONL]
format=%(message)s
class=wagl.logs.FormatJSONL

luigi-ARD-singlefile.cfg

[DataStandardisation]
workflow = NBAR
vertices = (3, 3)
modtran_exe = mod6c_cons
dsm_fname = /g/data/v10/eoancillarydata-2/elevation/tc_aus_3sec/dsm1sv1_0_Clean.h5:/SRTM/GA-DSM
tle_path = /g/data/v10/eoancillarydata/sensor-specific
land_sea_path = /g/data/v10/eoancillarydata/Land_Sea_Rasters
aerosol = {"pathname": "/g/data/v10/eoancillarydata-2/aerosol/AATSR/2.0/aerosol.h5"}
brdf = {"brdf_path": "/g/data/v10/eoancillarydata-2/BRDF/MCD43A1.006", "brdf_fallback_path": "/g/data/v10/eoancillarydata-2/BRDF_FALLBACK/MCD43A1.006", "ocean_mask_path": "/g/data/v10/eoancillarydata-2/ocean_mask/base_oz_tile_set_water_mask_geotif.tif"}
ozone_path = /g/data/v10/eoancillarydata-2/lookup_tables/ozone/ozone.h5
water_vapour = {"pathname": "/g/data/v10/eoancillarydata-2/water_vapour", "fallback_dataset": "/g/data/v10/eoancillarydata-2/water_vapour/pr_wtr.eatm.average.h5"}
dem_path = /g/data/v10/eoancillarydata-2/elevation/world_1deg/DEM_one_deg_20June2019.h5:/SRTM/GA-DSM
ecmwf_path = /g/data2/v10/eoancillarydata/mars/daily-data
invariant_height_fname = /g/data/v10/eoancillarydata/mars/daily-data/invariant/geo-potential-dem.tif
compression = BITSHUFFLE
h5_driver = 

[Package]
yamls_dir =
cleanup = true
cloud_buffer_distance = 0
cloud_shadow_buffer_distance = 0

[ARDP]
# root destination on S3
url_root = http://dea-public-data.s3-ap-southeast-2.amazonaws.com/ARD-Sample-Products/Beta

[GQATask]
output_yaml = {granule}.gqa.yaml
cleanup = 0
correlation_coefficient = 0.75
iterations = 1
standard_deviations = 1

[GverifyTask]
executable = /g/data/v10/projects/geo_assess/tool/image-gverify_v0.25c
ld_library_path = /g/data/v10/projects/geo_assess/tool/lib_v0.25:/g/data/v10/projects/PinkMatter/ExtraLibs/Jasper
gdal_data = /g/data/v10/projects/geo_assess/tool/lib_v0.25/gdal_data
pyramid_levels = 5
geotiff_csv = /g/data/v10/projects/geo_assess/tool/lib_v0.25/epsg_csv
# formats = image-gverify.res,image-gverify.log,image-gverify_base.jpeg,image-gverify_result.jpeg
thread_count = 4
null_value = 0
chip_size = 25
grid_size = 50
root_fix_qa_location = /g/data/v10/eoancillarydata/GCP/Fix_QA_points
timeout = 300
landsat_scenes_shapefile = /g/data/v10/eoancillarydata/wrs2-descending/wrs2_descending.shp
ocean_tile_list = {"Landsat": "/g/data/v10/eoancillarydata/ocean_tiles/ocean_list.csv", "Sentinel-2": "/g/data/v10/eoancillarydata/ocean_tiles/S2_ocean_tile.csv"}
reference_directory = /g/data/v10/eoancillarydata/GCP/GQA_v3/wrs2
backup_reference_directory = /g/data/v10/eoancillarydata/GCP/GLS2000_GCP_SCENE/wrs2

[core]
rpc_connect_timeout = 60.0
rpc_retry_attempts = 10
rpc_retry_wait = 60
logging_conf_file = /g/data/v10/private/modules/ard-pipeline/ARD_MODULE_VERSION/cfg/logging.cfg
# max-reschedules = 1

[scheduler]
record_task_history = true
state_path = luigi-state.pkl
disable_hard_timeout = 600

[task_history]
db_connection = sqlite:///luigi-task-hist.db

module-script

#%Module########################################################################
##
## ard-pipeline/20201030 modulefile
##
proc ModulesHelp { } {
        global version

        puts stderr "   This module loads the environment for producing ARD"
        puts stderr "   Version $version"
}

set version       20201030
set base          /g/data/v10/private/modules/ard-pipeline

set dea_env_b     /g/data/v10/public/modules/dea-env
set dea_env_v     20200612
set dea_env_m     dea-env/$dea_env_v

set odc_env_v     20200617
set odc_env_m     dea/$odc_env_v

module-whatis   "Loads wagl version $version"

if { [ module-info mode load ] } {

    if { ! [is-loaded dea-env] } {
        module load "$dea_env_m"
    } else {
        if { ! [is-loaded "$dea_env_m"] } {
            puts stderr "Module ard-pipeline/$version requires $dea_env_m"
            exit 1
        }
    }

    if { ! [is-loaded dea] } {
        module load "$odc_env_m"
    } else {
        if { ! [is-loaded "$odc_env_m"] } {
            puts stderr "Module ard-pipeline/$version requires $odc_env_m"
            exit 1
        }
    }

    prepend-path PYTHONPATH $base/$version/lib/python3.6/site-packages
    prepend-path PATH $base/$version/bin
    prepend-path CPATH $dea_env_b/$dea_env_v/include
    prepend-path CPATH $base/$version/include
    prepend-path LIBRARY_PATH $dea_env_b/$dea_env_v/lib
    prepend-path LIBRARY_PATH $base/$version/lib
    prepend-path LIBRARY_PATH $base/$version/lib64
    # prepend-path LD_LIBRARY_PATH $dea_env_b/$dea_env_v/lib  # THIS LINE NEEDS TO BE COMMENTED OUT WHEN COMPILING. ONCE DEPLOYED UNCOMMENT (THE DEPLOYED SCRIPT)
    prepend-path LD_LIBRARY_PATH $base/$version/lib
    prepend-path LD_LIBRARY_PATH $base/$version/lib64
    prepend-path MANPATH $base/$version/share/man
    # prepend-path INFOPATH $base/$version/share/info
    prepend-path PKG_CONFIG_PATH $base/$version/lib/pkgconfig
    prepend-path PKG_CONFIG_PATH $base/$version/lib64/pkgconfig
    setenv HDF5_PLUGIN_PATH $base/$version/hdf5-plugins
    setenv CPL_ZIP_ENCODING UTF-8
    setenv LANG en_AU.utf8
}

# Placed outside of the mode load logic to unload library
module load modtran/6.0.1
module load openmpi

# the following gets rid of this error "shmem: posix: file name search ..."
# see here for details https://github.com/underworldcode/underworld2/issues/415
setenv LD_PRELOAD $::env(OPENMPI_ROOT)/lib/libmpi.so

if { [ module-info mode remove ] } {
    remove-path PYTHONPATH $base/$version/lib/python3.6/site-packages/
    remove-path PATH $base/$version/bin
}

# setenv WAGL_CFG_ROOT "$base/$version/wagl-cfg"
setenv LUIGI_CONFIG_PATH $base/$version/cfg/luigi-ARD-singlefile.cfg

setup.bash

#!/usr/bin/bash                                                                 

ARD_MODULE_VERSION=20201030
MODULE_DIR=/g/data/v10/private/modules/modulefiles/ard-pipeline
BASE_INSTALL_DIR=/g/data/v10/private/modules/ard-pipeline

# install module script
echo "*********************"
echo "Installing: module script"
cp module-script "$MODULE_DIR"/"$ARD_MODULE_VERSION"
echo "*********************"

BASE_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[@]}")")"
BUILD_DIR="$BASE_DIR"/build
INSTALL_DIR="$BASE_INSTALL_DIR"/"$ARD_MODULE_VERSION"
CONFIG_DIR="$INSTALL_DIR"/cfg
HDF5_PLUGIN_DIR=$INSTALL_DIR/hdf5-plugins
LUIGI_CFG="$(<luigi-ARD-singlefile.cfg)"

mkdir "$BUILD_DIR"
cd "$BUILD_DIR" || exit 1

                                                                                
git clone https://github.com/sixy6e/idl-functions.git                           
git clone https://github.com/spotify/luigi.git                                  
git clone --branch wagl-5.5.1 https://github.com/GeoscienceAustralia/wagl.git   
git clone --branch eodatasets-0.12 https://github.com/GeoscienceAustralia/eo-datasets.git eod1
git clone --branch eodatasets3-0.14.2 https://github.com/GeoscienceAustralia/eo-datasets.git
git clone --branch tesp-0.8.0 https://github.com/OpenDataCubePipelines/tesp.git 
git clone --branch swfo-0.0.2 https://github.com/OpenDataCubePipelines/swfo.git 
git clone --branch eugl-0.3.1 https://github.com/OpenDataCubePipelines/eugl.git  
git clone --branch gost-0.0.3 https://github.com/OpenDataCubePipelines/gost.git  


# HDF5 and other utils
# git clone https://github.com/h5py/h5py.git
git clone https://github.com/Blosc/c-blosc.git
git clone https://github.com/Blosc/hdf5-blosc.git
git clone https://github.com/kiyo-masui/bitshuffle.git
git clone https://github.com/LLNL/zfp
git clone https://github.com/LLNL/H5Z-ZFP.git
                                                                                

module load ard-pipeline/"$ARD_MODULE_VERSION"
                                                                                
# add the install path to the PYTHONPATH                                        
export PYTHONPATH="$PYTHONPATH":"$PY_SITE_PACKAGES_DIR"

# set the pythonuserbase; allows the install to use --user rather than the complicated --install-option="--prefix=$PREFIX_PATH"
# the reason for using 'pip install . --user' instead of 'python setup.py install --prefix='
# was to avoid generating python-eggs which was causing havoc when using multiple pythonpaths
# and eggs seemed to be their own path on the pythonpath var.
# so when importing modules, it might not be the version you're expecting
export PYTHONUSERBASE="$INSTALL_DIR"

PY_BIN="$(which python)"
ENV_BASE_DIR="$(dirname "$(dirname "$PY_BIN")")"
PY_VERSION=$(python --version | cut -d" " -f2 | cut -c 1-3)
PY_SITE_PACKAGES_DIR="$INSTALL_DIR"/lib/python"$PY_VERSION"/site-packages
EASY_INSTALL_PTH="$PY_SITE_PACKAGES_DIR"/easy-install.pth
HDF5_DIR="$ENV_BASE_DIR"


mkdir -p "$PY_SITE_PACKAGES_DIR"
mkdir -p "$HDF5_PLUGIN_DIR"
mkdir -p "$CONFIG_DIR"

EOD_PKGS=( "eod1" "eo-datasets" )
for PKG in "${EOD_PKGS[@]}"
do
  cd "$PKG" || exit 1
  echo "*********************"
  echo "Installing: $PKG"
  pip install . --user
  echo "*********************"
  cd "$BUILD_DIR" || exit 1
done

PKGS=( "luigi" "idl-functions" "wagl" "eugl" "tesp" "swfo" "gost" )
for PKG in "${PKGS[@]}"
do
  cd "$PKG" || exit 1
  echo "*********************"
  echo "Installing: $PKG"
  python setup.py install --prefix="$INSTALL_DIR"
  echo "*********************"
  cd "$BUILD_DIR" || exit 1
done


# HDF5 LZF plugin
# echo "*********************"
# echo "Installing: HDF5 LZF plugin"
# cd h5py/lzf || exit 1
# gcc -O2 -fPIC -shared lzf/*.c lzf_filter.c -lhdf5 -o "$HDF5_PLUGIN_DIR"/liblzf_filter.so
# echo "*********************"
# cd "$BUILD_DIR" || exit 1


# HDF5 LZF and bitshuffle plugins
echo "*********************"
echo "Installing: HDF5 LZF and bitshuffle plugins"
cd bitshuffle || exit 1
python setup.py install --h5plugin --h5plugin-dir="$HDF5_PLUGIN_DIR" --prefix="$INSTALL_DIR"
echo "*********************"
cd "$BUILD_DIR" || exit 1


# blosc
echo "*********************"
echo "Installing: blosc"
cd c-blosc || exit 1
mkdir build
cd build
cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DDEACTIVATE_SNAPPY=OFF ../
make
make install
echo "*********************"
cd "$BUILD_DIR" || exit 1


# HDF5 blosc plugin
echo "*********************"
echo "Installing: HDF5 blosc plugin"
cd hdf5-blosc || exit 1
mkdir build
cd build || exit 1
gcc -O3 -fPIC -shared ../src/blosc_plugin.c ../src/blosc_filter.c -o "$HDF5_PLUGIN_DIR"/libH5Zblosc.so -lblosc -L"$ENV_BASE_DIR"/lib -lhdf5
echo "*********************"
cd "$BUILD_DIR" || exit 1


# zfp
echo "*********************"
echo "Installing: zfp"
cd zfp || exit 1
mkdir build
cd build || exit 1
cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" ../
make
make install
echo "*********************"
cd "$BUILD_DIR" || exit 1


# zfp HDF5 plugin
echo "*********************"
echo "Installing: zfp HDF5 plugin"
cd H5Z-ZFP || exit 1
make CC=gcc HDF5_HOME="$ENV_BASE_DIR" ZFP_HOME="$INSTALL_DIR" PREFIX="$INSTALL_DIR" install
cp "$INSTALL_DIR"/plugin/libh5zzfp.so "$HDF5_PLUGIN_DIR"
echo "*********************"
cd "$BUILD_DIR" || exit 1


# jq
echo "*********************"
echo "Installing: jq"
wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O jq
chmod +x jq
cp jq "$INSTALL_DIR"/bin
echo "*********************"


# back to base level in this repo
cd "$BASE_DIR" || exit 1


# config files (logging and luigi)
echo "*********************"
echo "Installing: logging and luigi config files"
cp logging.cfg "$CONFIG_DIR"
echo "${LUIGI_CFG/ARD_MODULE_VERSION/$ARD_MODULE_VERSION}" > "$CONFIG_DIR"/luigi-ARD-singlefile.cfg
echo "*********************"