An example script to cotter, calibrate and image an MWA observation.

ao-cal.sh

#!/usr/bin/env bash

# This script requires at least one argument:
# - the 10-digit-long obsid to be calibrated
# - (optional) the associated ASVO job number for downloading (needed only if the data doesn't already exist)

# This script is assumed to be running in a "timestamp directory", with the visibilties available or
# to be downloaded in the directory above.

if [[ $# -eq 0 ]]; then
    # Try to guess the obsid on the PWD.
    OBSID=$(echo "$PWD" | awk '{n=split($0,a,"/"); print a[n]}')
    echo "I reckon we're calibrating $OBSID"
elif [[ $# -gt 0 ]]; then
    OBSID=$1
fi

set -eux

NUM_SOURCES=100
TIME_AVERAGE=4
FREQ_AVERAGE=40
EDGEWIDTH=160
CALIBRATE_ITERATIONS=100
MAX_MEMORY_GIGABYTES=40
CALIBRATION_SOLUTION_RENAMED="$OBSID"_timeres"$TIME_AVERAGE"_freqres"$FREQ_AVERAGE"_"$NUM_SOURCES"sources_niters"$CALIBRATE_ITERATIONS".bin

# If a calibration solution and image are already here, then we don't need to do anything further.
[ -r "$CALIBRATION_SOLUTION_RENAMED" ] && [ -r wsclean-image.fits ] && echo "ao-cal.sh for obsid $OBSID completed successfully." && exit 0

# Download the latest metadata associated.
wget "http://mwa-metadata01.pawsey.org.au/metadata/fits/?obs_id=$OBSID" -O "$OBSID".metafits.latest

# Download visibilities or a measurement set only if necessary.
if test -n "$(find . -maxdepth 1 -name "$OBSID.ms" -print -quit)"; then
    echo "Measurement set already available; no need for download."
elif [ -r "$OBSID"_ms.zip ]; then
    echo "Measurement set already downloaded; unzipping."
    unzip -o "$OBSID"_ms.zip
elif test -n "$(find . -maxdepth 1 -name "*gpubox*.fits" -print -quit)"; then
    echo "Visibilities already available; no need for download."
elif [ -r "$OBSID"_vis.zip ]; then
    echo "Visibilities already downloaded; unzipping."
    unzip -o "$OBSID"_vis.zip
elif [[ $# -gt 1 ]]; then
    echo "Downloading from ASVO."
    time mwa_client -w "$2"
    unzip -o "$OBSID"*.zip
else
    echo "No measurement set available, and no ASVO download job number supplied! Exiting."
    exit 1
fi

# [ -r "$OBSID"_ms.zip ] && rm "$OBSID"_ms.zip
# [ -r "$OBSID"_vis.zip ] && rm "$OBSID"_vis.zip
touch .heracles_download_complete

# Variables for files.
METAFITS="$OBSID".metafits
SRCLIST=srclist_pumav3_EoR0aegean_EoR1pietro+ForA_"$OBSID"_aocal"$NUM_SOURCES".txt
MEASUREMENT_SET=$OBSID.ms
CALIBRATION_SOLUTION=solutions_timeres"$TIME_AVERAGE"_freqres"$FREQ_AVERAGE"_"$NUM_SOURCES"sources_niters"$CALIBRATE_ITERATIONS".bin

# Move the metafits we downloaded earlier; this ensures we're using the latest metafits.
mv "$OBSID".metafits.latest "$METAFITS"

# Generate a "calibrate"-compatible source list with Jack's code.
[ -r "$SRCLIST" ] || time /home/chj/Software/other/srclists/srclist_by_beam.py \
                          -n $NUM_SOURCES \
                          --srclist=/home/chj/Software/other/srclists/srclist_pumav3_EoR0aegean_EoR1pietro+ForA.txt \
                          --metafits="$METAFITS" \
                          --order="distance" \
                          --no_patch \
                          --cutoff=30 \
                          --aocalibrate

# Run the AO programs.
# Run cotter if we don't have a measurement set.
[ -r "$MEASUREMENT_SET" ] || time cotter \
                                  -absmem $MAX_MEMORY_GIGABYTES \
                                  -o "$MEASUREMENT_SET" \
                                  -m "$OBSID".metafits \
                                  -timeres $TIME_AVERAGE \
                                  -freqres $FREQ_AVERAGE \
                                  -edgewidth $EDGEWIDTH \
                                  -allowmissing \
                                  ./??????????_*gpubox*.fits
                                  # -use-dysco \

# Delete the visibilities.
# find . -maxdepth 1 -name "*gpubox*.fits" -exec rm {} \;

# Delete the non-renamed calibration solution if the renamed one doesn't exist.
# This state is expected if calibrate did not complete properly, so the
# non-renamed calibration-solution file can't be used reliably.
[ -r "$CALIBRATION_SOLUTION" ] && [ ! -r "$CALIBRATION_SOLUTION_RENAMED" ] && rm "$CALIBRATION_SOLUTION"

[ -r "$CALIBRATION_SOLUTION_RENAMED" ] || time calibrate \
                                               -applybeam \
                                               -mwa-path /usr/lib/python3.7/site-packages/mwa_pb/data \
                                               -absmem $MAX_MEMORY_GIGABYTES \
                                               -m "$SRCLIST" \
                                               -minuv 128 -maxuv 1300 \
                                               -i $CALIBRATE_ITERATIONS \
                                               "$MEASUREMENT_SET" \
                                               "$CALIBRATION_SOLUTION"

# Following a successful run of calibrate, move the calibration solution to
# the designated renamed form.
[ -r "$CALIBRATION_SOLUTION" ] && mv "$CALIBRATION_SOLUTION" "$CALIBRATION_SOLUTION_RENAMED"

time applysolutions \
    "$MEASUREMENT_SET" \
    "$CALIBRATION_SOLUTION_RENAMED"

time wsclean \
     -abs-mem $MAX_MEMORY_GIGABYTES \
     -name wsclean \
     -size 2048 2048 \
     -scale 30asec \
     -niter 1000000 \
     -auto-threshold 3 \
     "$MEASUREMENT_SET"

# rm -rf "$MEASUREMENT_SET" .heracles_download_complete
# [ -r "$OBSID"_flags.zip ] && rm "$OBSID"_flags.zip
# [ -r "$OBSID"_metafits_ppds.fits ] && rm "$OBSID"_metafits_ppds.fits

echo "ao-cal.sh for obsid $OBSID completed successfully."