Re-chunking GOES data

rechunk.sh

#!/usr/bin/env bash

source ~/.bash_functions
mod_py39
conda activate vdp-common

FILES=$(find css-geo/GOES-16-ABI-L1B-FULLD/2022/203 -type f)

# Define directory where outptus will be saved
OUTDIR=GOES-16-ABI-L1B-FULLD-rc1000

for infile in $FILES; do
  echo "Processing $(basename $infile)"
  # Define the output file by replacing the base directory of the input file
  # with the output directory.
  outfile=${infile/css-geo\/GOES-16-ABI-L1B-FULLD/$OUTDIR}
  mkdir -p $(dirname $outfile)
  if [ -f $outfile ]; then
    echo "File exists. Skipping..."
    continue
  fi
  # Re-chunk in the X and Y dimensions, setting both X and Y chunk sizes to 100
  ncks $infile \
    --chunk_dimension x,1000 \
    --chunk_dimension y,1000 \
    $outfile
done

Change $outfile to $outfile & and add wait at the very bottom of the file (outside the loop) to process these in parallel. But make sure you're on a compute node, not an interactive node!