Amazon S3 - Incremental copy using s3cmd

s3-copy.sh

#!/bin/bash

set -e

CFG="$1"
SRC="$2"
DST="$3"
FOLDER="$4"

echo "Getting ${FOLDER} list for ${SRC}"
s3cmd ls "s3://${SRC}/${FOLDER}/" --list-md5 --config="${CFG}" > "${SRC}-${FOLDER}.txt"

echo "Getting ${FOLDER} list for ${DST}"
s3cmd ls "s3://${DST}/${FOLDER}/" --list-md5 --config="${CFG}" > "${DST}-${FOLDER}.txt"

if [[ -f "${SRC}-${FOLDER}.txt" ]]; then
  cut -c30-61 "${SRC}-${FOLDER}.txt" > "${SRC}-${FOLDER}.txt.md5"
  cut -c30-61 "${DST}-${FOLDER}.txt" > "${DST}-${FOLDER}.txt.md5"

  diff -u "${DST}-${FOLDER}.txt.md5" "${SRC}-${FOLDER}.txt.md5" \
          | grep '^\+[a-f|0-9]' \
          | cut -c2- > "${DST}-${FOLDER}-pending.txt.md5"

  for j in $(cat "${DST}-${FOLDER}-pending.txt.md5"); do
          KEY=$(grep "$j" "${SRC}-${FOLDER}.txt" | grep -o "${FOLDER}/.*");
          s3cmd cp --config="${CFG}" "s3://${SRC}/${KEY}" "s3://${DST}/${KEY}"
  done;
fi

echo "Done"

exit 0