mirror.sh

#!/bin/sh

URL=url_with_zips
LINKS=`curl $URL | grep -o "http:.*.zip"`

for link in $LINKS
do
   FILENAME=`echo $link | grep -o "[A-Za-z0-9_]*.zip" | grep -o "[A-Za-z0-9_]*"`
   TIME_LS=`ls --full-time $FILENAME.zip`
   if [ -f $FILENAME.zip ]; then
      echo Updating $FILENAME.zip from $link
      # file exists, perform an update check
      curl -Oz $FILENAME.zip $link
   else
     echo "New: $link"
     curl -O $link
   fi

   NEW_TIME_LS=`ls --full-time $FILENAME.zip`

   if [ "$TIME_LS" != "$NEW_TIME_LS" ]; then
     echo "Unzipping updated archive for $FILENAME"
     rm -rf $FILENAME # remove the old folder
     unzip $FILENAME.zip
   fi
done

./generate-index.sh