Request glacier restore from file

glacier_restore_files.sh

#!/bin/sh
bucket="zooniverse-cold-storage"
path_prefix="data/Galaxy-Zoo-SDSS/COS_stamps_large"

for x in `cat glacier_to_restore.txt`
  do
    bucket_path="${path_prefix}/${x}"
    echo "Begin restoring ${bucket_path}"
    aws s3api restore-object --restore-request '{"Days":5,"GlacierJobParameters":{"Tier":"Bulk"}}' --bucket ${bucket} --key ${bucket_path}
    echo "Done restoring $bucket_path"
  done

I've used the following to generate the list of files to restore from glacier using the script above.

aws s3 sync --force-glacier-transfer --ignore-glacier-warnings s3://zooniverse-cold-storage/data/Galaxy-Zoo-SDSS/COS_stamps_large/ . &> sync_req.log

# changing the std* redirects above will probably fix the line ending issue below
# https://stackoverflow.com/questions/811193/how-to-convert-the-m-linebreak-to-normal-linebreak-in-a-file-opened-in-vim
# YMMV - i had to convert the log line endings using vi :%s/CTRLV CTRLM//g

# now just get the failure messages
cat sync_req.log | grep 'download failed' | grep fits.fz > sync_failed.log

# take the sync failure log and format it up for the path names
awk {'print $5'} < sync_failed.log | sed -e 's/.\///g' > glacier_to_restore.txt