Download data or request from glacier storage

download_or_restore_from_galcier.sh

while read row; do
  #echo $row
  file_name=$(echo $row | awk '{print $4}')
  #echo $file_name
  #test if the file is already downloaded
  file -E $file_name &> /dev/null
  LOCAL_FILE_TEST_RESULT=$?
  if [ $LOCAL_FILE_TEST_RESULT -eq 0 ]; then
    #echo "${file_name} exists locally - skipping"
    continue
  fi

  # test if we can download the file
  bucket_path="data/Galaxy-Zoo-SDSS/COS_stamps_large/${file_name}"
  bucket="zooniverse-cold-storage"
  s3_path="s3://${bucket}/${bucket_path}"
  aws s3 cp --quiet $s3_path .
  DOWNLOAD_REQ_RESULT=$?
  if [ $DOWNLOAD_REQ_RESULT -eq 0 ]; then
    echo "downloaded ${file_name}"
    echo $file_name > downloaded.log
  else
    aws s3api restore-object --restore-request '{"Days":5,"GlacierJobParameters":{"Tier":"Bulk"}}' --bucket ${bucket} --key ${bucket_path}
    RESTORE_REQUEST_RESULT=$?
    if [ $RESTORE_REQUEST_RESULT -eq 0 ]; then
      echo "requested a restore of ${file_name}"
      echo $file_name > restore_requests.log
    else
      echo "failed to request restore ${file_name}"
      echo $file_name > failed_restores.log
    fi
  fi
done <file_list.txt