Upload the failed files

re-upload.sh

Goto s3sync directory.
$ cd s3sync
$ sudo vim re-upload.sh

#!/bin/bash
##.........................................................##
## script to detect failed uploads of other date directories
## and re-try                                              ##
##.........................................................##

## AWS Credentials required for s3sync ##
export AWS_ACCESS_KEY_ID=ABHJGDVABU5236DVBJD
export AWS_SECRET_ACCESS_KEY=hgvgvjhgGYTfs/I5sdn+fsbfsgLKjs
export SSL_CERT_DIR=/etc/ssl/certs

# Get the previous date 
today_date=$(date -d "1 days ago" +%Y%m%d)
year=$(date -d "1 days ago" +%Y%m%d|head -c 4|tail -c 4)
month=$(date -d "1 days ago" +%Y%m%d|head -c 6|tail -c 2)
yday=$(date -d "1 days ago" +%Y%m%d|head -c 8|tail -c 2)

# Set the path of data
basePath="/home/ubuntu/data"
datePath="$year/$month/$yday"
fullPath="$basePath/$datePath"
echo "Path checked for: $fullPath"

# Declare the watch path and  S3 destination path
watchPath='/home/ubuntu/data'
s3path='bucket-data:'


# check for left files (failed uploads)
if [ "$(ls -A $fullPath)" ]; then
        for i in `ls -a $fullPath/*.*`
        do
        echo "Left over file: $i";
                if [ -f "$i" ]
                then out=${i#$watchPath};
                outPath=${out#"/"};
                echo "Uploading to $s3path/$outPath";
                ruby /home/ubuntu/s3sync/s3cmd.rb --ssl put $s3path$outPath $i  x-amz-storage-class:REDUCED_REDUNDANCY; #USE s3cmd : File
                RETVAL=$?
                [ $RETVAL -eq 0 ] && echo "[SUCCESS] Upload successful! " && 
                                sudo rm -rf $i &&
                                echo "[SUCCESS] Deletion successful!"
                [ $RETVAL -ne 0 ] && echo "[ERROR] Upload failed!!"
                else echo "[CLEAN] no files exist!!";
                     exit 1
                fi
        done
else
        echo "$fullPath is empty";
        sudo rm -rf $fullPath;
        echo "Successfully deleted $fullPath"
        exit 1
fi

# post failed uploads -- delete empty dirs
if [ "$(ls -A $fullPath)" ]; then
        echo "Man!! Somethingz FISHY! All (failed)uploaded files will be deleted. Are there files yet!??";
        echo "Man!! I cannot delete it then! Please go check $fullPath";
else
        echo "$fullPath is empty after uploads";
        sudo rm -rf $fullPath;
        echo "Successfully deleted $fullPath"
fi