Created
January 27, 2015 12:22
-
-
Save jaygooby/4e1a38311b2dc4247343 to your computer and use it in GitHub Desktop.
Gist of http://ankaerith.livejournal.com/147007.html (some background here http://s3sync.net/forum/index2214.html?topic=251.0) by Matthew Staver
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/sh | |
# | |
# S3rep.sh version 1.0 | |
# Copyright 2009 Matthew Staver | |
# | |
# This program is free software: you can redistribute it and/or modify | |
# it under the terms of the GNU General Public License as published by | |
# the Free Software Foundation, either version 2 of the License, or | |
# (at your option) any later version. | |
# | |
# This program is distributed in the hope that it will be useful, | |
# but WITHOUT ANY WARRANTY; without even the implied warranty of | |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
# GNU General Public License for more details. | |
# | |
# You should have received a copy of the GNU General Public License | |
# along with this program. If not, see <http://www.gnu.org/licenses/> | |
pid="$$" | |
# Set type to first argument. | |
type="$1" | |
# Set path to the rest of the arguments. | |
path="${*:2:$#}" | |
# Default incron mask, if you never plan to use deletion=1 and your | |
# directories are fairly static you can set this to: | |
# | |
# mask="IN_CREATE,IN_CLOSE_WRITE,IN_MOVED_TO" | |
# | |
# This may slightly reduce system load, but your incrontab file will | |
# not be cleaned of watched directories that are moved or deleted. | |
# I highly recommend you leave this with the default setting. | |
mask="IN_CREATE,IN_DELETE,IN_CLOSE_WRITE,IN_MOVED_TO,IN_MOVED_FROM" | |
# The location of your spool/queue directory. | |
queue="/var/spool/s3rep" | |
lockfile=/var/run/s3rep.pid | |
# The location of s3sync | |
s3path="/root/s3sync" | |
# The location of your incrontab file | |
incrontab="/var/spool/incron/root" | |
# The s3bucket you want to use | |
bucket="yourbucket" | |
# Set this to 0 to turn off syslog logging | |
syslog=1 | |
# Set this to 1 to enable deletion from S3 of removed files/directories | |
deletion=0 | |
function log () { | |
[ $syslog -eq 1 ] && logger -t "s3rep[$pid]" "$@" || cat > /dev/null | |
} | |
case "$type" in | |
'IN_CREATE') | |
exit 0 | |
;; | |
'IN_CREATE,IN_ISDIR' | 'IN_MOVED_TO,IN_ISDIR' | 'IN_CLOSE_WRITE' | 'IN_MOVED_TO' | '-a') | |
# If a file or directory is created, edited, or moved into a watched directory | |
# symlink the it in the queue directory. If no uploads are currently running, | |
# call this script with the UPLOAD option to start processing the queue directory. | |
symname="$pid`date +%N`" | |
ln -s "$path" "$queue/$symname" | |
log "Queued: '$path' in '$queue/$symname'" | |
[ ! -e $lockfile ] && $0 UPLOAD & | |
;; | |
'IN_DELETE' | 'IN_MOVED_FROM') | |
# If a file is deleted or moved out of a watched directory: Check to | |
# see if deletion is enabled in the script. If so, delete it from S3. | |
[ $deletion -eq 1 ] && $s3path/s3cmd.rb --ssl --verbose delete $bucket:"$path" 2>&1 | log | |
;; | |
'IN_DELETE,IN_ISDIR' | 'IN_MOVED_FROM,IN_ISDIR' | '-d') | |
# If a directory is deleted or removed from a watched directory: Remove | |
# the directory from the watched list in the incrontab file. Check | |
# to see if deletion is enabled in the script and if so, remove all files | |
# from within the watched directory. | |
sed -i "\|^${path// /\\\ }|d" $incrontab | |
log "Removing '$path' and all sub-folders from watch list '$incrontab'" | |
[ $deletion -eq 1 ] && $s3path/s3cmd.rb --ssl --verbose deleteall $bucket:"$path" 2>&1 | log | |
;; | |
'UPLOAD') | |
# Check for lockfile and being uploads if it doesn't exist. | |
if ( set -o noclobber; echo "$$" > "$lockfile") 2> /dev/null; | |
then | |
# Trap exits and clean up lockfile. | |
trap 'rm -f "$lockfile"; exit $?' INT TERM EXIT | |
# Pop oldest symlink, store it, and unlink it. | |
symlink=$(ls -rt1 $queue | sed -n -e '1,1p') | |
linkpath="$(readlink "$queue/$symlink")" | |
unlink "$queue/$symlink" | |
# If it is a directory s3sync the files inside. | |
if [ -d "$linkpath" ]; | |
then | |
log "Syncing '$linkpath' and all sub-folders to S3 bucket: '$bucket'" | |
$s3path/s3sync.rb --ssl -r --verbose "$linkpath/" $bucket:"$linkpath" 2>&1 | log | |
# Recurse directories, stream edit them to escape any spaces with \, | |
# append each directory to the incrontab file. NB: For some reason | |
# incron does not pass trailing spaces in triggered events. I've | |
# added a trailing \ to the incrontab and this appears to resolve this. | |
# I assume it causes incron to pass the trailing space and the \, and | |
# bash ignores the \. | |
find "$linkpath" -type d |sed -e 's| |\\ |g' -e "s|\$| $mask $0 \$% \$@/\$#\\\|" >>$incrontab | |
log "Added '$linkpath' and all sub-folders to watch list '$incrontab'" | |
else | |
# Upload single file vis s3cmd put. | |
[ -e "$linkpath" ] && $s3path/s3cmd.rb --ssl --verbose put $bucket:"$linkpath" "$linkpath" 2>&1 | log | |
fi | |
# Clean up lockfile | |
rm -f "$lockfile" | |
trap - INT TERM EXIT | |
# Continue processing queue directory if symlinks remain. | |
if [ "$(ls $queue|wc -l)" -gt "0" ]; then | |
exec $0 UPLOAD | |
fi | |
else | |
log "Upload $(cat $lockfile) in progress. Failed to acquire lockfile: $lockfile." | |
fi | |
;; | |
*) | |
echo "Usage: s3rep.sh [OPTION] DIRECTORY" | |
echo | |
echo "Options:" | |
echo " -a add DIRECTORY and sub-folders to incrontab and sync to S3" | |
echo " NOTE: Do not include a trailing / after DIRECTORY" | |
echo " -d remove DIRECTORY and all sub-folders from incrontab and" | |
echo " if deletion is enabled, remove them from S3" | |
echo | |
exit 1 | |
esac | |
exit 0 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment