Created
January 14, 2013 15:24
-
-
Save pyh/4530759 to your computer and use it in GitHub Desktop.
Very high-tech glazier uploader. For large files you would need these 2 patches: https://rt.cpan.org/Public/Bug/Display.html?id=81219 and https://rt.cpan.org/Public/Bug/Display.html?id=81864
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/perl | |
use strict; | |
use warnings; | |
use Net::Amazon::Glacier; | |
use File::Find; | |
use Digest::SHA qw/sha256_hex/; | |
use File::Slurp; | |
use Net::Amazon::TreeHash; | |
use File::Sync qw(fsync); | |
our $GPG_KEY = 'key_id....'; | |
our $backupdir = "/somedir"; | |
our $toplevel = "tld-dir-for-glazier"; | |
open(LOG, "<upload.journal"); | |
my $uploaded_files = {}; | |
while(<LOG>){ | |
chomp; | |
my($hash,$amazon_id,$size,$path)=split(/\s/,$_,4); | |
$path=$size if !$path; | |
$uploaded_files->{$path}={hash=>$hash,amazon_id=>$amazon_id}; | |
} | |
open(LOG, ">>upload.journal"); | |
our $glacier = Net::Amazon::Glacier->new( | |
'us-east-1', | |
'something-read-the-docs', | |
'something', | |
); | |
use Data::Dumper; | |
print Dumper($glacier->list_vaults()); | |
find({wanted=>\&wanted, no_chdir=>1}, $backupdir); | |
sub wanted { | |
return if /\.(pp2|db|thm|pp3|pto|mk|html|exr|swf|xml)$/i; | |
return if !-f $_; | |
if (-s $_ > 6280*1024*1024){ | |
print "Too large $_\n"; | |
return; | |
} | |
my $desc = $_; | |
$desc=~s/^\E$backupdir\Q/$toplevel/; | |
$desc=~s/ä/a/g; | |
$desc=~s/ö/o/g; | |
my $file = $_; | |
if($uploaded_files->{$desc}){ | |
# print "$desc already uploaded\n"; | |
return; | |
my $localhash = sha256_hex(read_file($file)); | |
if ($localhash eq $uploaded_files->{$desc}->{hash}){ | |
print "$desc already uploaded\n"; | |
return; | |
}else{ | |
die("$desc hash mishmatch\n"); | |
} | |
} | |
print "Encrypting $file\n"; | |
if ( encrypt($file) == 0 ){ | |
my $content_sha = Digest::SHA->new("sha256"); | |
$content_sha->addfile( $file ); | |
my $localhash = $content_sha->hexdigest; | |
if( my $amazon_id = $glacier->upload_archive( "backups", "/tmp/$$.gpg", $desc ) ){ | |
print LOG "$localhash $amazon_id ", -s "/tmp/$$.gpg", " $desc\n"; | |
fsync(\*LOG); | |
print "Uploaded ", -s "/tmp/$$.gpg", " $localhash $amazon_id $desc\n"; | |
}else{ | |
die("Could not upload $desc"); | |
} | |
}else{ | |
die("Enc error: $?\n"); | |
} | |
} | |
#encrypt(); | |
sub encrypt { | |
unlink("/tmp/$$.gpg"); | |
system("gpg", "-r", $GPG_KEY, "--output", "/tmp/$$.gpg", "--encrypt", $_[0]); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment