Created
March 15, 2017 18:04
-
-
Save jonathan-wheeler/ec075b00f4c3d4cdb1bca413a043b63a to your computer and use it in GitHub Desktop.
Lambda function to synchronize no objects between two diffrent S3 Buckets
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// adapted from http://docs.aws.amazon.com/lambda/latest/dg/with-s3-example-deployment-pkg.html | |
// dependencies | |
var async = require('async'); | |
var AWS = require('aws-sdk'); | |
// set environment vars | |
var DESTINATION_BUCKET = process.env.DESTINATION_BUCKET | |
// get reference to S3 client | |
var s3 = new AWS.S3(); | |
exports.handler = function(event, context, callback) { | |
if (event.Records == null) { | |
context.fail('Error', "Event has no records."); | |
return; | |
} | |
// Process all records in the event asynchronously. | |
async.each(event.Records, processRecord, function (err) { | |
if (err) { | |
context.fail('Error', "One or more objects could not be copied."); | |
} else { | |
context.succeed(); | |
} | |
}); | |
console.log("done") | |
}; | |
function processRecord(record, callback) { | |
// Read options from the event. | |
var srcBucket = record.s3.bucket.name; | |
// Object key may have spaces or unicode non-ASCII characters. | |
var srcKey = | |
decodeURIComponent(record.s3.object.key.replace(/\+/g, " ")); | |
var dstBucket = DESTINATION_BUCKET; | |
var dstKey = srcKey; | |
// Sanity check: validate that source and destination are different buckets. | |
if (srcBucket == dstBucket) { | |
callback("Source and destination buckets are the same."); | |
return; | |
} | |
s3.copyObject({ | |
Bucket: dstBucket, | |
CopySource: `${srcBucket}/${srcKey}`, | |
Key: dstKey, | |
}, function (err) { | |
if (err) { | |
console.error( | |
'Unable to Copy ' + srcBucket + '/' + srcKey + | |
' and upload to ' + dstBucket + '/' + dstKey + | |
' due to an error: ' + err | |
); | |
callback(err); | |
} else { | |
console.log( | |
'Successfully copied ' + srcBucket + '/' + srcKey + | |
' and uploaded to ' + dstBucket + '/' + dstKey | |
); | |
callback(); | |
} | |
}) | |
} | |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "s3sync ", | |
"description": "sync new s3 objects to another bucket.", | |
"version": "1.0.0", | |
"author": "Jonny Wheeler [email protected]", | |
"license": "MIT", | |
"dependencies": { | |
"aws-sdk": "^2.7.9", | |
"async": "^2.1.2" | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"Version": "2012-10-17", | |
"Statement": [ | |
{ | |
"Effect": "Allow", | |
"Action": [ | |
"s3:ListBucket" | |
], | |
"Resource": [ | |
"arn:aws:s3:::EXISTING-BUCKET" | |
] | |
}, | |
{ | |
"Effect": "Allow", | |
"Action": [ | |
"s3:GetObject" | |
], | |
"Resource": [ | |
"arn:aws:s3:::EXISTING-BUCKET/*" | |
] | |
}, | |
{ | |
"Effect": "Allow", | |
"Action": [ | |
"s3:ListBucket" | |
], | |
"Resource": [ | |
"arn:aws:s3:::NEW-BUCKET" | |
] | |
}, | |
{ | |
"Effect": "Allow", | |
"Action": [ | |
"s3:PutObject" | |
], | |
"Resource": [ | |
"arn:aws:s3:::NEW-BUCKET/*" | |
] | |
} | |
] | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment