Created
February 6, 2014 18:35
-
-
Save newtriks/8849948 to your computer and use it in GitHub Desktop.
NodeJS module to download multiple files from Amazon S3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env node | |
var Q = require('q'), | |
FS = require('fs'), | |
PB = require('progress'), | |
AWS = require('aws-sdk'), | |
conf = new require('../config')(); | |
AWS.config.update(conf.aws_credentials); | |
var s3 = new AWS.S3(); | |
function downloadFile(filename) { | |
var deferred = Q.defer(), | |
output = conf.inputFolder + filename, | |
stream = FS.createWriteStream(output), | |
params = { | |
Bucket: conf.S3bucket, | |
Key: filename | |
}; | |
var bar; | |
s3.getObject(params) | |
.on('httpHeaders', function (statusCode, headers, resp) { | |
var len = parseInt(headers['content-length'], 10); | |
bar = new PB(' ' + filename + ': [:bar] :percent :etas', { | |
complete: '=', | |
incomplete: ' ', | |
width: 20, | |
total: len | |
}); | |
}) | |
.on('httpData', function (chunk) { | |
stream.write(chunk); | |
bar.tick(chunk.length); | |
}) | |
.on('httpDone', function (response) { | |
if (response.error) { | |
deferred.reject(response.error); | |
} else { | |
deferred.resolve(output); | |
} | |
stream.end(); | |
}) | |
.send(); | |
return deferred.promise; | |
} | |
module.exports = { | |
'getFiles': function (files) { | |
return Q.all(files.map(downloadFile)); | |
} | |
}; | |
// *********** --------------------- *********** // | |
// *********** example usage: app.js *********** // | |
// *********** --------------------- *********** // | |
// #!/usr/bin/env node | |
// var Q = require('q'), | |
// fileAPI = require('./lib/file'), | |
// filenames = [ | |
// 'files/one.pdf', | |
// 'files/two.pdf', | |
// 'files/three.ppt', | |
// 'files/four.ppt' | |
// ]; | |
// fileAPI.getFiles(filenames) | |
// .then(console.log) | |
// .fail(function (error) { | |
// console.error('Error: ' + error.statusCode + ' - ' + error.message); | |
// }); | |
// *********** // example usage: app.js *********** // | |
// *********** -------------- *********** // | |
// *********** example output *********** // | |
// *********** -------------- *********** // | |
// files/one.pdf: [===================] 100% 0.0s | |
// files/two.pdf: [===================] 100% 0.0s | |
// files/three.ppt: [===================] 100% 0.0s | |
// conversions/four.ppt: [===================] 100% 0.0s | |
// *********** // example output *********** // |
thanks for the gist, do you have any suggestions on how the above code can be used with koajs or expressjs allowing download of multiple files?
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Nice piece of code.. Thanks for sharing !