Skip to content

Instantly share code, notes, and snippets.

@dustMason
Created May 30, 2014 00:17
Show Gist options
  • Save dustMason/5647440148a53083bba3 to your computer and use it in GitHub Desktop.
Save dustMason/5647440148a53083bba3 to your computer and use it in GitHub Desktop.
s3 tar streamer attempt
var knox = require("knox");
var mpu = require("knox-mpu");
var s3lister = require("s3-lister");
var async = require("async");
var stream = require("stream");
var tar = require("tar");
var bufferSize = 5;
var client = knox.createClient({
key: process.env.S3_ACCESS_KEY_ID,
secret: process.env.S3_SECRET_KEY,
bucket: process.env.S3_BUCKET
});
var lister = new s3lister(client, { prefix: 'departments' });
var keys = [];
var tarStream = tar.Pack();
lister.on('data', function(data) {
console.log(data.Key);
keys.push(data.Key);
});
lister.on('end', function() {
console.log('==> Done!, looping over keys...');
async.mapSeries(keys, function(key) {
client.get(key).on('response', function(res){
console.log(res.statusCode);
res.pipe(tarStream);
}).end();
}, function(results) {
console.log(results);
});
});
lister.on('error', function(err) {
console.log('==> Error!', err);
});
var upload = new mpu({
client: client,
objectName: 'departments.tar',
stream: tarStream
}, function(err, body) {
console.log(body);
});
events.js:72
throw er; // Unhandled 'error' event
^
Error: write after end
at writeAfterEnd (_stream_writable.js:133:12)
at WriteStream.Writable.write (_stream_writable.js:181:5)
at Pack.<anonymous> (/Users/jordan/Code/nuts.com/image-archiver/node_modules/knox-mpu/lib/multipartupload.js:246:28)
at Pack.EventEmitter.emit (events.js:95:17)
at EntryWriter.<anonymous> (/Users/jordan/Code/nuts.com/image-archiver/node_modules/tar/lib/pack.js:191:8)
at EntryWriter.EventEmitter.emit (events.js:95:17)
at BlockStream.<anonymous> (/Users/jordan/Code/nuts.com/image-archiver/node_modules/tar/lib/entry-writer.js:29:8)
at BlockStream.EventEmitter.emit (events.js:95:17)
at BlockStream._emitChunk (/Users/jordan/Code/nuts.com/image-archiver/node_modules/tar/node_modules/block-stream/block-stream.js:145:10)
at BlockStream.flush (/Users/jordan/Code/nuts.com/image-archiver/node_modules/tar/node_modules/block-stream/block-stream.js:70:8)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment