Streamer.js

Streamer.js

var knox = require("knox");
var mpu = require("knox-mpu");
var s3lister = require("s3-lister");
var async = require("async");
var fs = require("fs");
var archiver = require("archiver");

var client = knox.createClient({
  key: process.env.S3_ACCESS_KEY_ID,
  secret: process.env.S3_SECRET_KEY,
  bucket: process.env.S3_BUCKET
});

var lister = new s3lister(client, { prefix: 'departments' });
var keys = [];

lister.on('data', function(data) {
  keys.push(data.Key);
});

lister.on('end', function() {
  console.log('==> Done!, looping over '+keys.length+' keys...');

  var tarStream = archiver('tar');
  var upload = new mpu({
    client: client,
    objectName: 'departments.tar',
    stream: tarStream
  }, function(err, body) {
    console.log("Upload callback", err, body);
  });
  upload.on('uploading', console.log);
  upload.on('uploaded', console.log);
  upload.on('error', console.log);
  upload.on('initiated', console.log);
  upload.on('completed', console.log);

  // for testing:
  keys = keys.slice(1, 120);

  async.mapSeries(keys, function(key, callback) {

    console.log("Getting key", key);
    client.getFile(key, function(err, res) {
      if (err) {
        console.log(err);
      } else {
        tarStream.append(res, { name: key });
        res.on("end", callback);
      }
    });

  }, function() {
    console.log("Callback on async.mapSeries");
    tarStream.finalize();
  });
});

lister.on('error', function(err) {
  console.log('==> Error!', err);
});