Download all files from an amazon S3 bucket using node.

S3Downloader.js

// Config
var bucketName  = 'xxx';
var saveFileDir = 'files';
var creds       = {
  key     : 'xxx',
  secret  : 'xxx',
  bucket  : bucketName
};


// Dependencies
var knox        = require('knox');
var fs          = require('fs');
var mkPath      = require('mkpath');


// Create the directory in which the requested files will be saved
mkPath(saveFileDir, function(err){
  if(err) console.log(err);
});


// Connect knox client, list all files
var client = knox.createClient(creds);
client.list({}, function(listErr, data){

  if(listErr) console.log('Error: ', listErr);
  for(var i=0; i<data.Contents.length; i++){

    var key = data.Contents[i].Key;
    client.getFile(key, function(getErr, res){

      if(getErr) console.log('Error: ', getErr);
      else{

        // Truncate file name and create a file path
        var fileName = res.req.path.replace('/' + bucketName + '/', '')
        var filePath = saveFileDir + '/' + fileName;

        // Define proper encoding
        var encoding = 'utf8';
        if(fileName.match('(.jpg|.jpeg|.png|.pdf|.gif)')) encoding = 'binary'
        res.setEncoding(encoding);

        // Set up a buffer to collect file parts
        var buffer = '';
        res.on('data', function(chunk){
          buffer += chunk;
        });

        // Write the file to the filePath
        res.on('end', function(){
          fs.writeFile(filePath, buffer, encoding, function(writeErr){
            if(writeErr) console.log('Error: ', writeErr);
            else console.log('Succesfully wrote ' + filePath);
          });
        });
      }
    });

  }
});