Script to replicate vault storage from S3 to consul. Expects consul-agent on localhost:8500 and for AWS credentials to be set via environment/credential file/EC2 role

vlt-aws2consul.js

var _ = require('lodash'),
    Promise = require('bluebird'),
    consul = require('consul'),
    logger = require('log4js').getDefaultLogger(),
    AWS = require('aws-sdk');


var bucket = 'my.vault.main',
    prefix = '',
    region = 'us-west-2',
    cprefix = 'local/vlt/vlt-main';

var s3 = new Promise.promisifyAll(new AWS.S3({
    apiVersion: '2012-06-01',
    region: region,
    sslEnabled: true
}));

var consul = require('consul')(),
    kv = Promise.promisifyAll(consul.kv);

function listObjects() {
    logger.debug("Scanning bucket [" + [bucket,prefix].join("/") + "]");
    var params = {
        Bucket: bucket,
        Prefix: prefix
    };
    var res = {};
    var more = true;

    return s3.listObjectsAsync(params).then(function(data) {
        more = data.IsTruncated;
        _.each(data.Contents, function(obj) {
            res[obj.Key] = {Key:obj.Key,LastModified:obj.LastModified,ETag:obj.ETag,Size:obj.Size};
        });
        if (more) {
            params.Marker = _.last(data.Contents)['Key'];
            return Promise.delay(1).then(getBatch());
        } else {
            return Promise.resolve();
        }
    }).then(function() {
        logger.info("" + [bucket,prefix].join("/") +  " contains " + Object.keys(res).length + " keys");
        return Promise.resolve(res);
    }).catch(function(err) {
        logger.error(err);
    });
}

function clearconsul() {
    logger.info("Deleting old " + cprefix + " namespace");
    return kv.delAsync({key:cprefix,recurse:true}).catch(function(err) {
        logger.warn(err);
        return Promise.resolve();
    })
}

function write2consul(keys) {
    return Promise.map(_.keys(keys), function(key) {
        key = keys[key];
        logger.debug("Writing " + [cprefix,key.Key].join("/"));
        s3.getObjectAsync({Bucket: bucket, Key: key.Key}).then(function(data) {
            return kv.setAsync({key:[cprefix,key.Key].join("/"), value:data.Body});
        }).catch(function(err) {return Promise.reject(err)});
    }, {concurrency: 10});
}

clearconsul().then(listObjects).then(write2consul).catch(function(err) {
    logger.error(err);
}).done(function() {
    logger.info("Finished");
});