awslambda-throttled-worker

index.js

'use strict';
var _ = require('lodash');
var aws = require('aws-sdk');
var request = require('request');

var sqsUrl: 'sqs://';

var defaults = {
    timeout: 300, // t
    buffer: 15, // b
    limit: 10, // l
};

function partition() {
    // n = (t-b)/l;
    // partition data by n as tasks and write tasks msg to sqs
}

function initialize() {
    // get sqs msg
    // build key based on x... (msg id)?
    // start s3 part on key
}

function recurse(options) {
    // aws.lambda invoke self with options
}

function wrapup(err) {
    // if (err) cancel s3 part
    // else close s3 part
    // delete sqs msg batch
    // any other cleanup tasks...

    // call myself using aws js sdk passing in options arg
    return recurse({sqsUrl: options.sqsUrl, worker: options.worker});
}

function work() {
    // do while tasks.length = 0
        options.worker(task);
        // on error tasks -> dead letter

    // cleanup external dependencies
    wrapup();
}

function start(options) {
    // initial setup; happens only once
    if(options.data) {
        partition();
        return recurse({sqsUrl: options.sqsUrl, worker: options.worker});
    }

    // prime external dependencies
    initialize();

    // invoke work n times (limit) per second
    setInterval(function () {
        _.times(options.limit || defaults.limit, work);
    }, 1000);
}

start({
    tasks: 'Array | s3:// | Buffer | String', 
    worker: function (task) { 
        // worker implementation i.e. stream google maps api req
        return stream;
    }
});

//todo: add logging
//todo: add encryption
//todo: add notification i.e. email with signed url...?