Run 10000 async calls through a pipeline'd sequencer

example-10k-sequence-pipeline.js

//
// Example #3 : sequence 10000 async calls in a pipeline, with max 500 open
// 

function pipeline(arr, maxOpenCalls, finalCallback) {

  var openCalls = 0;

  function callback() {
    if (--openCalls < maxOpenCalls) {
      launch();
    }
  }

  function launch() {
    if (arr.length != 0) {
      do {
        openCalls++;
        arr.shift()(callback);
      } while (openCalls < maxOpenCalls);
    } else if ((arr.length == 0) && (openCalls == 0)){
      console.log("Finished all calls");
      finalCallback();
    } else {
      console.log("Too many in flight, waiting");
    }
  }

  var initial = (maxOpenCalls > arr.length) ? maxOpenCalls : arr.length;

  for (var i=0; i<initial; i++) {
    launch();
  }
}

//
// Build an array with 10000 sequencable async calls
//

var asyncCalls = [];

for (var i=0;i<10000;i++) {
  (function(no) {
      asyncCalls.push(
        function(callback) {
          console.log("Call #" + no);
          setTimeout(callback, 200); // Simulate async
        }
      );
    })(i);
}

pipeline(asyncCalls, 500, function() { console.log("All done"); });