JavaScript で1つ以上の引数取って結果をキャッシュしつつ非同期で返すやつの挙動がおかしい

is_prime.js

var isPrime = (function() {
  var cache = {},
      callbackList = {};

  return function(args, callback) {
    if (typeof args === 'number') {
      args = [args];
    }

    var argsCount = args.length,
        results = {};

    var doCallback = function() {
      callback(results);
    };

    if (argsCount === 0) {
      doCallback();
      return;
    }

    var checkCalculated = function(number, result) {
      results[number] = result;

      argsCount -= 1;
      if (argsCount <= 0) {
        doCallback();
      }
    };

    var onCalculate = function(number, result) {
      // Mark this number as calculated.
      cache[number] = result;

      // Get the list of callback checks waiting for this number
      var callbacks = callbackList[number];
      delete callbackList[number];

      // Check all callbacks waiting for this number
      for (var i = 0; i < callbacks.length; i += 1) {
        callbacks[i](number, result);
      }
    };

    var isPrime = function (number) {
      if (isNaN(number) || !isFinite(number) || number % 1 || number < 2) {
        return false;
      }
      var sqrt = Math.sqrt(number);
      for (var i = 2; i <= sqrt; i += 1) {
        if (number % i === 0) {
          return false;
        }
      }
      return true;
    };

    var calculate = function(number) {
      var result = cache[number];
      if (typeof result !== 'undefined') {
        checkCalculated(number, result);
        return;
      }

      var callbacks = callbackList[number] || (callbackList[number] = []);
      callbacks.push(checkCalculated);

      // Calculate it only for the first request
      if (callbacks.length > 1) {
        return;
      }

      setTimeout(function() {
        onCalculate(number, isPrime(number));
      }, 0);
    };

    for (var i = 0; i < argsCount; i += 1) {
      calculate(args[i]);
    }
  };
}());

最後の引数以外に既に計算した（キャッシュされた）数を与えると結果が返って来ない

isPrime([11], function(results) { console.log(results); })
// => {11: true}

isPrime([11], function(results) { console.log(results); })
// => {11: true}

isPrime([23, 31], function(results) { console.log(results); })
// => {23: true, 31: true}

isPrime([23, 31], function(results) { console.log(results); })
// 結果が返って来ない