node.js http.agent の agent.maxSockets の上限を避ける

gistfile1.js

var http, url, path;

http = require('http');
url  = require('url');
path = require('path');

function httpRequest (/* getURL, method, options, onResponse */) {
        var that, watcher, helper,
            args, getURL, uri, uriPath, method, options, headers, body, requestOptions;

        that       = this;
        args       = Array.prototype.slice.apply(arguments);
        getURL     = args.shift();
        onResponse = args.pop();
        method     = args.shift() || 'GET';
        method     = method.toUpperCase();
        options    = args.shift() || {};
        if (options.body) {
            body = options.body || null;
            delete options.body;
        }
        headers   = options;

        uri     = url.parse(getURL());
        uriPath = (! uri.pathname)                   ? '/'
                : (uri.pathname.slice(0, 1) === '/') ? uri.pathname
                :                                      '/' + uri.pathname;
        if (uri.search) uriPath += uri.search;

        if (method === 'POST') {
            headers['content-type'] = headers['content-type'] || 'application/x-www-form-urlencoded';
            headers['content-length'] = (body) ? Buffer.byteLength(body) : 0;
        }

        if (! headers.referer && this.referer) headers.referer = this.referer;
        if (! headers.cookie  && this.cookie)  headers.cookie  = this.cookie;

        requestOptions = {
            method  : method,
            host    : uri.host,
            port    : (uri.port) ? uri.port : (uri.protocol === 'https') ? '443' : '80',
            path    : uriPath,
            headers : headers
        };

        watcher = function () {
                var agent, intervalID, connectNum, connectMax;
                agent      = http.getAgent(requestOptions.host, requestOptions.port);
                connectMax = agent.maxSockets;
                intervalID = setInterval(function () {
                        if (agent.sockets.length < connectMax) {
                                helper();
                                clearInterval(intervalID);
                                intervarID = undefined;
                                return;
                        }
                }, 10);
        };

        helper = function () {
                var req;
                req  = http.request(requestOptions);
                req.on('error', function (error) {
                        console.log(error);
                        process.exit(1);
                });
                req.on('response', function (response) {
                        var statusCode, responseHeaders, _onResponse;
                        statusCode      = response.statusCode;
                        responseHeaders = response.headers;
                        if (responseHeaders['set-cookie']) that.cookie = responseHeaders['set-cookie'][0];
                        that.referer = uri.href;
                
                        if (that.redirect && statusCode >= 300 && statusCode < 400 && responseHeaders.location) {
                            _onResponse = onResponse;
                            httpRequest(
                                function () { return responseHeaders.location; }, 'GET',
                                { referer : that.referer, cookie : that.cookie },
                                function (response, request) {
                                        _onResponse(response, request);
                                }
                            )
                            return;
                        }
                
                        onResponse(response, [ uri.href, headers, body ]);
                });
                if (body) req.write(body);
                req.end();
        };

        watcher();
}


exports.httpRequest = httpRequest;

[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ].forEach(function (n, i) {
        var uri = [ 'http://image.foo.com/image/p', n, '.jpg' ].join('');
        var client = {};
        client.httpRequest = httpRequest;
        client.httpRequest(function () { return uri; }, function (response, _request) {
            var target, wStream;
            target = path.basename(uri);
            wStream = fs.createWriteStream(target);
            wStream.on('error', function (err) {
                return console.log(err);
            });
            response.on('end', function () {
                 return console.log('response "end"');
            });
            response.pipe(wStream);
        });
});