-
-
Save mrlannigan/3872261 to your computer and use it in GitHub Desktop.
Node.js http timeout
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
node_modules |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var http = require('http'), | |
async = require('async'), | |
extCount = 0, | |
concurrency = 20; | |
var extensions = ['com', 'ac', 'ad', 'ae', 'com.af', 'com.ag', 'com.ai', 'am', 'it.ao', | |
'com.ar', 'as', 'at', 'com.au', 'az', 'ba', 'com.bd', 'be', 'bf', 'bg', 'com.bh', 'bi', | |
'bj', 'com.bn', 'com.bo', 'com.br', 'bs', 'co.bw', 'com.by', 'com.bz', 'ca', 'com.kh', | |
'cc', 'cd', 'cf', 'cat', 'cg', 'ch', 'ci', 'co.ck', 'cl', 'cm', 'cn', 'com.co', 'co.cr', | |
'com.cu', 'cv', 'cz', 'de', 'dj', 'dk', 'dm', 'com.do', 'dz', 'com.ec', 'ee', 'com.eg', | |
'es', 'com.et', 'fi', 'com.fj', 'fm', 'fr', 'ga', 'gd', 'ge', 'gf', 'gg', 'com.gh', | |
'com.gi', 'gl', 'gm', 'gp', 'gr', 'com.gt', 'gy', 'com.hk', 'hn', 'hr', 'ht', 'hu', | |
'co.id', 'iq', 'ie', 'co.il', 'im', 'co.in', 'io', 'is', 'it', 'je', 'com.jm', 'jo', | |
'co.jp', 'co.ke', 'com.kh', 'ki', 'kg', 'co.kr', 'com.kw', 'kz', 'la', 'com.lb', | |
'com.lc', 'li', 'lk', 'co.ls', 'lt', 'lu', 'lv', 'com.ly', 'co.ma', 'md', 'me', 'mg', | |
'mk', 'ml', 'mn', 'ms', 'com.mt', 'mu', 'mv', 'mw', 'com.mx', 'com.my', 'co.mz', | |
'com.na', 'ne', 'com.nf', 'com.ng', 'com.ni', 'nl', 'no', 'com.np', 'nr', 'nu', 'co.nz', | |
'com.om', 'com.pa', 'com.pe', 'com.ph', 'com.pk', 'pl', 'pn', 'com.pr', 'ps', 'pt', | |
'com.py', 'com.qa', 'ro', 'rs', 'ru', 'rw', 'com.sa', 'com.sb', 'sc', 'se', 'com.sg', | |
'sh', 'si', 'sk', 'com.sl', 'sn', 'sm', 'so', 'st', 'com.sv', 'td', 'tg', 'co.th', | |
'com.tj', 'tk', 'tl', 'tm', 'to', 'com.tn', 'com.tr', 'tt', 'com.tw', 'co.tz', 'com.ua', | |
'co.ug', 'co.uk', 'us', 'com.uy', 'co.uz', 'com.vc', 'co.ve', 'vg', 'co.vi', 'com.vn', | |
'vu', 'ws', 'co.za', 'co.zm', 'co.zw']; | |
// THE QUEUE | |
var downloadQueue = async.queue(function(task, callback) { | |
http.get({host: task.url}, function(res) { | |
console.log(task.url + ' : ' + res.statusCode); | |
// - You must always insure the callback is fired. | |
// - You don't have to call the queue callback in the error because | |
// the current http callback will always fire, regardless of connection status. | |
callback(); | |
}).on('error', function(e) { | |
console.log(task.url + ' error: ' + e.message); | |
}); | |
}, concurrency); | |
downloadQueue.saturated = function() { | |
console.log('downloadQueue: is saturated'); | |
} | |
downloadQueue.empty = function() { | |
console.log('downloadQueue: is now empty'); | |
} | |
downloadQueue.drain = function() { | |
console.log('downloadQueue: has finished processing all queued tasks'); | |
} | |
//Do a lot of http get requests to different google pages | |
for (var i = 0; i < 1000; i++){ | |
downloadQueue.push({url: 'www.google.' + extensions[extCount]}); | |
//Increment the extension counter | |
extCount++; | |
//Reset the extension counter if necessary | |
if(extCount == extensions.length){ | |
extCount = 0; | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "gist-3872261", | |
"version": "0.0.1", | |
"dependencies": { | |
"async": "*" | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment