Skip to content

Instantly share code, notes, and snippets.

@averagesecurityguy
Last active August 29, 2015 14:05
Show Gist options
  • Save averagesecurityguy/666658c8ad41ca68ebc6 to your computer and use it in GitHub Desktop.
Save averagesecurityguy/666658c8ad41ca68ebc6 to your computer and use it in GitHub Desktop.
URL Scanner in Node
var fs = require('fs');
var readline = require('readline');
var http = require('http');
var timeout = 2; // Timeout in seconds.
var rd = readline.createInterface({
input: fs.createReadStream('tenthousand_urls.txt'),
output: process.stdout,
terminal: false
});
function processResponse(res) {
var code = res.statusCode;
var requestedHost = res.req.getHeader('host');
var redirectedHost = res.headers['location'];
if (code >= 200 && code <= 299) {
console.log('2XX: ' + requestedHost);
} else if (code >= 300 && code <= 399) {
console.log('3XX: ' + requestedHost + ' -> ' + redirectedHost);
} else if (code >= 400 && code <= 499) {
console.log('4XX: ' + requestedHost);
} else if (code >= 500 && code <= 599) {
console.log('5XX: ' + requestedHost);
} else {
console.log(code + ': ' + requestedHost);
}
}
function getUrl(hostname, port) {
var options = {method: 'HEAD', hostname: hostname, port: port};
var req = http.request(options);
req.on('response', processResponse);
req.on('error', function(e) {});
req.setTimeout(timeout * 1000, req.abort);
req.end();
}
rd.on('line', function(line) {
switch(line) {
case line[0] === '#':
break;
case line === '':
break;
case line.slice(0, 4) !== 'http':
break;
default:
if (line.slice(0, 5) === 'http:') {
getUrl(line.slice(7), 80);
}
if (line.slice(0, 5) === 'https') {
getUrl(line.slice(8), 443);
}
}
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment