Parsing crawler proxy with Hapi

crawler-parser-proxy.js

var _ = require("lodash"),
	Hapi = require("hapi"),
	Nipple = require("nipple"),
	jsdom = require("jsdom"),
	Url = require("url");

var parsers = {
	"default": function (url, document, cb) {
		cb(null, {
			title: document.title,
			text: document.body.textContent.replace(/\s+/g, " ").trim()
		});
	}
};

function parse(url, responseBody, cb) {
	var domain = Url.parse(url).hostname;
	if (domain.indexOf("www.") === 0) domain = domain.substr(4);

	var parser = parsers[domain] || parsers["default"];
	jsdom.env(responseBody.toString(), function (e, win) {
		if (e) {
			return cb(e);
		}

		// clean out non-text content tags
		_(["script", "link", "img", "style", "meta", "iframe", "noscript"]) // iframe inside noscript??
			.map(function (t) {
				return _.toArray(win.document.body.getElementsByTagName(t))
			})
			.flatten()
			.forEach(function (el) {
				el.parentNode.removeChild(el);
			});

		parser(url, win.document, function (e, parsed) {
			cb(e, parsed);
		});
	});
}

var server = new Hapi.Server({});

server.connection({port: 3128});

server.method('getUrl', function (url, cb) {
	console.log("Downloading", url);
	Nipple.get(url, function (err, res, payload) {
		cb(err, payload);
	});
}, {
	cache: {
		expiresIn: 10 * 1000
	}
});

server.route({
	method: '*',
	path: '/{p*}',
	handler: function (req, reply) {
		var url = req.url.href;
		req.server.methods.getUrl(url, function (err, payload) {

			if (err) {
				return reply(err);
			}

			console.log("Parsing", url);
			parse(url, payload, function (err, parsed) {
				reply(err || parsed);
			});

		});
	}
});

server.start(function () {
	console.log("Listening");
});