CasperJS Scraper Script for Metacritic

metascraper

var fs = require('fs'),
    _ = require('lodash'),
    casper = require('casper').create({
        clientScripts: ["jquery.js", "moment.js"]
    });

var games = [],
    pageCount = 146;

var fetchGamesFromPage = function(page) {

    return page.evaluate(function() {

        var gameList = [];

        $('.product_row').each(function(index, el) {

            var game = {
                title: $.trim($(el).find('.product_title a').text().replace(/\n/g, " ").replace(/\s{2,}/g, ' ')),
                rating: parseInt($(el).find('.product_score div').text()),
                userRating: parseInt($(el).find('.product_userscore_txt .textscore').text()),
                released: moment.utc($.trim($(el).find('.product_date').text())).format()
            };

            game.platform = game.title.split(" ").pop().substr(1, game.title.split(" ").pop().length - 2);
            game.title = game.title.substr(0, game.title.lastIndexOf(" "));
            gameList.push(game);
        
        });

        return gameList;

    });
};

var openPage = function(index) {
    casper.thenOpen('http://www.metacritic.com/browse/games/score/metascore/all?sort=desc&page=' + index, function() {
        console.log('Fetching page ' + index);
        games.push(fetchGamesFromPage(this));
        this.emit('games.fetched', index);
    });
};

casper.start('http://www.metacritic.com/browse/games/score/metascore/all/all?sort=desc&page=0', function() {
    console.log('Fetching ' + pageCount + ' pages:');
    games.push(fetchGamesFromPage(this));
    for (i = 0; i < pageCount; i++) {
        openPage(i + 1);
    }
});

casper.on('games.fetched', function(index) {
    if (pageCount === index) {
        console.log('Finished! Games processed: ' + _.flatten(games).length);
        fs.write('games.json', JSON.stringify(_.flatten(games), null, '  '), 'w');
    }
});

casper.run();