Skip to content

Instantly share code, notes, and snippets.

@founddrama
Last active August 29, 2015 14:05
Show Gist options
  • Save founddrama/17f27a7630a5683f7bdb to your computer and use it in GitHub Desktop.
Save founddrama/17f27a7630a5683f7bdb to your computer and use it in GitHub Desktop.
Oh nothing just screenscraping Untappd for my beer data because they're so stingy with their API keys.
var casper = require('casper').create(),
username = casper.cli.args[0],
passwod = casper.cli.args[1],
UNTAPPD = 'https://untappd.com';
// in which we get all the beers
// in which we assume `$` == `jQuery` already on this page
function collectBeers(UNTAPPD) {
var beers = [],
items = $('.beer-item'),
rx = /([\d\.]+)/;
function getNumber($el) {
var result = rx.exec($el.text().trim());
if (result) return Number(result[1]);
else return '?';
}
items.each(function(i, el) {
var $el = $(el),
beer = {
'BID': $el.data('bid'),
'Name': $el.find('.name').text().trim().replace(/,/g, ''),
'Brewery': $el.find('.brewery').text().trim().replace(/Company/g, 'Co.').replace(/,/g, ''),
'Style': $el.find('.style').text().trim(),
'My Rating': getNumber($el.find('.ratings .you p:contains("Your Rating")')),
'Global Rating': getNumber($el.find('.ratings .you p:contains("Global Rating")')),
'ABV': getNumber($el.find('.abv')),
'IBU': getNumber($el.find('.ibu')),
'Checkins': getNumber($el.find('.check-ins')),
'First Checkin': $el.find('.date:contains("First") a').text().trim(),
'Latest Checkin': $el.find('.date:contains("Recent") a').text().trim(),
'URL': UNTAPPD + $el.find('.label').attr('href')
};
beers.push(beer);
});
return beers;
}
function beers2CSV(beers) {
var keys = Object.keys(beers[0]);
csv = [keys.concat(['Notes'])];
beers.forEach(function(b) {
var beer = keys.map(function(k) {
return b[k];
}).concat('');
csv.push(beer.join(','));
});
return csv.join('\n');
}
casper.start(UNTAPPD + '/login', function() {
this.fillSelectors('form', {
'#username': username,
'#password': password
}, true);
});
casper.thenOpen(UNTAPPD + '/user/' + username + '/beers', function() {
this.log(this.getCurrentUrl(), 'debug');
});
casper.then(function() {
this.log('Retrieving beer data...', 'debug');
var MORE_BEERS_SELECTOR = '.button.more_beers',
LOADING_ICON = '.stream-loading',
counter = 0,
getMoreBeers = function() {
this.log('Getting page ' + (++counter) + '...', 'debug');
this.click(MORE_BEERS_SELECTOR);
this.waitUntilVisible(LOADING_ICON, hasLoadedAll, function() {
this.log(LOADING_ICON + ' never appeared; didn\'t load?', 'error').exit();
}, 500);
}.bind(this),
hasLoadedAll = function() {
this.waitWhileVisible(LOADING_ICON, function() {
if (this.visible(MORE_BEERS_SELECTOR)) {
getMoreBeers();
} else {
this.log('Evaluating page contents...', 'debug');
var allBeers = this.evaluate(collectBeers, UNTAPPD);
this.echo(beers2CSV(allBeers)).exit();
}
});
}.bind(this);
getMoreBeers();
});
casper.run();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment