Skip to content

Instantly share code, notes, and snippets.

@qcom
Created July 9, 2013 21:21
Show Gist options
  • Save qcom/5961400 to your computer and use it in GitHub Desktop.
Save qcom/5961400 to your computer and use it in GitHub Desktop.
var casper = require('./util/augment/extend')(require('casper').create({}));
var fs = require('fs');
var check = require('./util/check');
var sumIn = require('../mans/util/sumIn');
casper.start();
var n = 0;
var mans;
casper.label('start');
casper.then(function() {
++n;
mans = require('./mans.json');
var files = fs.list('./output').slice(2);
var todo = [];
this.each(Object.keys(mans), function(self, manKey) {
var file = manKey + '.json';
if (files.indexOf(manKey + '.json') !== -1) {
var man = require('./output/' + file);
var scrapeCount = man.length;
var totalCount = sumIn(mans[manKey]);
if (scrapeCount !== totalCount) {
todo.push(manKey);
}
} else {
todo.push(manKey);
}
});
});
casper.then(function() {
if (n < 5) {
this.goto('start');
}
});
casper.run(function() {
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment