Skip to content

Instantly share code, notes, and snippets.

@abhididdigi
Last active August 29, 2015 14:12
Show Gist options
  • Save abhididdigi/22d0275985112df76cf1 to your computer and use it in GitHub Desktop.
Save abhididdigi/22d0275985112df76cf1 to your computer and use it in GitHub Desktop.
Scrapping Topcoder with CasperJS
var casper = require('casper').create();
var startID = 1964;
var endID = 13533;
casper.start();
while (startID <= endID) {
scrapPages(startID);
startID++;
}
function scrapPages(i) {
var str = "Starting to get the HTML for the problem" + i;
var url = 'http://community.topcoder.com/stat?c=problem_statement&pm=' + i;
casper.thenOpen(url, function() {
var result = this.getHTML('td.problemText');
casper.then(function() {
var fileName = 'problem' + i + ".html";
require('fs').write(fileName, result, 'w');
this.echo("writing it to filename: " + fileName)
});
});
}
casper.run();
//Invocation : casperjs /path_to_this_file/casper_scrapping.js
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment