Created
May 28, 2016 06:31
-
-
Save pastleo/9351ba705693dd97640f7e268697cda8 to your computer and use it in GitHub Desktop.
freecodecamp crawler using promise, generator and coroutine
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var request = require('request-promise'); | |
var co = require('co'); | |
var fs = require('fs'); | |
var cheerio = require('cheerio'); | |
var dateFormat = require('dateformat'); | |
var usernames = ['jd615645', 'jayhung97724']; | |
var fcc_info = []; | |
var parser_time = dateFormat(new Date(), 'yyyy-mm-dd HH:MM:ss'); | |
co(function*(){ | |
var fcc_info = yield usernames.map(function(name) { | |
return request({ | |
url: 'https://www.freecodecamp.com/' + name, | |
method: 'GET' | |
}); | |
}); | |
return fcc_info.map(function(data,i) { | |
$ = cheerio.load(data); | |
var source = parseInt($('h1.text-primary').text().match(/[0-9]{2}/)); | |
var challenges = $('table tr'); | |
var challenge = []; | |
for (var j = 0; j < challenges.length; j++) { | |
var name = $(challenges[j]).find('td:first-child').text(); | |
var time = $(challenges[j]).find('td:nth-child(2)').text(); | |
challenge.push({ | |
name: name, | |
time: time | |
}); | |
} | |
return { | |
username: usernames[i], | |
source: source, | |
challenges: challenge | |
}; | |
}); | |
}).then( | |
fcc_info => fs.writeFileSync("result.json", JSON.stringify(fcc_info)), | |
err => console.error(err) | |
); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "try_promise", | |
"version": "1.0.0", | |
"description": "", | |
"main": "index.js", | |
"scripts": { | |
"test": "echo \"Error: no test specified\" && exit 1" | |
}, | |
"author": "", | |
"license": "ISC", | |
"dependencies": { | |
"cheerio": "^0.20.0", | |
"co": "^4.6.0", | |
"dateformat": "^1.0.12", | |
"request-promise": "^3.0.0" | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment