Skip to content

Instantly share code, notes, and snippets.

@davidsharp
Last active April 19, 2017 12:31
Show Gist options
  • Select an option

  • Save davidsharp/e46113881d912910cb8fdbb94d4910b3 to your computer and use it in GitHub Desktop.

Select an option

Save davidsharp/e46113881d912910cb8fdbb94d4910b3 to your computer and use it in GitHub Desktop.
Quickly ripped apart `puni-scrape.js` to throw together a scraper to get a markdown react.amsterdam schedule
const cheerio = require('cheerio'),
request = require('request'),
dance = require('breakdance'),
fs = require('fs'),
path = require('path'),
url = require('url')
const save_loc = path.resolve(process.argv[1],'..');
const r_a = 'https://react.amsterdam';
const makeRequest = (_url,_loc) => (request(_url, function (error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
let shedule=$('section.shedule ul');
let times = []
let rows = {} //{time,react={speaker,description},native,lightning}
shedule.each((i,e)=>{
$('li',e).each((_i,_e)=>{
let time = $('.shedule-list__time',_e).text().split(' ').join('').split('.').join(':')
if(!time)return;
if(!rows[time]){
times.push(time);
rows[time]={time}
rows[time].react={
speaker:($('.shedule-list__title',_e).text().length?$('.shedule-list__title',_e).text().split('\n').join('')+' – ':'')+$('.shedule-list__speaker',_e).text().split('\n').join(''),
desc:$('.shedule-list__p',_e).text().split('\n').join('')
}
}else{
rows[time].native={
speaker:($('.shedule-list__title',_e).text().length?$('.shedule-list__title',_e).text().split('\n').join('')+' – ':'')+$('.shedule-list__speaker',_e).text().split('\n').join(''),
desc:$('.shedule-list__p',_e).text().split('\n').join('')
}
}
})
});
schedule=times.reduce((array,c)=>(
[...array,
`| **${c}** | **${rows[c].react.speaker}** | **${rows[c].native?rows[c].native.speaker:' '}** |`,
`| | ${rows[c].react.desc} | ${rows[c].native?rows[c].native.desc:' '} |`
]
),['| TIME | REACT | NATIVE |','|---|---|---|'])
fs.writeFile(path.resolve(_loc,'react-amsterdam-timetable.md'), schedule.join('\n'), function(err) {
if(err)console.log(err);
else console.log('Complete! '+path.resolve(_loc,'react-amsterdam-timetable.md'));
});
}else{console.log(error)}
}) )
//main
makeRequest(r_a,save_loc)
@davidsharp

Copy link
Copy Markdown
Author

Did a second pass which is likely more error-prone, misses out the lightning talks, however, now we have two lists for track comparisons

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment