scrape top 100 lyrics

just-leaving-this-here.js

var request = require('request')
var cheerio = require('cheerio')
var fs = require('fs')

var pages = []

request('http://lyrics.wikia.com/LyricWiki:Top_100', function (error, response, html) {
  if (!error && response.statusCode == 200) {
    var $ = cheerio.load(html);
    //handle the 'page not found' page?
    $('li b a').each(function(i, element){
      pages.push('http://lyrics.wikia.com' + $(element).attr('href'))
      //
    });

    console.log(pages)

    //start scraping
    doit()
  }
});

function doit(){
  setTimeout(function(){
    scrape()
    if(pages.length) doit()
  }, 5000)
}

function scrape(){
  // grab a page off the stack (queue? whatever....)
  var page = pages.pop()

  console.log(page, 'SPIDER')
  // handle un-created pages
  if(page.indexOf('redlink') === -1) {


    request(page, function (error, response, html) {
      if (!error && response.statusCode == 200) {
        var $ = cheerio.load(html);

        // grab those sweet lyrics
        var text = $('.lyricbox').html()

        // remove script tags for tracking junk
        text = text.replace(/<script>.+<\/script>/g, '')

        // remove weird comment thing, split on break
        lines = text.slice(0, text.indexOf('<!')).split('<br>')
        
        // TODO: fix filename formatting
        fs.writeFileSync(page.replace(/\W/g, "-") + '.txt', lines.join("\n"))
        console.log("SUCCESS")
      }
    });
  }
}