A naive (WIP & incomplete) PuniPuniJapan scraper

puni-scrape.js

const cheerio = require('cheerio'),
      request = require('request'),
      dance   = require('breakdance'),
      fs      = require('fs'),
      path    = require('path'),
      url     = require('url')

const save_loc = path.resolve(process.argv[1],'..','puni-scrape'); //configurable?
const puni = 'http://www.punipunijapan.com';

requestCategory = (_url,_loc,pageNo) => (request(pageNo?(_url+'/page/'+pageNo+'/'):_url, function (error, response, html) {
  if (!error && response.statusCode == 200) {
    var $ = cheerio.load(html);
    let posts=$('#post-title')
    posts.each((i,e)=>{//could just use '#post-title a', here
      //console.log($(e).text(),'--->',$('a',e).attr('href'));
      let title=$('a',e).attr('href').split('/').filter(c=>c.length>0).pop()
      requestLesson(title,$('a',e).attr('href'),_loc)
    });
    if(!posts.length<10){
      requestCategory(_url,_loc,pageNo?pageNo++:2)
    }
  }else{console.log(error)}
}) )

const requestLesson = (title,url,loc) => {
  request(url, function (error, response, html) {
    if (!error && response.statusCode == 200) {
      var $ = cheerio.load(html);
      $('#home-post').each((i,e)=>{
        //console.log($(e).text());
        fs.writeFile(path.resolve(loc,title+'.md'), dance($(e).html()), function(err) {
          if(err)console.log(err);
          else console.log('Complete! '+path.resolve(loc,title+'.txt'));
        });
      });
    }
  });
}


//multiple categories //grammar phrases vocabulary
saveCategory = (category) => {
  const cat_loc=path.resolve(save_loc,category)
  try{fs.mkdirSync(cat_loc)}catch(e){console.log(cat_loc,' exists')}
  requestCategory(url.resolve(puni,'category/'+category),path.resolve(save_loc,category))
}

//main
['japanese-grammar','japanese-phrases','japanese-vocabulary'].map(c=>saveCategory(c,save_loc))

In this quick 0.0.0 version, it scrapes the first grammar lessons page, checks for any lessons and copies the text out of the main body and dumps it into a .txt.

Ideally, I'd be getting all pages, filtering out un-needed bits, preserving image links (or scraping them out too), linking to embedded YouTube vids and retaining emphasis on example phrases. Then pushing it through a markdown converter (or hand-cranking it) and dumping it into an .mdfile.

I really need to add something to only request things I've already gotten, and limit the requests, rather than getting one of everything.

Also, it still does nothing smart with the actual pages, which really is the whole point.

Using Breakdance, and saving as markdown, it basically does all the heavy lifting, but I'd like to pluck out YouTube links where I can.

The images are all still linked to the web, rather than pulling down the images to use locally. Another change I'd like to make.