lelivros crawler

index.js

var fs = require('fs'),
    readline = require('readline'),
    stream = require('stream'),
    request = require('request'),
    sh = require('shelljs'),
    cheerio = require('cheerio'),
    im = require('imagemagick'),
    booksLink = [],
    booksName = [],
    catalog = sh.cat('./catalog.txt'),
    _catalog = [],
    booksPath = './books/';

require('shelljs/global');
//request.debug = true;


scrapLinks('http://lelivros.link/book/');

function scrapLinks( url ) { 

    request(url, function( err, response, body ) {

        // Trata erro
        errorHandling(err);

        if ( response.statusCode === 200 ) {

            $ = cheerio.load( body );

            $('.button[href*="book/"]').each(function() {

                var $this = $(this),
                    href = $this.attr('href'),
                    bookName = $this.prev().find('h3').text();

                if ( bookName ) {
                    booksLink.push( href );
                    booksName.push( slugify( bookName ) );
                    _catalog.push( slugify( bookName ) );
                }

            
            });

            var nextPage = $('.wp-pagenavi .current').next().attr('href');

            // Enquanto tiver proxima pagina 
            if ( nextPage ) {
                scrapLinks( nextPage );
            } else  {
                console.log('salvando catalogo...');
                _catalog.join('\n').to('./catalog.txt');
            }
            
        }

    });

}


(function downloadBook() {

    if ( booksLink.length ) {

        // Se ja existir no catalogo, nao faz o download
        if ( sh.test('-e', booksPath + booksName[0] ) ) { 
            console.log('Ja existe ', booksPath + booksName[0]);
            booksName = booksName.slice(1);
            booksLink = booksLink.slice(1);
            downloadBook();
            return false;
        }

        _request(booksLink[0], function( $ ) {
        
            var bookName = $('.entry-title').text(),
                bookThumb = $('.images img').attr('src'),
                files = $('a[href*="baixaragora.jegueajato"]'),
                //category = booksPath + slugify($('.posted_in a').text()) + '/',
                dirName = booksPath + slugify( bookName ),
                fileName = dirName + '/' + slugify(bookName);

            // Verifica se a categoria existe
            //if ( !sh.test('-e', category) ) { 
                //sh.mkdir('-p', category);
            //}

            // Verifica se o path existe
            if ( !sh.test('-e', dirName) ) { 
                sh.mkdir('-p', dirName);
            }

            // Salva arquivos no formato pdf, mobi, epub
            saveBook( files );

            // Salva thumb do livro
            if ( bookThumb ) {
                download(bookThumb, fileName + getExtension( bookThumb ), function( fileName ) {
                    cropImage( fileName );
                });
            }

            var readme = bookName + '\n' + $('.panel.entry-content p').text().trim();
            readme.to( fileName + '.txt' );

            booksName = booksName.slice(1);
            booksLink = booksLink.slice(1);
            downloadBook();

        });
    
    } else {
        setTimeout(function() {
            downloadBook();
        }, 10);
    } 

}());


function saveBook( files ) {

    files.each(function() {

        var link = $(this).attr('href');

        if ( link ) {

            var file = fileName + getExtension( link );

            download(link, file, function( fileName ) {
                console.log(fileName);
            });

        }

    });
}

function getExtension( fileName ) {
    return '.' + fileName.split('.').pop();
}


function cropImage( fileName ) {

    if ( !fileName ) 
        return;

    im.crop({
        srcPath: fileName,
        dstPath: fileName,
        width: 333,
        height: 500,
        quality: 0.8,
    }, function(err, stdout, stderr) { });

}


function _request( url, cb ) {

    request(url, function( err, response, body ) {

        // Trata erro
        errorHandling(err);

        if ( response.statusCode === 200 ) {

            $ = cheerio.load( body );

            cb( $ );

        }
    });

}

function download( uri, fileName, callback ) {

    request.head(uri, function(err, res, body){
        request( uri ).pipe( fs.createWriteStream(fileName) ).on('close', function() { callback(fileName); });
    });
}


function slugify( str ) {
 
    str = str.toLowerCase()
        .replace(/[\u00C0-\u00C5]/ig, 'a')
        .replace(/[\u00C8-\u00CB]/ig, 'e')
        .replace(/[\u00CC-\u00CF]/ig, 'i')
        .replace(/[\u00D2-\u00D6]/ig, 'o')
        .replace(/[\u00D9-\u00DC]/ig, 'u')
        .replace(/[\u00D1]/ig, 'n')
        .replace(/ç/ig, 'c')
        .replace(/[^a-z0-9 ]+/gi, '')
        .trim().replace(/[ ]/g, '-')
        .replace(/[\-]{2}/g, '-')
        .replace(/--/g, '-');
 
    return ( str.replace(/[^a-z\- ]*/gi, '') );
 
}

function errorHandling( err ) {
    if ( err ) { 
        console.log( err );
        throw err;
    }
}

package.json

{
  "name": "lelivros",
  "version": "1.0.0",
  "main": "index.js",
  "devDependencies": {},
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "Franklin Javier",
  "license": "ISC",
  "dependencies": {
    "cheerio": "^0.18.0",
    "graceful-fs": "^3.0.5",
    "imagemagick": "^0.1.3",
    "request": "^2.53.0",
    "shelljs": "^0.4.0"
  }
}