小说爬虫.js

小说爬虫.js

var request = require('request')
var cheerio = require('cheerio')
var iconv = require('iconv-lite');

async function get(url) {
    return new Promise((resolve) => {
        request.get(url, {
            encoding: null,
            gzip: true
        }, (err, resp, body) => {
            body = iconv.decode(body, 'gbk')

            body = body.replace(/<\/p>/gi,'</p>\n')

            let $ = cheerio.load(body)

            let title = $('#content h1').text()
            console.log('title',title);
            
            let article = $('#content').text().replace(title,'')

            let result = `\n\t${title}\n\n${article}\n\n\n\n`
            let next = $('li>a.next').attr('href')
            
            console.log('next',next);
            
            resolve([result, next])
        })
    })
}

async function start() {
    let fs = require('fs')
    let url = '/wapbook/97027_158708817.html'
    while(url){
        let result = await get(`https://m.boquge.com${url}`)
        fs.appendFileSync('books2.txt', result[0], 'utf-8')
        url = result[1]
    }
}

start()