小说列表爬虫.js

小说列表爬虫.js

var request = require('request')
var cheerio = require('cheerio')

async function get(page) {
    return new Promise((resolve) => {
        request.get(`https://www.80txt.com/sort3/${page}.html`, {
            encoding: 'utf-8',
            gzip: true
        }, (err, resp, body) => {
            let $ = cheerio.load(body)
            let titles = $('div.title_box>div.book_bg>a')
            let results = []
            titles.each((index, ele) => {
                let title = $(ele).text()
                title = title.split('TXT')[0].trim()
                console.log(title);
                results.push(title)
            });
            resolve(results)
        })
    })
}

async function start() {
    let fs = require('fs')
    for (let i = 1; i < 333; i++) {
        let results = await get(i)
        fs.appendFileSync('books2.txt', results.join('\n'), 'utf-8')
    }
}

start()