-
-
Save robertoentringer/ed8f2680ddcb08651012bd6b56df30e2 to your computer and use it in GitHub Desktop.
Scrapes the main page of HackerNews and returns an array of objects using Puppeteer and Cheerio
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const puppeteer = require('puppeteer'); | |
const cheerio = require('cheerio'); | |
async function run() { | |
const browser = await puppeteer.launch(); | |
const page = await browser.newPage(); | |
await page.goto('https://news.ycombinator.com'); | |
let content = await page.content(); | |
var $ = cheerio.load(content); | |
$('span.comhead').each(function(i, element){ | |
var a = $(this).prev(); | |
var rank = a.parent().parent().text(); | |
var title = a.text(); | |
var url = a.attr('href'); | |
var subtext = a.parent().parent().next().children('.subtext').children(); | |
var points = $(subtext).eq(0).text(); | |
var username = $(subtext).eq(1).text(); | |
var comments = $(subtext).eq(2).text(); | |
var metadata = { | |
rank: parseInt(rank), | |
title: title, | |
url: url, | |
points: parseInt(points), | |
username: username, | |
comments: parseInt(comments) | |
}; | |
console.log(metadata); | |
}); | |
browser.close(); | |
} | |
run(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment