Demo Puppetter And Node.js Chrome Headless Control For Alexander Nwala

README.md

Requires

Node.js download

Usage

1. npm install
2. npm run go

Other Info

Comments in js file and have fun!

demoPuppeteer.js

const puppeteer = require('puppeteer')
const fs = require('fs-extra') // do not use the built in fs
const Promise = require('bluebird') // do not use the built in Promise

let browser = null // need this reference for later

/*
 consolidates

 twitterGetTweetIfExist
 twitterGetLinksFromTweetDiv
 isVideoAdaptiveMediaInTweet
 */
function tweetEval () {
  // make it an array for simplicity
  let ar = Array.from(document.querySelectorAll('.tweet'))
  return ar.map(elem => { // elem is an alive element and can use the web javascript api https://developer.mozilla.org/en-US/docs/Web/API
    // again array for simplicity and to easily serialize the attributes
    let attrs = Array.from(elem.attributes).map(someAt => ({name: someAt.name, value: someAt.value}))
    // find the element that has the data-expanded-url attribute
    let dataExpandedUrl = elem.querySelector('[data-expanded-url]')
    // find the element that has the adaptive media photocontainer
    let adaptiveImage = elem.querySelector('.AdaptiveMedia-photoContainer')
    // if not null or undefined
    if (dataExpandedUrl) {
      // get the attributes
      dataExpandedUrl = dataExpandedUrl.attributes
      // if not null or undefined
      if (dataExpandedUrl) {
        // make array for simplicity
        dataExpandedUrl = Array.from(dataExpandedUrl)
          .filter(someAt => someAt.name === 'data-expanded-url') // filter the attributes looking for x
          .map(someAt => someAt.value) // take the value
        if (dataExpandedUrl.length === 1) { // since there should only be one element in the array check for 1
          dataExpandedUrl = dataExpandedUrl[0]  // this is the url
        } else { // otherwise make it null i.e not found
          dataExpandedUrl = null
        }
      }
    }
    // if not null or undefined
    if (adaptiveImage) {
      // find the img
      let img = adaptiveImage.querySelector('img')
      // if not null or undefined
      if (img) {
        img = img.src // img is now the string src attribute
      }
      // its also on the attributes!
      let dataImageUrl = adaptiveImage.attributes
      // if not null or undefined
      if (dataImageUrl) {
        // array for simplicity
        dataImageUrl = Array.from(dataImageUrl)
          .filter(someAt => someAt.name === 'data-image-url' || someAt.name === 'data-element-context')
          .map(someAt => ({
            name: someAt.name,
            value: someAt.value
          }))
        if (dataImageUrl.length === 0) {
          dataImageUrl = null
        }
      }
      adaptiveImage = {img, dataImageUrl}
    }
    // you can do this one <3
    let adaptiveMedia = elem.querySelector('.AdaptiveMedia-videoContainer')
    adaptiveMedia = adaptiveMedia !== null && adaptiveMedia !== undefined
    // return an object
    return {
      attrs,
      text: elem.querySelector('.tweet-text').textContent,
      tweetTime: elem.querySelector('.tweet-timestamp').title,
      dataExpandedUrl,
      adaptiveImage,
      adaptiveMedia
    }
  })
}

function scrollDocumentHeight () {
  window.scrollTo(0, document.body.scrollHeight)
}

function clickShowMore () {
  let showMoreSignature = ['.ThreadedConversation-moreRepliesLink', '.show-more-link'] // your selectors were wrong
  let showMore
  let i = 0
  let len
  showMoreSignature.forEach(sms => {
    showMore = document.querySelectorAll(sms)
    len = showMore.length
    for (; i < len; i++) {
      showMore[i].click()
    }
  })
}

/*
 !! USE THE PROMISE LIBRARY I HAVE SUPPLIED IN THIS CODE !!
 see Promise https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
 see async   https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function
 see await  https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await
 */
async function run () {
  browser = await puppeteer.launch({headless: false})
  const page = await browser.newPage()
  await page.setViewport({width: 840, height: 380})
  await page.goto('https://twitter.com/acnwala', {waitUntil: 'networkidle'})
  // evaluation takes actual functions yay!!
  await page.evaluate(clickShowMore)
  await Promise.delay(5000) // this is how you add pythons sleep
  const tweets = await page.evaluate(tweetEval)
  tweets.forEach(it => {
    console.log(it)
  })

  // you gotta use PageDown not Spacebar. It moves the same
  await page.press('PageDown', {delay: 1000})
  await Promise.delay(5000)
  await page.evaluate(scrollDocumentHeight)
  await fs.writeJSON('alexandersTweets.json', tweets)
  browser.close()
}

run()
  .then(() => {
    console.log('finished')
  })
  .catch(error => {
    console.error(error)
    if (browser) {
      browser.close()
    }
  })

package.json

{
  "name": "demoPuppeteer",
  "version": "1.0.0",
  "main": "demoPuppeteer.js",
  "author": "John Berlin",
  "license": "MIT",
  "dependencies": {
    "bluebird": "^3.5.0",
    "fs-extra": "^4.0.2",
    "puppeteer": "^0.10.2"
  },
  "scripts": {
    "go": "node --harmony demoPuppeteer.js"
  }
}

screenShot.js

const puppeteer = require('puppeteer')
const Promise = require('bluebird')
const fs = require('fs-extra')

async function doIt () {
  const browser = await puppeteer.launch({headless: false})//launch({executablePath: 'google-chrome-unstable'})
  const page = await browser.newPage()
  await page.setViewport({
    width: 1920,
    height: 1080,
    deviceScaleFactor: 0,
    mobile: false,
    fitWindow: false
  })
  await page.goto('http://web.archive.org/web/20170829110001/http://www.foxnews.com/', {waitUntil: 'load'})
  console.log('loaded')
  await page.screenshot({path: 'example.png'})
  browser.close()
}

doIt()
  .then(() => {

  })
  .catch(error => {
    console.error(error)
  })