2017-07-25: Swimming In A Ocean Of JavaScript. Using Nodejs, Electron And Chrome To Preserve The Web

ncdxjExample.js

const CDXJ = require('cdxj')

// To read the cdxj file entirely
// and receive an array of cdxj entries
async function getMeSomeCDXJ () {
  let cdxj = await CDXJ.readCDXJ('<path-to-cdxj-file>')
  cdxj.forEach(cdxjEntry => {
     console.log(`surt: ${cdxjEntry.surt}`)
     console.log(`datetime: ${cdxjEntry.dt}`)
     console.log(`json data: ${cdxjEntry.json}`)
  })
}

// Async/Await not your thing use the Promise API
CDXJReader.readCDXJ('<path-to-cdxj-file>').then(cdxj => {
  cdxj.forEach(cdxjEntry => {
    console.log(`surt: ${cdxjEntry.surt}`)
    console.log(`datetime: ${cdxjEntry.dt}`)
    console.log(`json data: ${cdxjEntry.json}`)
  })
})

// If you do not want to read the cdxj file in its entirety
// then you can use the Readable Stream API
const cdxjStream = CDXJ.createReadStream('<path-to-cdxj-file>')

// Be sure to add the data event listener 
// to receive each cdxj entry
cdxjStream.on('data', cdxjEntry => { 
  console.log(`surt: ${cdxjEntry.surt}`)
  console.log(`datetime: ${cdxjEntry.dt}`)
  console.log(`json data: ${cdxjEntry.json}`) 
})

nwExample1.js

/* 
 Simply require node-warc to start parsing 
 both .warc and .warc.gz
 use .warc.gz parser directly require('node-warc').WARCGzParser
 use .warc parser directly require('node-warc').WARCParser
*/
const AutoWARCParser = require('node-warc')
// Creation
const parser = new AutoWARCParser('<path-to-warcfile>')
// Add listeners for the parser events
parser.on('record', record => { console.log(record) })
parser.on('done', finalRecord => { console.log(finalRecord) })
parser.on('error', error => { console.error(error) })
// Finally call start to begin parsing!
parser.start()

squidWarcExampleConf.js

{
  // supports page-only, page-same-domain, page-all-links
  "mode": "page-same-domain",
 
 // an array of seeds or a single seed
  "seeds": [
    "http://acid.matkelly.com"
  ],
  
  "warc": {
    // currently this is the only option supported
    "naming": "url", 
    // where do you want the WARCs to be placed
    // optional defaults to cwd
    "output": "path" 
  },
  
  // Chrome is an HTTP/2 capable browser since v49 
  // Firefox since v52 http://caniuse.com/#search=http2
  // No known replay system can replay 
  // archived HTTP/2 request/responses  
  "noHTTP2": true, // optional defaults to false
  
   // Chrome instance we are to connect to is running on 
  "connect": {
    "host": "localhost",
    "port": 9222
  },
  
  // time is in milliseconds
  "timeouts": {
    // wait at maxium 8s for Chrome to navigate to a page
    "navigationTimeout": 8000,
    // wait 7 seconds after page load
    "waitAfterLoad": 7000
  },
  
 // optional auto scrolling of the page
 // time is in milliseconds and indicates the 
 // duration of the scroll in proportion to 
 // page size. Higher values means longer 
 // smooth scrolling, shorter values means 
 // faster smooth scroll
 "scroll": 4000
}