extract URLs from a stream (newline-separated)

extract.js

#!/usr/bin/env node

/* 
  Installation: npm install linewise
  Usage:
  
  cat urls.json | node extract.js > urls.txt

  Assuming urls.json contains lines with zero or more urls per line, this will 
  extract all the urls and produce a file with one url per line.

  You may want to tweak the ending, e.g.
   [^"}\s\]\)\n\\]+

  which assumes that urls end with ", }, whitespace, ], \n or \.

*/

var re_weburl = /(\w+:\/\/[^"}\s\]\)\n\\]+)/gi;

var nls = require('linewise').getPerLineBuffer();

nls.on('data', function(data) {
  var matches = data.match(re_weburl);

  if(matches) {
    console.log(matches.join('\n'));
  }
});

process.stdin.pipe(nls);

nls.resume();
process.stdin.resume();