johnmurch · May 6, 2022 23:47
diff --git a/results.csv b/results.csv
diff --git a/scrapeSeekingAlpha.js b/scrapeSeekingAlpha.js
 //****************************************************************************
 //****************************************************************************
 //*** Bountify - https://bountify.co/scraper-for-seeking-alpha             ***
 //****************************************************************************
 //****************************************************************************

 //How to Run
 //1. Install Node.js and NPM
 //2. Install Packages
 //Required Packages
 //npm install request
 //npm install async
 //npm install underscore
 //npm cheerio
 //3. node scrapeSeekingAlpha.js
 //This should generate results.csv

 var fs = require('fs'),
    request = require('request'),
    async = require('async'),
    _ = require('underscore'),
    cheerio = require('cheerio');

 var outputCSV = [];

 //******************************************************
 // EACH CATGORY URL (Top 100)
 // http://seekingalpha.com/leader-board/opinion-leaders
 //******************************************************
 var urls = [
 "http://seekingalpha.com/opinion-leaders/long-ideas",
 "http://seekingalpha.com/opinion-leaders/long-ideas/2",
 "http://seekingalpha.com/opinion-leaders/long-ideas/3",
 "http://seekingalpha.com/opinion-leaders/long-ideas/4",
 "http://seekingalpha.com/opinion-leaders/short-ideas",
 "http://seekingalpha.com/opinion-leaders/short-ideas/2",
 "http://seekingalpha.com/opinion-leaders/short-ideas/3",
 "http://seekingalpha.com/opinion-leaders/short-ideas/4",
 "http://seekingalpha.com/opinion-leaders/quick-picks-lists",
 "http://seekingalpha.com/opinion-leaders/quick-picks-lists/2",
 "http://seekingalpha.com/opinion-leaders/quick-picks-lists/3",
 "http://seekingalpha.com/opinion-leaders/quick-picks-lists/4",
 "http://seekingalpha.com/opinion-leaders/fund-holdings",
 "http://seekingalpha.com/opinion-leaders/fund-holdings/2",
 "http://seekingalpha.com/opinion-leaders/fund-holdings/3",
 "http://seekingalpha.com/opinion-leaders/fund-holdings/4",
 "http://seekingalpha.com/opinion-leaders/insider-ownership",
 "http://seekingalpha.com/opinion-leaders/insider-ownership/2",
 "http://seekingalpha.com/opinion-leaders/insider-ownership/3",
 "http://seekingalpha.com/opinion-leaders/insider-ownership/4",
 "http://seekingalpha.com/opinion-leaders/ipo-analysis",
 "http://seekingalpha.com/opinion-leaders/ipo-analysis/2",
 "http://seekingalpha.com/opinion-leaders/ipo-analysis/3",
 "http://seekingalpha.com/opinion-leaders/ipo-analysis/4",
 "http://seekingalpha.com/opinion-leaders/dividend-investing-strategy",
 "http://seekingalpha.com/opinion-leaders/dividend-investing-strategy/2",
 "http://seekingalpha.com/opinion-leaders/dividend-investing-strategy/3",
 "http://seekingalpha.com/opinion-leaders/dividend-investing-strategy/4",
 "http://seekingalpha.com/opinion-leaders/income-investing-strategy",
 "http://seekingalpha.com/opinion-leaders/income-investing-strategy/2",
 "http://seekingalpha.com/opinion-leaders/income-investing-strategy/3",
 "http://seekingalpha.com/opinion-leaders/income-investing-strategy/4",
 "http://seekingalpha.com/opinion-leaders/dividend-ideas",
 "http://seekingalpha.com/opinion-leaders/dividend-ideas/2",
 "http://seekingalpha.com/opinion-leaders/dividend-ideas/3",
 "http://seekingalpha.com/opinion-leaders/dividend-ideas/4",
 "http://seekingalpha.com/opinion-leaders/dividend-quick-picks-lists",
 "http://seekingalpha.com/opinion-leaders/dividend-quick-picks-lists/2",
 "http://seekingalpha.com/opinion-leaders/dividend-quick-picks-lists/3",
 "http://seekingalpha.com/opinion-leaders/dividend-quick-picks-lists/4",
 "http://seekingalpha.com/opinion-leaders/bonds",
 "http://seekingalpha.com/opinion-leaders/bonds/2",
 "http://seekingalpha.com/opinion-leaders/bonds/3",
 "http://seekingalpha.com/opinion-leaders/bonds/4",
 "http://seekingalpha.com/opinion-leaders/reits",
 "http://seekingalpha.com/opinion-leaders/reits/2",
 "http://seekingalpha.com/opinion-leaders/reits/3",
 "http://seekingalpha.com/opinion-leaders/reits/4",
 "http://seekingalpha.com/opinion-leaders/retirement",
 "http://seekingalpha.com/opinion-leaders/retirement/2",
 "http://seekingalpha.com/opinion-leaders/retirement/3",
 "http://seekingalpha.com/opinion-leaders/retirement/4",
 "http://seekingalpha.com/opinion-leaders/market-outlook",
 "http://seekingalpha.com/opinion-leaders/market-outlook/2",
 "http://seekingalpha.com/opinion-leaders/market-outlook/3",
 "http://seekingalpha.com/opinion-leaders/market-outlook/4",
 "http://seekingalpha.com/opinion-leaders/gold-and-precious-metals",
 "http://seekingalpha.com/opinion-leaders/gold-and-precious-metals/2",
 "http://seekingalpha.com/opinion-leaders/gold-and-precious-metals/3",
 "http://seekingalpha.com/opinion-leaders/gold-and-precious-metals/4",
 "http://seekingalpha.com/opinion-leaders/commodities",
 "http://seekingalpha.com/opinion-leaders/commodities/2",
 "http://seekingalpha.com/opinion-leaders/commodities/3",
 "http://seekingalpha.com/opinion-leaders/commodities/4",
 "http://seekingalpha.com/opinion-leaders/economy",
 "http://seekingalpha.com/opinion-leaders/economy/2",
 "http://seekingalpha.com/opinion-leaders/economy/3",
 "http://seekingalpha.com/opinion-leaders/economy/4",
 "http://seekingalpha.com/opinion-leaders/forex",
 "http://seekingalpha.com/opinion-leaders/forex/2",
 "http://seekingalpha.com/opinion-leaders/forex/3",
 "http://seekingalpha.com/opinion-leaders/forex/4",
 "http://seekingalpha.com/opinion-leaders/real-estate",
 "http://seekingalpha.com/opinion-leaders/real-estate/2",
 "http://seekingalpha.com/opinion-leaders/real-estate/3",
 "http://seekingalpha.com/opinion-leaders/real-estate/4",
 "http://seekingalpha.com/opinion-leaders/demographics",
 "http://seekingalpha.com/opinion-leaders/demographics/2",
 "http://seekingalpha.com/opinion-leaders/demographics/3",
 "http://seekingalpha.com/opinion-leaders/demographics/4",
 "http://seekingalpha.com/opinion-leaders/basic-materials",
 "http://seekingalpha.com/opinion-leaders/basic-materials/2",
 "http://seekingalpha.com/opinion-leaders/basic-materials/3",
 "http://seekingalpha.com/opinion-leaders/basic-materials/4",
 "http://seekingalpha.com/opinion-leaders/conglomerates",
 "http://seekingalpha.com/opinion-leaders/conglomerates/2",
 "http://seekingalpha.com/opinion-leaders/conglomerates/3",
 "http://seekingalpha.com/opinion-leaders/conglomerates/4",
 "http://seekingalpha.com/opinion-leaders/consumer-goods",
 "http://seekingalpha.com/opinion-leaders/consumer-goods/2",
 "http://seekingalpha.com/opinion-leaders/consumer-goods/3",
 "http://seekingalpha.com/opinion-leaders/consumer-goods/4",
 "http://seekingalpha.com/opinion-leaders/financial",
 "http://seekingalpha.com/opinion-leaders/financial/2",
 "http://seekingalpha.com/opinion-leaders/financial/3",
 "http://seekingalpha.com/opinion-leaders/financial/4",
 "http://seekingalpha.com/opinion-leaders/healthcare",
 "http://seekingalpha.com/opinion-leaders/healthcare/2",
 "http://seekingalpha.com/opinion-leaders/healthcare/3",
 "http://seekingalpha.com/opinion-leaders/healthcare/4",
 "http://seekingalpha.com/opinion-leaders/industrial-goods",
 "http://seekingalpha.com/opinion-leaders/industrial-goods/2",
 "http://seekingalpha.com/opinion-leaders/industrial-goods/3",
 "http://seekingalpha.com/opinion-leaders/industrial-goods/4",
 "http://seekingalpha.com/opinion-leaders/services",
 "http://seekingalpha.com/opinion-leaders/services/2",
 "http://seekingalpha.com/opinion-leaders/services/3",
 "http://seekingalpha.com/opinion-leaders/services/4",
 "http://seekingalpha.com/opinion-leaders/technology",
 "http://seekingalpha.com/opinion-leaders/technology/2",
 "http://seekingalpha.com/opinion-leaders/technology/3",
 "http://seekingalpha.com/opinion-leaders/technology/4",
 "http://seekingalpha.com/opinion-leaders/utilities",
 "http://seekingalpha.com/opinion-leaders/utilities/2",
 "http://seekingalpha.com/opinion-leaders/utilities/3",
 "http://seekingalpha.com/opinion-leaders/utilities/4",
 "http://seekingalpha.com/opinion-leaders/portfolio-strategy-asset-allocation",
 "http://seekingalpha.com/opinion-leaders/portfolio-strategy-asset-allocation/2",
 "http://seekingalpha.com/opinion-leaders/portfolio-strategy-asset-allocation/3",
 "http://seekingalpha.com/opinion-leaders/portfolio-strategy-asset-allocation/4",
 "http://seekingalpha.com/opinion-leaders/etf-long-short-ideas",
 "http://seekingalpha.com/opinion-leaders/etf-long-short-ideas/2",
 "http://seekingalpha.com/opinion-leaders/etf-long-short-ideas/3",
 "http://seekingalpha.com/opinion-leaders/etf-long-short-ideas/4",
 "http://seekingalpha.com/opinion-leaders/etf-analysis",
 "http://seekingalpha.com/opinion-leaders/etf-analysis/2",
 "http://seekingalpha.com/opinion-leaders/etf-analysis/3",
 "http://seekingalpha.com/opinion-leaders/etf-analysis/4",
 "http://seekingalpha.com/opinion-leaders/etf-quick-picks-and-lists",
 "http://seekingalpha.com/opinion-leaders/etf-quick-picks-and-lists/2",
 "http://seekingalpha.com/opinion-leaders/etf-quick-picks-and-lists/3",
 "http://seekingalpha.com/opinion-leaders/etf-quick-picks-and-lists/4",
 "http://seekingalpha.com/opinion-leaders/closed-end-funds",
 "http://seekingalpha.com/opinion-leaders/closed-end-funds/2",
 "http://seekingalpha.com/opinion-leaders/closed-end-funds/3",
 "http://seekingalpha.com/opinion-leaders/closed-end-funds/4",
 "http://seekingalpha.com/opinion-leaders/mutual-funds",
 "http://seekingalpha.com/opinion-leaders/mutual-funds/2",
 "http://seekingalpha.com/opinion-leaders/mutual-funds/3",
 "http://seekingalpha.com/opinion-leaders/mutual-funds/4"
 ]
 var authors = [];

 async.forEach(urls,function(url,urlcb){
  request(url, function(error, response, body) {
      if (!error && response.statusCode == 200) {
        var $ = cheerio.load(body);
        ////*[@id="author_articles_wrapper"]/div/div[1]/ul
        $('.ld_top_list li a').each(function(i, elm) {          
          authors.push($(this).attr('href'));
        }); 
        $('.ld_more_list li a').each(function(i, elm) {          
          authors.push($(this).attr('href'));
        }); 
        console.log("FETCH: "+ url);
        urlcb();
      }else{
        console.log("Check Connection: Issue viewing: "+ url);
        console.log(error);
        process.exit(0);
      }
  });
 },function(){
  var uniqAuthors = _.uniq(authors);

  //@todo - If you want to output each author to text file
  // var wstream = fs.createWriteStream('authors.txt',{'flags': 'a'});
  // async.forEach(uniqAuthors,function(a,acb){
  //   wstream.write(a+"\n");
  // },function(){
  //   wstream.end();
  //   console.log("DONE");
  //   process.exit(0);
  // });

    async.forEach(uniqAuthors,function(name,pcb){
      var url = 'http://seekingalpha.com' + name;
      request(url, function(error, response, body) {
          if (!error && response.statusCode == 200) {
            console.log("Fetching: "+url);
            var $ = cheerio.load(body);
            var author = $("#author_full_name").text();
            var profile = $('.profile_item_mini_text').text().toLowerCase();
            // console.log(name+"|"+profile);
            var boolHedgeFund = contains(profile, 'hedge fund');
            var boolFormer = contains(profile, 'former');
            var boolEx = contains(profile, 'ex');
            var boolPortfolioManager = contains(profile, 'portfolio manager');

            var count=0;
            var keywords =[];
            if(boolHedgeFund){count++;keywords.push('hedge fund')}
            if(boolFormer){count++;keywords.push('former')}
            if(boolEx){count++;keywords.push('ex')}
            if(boolPortfolioManager){count++;keywords.push('portfolio manager')}

            var capture = {
              author:author,
              url:url,
              keywords:keywords,
              // profile: profile,
              count:count
            }
            if(count>0){
              outputCSV.push(capture);
              pcb();
            }else{
              pcb();
            }
          }else{
            pcb();
          }
      });
    },function(){
      console.log("DONE");
      var wstream = fs.createWriteStream('results.csv',{'flags': 'w'});
      wstream.write("Author,URL,Count,Keywords\n");
      async.forEach(outputCSV,function(a,acb){
        wstream.write(JSON.stringify(a.author)+","+a.url+","+a.count+","+JSON.stringify(a.keywords.join())+"\n");
      },function(){
        wstream.end();
        console.log("DONE");
        process.exit(0);
      });
    })
 })

 function contains(r,s){ 
  return r.indexOf(s) !== -1; 
 }
	//****************************************************************************
	//****************************************************************************
	//* Bountify - https://bountify.co/scraper-for-seeking-alpha *
	//****************************************************************************
	//****************************************************************************

	//How to Run
	//1. Install Node.js and NPM
	//2. Install Packages
	//Required Packages
	//npm install request
	//npm install async
	//npm install underscore
	//npm cheerio
	//3. node scrapeSeekingAlpha.js
	//This should generate results.csv

	var fs = require('fs'),
	request = require('request'),
	async = require('async'),
	_ = require('underscore'),
	cheerio = require('cheerio');

	var outputCSV = [];

	//******************************************************
	// EACH CATGORY URL (Top 100)
	// http://seekingalpha.com/leader-board/opinion-leaders
	//******************************************************
	var urls = [
	"http://seekingalpha.com/opinion-leaders/long-ideas",
	"http://seekingalpha.com/opinion-leaders/long-ideas/2",
	"http://seekingalpha.com/opinion-leaders/long-ideas/3",
	"http://seekingalpha.com/opinion-leaders/long-ideas/4",
	"http://seekingalpha.com/opinion-leaders/short-ideas",
	"http://seekingalpha.com/opinion-leaders/short-ideas/2",
	"http://seekingalpha.com/opinion-leaders/short-ideas/3",
	"http://seekingalpha.com/opinion-leaders/short-ideas/4",
	"http://seekingalpha.com/opinion-leaders/quick-picks-lists",
	"http://seekingalpha.com/opinion-leaders/quick-picks-lists/2",
	"http://seekingalpha.com/opinion-leaders/quick-picks-lists/3",
	"http://seekingalpha.com/opinion-leaders/quick-picks-lists/4",
	"http://seekingalpha.com/opinion-leaders/fund-holdings",
	"http://seekingalpha.com/opinion-leaders/fund-holdings/2",
	"http://seekingalpha.com/opinion-leaders/fund-holdings/3",
	"http://seekingalpha.com/opinion-leaders/fund-holdings/4",
	"http://seekingalpha.com/opinion-leaders/insider-ownership",
	"http://seekingalpha.com/opinion-leaders/insider-ownership/2",
	"http://seekingalpha.com/opinion-leaders/insider-ownership/3",
	"http://seekingalpha.com/opinion-leaders/insider-ownership/4",
	"http://seekingalpha.com/opinion-leaders/ipo-analysis",
	"http://seekingalpha.com/opinion-leaders/ipo-analysis/2",
	"http://seekingalpha.com/opinion-leaders/ipo-analysis/3",
	"http://seekingalpha.com/opinion-leaders/ipo-analysis/4",
	"http://seekingalpha.com/opinion-leaders/dividend-investing-strategy",
	"http://seekingalpha.com/opinion-leaders/dividend-investing-strategy/2",
	"http://seekingalpha.com/opinion-leaders/dividend-investing-strategy/3",
	"http://seekingalpha.com/opinion-leaders/dividend-investing-strategy/4",
	"http://seekingalpha.com/opinion-leaders/income-investing-strategy",
	"http://seekingalpha.com/opinion-leaders/income-investing-strategy/2",
	"http://seekingalpha.com/opinion-leaders/income-investing-strategy/3",
	"http://seekingalpha.com/opinion-leaders/income-investing-strategy/4",
	"http://seekingalpha.com/opinion-leaders/dividend-ideas",
	"http://seekingalpha.com/opinion-leaders/dividend-ideas/2",
	"http://seekingalpha.com/opinion-leaders/dividend-ideas/3",
	"http://seekingalpha.com/opinion-leaders/dividend-ideas/4",
	"http://seekingalpha.com/opinion-leaders/dividend-quick-picks-lists",
	"http://seekingalpha.com/opinion-leaders/dividend-quick-picks-lists/2",
	"http://seekingalpha.com/opinion-leaders/dividend-quick-picks-lists/3",
	"http://seekingalpha.com/opinion-leaders/dividend-quick-picks-lists/4",
	"http://seekingalpha.com/opinion-leaders/bonds",
	"http://seekingalpha.com/opinion-leaders/bonds/2",
	"http://seekingalpha.com/opinion-leaders/bonds/3",
	"http://seekingalpha.com/opinion-leaders/bonds/4",
	"http://seekingalpha.com/opinion-leaders/reits",
	"http://seekingalpha.com/opinion-leaders/reits/2",
	"http://seekingalpha.com/opinion-leaders/reits/3",
	"http://seekingalpha.com/opinion-leaders/reits/4",
	"http://seekingalpha.com/opinion-leaders/retirement",
	"http://seekingalpha.com/opinion-leaders/retirement/2",
	"http://seekingalpha.com/opinion-leaders/retirement/3",
	"http://seekingalpha.com/opinion-leaders/retirement/4",
	"http://seekingalpha.com/opinion-leaders/market-outlook",
	"http://seekingalpha.com/opinion-leaders/market-outlook/2",
	"http://seekingalpha.com/opinion-leaders/market-outlook/3",
	"http://seekingalpha.com/opinion-leaders/market-outlook/4",
	"http://seekingalpha.com/opinion-leaders/gold-and-precious-metals",
	"http://seekingalpha.com/opinion-leaders/gold-and-precious-metals/2",
	"http://seekingalpha.com/opinion-leaders/gold-and-precious-metals/3",
	"http://seekingalpha.com/opinion-leaders/gold-and-precious-metals/4",
	"http://seekingalpha.com/opinion-leaders/commodities",
	"http://seekingalpha.com/opinion-leaders/commodities/2",
	"http://seekingalpha.com/opinion-leaders/commodities/3",
	"http://seekingalpha.com/opinion-leaders/commodities/4",
	"http://seekingalpha.com/opinion-leaders/economy",
	"http://seekingalpha.com/opinion-leaders/economy/2",
	"http://seekingalpha.com/opinion-leaders/economy/3",
	"http://seekingalpha.com/opinion-leaders/economy/4",
	"http://seekingalpha.com/opinion-leaders/forex",
	"http://seekingalpha.com/opinion-leaders/forex/2",
	"http://seekingalpha.com/opinion-leaders/forex/3",
	"http://seekingalpha.com/opinion-leaders/forex/4",
	"http://seekingalpha.com/opinion-leaders/real-estate",
	"http://seekingalpha.com/opinion-leaders/real-estate/2",
	"http://seekingalpha.com/opinion-leaders/real-estate/3",
	"http://seekingalpha.com/opinion-leaders/real-estate/4",
	"http://seekingalpha.com/opinion-leaders/demographics",
	"http://seekingalpha.com/opinion-leaders/demographics/2",
	"http://seekingalpha.com/opinion-leaders/demographics/3",
	"http://seekingalpha.com/opinion-leaders/demographics/4",
	"http://seekingalpha.com/opinion-leaders/basic-materials",
	"http://seekingalpha.com/opinion-leaders/basic-materials/2",
	"http://seekingalpha.com/opinion-leaders/basic-materials/3",
	"http://seekingalpha.com/opinion-leaders/basic-materials/4",
	"http://seekingalpha.com/opinion-leaders/conglomerates",
	"http://seekingalpha.com/opinion-leaders/conglomerates/2",
	"http://seekingalpha.com/opinion-leaders/conglomerates/3",
	"http://seekingalpha.com/opinion-leaders/conglomerates/4",
	"http://seekingalpha.com/opinion-leaders/consumer-goods",
	"http://seekingalpha.com/opinion-leaders/consumer-goods/2",
	"http://seekingalpha.com/opinion-leaders/consumer-goods/3",
	"http://seekingalpha.com/opinion-leaders/consumer-goods/4",
	"http://seekingalpha.com/opinion-leaders/financial",
	"http://seekingalpha.com/opinion-leaders/financial/2",
	"http://seekingalpha.com/opinion-leaders/financial/3",
	"http://seekingalpha.com/opinion-leaders/financial/4",
	"http://seekingalpha.com/opinion-leaders/healthcare",
	"http://seekingalpha.com/opinion-leaders/healthcare/2",
	"http://seekingalpha.com/opinion-leaders/healthcare/3",
	"http://seekingalpha.com/opinion-leaders/healthcare/4",
	"http://seekingalpha.com/opinion-leaders/industrial-goods",
	"http://seekingalpha.com/opinion-leaders/industrial-goods/2",
	"http://seekingalpha.com/opinion-leaders/industrial-goods/3",
	"http://seekingalpha.com/opinion-leaders/industrial-goods/4",
	"http://seekingalpha.com/opinion-leaders/services",
	"http://seekingalpha.com/opinion-leaders/services/2",
	"http://seekingalpha.com/opinion-leaders/services/3",
	"http://seekingalpha.com/opinion-leaders/services/4",
	"http://seekingalpha.com/opinion-leaders/technology",
	"http://seekingalpha.com/opinion-leaders/technology/2",
	"http://seekingalpha.com/opinion-leaders/technology/3",
	"http://seekingalpha.com/opinion-leaders/technology/4",
	"http://seekingalpha.com/opinion-leaders/utilities",
	"http://seekingalpha.com/opinion-leaders/utilities/2",
	"http://seekingalpha.com/opinion-leaders/utilities/3",
	"http://seekingalpha.com/opinion-leaders/utilities/4",
	"http://seekingalpha.com/opinion-leaders/portfolio-strategy-asset-allocation",
	"http://seekingalpha.com/opinion-leaders/portfolio-strategy-asset-allocation/2",
	"http://seekingalpha.com/opinion-leaders/portfolio-strategy-asset-allocation/3",
	"http://seekingalpha.com/opinion-leaders/portfolio-strategy-asset-allocation/4",
	"http://seekingalpha.com/opinion-leaders/etf-long-short-ideas",
	"http://seekingalpha.com/opinion-leaders/etf-long-short-ideas/2",
	"http://seekingalpha.com/opinion-leaders/etf-long-short-ideas/3",
	"http://seekingalpha.com/opinion-leaders/etf-long-short-ideas/4",
	"http://seekingalpha.com/opinion-leaders/etf-analysis",
	"http://seekingalpha.com/opinion-leaders/etf-analysis/2",
	"http://seekingalpha.com/opinion-leaders/etf-analysis/3",
	"http://seekingalpha.com/opinion-leaders/etf-analysis/4",
	"http://seekingalpha.com/opinion-leaders/etf-quick-picks-and-lists",
	"http://seekingalpha.com/opinion-leaders/etf-quick-picks-and-lists/2",
	"http://seekingalpha.com/opinion-leaders/etf-quick-picks-and-lists/3",
	"http://seekingalpha.com/opinion-leaders/etf-quick-picks-and-lists/4",
	"http://seekingalpha.com/opinion-leaders/closed-end-funds",
	"http://seekingalpha.com/opinion-leaders/closed-end-funds/2",
	"http://seekingalpha.com/opinion-leaders/closed-end-funds/3",
	"http://seekingalpha.com/opinion-leaders/closed-end-funds/4",
	"http://seekingalpha.com/opinion-leaders/mutual-funds",
	"http://seekingalpha.com/opinion-leaders/mutual-funds/2",
	"http://seekingalpha.com/opinion-leaders/mutual-funds/3",
	"http://seekingalpha.com/opinion-leaders/mutual-funds/4"
	]
	var authors = [];

	async.forEach(urls,function(url,urlcb){
	request(url, function(error, response, body) {
	if (!error && response.statusCode == 200) {
	var $ = cheerio.load(body);
	////*[@id="author_articles_wrapper"]/div/div[1]/ul
	$('.ld_top_list li a').each(function(i, elm) {
	authors.push($(this).attr('href'));
	});
	$('.ld_more_list li a').each(function(i, elm) {
	authors.push($(this).attr('href'));
	});
	console.log("FETCH: "+ url);
	urlcb();
	}else{
	console.log("Check Connection: Issue viewing: "+ url);
	console.log(error);
	process.exit(0);
	}
	});
	},function(){
	var uniqAuthors = _.uniq(authors);

	//@todo - If you want to output each author to text file
	// var wstream = fs.createWriteStream('authors.txt',{'flags': 'a'});
	// async.forEach(uniqAuthors,function(a,acb){
	// wstream.write(a+"\n");
	// },function(){
	// wstream.end();
	// console.log("DONE");
	// process.exit(0);
	// });

	async.forEach(uniqAuthors,function(name,pcb){
	var url = 'http://seekingalpha.com' + name;
	request(url, function(error, response, body) {
	if (!error && response.statusCode == 200) {
	console.log("Fetching: "+url);
	var $ = cheerio.load(body);
	var author = $("#author_full_name").text();
	var profile = $('.profile_item_mini_text').text().toLowerCase();
	// console.log(name+"\|"+profile);
	var boolHedgeFund = contains(profile, 'hedge fund');
	var boolFormer = contains(profile, 'former');
	var boolEx = contains(profile, 'ex');
	var boolPortfolioManager = contains(profile, 'portfolio manager');

	var count=0;
	var keywords =[];
	if(boolHedgeFund){count++;keywords.push('hedge fund')}
	if(boolFormer){count++;keywords.push('former')}
	if(boolEx){count++;keywords.push('ex')}
	if(boolPortfolioManager){count++;keywords.push('portfolio manager')}

	var capture = {
	author:author,
	url:url,
	keywords:keywords,
	// profile: profile,
	count:count
	}
	if(count>0){
	outputCSV.push(capture);
	pcb();
	}else{
	pcb();
	}
	}else{
	pcb();
	}
	});
	},function(){
	console.log("DONE");
	var wstream = fs.createWriteStream('results.csv',{'flags': 'w'});
	wstream.write("Author,URL,Count,Keywords\n");
	async.forEach(outputCSV,function(a,acb){
	wstream.write(JSON.stringify(a.author)+","+a.url+","+a.count+","+JSON.stringify(a.keywords.join())+"\n");
	},function(){
	wstream.end();
	console.log("DONE");
	process.exit(0);
	});
	})
	})

	function contains(r,s){
	return r.indexOf(s) !== -1;
	}