Ingress portal level crawler with PhantomJS.

ingress-crawl-portal-level.js

/**
 * @file Ingress portal level crawler with PhantomJS
 * @author teckl (https://github.com/teckl)
 * @version 0.0.1
 * @license MIT
 * @see {@link https://github.com/nibogd/tg-ingress-scorebot|GitHub }
 * @see {@link https://github.com/jonatkins/ingress-intel-total-conversion|GitHub }
 */

"use strict";
//Initialize

var fs          = require('fs');
var system      = require('system');

//var args        = system.args;
var cookiespath = 'iced_cookies';
var cookieSACSID, cookieCSRF;

var settingsfile = fs.open('bot.json', 'r');
var settings = JSON.parse(settingsfile.read());
settingsfile.close();
var l            = settings['gmail'];
var p            = settings['password'];
var area         = settings['link'];
var my_team      = settings['my_team'];
var v = 10000;

/*global phantom */
/*global idleReset */

/**
 * Counter for number of screenshots
 */
var version      = '0.0.1';

/**
 * Delay between logging in and checking if successful
 * @default
 */
var loginTimeout = 10 * 1000;

/**
 * twostep auth trigger
 */
var twostep      = 0;
var page         = require('webpage').create();
// page.onConsoleMessage = function () {};
// page.onError  = function () {};

/** @function setVieportSize */

page.viewportSize = {
  width: 42,
  height: 42
}

var resourceCounter = 0;
page.onResourceRequested = function (requestData, networkRequest) {
  console.log(resourceCounter++ + '.\t', requestData.url);
};

page.onResourceReceived = function(response) {
  if (/ingress.com\/r\/getPlexts/.test(response.url)) {
    console.log('Response (#' + response.id + ', stage "' + response.stage + '"): ' + JSON.stringify(response));
  }
};

page.onResourceError = function (resourceError) {
  system.stderr.writeLine('Unable to load resource (#' + resourceError.id + 'URL:' + resourceError.url + ')');
  system.stderr.writeLine('Error code: ' + resourceError.errorCode + '. Description: ' + resourceError.errorString);
};


function captureAjaxResponsesToConsole() {
  // logs ajax response contents to console so sublime's onConsoleMessage can use the contents
  // credit to Ionuț G. Stan
  // http://stackoverflow.com/questions/629671/how-can-i-intercept-xmlhttprequests-from-a-greasemonkey-script
  page.evaluate(function() {
    (function(open) {
      XMLHttpRequest.prototype.open = function(method, url, async, user, pass) {
        this.addEventListener("readystatechange", function() {
          if (this.readyState === 4) {
            var res = {'response':this.responseText, 'url' : url};
            console.log(JSON.stringify(res));
          }
        }, false);
        open.call(this, method, url, async, user, pass);
      };
    })(XMLHttpRequest.prototype.open);
    return 1;
  });
}

window.parsePortalEntities = function(data) {
  var entities = JSON.parse(data);
  var m = entities.result.map;
  var portal_count = 0, team_count = 0, p8_count = 0;
  for (var id in m) {
    var val = m[id];
    var gameEntities = val.gameEntities;
    for (var i in gameEntities) {
      var ent = gameEntities[i]
      if (ent[2][0] == 'p') {
        var level = parseInt(ent[2][4])||0;
        var team     = ent[2][1];
        var health   = ent[2][5];
        var resCount   = ent[2][6];
        var image    = ent[2][7];
        var title    = ent[2][8];
        var timestamp  = ent[2][13];
        announce('level : ' + level + ' team : ' + team  + ' title : ' + title);

        var detail_data = 'time:' + getDateTime(2) + '\tmetric:portal.detail' + '\tteam:' + team + '\tresCount:' + resCount + '\tlevel:' + level + '\ttitle:' + title + '\n';
        announce(detail_data);
        fs.write('portal_detail.ltsv', detail_data, 'a');

        portal_count++;
        if (team == my_team) {
          team_count++;
          if (level >= 8) {
            p8_count++;
          }
        }
      }
    }
  }

  var portal_level_percent = Math.round(p8_count / portal_count * 100);
  var team_percent     = Math.round(team_count / portal_count * 100);
  console.log('portal_count : ' + portal_count + '  p8_count : ' + p8_count + ' portal_level_percent : ' + portal_level_percent + ' team_percent : ' + team_percent);
  var data = {
    date : getDateTime(2),
    portal_count : portal_count,
    team_count : team_count,
    team_percent : team_percent,
    p8_count : p8_count,
    p8_percent : portal_level_percent,
  };
  fs.write('portal_summary.json', JSON.stringify(data, null, '  '), 'a');

}

page.onConsoleMessage = function (msg) {
  var res;
  try {
    res = JSON.parse(msg);
    console.log('URL:' + res.url);
    if (/\/r\/getEntities/.test(res.url)) {
      parsePortalEntities(res.response);
    }
    if (/\/r\/getPlexts/.test(res.url)) {
      console.log('getPlexts: ' + res.response);
    }
  } catch (e) {
    console.log(msg);
    console.log(e);
  }
};



//Functions

/**
 * console.log() wrapper
 * @param {String} str - what to announce
 */
function announce(str) {
  console.log(getDateTime(0) + ': ' + str);
}

/**
 * Returns Date and time
 * @param {number} format - the format of output, 0 for DD.MM.YYY HH:MM:SS, 1 for YYYY-MM-DD--HH-MM-SS (for filenames)
 * @returns {String} date
 */
function getDateTime(format) {
  var now   = new Date();
  var year  = now.getFullYear();
  var month   = now.getMonth()+1;
  var day   = now.getDate();
  var hour  = now.getHours();
  var minute  = now.getMinutes();
  var second  = now.getSeconds();
  if(month.toString().length === 1) {
    month = '0' + month;
  }
  if(day.toString().length === 1) {
    day = '0' + day;
  }
  if(hour.toString().length === 1) {
    hour = '0' + hour;
  }
  if(minute.toString().length === 1) {
    minute = '0' + minute;
  }
  if(second.toString().length === 1) {
    second = '0' + second;
  }
  var dateTime;
  if (format === 1) {
    dateTime = year + '-' + month + '-' + day + '--' + hour + '-' + minute + '-' + second;
  } else if (format === 2) {
    dateTime = year + '-' + month + '-' + day + 'T' + hour + ':' + minute + ':' + second;
  } else {
    dateTime = day + '.' + month + '.' + year + ' '+hour+':'+minute+':'+second;
  }
  return dateTime;
}


/**
 * Quit if an error occured
 * @param {String} err - the error text
 */
function quit(err) {
  if (err) {
    announce('crawler crashed. Reason: ' + err + ' T_T');
  } else {
    announce('Quit');
  }
  phantom.exit();
}


/**
 * Log in to google. Doesn't use post, because URI may change.
 * Fixed in 3.0.0 -- obsolete versions will not work (google changed login form)
 * @param l - google login
 * @param p - google password
 */
function login(l, p) {
  page.evaluate(function (l) {
    document.getElementById('Email').value = l;
  }, l);
  page.evaluate(function () {
    document.querySelector("#next").click();
  });
  window.setInterval(function () {
    page.evaluate(function (p) {
      document.getElementById('Passwd').value = p;
    }, p);
    page.evaluate(function () {
      document.querySelector("#next").click();
    });
    page.evaluate(function () {
      document.getElementById('gaia_loginform').submit();
    });
  }, loginTimeout / 10);
}

/**
 * Check if logged in successfully, quit if failed, accept appEngine request if needed and prompt for two step code if needed.
 */
function checkLogin() {

  announce('URI is now ' + page.url.substring(0,40) + '...');

  if (page.url.substring(0,40) === 'https://accounts.google.com/ServiceLogin') {quit('login failed: wrong email and/or password');}
    if (page.url.substring(0,40) === 'https://appengine.google.com/_ah/loginfo') {
      announce('Accepting appEngine request...');
      page.evaluate(function () {
      document.getElementById('persist_checkbox').checked = true;
      document.getElementsByTagName('form').submit();
    });
  }

  if (page.url.substring(0,40) === 'https://accounts.google.com/SecondFactor') {
    announce('Using two-step verification, please enter your code:');
    twostep = system.stdin.readLine();
  }

  if (twostep) {
    page.evaluate(function (code) {
      document.getElementById('smsUserPin').value = code;
    }, twostep);
    page.evaluate(function () {
      document.getElementById('gaia_secondfactorform').submit();
    });
  }
}

/**
 * Does all stuff needed after login/password authentication
 * @since 3.1.0
 */
function afterPlainLogin() {
  window.setTimeout(function () {
    announce('Verifying login...');
    checkLogin();
    announce('before_storeCookies...');
    window.setTimeout(function () {
      page.open(area, function () {
        storeCookies();
        setTimeout(function () {
          setTimeout(main, v);
        }, loginTimeout);
      });
    }, loginTimeout);
  }, loginTimeout);
}

/**
 * Checks if user is signed in by looking for the "Sign in" button
 * @returns {boolean}
 * @author mfcanovas (github.com/mfcanovas)
 * @since 3.2.0
 */
function isSignedIn() {
  return page.evaluate(function() {
  var btns = document.getElementsByClassName('button_link');
  for(var i = 0; i<btns.length;i++) {
    if(btns[i].innerText.trim() === 'Sign in') return false;
  }
  return true;
  });

}

/**
 * Main function. Wrapper for others.
 */
function main() {
  var data = page.evaluate(function() {return document.getElementById('rs_score_history_scores')});
  window.setTimeout(function() {
    announce('Crawler finished');
    fs.write('crawled.ice', data.innerHTML, 'w');
    phantom.exit();
  }, 1000);
}

/**
 * Checks if cookies file exists. If so, it sets SACSID and CSRF vars
 * @returns {boolean}
 * @author mfcanovas (github.com/mfcanovas)
 * @since 3.2.0
 */
function cookiesFileExists() {
  if(fs.exists(cookiespath)) {
  var stream = fs.open(cookiespath, 'r');

  while(!stream.atEnd()) {
    var line = stream.readLine();
    var res = line.split('=');
    if(res[0] === 'SACSID') {
      cookieSACSID = res[1];
    } else if(res[0] === 'csrftoken') {
      cookieCSRF = res[1];
    }
  }
  stream.close();
  return true;
  } else {
    return false;
  }
}

/**
 * Remove cookies file if exists
 * @author mfcanovas (github.com/mfcanovas)
 * @since 3.2.0
 */
function removeCookieFile() {
  if(fs.exists(cookiespath)) {
  fs.remove(cookiespath);
  }
}

function storeCookies() {
  var cookies = page.cookies;
  announce('storeCookies...');
  fs.write(cookiespath, '', 'w');
  for(var i in cookies) {
  announce('Cookies_write...');
  fs.write(cookiespath, cookies[i].name + '=' + cookies[i].value +'\n', 'a');
  }
}

/**
 * Fires plain login
 */
function firePlainLogin() {
  cookieSACSID = '';
  cookieCSRF = '';
  page.open('https://www.ingress.com/intel', function (status) {

    if (status !== 'success') {quit('cannot connect to remote server');}
  
    var link = page.evaluate(function () {
      return document.getElementsByTagName('a')[0].href;
    });
  
    announce('Logging in...');
    page.open(link, function () {
      login(l, p);
      afterPlainLogin();
    });
  });
}

//MAIN SCRIPT

announce('Crawler loaded')
if (!cookiesFileExists()) {
  firePlainLogin();
} else {
  announce('Using stored cookie');
  addCookies(cookieSACSID, cookieCSRF);
  afterCookieLogin();
}


/**
 * Log in using cookies
 * @param {String} sacsid
 * @param {String} csrf
 * @since 3.1.0
 */
function addCookies(sacsid, csrf) {
  phantom.addCookie({
    name: 'SACSID',
    value: sacsid,
    domain: 'www.ingress.com',
    path: '/',
    httponly: true,
    secure: true
  });
  phantom.addCookie({
    name: 'csrftoken',
    value: csrf,
    domain: 'www.ingress.com',
    path: '/'
  });
}


/**
 * Does all stuff needed after cookie authentication
 * @since 3.1.0
 */
function afterCookieLogin() {
  page.open(area, function (status) {
    if (status === 'success') {
      captureAjaxResponsesToConsole();
    }

    if(!isSignedIn()) {
      removeCookieFile();
      if(l && p) {
        firePlainLogin();
        return;
      } else {
        quit('User not logged in');
      }
    }
    setTimeout(main, loginTimeout);
  });
}

td-agent.conf

<source>
  type tail
  format ltsv
#  time_format %d/%b/%Y:%H:%M:%S %z
  path /home/hoge/portal_detail.ltsv
  pos_file /var/log/td-agent/portal_detail.ltsv.pos
  tag data.portal.detail
  refresh_interval 3m
</source>
<match debug.**>
  type stdout
</match>

<match data.portal.detail>
  type copy
  <store>
    type stdout
  </store>
  <store>
    type datacounter
    count_interval 3m
    count_key level
    pattern1 p8 8
    pattern2 p7 7
    pattern3 p6 6
    pattern4 p5 5
    outcast_unmatched yes
    tag datacount.portal.level
  </store>
  <store>
    type datacounter
    count_interval 3m
    count_key team
    pattern1 Resistance R
    pattern2 Enlightened E
    outcast_unmatched yes
    tag datacount.portal.team
  </store>
</match>


<match datacount.portal.level>
  type copy
  <store>
    type stdout
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p8.count","value" => record["data.portal.detail_p8_count"].to_i}]
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p7.count","value" => record["data.portal.detail_p7_count"].to_i}]
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p6.count","value" => record["data.portal.detail_p6_count"].to_i}]
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p8.rate","value" => record["data.portal.detail_p8_rate"].to_i}]
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p7.rate","value" => record["data.portal.detail_p7_rate"].to_i}]
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p6.rate","value" => record["data.portal.detail_p6_rate"].to_i}]
  </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p8.percentage","value" => record["data.portal.detail_p8_percentage"].to_i}]
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p7.percentage","value" => record["data.portal.detail_p7_percentage"].to_i}]
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p6.percentage","value" => record["data.portal.detail_p6_percentage"].to_i}]
  </store>
</match>

<match datacount.portal.team>
  type copy
  <store>
    type stdout
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.Resistance.count","value" => record["data.portal.detail_Resistance_count"].to_i}]
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.Enlightened.count","value" => record["data.portal.detail_Enlightened_count"].to_i}]
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.Resistance.rate","value" => record["data.portal.detail_Resistance_rate"].to_i}]
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.Enlightened.rate","value" => record["data.portal.detail_Enlightened_rate"].to_i}]
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.Resistance.percentage","value" => record["data.portal.detail_Resistance_percentage"].to_i}]
  </store>
  <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.Enlightened.percentage","value" => record["data.portal.detail_Enlightened_percentage"].to_i}]
  </store>
</match>

<match stat.datacount.portal.**>
  type copy
  <store>
    type stdout
  </store>
  <store>
    type dd
    dd_api_key your_datadoc_api_key
  </store>
</match>