Load jQuery in Google SERPs & Mozbar Scraping

load-jQuery-in-Google-Serps-Mozbar-Scraping

	// *** Selector - instead find moz iframe, then .closest('li') for more reliability
  
  
  	//	*** Work with Oop Version and Test Driven Development		
  
  
	/*
	
		Default save 
			Serps-%%search_term%%-pg%%pg_number%%-%%date%%
			// var search_term = '';
			// .replace(/%%search_term%%/, search_term) etc
	
		Include spreadsheet header?
			Same var options as above. 
			header options
			Blank row
			Data
	
	

		1. Save html title - shows keyword matches
			// $('iframe.SEOmoz-iframe:not(0)') // if not in serps, get current page data - first iframe.  Also check if logged in, and provide link to log in. 
			
		2. Get Better titles for all metrics -  done. 
		
		3. Headers to CSV - account for columns
		4. Filename and header options - 

	*/
  
	// Loggin in gives me access to way, way more data.  - suggest login if not.
	// Currently I've only got it set on non-logged in data. 

	// each item, .user.proEnabled

	// check if moz loaded, if not send warning.  Or make sure all 10 loaded, if not set interval until loaded
  
    // currently 11,000 characters. 
  // 8,000 - lame compression
  // 5,000 - good compression - test as is in all browsers? See how it works
  // convert to only csv and you should be able to do it a lot shorter. 
  // push to array rather than json and you should do better
  // remove all debugging - 
  // remove dom - that'll all help a ton - if it's actually needed.  200 characters, not a chance
  
   // 6,500 worked like a charm for bookmarklet.  20,000 should be easy. 
  
  /*
  
	1. Load datatables - use for table creation, csv download.  Use css?
		- then my plugin for highlighting everything. 
		- Then interface. 
		
	2. Charting - use plugin?  Or use table charting I'm creating?  - later if really needed
	
	Resources
	1. jsdelivr for CDN - unless google as on page and probably already cached?
		- datatables
		- CDNJS - toggles (use css as well?)
		- fontawesome - from bootstrap cdn.  - would I actually use it?
			- load all after jquery async - find loader in jsdelivr library - //cdn.jsdelivr.net/jquery.infiniteajaxscroll/1.0.1/images/loader.gif
			// - lots of options in jsdelivr library - wp-advanced-ajax-page-loader - just use as image to resize - spinning earth at top, small spinners if needed
		- jquery ui for drag and multiple select last
	2. Create interface yourself - quickest way - that way I can show libraries loading. - keep simple then use other css
		- use ajax loading image icon - or pure css?  
	3. 
  
  
  */
  
  
   
  // *** Selector - instead find moz iframe, then .closest('li') for more reliability
  
  // Loggin in gives me access to way, way more data. 
  // Currently I've only got it set on non-logged in data. 
  
  // each item, .user.proEnabled

  // check if moz loaded, if not send warning.  Or make sure all 10 loaded, if not set interval until loaded
  
  
  /*
  
    // organize code. 
    1. Scrape Moz data - get into json array
    2. Object - scrape serps
    3. Data manipulation
      - Json, CSV, Html tables, combine moz and scraped data, and seoquake data. 
    4. Interface?
    
    // check for author snippets - by, image, link
    
    // reactivate table - download options as well - http://www.phpied.com/reactivetable-bookmarklet/
    
    // grab ads?
    var ads_data = [];
    $('li.ads-ad').each(function(i){
      ads_data[i] = {};
      ads_data[i].title = $(this).find('h3:first').text(); // keyword term match in bold = <b></b>
      ads_data[i].url = $(this).find('.ads-visurl cite').text();
      ads_data[i].description = $(this).find('.ads-creative').text();
      ads_data[i].local_info = ($(this).find('._Tu').length) ? $(this).find('._Tu').text() : '';
    });
    console.log(ads_data);  // currently grabs from sidebar, untested for others
    
  */
/*
 // resources to link to - option to open all? 
 // individual links for sites
 // http://www.highposition.com/blog/the-ultimate-list-of-seo-tool-bookmarklets/
 // http://moz.com/blog/some-nifty-seo-bookmarklets-to-make-you-more-efficient

// bulk tools to copy into - bulk copy and past urls into specific tools.  Tools to scrape that data and combine it.
  // - link adding on all urls as get equests &urls=google.com,moz.com,  then bookmarklet to paste into tool and run?  Awesomesauce. Then scrape. 
// social - http://tools.seochat.com/tools/article-popularity-tool/
// bulk tools - http://www.seoreviewtools.com/seo-tools/social-media-tools/
// http://www.tomanthony.co.uk/tools/simple-bulk-social-counter/process.php
// - compare data?
*/


/*    Fix headers - associative array, and header_order numeric.  Get Headers from massive legend if possible */


  
  // Serp Savvy
  var ss_debug = true; 
  
  if(typeof jQuery !== 'undefined'){
    start_bookmarklet();
  }else{
    var jqueryScript = document.createElement('script');
    jqueryScript.type = 'text/javascript'; 
    jqueryScript.src = '//ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js'; 
    var head = document.getElementsByTagName('script')[0]; head.appendChild(jqueryScript);
    jqueryScript.onload = start_bookmarklet;
  }
  
  function start_bookmarklet(){
    if(typeof ss_debug === 'undefined'){ ss_debug = true; }
    if(ss_debug){ console.log('jquery loaded'); }
    
    bookmarklet_code(jQuery);
  }
  
  function get_serp_data($serpItem){
    serp_data = {};
    serp_data.title = $serpItem.find('h3:first').text();
    serp_data.html_title = $serpItem.find('h3:first a').html().replace(/\"/g, '');
    serp_data.url = $serpItem.find('.s .f cite').text();
    serp_data.description = $serpItem.find('.s span.st').text();  
    return serp_data;
  }
  
  function get_local_serps($serp_box){
    $serp_box.find('.vsc.vscl').each(function(){
      console.log($(this));
    });
  }
  
  // when I need to run after jquery loaded for testing
  // bookmarklet_code(jQuery);
  function bookmarklet_code($){
      // run jQuery here
      if(ss_debug){ console.log('in jQuery bookmarklet body'); }
      
	  var options = {
		csv_header: true,
		csv_header_option:'', // '%%search_term%% %%date%%' etc
		filename_option: '',
		header_date_format: '',
		file_date_format: ''
	  };

	  var now = new Date();
	  
	  var main_vars = {
		search_term: '',
		date: now.getMonth() + '/' + now.getDate() + '/' + now.getFullYear(),		
		filename: '',		// converted from variable above into text
		csv_header: '',		// number of columns needs to be accounted for if headers added
		filename_date: now.getFullYear() + "-" + now.getMonth() + "-" + now.getDate(),
		filename_search_term: '',
		header_legend:{ "index":"Index", "url":"Url", "title":"Title", "html_title":"HTML Title", "domain":"Root url", "description":"Description", "pda":"DA", "upa":"PA", "description":"Description","ut":"Title","uu":"Canonical URL","ufq":"Subdomain","upl":"Root Domain","ueid":"External Equity Links","feid":"Subdomain External Links","peid":"Root Domain External Links","ujid":"Equity Links","uifq":"Subdomains Linking","uipl":"Root Domains Linking","uid":"Links","fid":"Subdomain Subdomains Linking","pid":"Root Domain Root Domains Linking","umrpumrr":"MozRank","fmrpfmrr":"Subdomain MozRank","pmrppmrr":"Root Domain MozRank","utrputrr":"MozTrust","ftrpftrr":"Subdomain MozTrust","ptrpptrr":"Root Domain MozTrust","uemrpuemrr":"External MozRank","fejpfejr":"Subdomain External Link Equity","pejppejr":"Root Domain External Link Equity","fjpfjr":"Subdomain Link Equity","pjppjr":"Root Domain Link Equity","us":"HTTP Status Code","fuid":"Links to Subdomain","puid":"Links to Root Domain","fipl":"Root Domains Linking to Subdomain","upa":"Page Authority","pda":"Domain Authority","ulc":"Time last crawled", "umrp":"MozRank", "umrr":"MozRank raw", "fmrp":"Subdomain MozRank", "fmrr":"Subdomain MozRank raw", "pmrp":"Root Domain MozRank", "pmrr":"Root Domain MozRank raw", "utrp":"MozTrust", "utrr":"MozTrust raw", "ftrp":"Subdomain MozTrust", "ftrr":"Subdomain MozTrust raw", "ptrp":"Root Domain MozTrust", "ptrr":"Root Domain MozTrust raw", "uemrp":"External MozRank", "uemrr":"External MozRank raw", "fejp":"Subdomain External Link Equity", "fejr":"Subdomain External Link Equity raw", "pejp":"Root Domain External Link Equity", "pejr":"Root Domain External Link Equity raw", "fjp":"Subdomain Link Equity", "fjr":"Subdomain Link Equity raw" }
	  };
	  
	  // need to be devided into normalized and raw
	  /*
		
	  

		
	  */
	  
      
	  // key: header name
      var header_obj = {  };
	  var headers = {};

      var csv_args = {
        line_open:'"',
        line_close:'"',
        cell_delineator:'","',
        row_delineator: '\r\n',
        headers_ordered:['index', 'url', 'title', 'domain', 'description', 'html_title'] 
      };
      	  // doesn't have to be complete .replace(/\-|_/, ' ');
	  
	  // key keyword search term - compare with titles
	  if($('#gbqfq').length){
		main_vars.search_term = $('#gbqfq').val();
	  }else if($('input[name="q"]').length){
		main_vars.search_term = $('input[name="q"]').val();
	  }else{
		main_vars.search_term = $('input[type="text"]:first').val();
		console.log('getting first text input val as selector has changed');
	  }

	  // Main vars - filename, search term, 
	  main_vars.filename_search_term = main_vars.search_term.replace(/[^a-zA-Z0-9]+/g, '_');
	  main_vars.filename = 'Serps-'+main_vars.filename_search_term+'-'+main_vars.filename_date;
      main_vars.csv_header = 'Serps - '+main_vars.search_term+' - '+main_vars.date;
	  
      // get exact url, and title seperate
      var serp_data = [];
      var local_data = [];
      
      if($('#search ol#rso li.g').length < 10){
        alert('moz data hasnt been fully loaded, try again.  If it has completely loaded, then theres a selector error that needs to be fixed.');
        return; 
      }
      $('#search ol#rso li.g').each(function(i){
        // more accurate way to attach to moz data grab each iframe one at a time
        
        if($(this).attr('id') === 'lclbox'){
          // local results
          // local_serp_data = get_local_serps($(this));
        }else{
          serp_data.push( get_serp_data($(this)) );
        }

      });
      
      if(ss_debug){
        console.log('serp scraped data - ');
        console.log(serp_data);
      }

      // set this as checkbox options? 
      var keepAttrs = ['upa', 'pda', 'fmrp', 'umrp'];
      keepAttrs = [];
      
      var srcString = '';
      
      var mozAttrs = [];
      $('#search iframe').each(function(){
        srcString = $(this).attr('src');
        srcString = srcString.substring(srcString.indexOf("#")+1);
        mozAttrs.push($.parseJSON(srcString));
      });

      if(ss_debug){
        console.log('moz Attrs - ');
        console.log(mozAttrs);
      }
      
      var jsonData = [];
      jsonData = filter_data_to_json(mozAttrs, keepAttrs);
      
      if(ss_debug){
        console.log('mozAttrs filtered to json data - after keepAttrs defined');
        console.log(jsonData);
      }
      var jsonDataCombined = {};
      if(typeof serp_data === 'object' && serp_data && typeof jsonData === 'object' && jsonData){
        jsonDataCombined = combine_json_arrays_by_index(jsonData, serp_data);
      }else{
        jsonDataCombined = jsonData;
      }
      
      console.log('serp data and moz attrs combined - '); 
	  console.log(serp_data);
      console.log(jsonDataCombined);
	  console.log(headers);
      // console.log( JSON.stringify(jsonData) );
      // combine json array serp data and json array moz data
      
      // these are just keys. need to convert to titles. Also order headers
	  var temp = '';
      for(header_key in jsonDataCombined[1]){
        if(jsonDataCombined[1].hasOwnProperty(header_key)){
			if(typeof headers[header_key] === 'undefined' || !headers[header_key]){
				if(typeof main_vars.header_legend[header_key] !== 'undefined' && main_vars.header_legend[header_key]){
					console.log(main_vars.header_legend[header_key]);
					headers[header_key] = main_vars.header_legend[header_key];
				}else{
					alert('header key = headers key'+header_key);
					temp = header_key.replace(/\-|_/, ' ');
					headers[header_key] = temp.charAt(0).toUpperCase() + temp.slice(1);
				}
			}
          // alert(header_key + ' val = ' + header_obj[header_key]);
        }else{
          alert('creating headers property doesnt exists');
        }
      }

      console.log('headers attempt - ');
      console.log(headers);

      var json_filtered = filter_json_for_csv(jsonDataCombined, '"', "'");


     
      console.log('all json data filtered - ');
      console.log(json_filtered);
      var csv_data = JsonToCSV(json_filtered, headers, csv_args, main_vars);
      // console.log(csv_data);
      
	  
      if(!$('#bookmarklet_container').length){
        var template = {
          title: '<div id="ss-title" style="padding:5px 0 10px;font-weight:bold;text-align:center;width:100%;font-size:1.1em;">Serp Savvy</div>',
          download_moz_csv:'<div id="download_moz_csv" style="padding:5px; margin:auto; border:1px solid #aaa; background:#f5f5f5; border-radius:3px; cursor:pointer; box-shadow:1px 1px 2px #fff inset; font-size:12px; text-align:center; color:#444; " >Download <br /> Moz CSV</div>'
        };
        
         $bookmarklet_container = $('<div id="bookmarklet_container" style="position:absolute; top:130px; left:5px; width:100px; z-index:1000; padding:10px; border:1px solid #888; box-shadow:1px 1px 1px rgba(0,0,0,.2); border-radius:5px; background:rgba(255,255,255,.9); ">  </div>');
        $bookmarklet_container.append(template.title); 
        $bookmarklet_container.append(template.download_moz_csv); 
         
        $bookmarklet_container.hide();
        $bookmarklet_container.appendTo($('body')).slideDown(200);
        $('#download_moz_csv').click(function(){
		  csvToPrint(csv_data, main_vars);
        });     
      }

      
      
      
      // then convert to html table, combine with seoquake and csv
  }
  
  function filter_data_to_json(mozAttrs, keepAttrs){
    // if no keep attrs defined, return them all
    
    var jsonDataFiltered = [];
    
    if(ss_debug){ console.log('keep attrs, then moz data - '); console.log(keepAttrs); console.log(mozAttrs);  }
    
    
    // cycle through each serp url
    var max = mozAttrs.length;
    
    
    for(i=0;i<max;i++){
      jsonDataFiltered[i] = filter_keep_attrs(mozAttrs[i].lsData, keepAttrs);
      // console.log('json data for each array elem, in filtering ');
      // console.log( jsonDataFiltered[i]);
      // console.log('i - '+i);
      // console.log(mozAttrs[i].lsData);
      jsonDataFiltered[i].domain = mozAttrs[i].host;
      jsonDataFiltered[i].index = mozAttrs[i].serp;
    }
    
    console.log('filtered before return - ');
    console.log(jsonDataFiltered);
    
    return jsonDataFiltered; 
  }
  
  function filter_keep_attrs(mozLsData, keepAttrs){
    
    var attrs = {};

    if(keepAttrs === 'undefined' || !keepAttrs || !keepAttrs.length){
      if(ss_debug){ console.log('return all moz data from filtering keepAttrs'); }
      return mozLsData;
    }else{
      amax = keepAttrs.length;
      for(a=0;a<amax;a++){
        if(typeof mozLsData[keepAttrs[a]] !== 'undefined'){
          attrs[keepAttrs[a]] = mozLsData[keepAttrs[a]];
        }
      }
      if(ss_debug){ console.log('keepAttrs filter finished. Return data - '); console.log(attrs); }
    }
    return attrs;  
  }

  function combine_json_arrays_by_index(json_one, json_two){
    var j = 0;
    console.log('combining two arrays');
    console.log(json_one);
    console.log(json_two);
    console.log(json_one.length);
    console.log(json_two.length);
    var maxj = (json_two.length > json_one.length) ? json_two.length : json_one.length;
    for(j=0;j<maxj;j++){
      if(typeof json_one[j] !== 'undefined' && typeof json_two[j] !== 'undefined'){
        json_one[j] = combine_objects(json_one[j], json_two[j]);
      }else if(json_two[j] !== 'undefined'){
        json_one[j] = json_two[j];
      }
    }
    return json_one;
  }

  function combine_json_arrays_by_attr(json_one, json_two, first_attr_match, second_attr_match){
    var j = 0;
    var k =0;
    var maxj = json_one.length;
    var maxk = json_two.length;
    // loop through first
    for(j=0;j<maxj;j++){
      
      // loop through second looking for match
      for(k=0;k<maxk;k++){
        if(json_one[j][first_attr_match] === json_two[k][second_attr_match]){
          json_one = combine_objects(json_one[j], json_two[k]);
        }
      }
    } // end json loops
    return json_one;
  }
  
  function combine_objects(json_one, json_two){
    if(typeof $ === 'function'){
      $.extend( json_one, json_two );
    }else{
      // if jquery could mess up prototype and not work, thus use jquery above
      for (var attrname in json_two) { json_one[attrname] = json_two[attrname]; }
    }  
    return json_one; 
  }

 // copied from http://www.zachhunter.com/2011/06/json-to-csv/
  function JsonToCSV(objArray, headers, user_args, main_vars) {

	if(typeof headers === 'undefined'){
		var headers = {};
		alert('headers undefined in JsonToCSV');
	}
  
      console.log('first row - ');
      console.log(objArray[0]);
      console.log(headers);
    
	// make sure all headers there
	for(header_name in objArray[1]){
		if(typeof headers[header_name] === 'undefined' || !headers[header_name]){
			if(typeof main_vars.header_legend[header_name] !== 'undefined' && main_vars.header_legend[header_name]){
				headers[header_name] = main_vars.header_legend[header_name];
			}else{
				headers[header_name] = header_name;
			}		
		}
	}
     console.log('headers just after appended - ');
	 console.log(headers);
      
	  // default arts
      var args = {
        line_open:'"',
        line_close:'"',
        cell_delineator:'","',
        row_delineator: '\r\n',
        headers_ordered:['index', 'url', 'title', 'domain', 'description', 'html_title'] 
      };

      if(typeof user_args !== 'undefined' && user_args){
        args = combine_objects(args, user_args);
      }
      
      // make sure headers filled
	  /*
      for(key in objArray[2]){
        if(typeof headers[key] === 'undefined'  || !headers[key]){
          
		  headers[key] = key;
        }
      }
	  */
    
      args.headers_ordered = check_filter_header_order(args.headers_ordered, headers);
      var maxh = args.headers_ordered.length; 
      
      console.log('headers - ');
      console.log(args.headers_ordered);
      console.log(headers);
      console.log('max = '+maxh);
      
      
      var array = typeof objArray != 'object' ? JSON.parse(objArray) : objArray;
      var str = '';
      
      array.unshift(headers); // add headers to first line
      // console.log(array[0]);
      // if(typeof headers !== 'undefined' && headers){
      //   str += args.line_open + args.headers_ordered.join(args.cell_delineator) + args.line_close + args.row_delineator;
      // }
      
      for (var i = 0; i < array.length; i++) {
          var line = '';
          /*
          // check if numeric array, if so use this
          for (var index in array[i]) {
              if (line != '') line += args.cell_delineator
  
              line += array[i][index];
          }
          */
          for(var h=0;h<maxh;h++){
            if (line != '') line += args.cell_delineator
            if(typeof array[i][args.headers_ordered[h]] === 'undefined' || !array[i][args.headers_ordered[h]]){
              console.log('adding to csv, col not there - '+args.headers_ordered[h]);
              console.log(array[i]);
              array[i][args.headers_ordered[h]] = args.headers_ordered[h];
            }
            line += array[i][args.headers_ordered[h]];
          }
  
          str += args.line_open + line + args.line_close + args.row_delineator;
      }

      return str;
  }
    
    function check_filter_header_order(ordered_headers, headers){
    
      console.log(headers);
      console.log(ordered_headers);
    
      var remaining_unordered_headers = [];
      var filtered_headers = [];
      var max = ordered_headers.length;
      var i = 0;
      for(i=0;i<max;i++){
        if( typeof headers[ordered_headers[i]] !== 'undefined' ){
          filtered_headers.push(ordered_headers[i]);
          delete headers[ordered_headers[i]]
        }
      }
      // add on any remaining headers not in ordered list
      for(header_name in headers){
        filtered_headers.push(header_name);
      }
      
      return filtered_headers;
    }

    function csvToPrint(str, main_vars){
      var a = document.createElement('a');
      var blob = new Blob([str], {'type':'application\/octet-stream'});
      a.href = window.URL.createObjectURL(blob);
      a.download = main_vars.filename+'.csv';
      a.click();
      return true;    
    }
    
    function filter_json_for_csv(objArray, pattern, replace_val){
      var i = 0;
      var max = objArray.length;
      var regex = new RegExp(pattern, "g");
      for (var i = 0; i < objArray.length; i++){
        for(attr in objArray[i]){
          if(typeof objArray[i][attr] === 'string'){
            objArray[i][attr] = objArray[i][attr].replace(regex, replace_val);
          }
        }
      }
      return objArray;
    }
  
/*

Legend - 

moz_legend = jQuery.parseJSON(moz_legend);
key_title = get_moz_legend_attrs(moz_legend, 'title');
console.log(key_title);
console.log(JSON.stringify(key_title));

function get_moz_legend_attrs(moz_legend, type){
	if(typeof type === 'undefined'){
		type = 'title';
	}
	var legend_attrs = {};
	var i = 0;
	var max = moz_legend.length; 
	for(i=0;i<max;i++){
		if(type==='description'){
			legend_attrs[moz_legend[i].responsefield] = moz_legend[i].description;
		}else{
			legend_attrs[moz_legend[i].responsefield] = moz_legend[i].urlmetric;
		}
		
	}
	return legend_attrs;
}

function check_if_raw_normalized(moz_legend){
	// regex = (\(\w{2,7}\))
	if(moz_legend.description.match(/\(\w{2,7}\)/).length > 1){
	
	}
}

fix_legend_normalized_raw(moz_legend){
	var moz_legend_fixed = [];
	var items = false;
	var i = 0;
	var max = moz_legend.length;
	for(i=0;i<max;i++){
		items = fix_legend_item_raw_normalized(moz_legend[i]);
		if(items){
			moz_legend_fixed = moz_legend_fixed.concat(items);
		}else{
			moz_legend_fixed.push(moz_legend[i]);
		}
	}
	return moz_legend_fixed;
}

function fix_legend_item_raw_normalized(moz_legend){
	
	matches = moz_legend.description.match(/\(\w{2,7}\)/g);
	console.log(matches);
	if(matches.length === 2){
		var description = moz_legend.urlmetric.replace(/in\sboth.*\s?/, '');
		var items = [
			{
				"urlmetric" : matches[0].replace(/\(|\)/g, ''),
				"responsefield" : moz_legend.urlmetric,
				"description" : description+'normalized'
			},{
				"urlmetric" : matches[1].replace(/\(|\)/g, ''),
				"responsefield" : moz_legend.urlmetric,
				"description" : description+'raw'
			}
		]
	}

}



// url-metrics - http://apiwiki.moz.com/url-metrics

// scrape the table explaining everything. - metric, response, description
// then make an array of values to keep and use. 


.lsData: {
  feid: 4154    // Subdomain external links
  fipl: 93
  fmrp: 4.334148958652751     // Mozrank - Subdomain - normalized
  fmrr: 9.136512895345806e-10 // Mozrank - Subdomain - raw
  ftrp: 4.794624262464523
  ftrr: 2.3974940417748502e-8
  fuid: 12209
  pda: 34.1025763780209       // DA
  peid: 4164
  pid: 93
  pmrp: 4.088676802875943     // Mozrank - Root Domain - normalized
  pmrr: 2.2168574202653488e-8 // Mozrank - Root Domain - raw
  ptrp: 4.400450399832809     // Moztrust - Root Domain - normalized
  ptrr: 3.701476990845632e-8  // Moztrust - Root Domain - raw
  puid: 12221
  ueid: 3830
  uemrp: 5.182651754750022
  uemrr: 6.855684662794894e-10
  uid: 4076
  uipl: 68
  ujid: 4051
  umrp: 5.236369443833013       // Mozrank - normalized
  umrr: 8.054739403803252e-10   // Mozrank - raw
  upa: 44.791172741485525       // PA
  utrp: 5.572420463893636       // MozTrust Normalized
  utrr: 6.788566425001243e-13   // MozTrust Raw
}


url-metrics table

{
"urlmetric":"Title",
"responsefield":"ut",
"description":"The title of the page, if available"
},

		// this doesnt take care of descriptions though 

		"umrp":"MozRank",
		"umrr":"MozRank raw",
		"fmrp":"Subdomain MozRank",
		"fmrr":"Subdomain MozRank raw",
		"pmrp":"Root Domain MozRank",
		"pmrr":"Root Domain MozRank raw",
		"utrp":"MozTrust",
		"utrr":"MozTrust raw",
		"ftrp":"Subdomain MozTrust",
		"ftrr":"Subdomain MozTrust raw",
		"ptrp":"Root Domain MozTrust",
		"ptrr":"Root Domain MozTrust raw",
		"uemrp":"External MozRank",
		"uemrr":"External MozRank raw",
		"fejp":"Subdomain External Link Equity",
		"fejr":"Subdomain External Link Equity raw",
		"pejp":"Root Domain External Link Equity",
		"pejr":"Root Domain External Link Equity raw",
		"fjp":"Subdomain Link Equity",
		"fjr":"Subdomain Link Equity raw",

moz_legend = '[{"urlmetric":"Title","responsefield":"ut","description":"The title of the page, if available"},{"urlmetric":"Canonical URL","responsefield":"uu","description":"The canonical form of the URL"},{"urlmetric":"Subdomain","responsefield":"ufq","description":"The subdomain of the URL (for example, apiwiki.moz.com)"},{"urlmetric":"Root Domain","responsefield":"upl","description":"The root domain of the URL (for example, moz.com)"},{"urlmetric":"External Equity Links","responsefield":"ueid","description":"The number of external equity links to the URL"},{"urlmetric":"Subdomain External Links","responsefield":"feid","description":"The number of external equity links to the subdomain of the URL"},{"urlmetric":"Root Domain External Links","responsefield":"peid","description":"The number of external equity links to the root domain of the URL"},{"urlmetric":"Equity Links","responsefield":"ujid","description":"The number of equity links (internal or external) to the URL"},{"urlmetric":"Subdomains Linking","responsefield":"uifq","description":"The number of subdomains with any pages linking to the URL"},{"urlmetric":"Root Domains Linking","responsefield":"uipl","description":"The number of root domains with any pages linking to the URL"},{"urlmetric":"Links","responsefield":"uid","description":"The number of links (equity or nonequity or not, internal or external) to the URL"},{"urlmetric":"Subdomain Subdomains Linking","responsefield":"fid","description":"The number of subdomains with any pages linking to the subdomain of the URL"},{"urlmetric":"Root Domain Root Domains Linking","responsefield":"pid","description":"The number of root domains with any pages linking to the root domain of the URL"},{"urlmetric":"MozRank","responsefield":"umrpumrr","description":"The MozRank of the URL, in both the normalized 10-point score (umrp) and the raw score (umrr)"},{"urlmetric":"Subdomain MozRank","responsefield":"fmrpfmrr","description":"The MozRank of the subdomain of the URL, in both the normalized 10-point score (fmrp) and the raw score (fmrr)"},{"urlmetric":"Root Domain MozRank","responsefield":"pmrppmrr","description":"The MozRank of the Root Domain of the URL, in both the normalized 10-point score (pmrp) and the raw score (pmrr)"},{"urlmetric":"MozTrust","responsefield":"utrputrr","description":"The MozTrust of the URL, in both the normalized 10-point score (utrp) and the raw score (utrr)"},{"urlmetric":"Subdomain MozTrust","responsefield":"ftrpftrr","description":"The MozTrust of the subdomain of the URL, in both the normalized 10-point score (ftrp) and the raw score (ftrr)"},{"urlmetric":"Root Domain MozTrust","responsefield":"ptrpptrr","description":"The MozTrust of the root domain of the URL, in both the normalized 10-point score (ptrp) and the raw score (ptrr)"},{"urlmetric":"External MozRank","responsefield":"uemrpuemrr","description":"The portion of the URL\'s MozRank coming from external links, in both the normalized 10-point score (uemrp) and the raw score (uemrr)."},{"urlmetric":"Subdomain External Link Equity","responsefield":"fejpfejr","description":"The portion of the MozRank of all pages on the subdomain coming from external links, in both the normalized 10-digit score (pejp) and the raw score"},{"urlmetric":"Root Domain External Link Equity","responsefield":"pejppejr","description":"The portion of the MozRank of all pages on the root domain coming from external links, in both the normalized 10-digit score (pejp) and the raw source"},{"urlmetric":"Subdomain Link Equity","responsefield":"fjpfjr","description":"The MozRank of all pages on the subdomain combined, in both the normalized 10-point score (fjp) and the raw score (fjr)"},{"urlmetric":"Root Domain Link Equity","responsefield":"pjppjr","description":"The MozRank of all pages on the root domain combined, in both the normalized 10-point score (pjp) and the raw score (pjr)"},{"urlmetric":"HTTP Status Code","responsefield":"us","description":"The HTTP status code recorded by Mozscape for this URL, if available"},{"urlmetric":"Links to Subdomain","responsefield":"fuid","description":"The total number of links (including internal and nofollow links) to the subdomain of the URL"},{"urlmetric":"Links to Root Domain","responsefield":"puid","description":"The total number of links, including internal and nofollow links, to the root domain of the URL"},{"urlmetric":"Root Domains Linking to Subdomain","responsefield":"fipl","description":"The number of root domains with at least one link to the subdomain of the URL"},{"urlmetric":"Page Authority","responsefield":"upa","description":"A normalized 100-point score representing the likelihood of a page to rank well in search engine results"},{"urlmetric":"Domain Authority","responsefield":"pda","description":"A normalized 100-point score representing the likelihood of a domain to rank well in search engine results"},{"urlmetric":"Time last crawled","responsefield":"ulc","description":"The time and date on which Mozscape last crawled the URL, returned in Unix epoch format"}]';


// broken infinite loop
function fix_legend_normalized_raw(moz_legend){
	var moz_legend_fixed = [];
	var items = false;
	var i = 0;
	var max = moz_legend.length;
	for(i=0;i<max;i++){
		items = fix_legend_item_raw_normalized(moz_legend[i]);
		if(items){
			moz_legend_fixed = moz_legend_fixed.push(items[0]);
			moz_legend_fixed = moz_legend_fixed.push(items[1]);
		}else{
			moz_legend_fixed.push(moz_legend[i]);
		}
	}
	return moz_legend_fixed;
}

function fix_legend_item_raw_normalized(moz_legend_item){
	
	console.log(moz_legend_item);
	if(typeof moz_legend_item.description !== 'undefined'){
		matches = moz_legend_item.description.match(/\(\w{2,7}\)/g);
		console.log(matches);
		if(matches.length === 2){
			var description = moz_legend_item.urlmetric.replace(/in\sboth.*\s?/, '');
			var items = [
				{
					"urlmetric" : matches[0].replace(/\(|\)/g, ''),
					"responsefield" : moz_legend_item.urlmetric,
					"description" : description+'normalized'
				},{
					"urlmetric" : matches[1].replace(/\(|\)/g, ''),
					"responsefield" : moz_legend_item.urlmetric,
					"description" : description+'raw'
				}
			]
		}	
	}else{
		return false;
	}
	return items;
}

moz_legend_fixed = fix_legend_normalized_raw(moz_legend);
console.log(moz_legend_fixed);
str = JSON.stringify(moz_legend_fixed);
console.log(str);


*/