Scrape some country data

addressScaper.js

// First run: `npm install request q`

var request = require('request');
var Q = require('q');

var COUNTRIES = ['US'];
var BASE_URL = 'http://i18napis.appspot.com/address/data/'

function scrape(countries, cb) {
  var datapromises = countries.map(function(country){
    var dfd = Q.defer();

    request(BASE_URL + country, function(err, response, body){
      if (err) {
        console.log('!!!!!! Fail: ', country);
        return dfd.fail();
      }

      dfd.resolve(JSON.parse(body));
    });

    return dfd.promise;
  });

  Q.spread(datapromises, function(){
    var countryData = [].slice.apply(arguments);
    cb(countryData.map(shittySchemaParse));
  })
}

function shittySchemaParse(country) {
  country.sub_keys = country.sub_keys.split('~');
  country.sub_names = country.sub_names.split('~');
  country.sub_zips = country.sub_zips.split('~');
  country.sub_zipexs = country.sub_zipexs.split('~');

  country.subs = {};
  country.sub_keys.forEach(function(key, idx){
    country.subs[key] = country.sub_names[idx];
  });

  return country;
}

scrape(COUNTRIES, function(countries) {
  console.log(countries);
});