Skip to content

Instantly share code, notes, and snippets.

@ahmadia

ahmadia/HMDA.ipynb

Created June 25, 2014 05:30
Show Gist options
  • Save ahmadia/54a91faec47aafb6cf28 to your computer and use it in GitHub Desktop.
Save ahmadia/54a91faec47aafb6cf28 to your computer and use it in GitHub Desktop.
Download ZIP
Playing with LeafletWidget and the HMDA dataset
Display the source blob
Display the rendered blob
Raw
HMDA.ipynb
{
"metadata": {
"name": "",
"signature": "sha256:7975f61ddb43e1254305af8d3528cc55475da4b548e1808c813feb4332f00625"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "heading",
"level": 2,
"metadata": {},
"source": [
"Initialization"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"%matplotlib inline"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 88
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"import sys\n",
"import csv\n",
"import json\n",
"import numpy as np\n",
"from collections import Counter, defaultdict\n",
"import leafletwidget as lw\n",
"import matplotlib as mpl\n",
"import matplotlib.cm\n",
"import matplotlib.colors\n",
"import matplotlib.pyplot as plt"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 425
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"lw.initialize_notebook()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"html": [
"<link rel=\"stylesheet\" href=\"//cdn.leafletjs.com/leaflet-0.7.2/leaflet.css\" />"
],
"metadata": {},
"output_type": "display_data",
"text": [
"<IPython.core.display.HTML object>"
]
},
{
"html": [
"<link rel=\"stylesheet\" href=\"//cdnjs.cloudflare.com/ajax/libs/leaflet.draw/0.2.3/leaflet.draw.css\" />"
],
"metadata": {},
"output_type": "display_data",
"text": [
"<IPython.core.display.HTML object>"
]
},
{
"javascript": [
"\n",
"\n",
"require.config({\n",
" paths: {\n",
" leaflet: \"//cdn.leafletjs.com/leaflet-0.7.2/leaflet\",\n",
" leaflet_draw: \"//cdnjs.cloudflare.com/ajax/libs/leaflet.draw/0.2.3/leaflet.draw\"\n",
" },\n",
" shim: {leaflet_draw: \"leaflet\"}\n",
"});\n",
"\n",
"require([\"widgets/js/widget\", \"leaflet\", \"leaflet_draw\"], function(WidgetManager, L) {\n",
"\n",
" function camel_case(input) {\n",
" // Convert from foo_bar to fooBar \n",
" return input.toLowerCase().replace(/_(.)/g, function(match, group1) {\n",
" return group1.toUpperCase();\n",
" });\n",
" }\n",
" \n",
" var LeafletLayerView = IPython.WidgetView.extend({\n",
" \n",
" initialize: function (parameters) {\n",
" LeafletLayerView.__super__.initialize.apply(this, arguments);\n",
" // Remove this line after testing...\n",
" this.model.on('displayed', this.test_display, this);\n",
" this.map_view = this.options.map_view;\n",
" },\n",
" \n",
" // Remove this method after testing...\n",
" test_display: function () {\n",
" },\n",
" \n",
" render: function () {\n",
" this.create_obj();\n",
" this.leaflet_events();\n",
" this.model_events();\n",
" },\n",
"\n",
" leaflet_events: function () {\n",
" },\n",
"\n",
" model_events: function () {\n",
" },\n",
"\n",
" get_options: function () {\n",
" var o = this.model.get('options');\n",
" var options = {};\n",
" var key;\n",
" for (var i=0; i<o.length; i++) {\n",
" key = o[i];\n",
" // Convert from foo_bar to fooBar that Leaflet.js uses\n",
" options[camel_case(key)] = this.model.get(key);\n",
" }\n",
" return options;\n",
" },\n",
"\n",
" });\n",
" WidgetManager.register_widget_view('LeafletLayerView', LeafletLayerView);\n",
"\n",
"\n",
" // UILayer\n",
" var LeafletUILayerView = LeafletLayerView.extend({\n",
" });\n",
" WidgetManager.register_widget_view('LeafletUILayerView', LeafletUILayerView);\n",
" \n",
" \n",
" var LeafletMarkerView = LeafletUILayerView.extend({\n",
" \n",
" create_obj: function () {\n",
" this.obj = L.marker(\n",
" this.model.get('location'),\n",
" this.get_options()\n",
" );\n",
" },\n",
"\n",
" model_events: function () {\n",
" LeafletMarkerView.__super__.model_events.apply(this, arguments);\n",
" var that = this;\n",
" this.model.on('change:opacity', function () {\n",
" that.obj.setOpacity(that.model.get('opacity'));\n",
" });\n",
" },\n",
"\n",
" });\n",
" WidgetManager.register_widget_view('LeafletMarkerView', LeafletMarkerView);\n",
" \n",
" \n",
" var LeafletPopupView = LeafletUILayerView.extend({\n",
" });\n",
" WidgetManager.register_widget_view('LeafletPopupView', LeafletPopupView);\n",
" \n",
" \n",
" // RasterLayer\n",
" var LeafletRasterLayerView = LeafletLayerView.extend({\n",
" });\n",
" WidgetManager.register_widget_view('LeafletRasterLayerView', LeafletRasterLayerView);\n",
" \n",
" \n",
" var LeafletTileLayerView = LeafletRasterLayerView.extend({\n",
" \n",
" create_obj: function () {\n",
" this.obj = L.tileLayer(\n",
" this.model.get('url'),\n",
" this.get_options()\n",
" );\n",
" },\n",
" \n",
" });\n",
" WidgetManager.register_widget_view('LeafletTileLayerView', LeafletTileLayerView);\n",
" \n",
" \n",
" var LeafletImageOverlayView = LeafletRasterLayerView.extend({\n",
" \n",
" create_obj: function () {\n",
" this.obj = L.imageOverlay(\n",
" this.model.get('url'),\n",
" this.model.get('bounds'),\n",
" this.get_options()\n",
" );\n",
" },\n",
" \n",
" });\n",
" WidgetManager.register_widget_view('LeafletImageOverlayView', LeafletImageOverlayView);\n",
" \n",
" \n",
" // VectorLayer\n",
" var LeafletVectorLayerView = LeafletLayerView.extend({\n",
" });\n",
" WidgetManager.register_widget_view('LeafletVectorLayerView', LeafletVectorLayerView);\n",
" \n",
" \n",
" var LeafletPathView = LeafletVectorLayerView.extend({\n",
" \n",
" model_events: function () {\n",
" LeafletPathView.__super__.model_events.apply(this, arguments);\n",
" var that = this;\n",
" var key;\n",
" var o = this.model.get('options');\n",
" for (var i=0; i<o.length; i++) {\n",
" key = o[i];\n",
" this.model.on('change:'+key, function () {\n",
" that.obj.setStyle(that.get_options())\n",
" })\n",
" }\n",
" },\n",
"\n",
" });\n",
" WidgetManager.register_widget_view('LeafletPathView', LeafletPathView);\n",
" \n",
" \n",
" var LeafletPolylineView = LeafletPathView.extend({\n",
" \n",
" create_obj: function () {\n",
" this.obj = L.polyline(\n",
" this.model.get('locations'),\n",
" this.get_options()\n",
" );\n",
" },\n",
" \n",
" });\n",
" WidgetManager.register_widget_view('LeafletPolylineView', LeafletPolylineView);\n",
" \n",
" \n",
" var LeafletPolygonView = LeafletPolylineView.extend({\n",
" \n",
" create_obj: function () {\n",
" this.obj = L.polygon(this.model.get('locations'), this.get_options());\n",
" },\n",
" \n",
" });\n",
" WidgetManager.register_widget_view('LeafletPolygonView', LeafletPolygonView);\n",
" \n",
" \n",
" var LeafletRectangleView = LeafletPolygonView.extend({\n",
" \n",
" create_obj: function () {\n",
" this.obj = L.rectangle(\n",
" this.model.get('bounds'),\n",
" this.get_options()\n",
" );\n",
" },\n",
"\n",
" });\n",
" WidgetManager.register_widget_view('LeafletRectangleView', LeafletRectangleView);\n",
" \n",
" \n",
" var LeafletCircleView = LeafletPathView.extend({\n",
" \n",
" create_obj: function () {\n",
" this.obj = L.circle(\n",
" this.model.get('location'), this.model.get('radius'),\n",
" this.get_options()\n",
" );\n",
" },\n",
" \n",
" });\n",
" WidgetManager.register_widget_view('LeafletCircleView', LeafletCircleView);\n",
" \n",
" \n",
" var LeafletCircleMarkerView = LeafletCircleView.extend({\n",
" \n",
" create_obj: function () {\n",
" this.obj = L.circleMarker(\n",
" this.model.get('location'), this.model.get('radius'),\n",
" this.get_options()\n",
" );\n",
" },\n",
" \n",
" });\n",
" WidgetManager.register_widget_view('LeafletCircleMarkerView', LeafletCircleMarkerView);\n",
" \n",
" \n",
" // LayerGroup\n",
" var LeafletLayerGroupView = LeafletLayerView.extend({\n",
" \n",
" create_obj: function () {\n",
" this.obj = L.layerGroup();\n",
" },\n",
" \n",
" });\n",
" WidgetManager.register_widget_view('LeafletLayerGroupView', LeafletLayerGroupView);\n",
" \n",
" \n",
" var LeafletFeatureGroupView = LeafletLayerGroupView.extend({\n",
" \n",
" create_obj: function () {\n",
" this.obj = L.featureGroup();\n",
" },\n",
" \n",
" });\n",
" WidgetManager.register_widget_view('LeafletFeatureGroupView', LeafletFeatureGroupView);\n",
" \n",
" \n",
" var LeafletMultiPolylineView = LeafletFeatureGroupView.extend({\n",
" });\n",
" WidgetManager.register_widget_view('LeafletMultiPolylineView', LeafletMultiPolylineView);\n",
" \n",
" \n",
" var LeafletGeoJSONView = LeafletFeatureGroupView.extend({\n",
" \n",
" create_obj: function () {\n",
" var style = this.model.get('style');\n",
" if ($.isEmptyObject(style)) {\n",
" style = function (feature) {\n",
" return feature.properties.style;\n",
" } \n",
" }\n",
" this.obj = L.geoJson(this.model.get('data'), {style: style});\n",
" },\n",
" });\n",
" WidgetManager.register_widget_view('LeafletGeoJSONView', LeafletGeoJSONView);\n",
"\n",
"\n",
" var LeafletMultiPolygonView = LeafletFeatureGroupView.extend({\n",
" });\n",
" WidgetManager.register_widget_view('LeafletMultiPolygonView', LeafletMultiPolygonView);\n",
"\n",
"\n",
" var LeafletControlView = IPython.WidgetView.extend({\n",
" \n",
" initialize: function (parameters) {\n",
" LeafletControlView.__super__.initialize.apply(this, arguments);\n",
" this.map_view = this.options.map_view;\n",
" },\n",
"\n",
" });\n",
" WidgetManager.register_widget_view('LeafletControlView', LeafletControlView);\n",
"\n",
"\n",
" var LeafletDrawControlView = LeafletControlView.extend({\n",
" \n",
" initialize: function (parameters) {\n",
" LeafletDrawControlView.__super__.initialize.apply(this, arguments);\n",
" this.map_view = this.options.map_view;\n",
" },\n",
"\n",
" render: function () {\n",
" this.layer_view = this.create_child_view(this.model.get('layer'), {map_view: this.map_view});\n",
" this.map_view.obj.addLayer(this.layer_view.obj);\n",
" this.create_obj();\n",
" },\n",
"\n",
" create_obj: function () {\n",
" var that = this;\n",
" var polyline = this.model.get('polyline');\n",
" if ($.isEmptyObject(polyline)) {polyline=false;}\n",
" var polygon = this.model.get('polygon');\n",
" if ($.isEmptyObject(polygon)) {polygon=false;}\n",
" var circle = this.model.get('circle');\n",
" if ($.isEmptyObject(circle)) {circle=false;}\n",
" var rectangle = this.model.get('rectangle');\n",
" if ($.isEmptyObject(rectangle)) {rectangle=false;}\n",
" var marker = this.model.get('marker');\n",
" if ($.isEmptyObject(marker)) {marker=false;}\n",
" this.obj = new L.Control.Draw({\n",
" edit: {featureGroup: this.layer_view.obj},\n",
" draw: {\n",
" polyline: polyline,\n",
" polygon: polygon,\n",
" circle: circle,\n",
" rectangle: rectangle,\n",
" marker: marker\n",
" }\n",
" });\n",
" this.map_view.obj.on('draw:created', function (e) {\n",
" var type = e.layerType;\n",
" var layer = e.layer;\n",
" var geo_json = layer.toGeoJSON();\n",
" geo_json.properties.style = layer.options;\n",
" that.send({\n",
" 'event': 'draw:created',\n",
" 'geo_json': geo_json\n",
" });\n",
" that.layer_view.obj.addLayer(layer);\n",
" });\n",
" this.map_view.obj.on('draw:edited', function (e) {\n",
" var layers = e.layers;\n",
" layers.eachLayer(function (layer) {\n",
" var geo_json = layer.toGeoJSON();\n",
" geo_json.properties.style = layer.options;\n",
" that.send({\n",
" 'event': 'draw:edited',\n",
" 'geo_json': geo_json\n",
" });\n",
" });\n",
" });\n",
" this.map_view.obj.on('draw:deleted', function (e) {\n",
" var layers = e.layers;\n",
" layers.eachLayer(function (layer) {\n",
" var geo_json = layer.toGeoJSON();\n",
" geo_json.properties.style = layer.options;\n",
" that.send({\n",
" 'event': 'draw:deleted',\n",
" 'geo_json': geo_json\n",
" });\n",
" });\n",
" });\n",
" },\n",
"\n",
" });\n",
" WidgetManager.register_widget_view('LeafletDrawControlView', LeafletDrawControlView);\n",
"\n",
"\n",
" var LeafletMapView = IPython.DOMWidgetView.extend({\n",
" \n",
" initialize: function (options) {\n",
" LeafletMapView.__super__.initialize.apply(this, arguments);\n",
" this.rendered = false;\n",
" },\n",
" \n",
" // Layer management\n",
" update_layers: function (old_list, new_list) {\n",
" this.do_diff(\n",
" old_list,\n",
" new_list, \n",
" $.proxy(this.remove_layer_model, this),\n",
" $.proxy(this.add_layer_model, this)\n",
" );\n",
" },\n",
" \n",
" remove_layer_model: function (child_model) {\n",
" var child_view = this.child_views[child_model.id];\n",
" this.obj.removeLayer(child_view.obj);\n",
" this.delete_child_view(child_model);\n",
" },\n",
" \n",
" add_layer_model: function (child_model) {\n",
" var child_view = this.create_child_view(child_model, {map_view: this});\n",
" this.obj.addLayer(child_view.obj);\n",
" },\n",
"\n",
" // Control Management\n",
" update_controls: function (old_list, new_list) {\n",
" this.do_diff(\n",
" old_list,\n",
" new_list, \n",
" $.proxy(this.remove_control_model, this),\n",
" $.proxy(this.add_control_model, this)\n",
" );\n",
" },\n",
"\n",
" remove_control_model: function (child_model) {\n",
" var child_view = this.child_views[child_model.id];\n",
" this.obj.removeControl(child_view.obj);\n",
" this.delete_child_view(child_model);\n",
" },\n",
"\n",
" add_control_model: function (child_model) {\n",
" var child_view = this.create_child_view(child_model, {map_view: this});\n",
" this.obj.addControl(child_view.obj);\n",
" },\n",
"\n",
" // Rendering\n",
" render: function () {\n",
" this.$el.width(this.model.get('width')).height(this.model.get('height'));\n",
" this.model.on('displayed', this.render_leaflet, this);\n",
" },\n",
" \n",
" render_leaflet: function () {\n",
" if (!this.rendered) {\n",
" var that = this;\n",
" this.create_obj();\n",
" this.update_layers([], this.model.get('layers'));\n",
" this.update_controls([], this.model.get('controls'));\n",
" this.leaflet_events();\n",
" this.model_events();\n",
" this.update_bounds();\n",
" this.rendered = true;\n",
" }\n",
" },\n",
"\n",
" create_obj: function () {\n",
" this.obj = L.map(this.$el.get(0), this.get_options());\n",
" },\n",
" \n",
" get_options: function () {\n",
" var o = this.model.get('options');\n",
" var options = {};\n",
" var key;\n",
" for (var i=0; i<o.length; i++) {\n",
" key = o[i];\n",
" // Convert from foo_bar to fooBar that Leaflet.js uses\n",
" options[camel_case(key)] = this.model.get(key);\n",
" }\n",
" return options;\n",
" },\n",
" \n",
" leaflet_events: function () {\n",
" var that = this;\n",
" this.obj.on('moveend', function (e) {\n",
" var c = e.target.getCenter();\n",
" that.model.set('center', [c.lat, c.lng]);\n",
" that.touch();\n",
" that.update_bounds();\n",
" });\n",
" this.obj.on('zoomend', function (e) {\n",
" var z = e.target.getZoom();\n",
" that.model.set('zoom', z);\n",
" that.touch();\n",
" that.update_bounds();\n",
" });\n",
" },\n",
" \n",
" update_bounds: function () {\n",
" var b = this.obj.getBounds();\n",
" this.model.set('_north', b.getNorth());\n",
" this.model.set('_south', b.getSouth());\n",
" this.model.set('_east', b.getEast());\n",
" this.model.set('_west', b.getWest());\n",
" this.touch();\n",
" },\n",
" \n",
" model_events: function () {\n",
" var that = this;\n",
" this.model.on('msg:custom', this.handle_msg, this);\n",
" this.model.on('change:layers', function(model, value, options) {\n",
" that.update_layers(model.previous('layers'), value);\n",
" });\n",
" this.model.on('change:controls', function(model, value, options) {\n",
" that.update_controls(model.previous('controls'), value);\n",
" });\n",
" this.model.on('change:zoom', function () {\n",
" that.obj.setZoom(that.model.get('zoom'));\n",
" that.update_bounds();\n",
" });\n",
" this.model.on('change:center', function () {\n",
" that.obj.panTo(that.model.get('center'));\n",
" that.update_bounds();\n",
" });\n",
" },\n",
" \n",
" handle_msg: function (content) {\n",
" switch(content.method) {\n",
" case 'foo':\n",
" break;\n",
" }\n",
" },\n",
" \n",
" });\n",
" WidgetManager.register_widget_view('LeafletMapView', LeafletMapView);\n",
"\n",
"\n",
"});"
],
"metadata": {},
"output_type": "display_data",
"text": [
"<IPython.core.display.Javascript object>"
]
}
],
"prompt_number": 23
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This is a big dataset!"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Number of records (+ header)\n",
"!wc -l hmda_lar-2012.csv"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 18691552 hmda_lar-2012.csv\r\n"
]
}
],
"prompt_number": 7
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We would like to present this data in an informative manner. We're going to take advantage of the fact that the data is coded by state and county to aggregate based on this information when it's available in the data."
]
},
{
"cell_type": "heading",
"level": 2,
"metadata": {},
"source": [
"Loading and aggregating the data"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"state_actions = defaultdict(Counter)\n",
"county_actions = defaultdict(Counter)\n",
"bad_records = []\n",
"with open('hmda_lar-2012.csv') as csv_file:\n",
" dialect = csv.Sniffer().sniff(csv_file.read(4096))\n",
" csv_file.seek(0)\n",
" reader = csv.reader(csv_file, dialect)\n",
" header_list = reader.next()\n",
" action_idx = header_list.index('action_taken_name')\n",
" county_code_idx = header_list.index('county_code')\n",
" state_code_idx = header_list.index('state_code')\n",
" state_name_idx = header_list.index('state_name')\n",
" \n",
" def parse_row_list(row_list):\n",
" action = row_list[action_idx]\n",
" county = int(row_list[county_code_idx])\n",
" state = int(row_list[state_code_idx]) \n",
" state_name = row_list[state_name_idx]\n",
" county_fips = state*1000 + county\n",
" return action, county_fips, state, state_name\n",
" \n",
" for i, row_list in enumerate(reader):\n",
" try:\n",
" action, county_fips, state, state_name = parse_row_list(row_list)\n",
" county_actions[county_fips][action] += 1\n",
" state_actions[state_name][action] += 1\n",
" except:\n",
" bad_records.append(row_list)\n",
" if (i+1) % 100000 == 0:\n",
" sys.stdout.write('.')\n",
" sys.stdout.flush()\n",
" # uncomment the line below to only run 100K records\n",
" # break\n",
"print ''\n",
"print 'Processed %d records' % i\n",
"print 'Found %d records with missing county data' % len(bad_records)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"."
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"\n",
"Processed 18691550 records\n",
"Found 352923 records with missing county data\n"
]
}
],
"prompt_number": 451
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can now take a look at some of our accumulated data for a given state."
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"state_actions['California']"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 452,
"text": [
"Counter({'Loan originated': 1390413, 'Loan purchased by the institution': 427704, 'Application denied by financial institution': 316777, 'Application withdrawn by applicant': 216891, 'Application approved but not accepted': 99626, 'File closed for incompleteness': 86565, 'Preapproval request approved but not accepted': 17, 'Preapproval request denied by financial institution': 15})"
]
}
],
"prompt_number": 452
},
{
"cell_type": "heading",
"level": 2,
"metadata": {},
"source": [
"Querying the Data Set"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"def query(county_id):\n",
" return county_actions[county_id]['Application denied by financial institution']"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 453
},
{
"cell_type": "heading",
"level": 2,
"metadata": {},
"source": [
"Computing the Query"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# http://eric.clst.org/wupl/Stuff/gz_2010_us_050_00_5m.json\n",
"with open('gz_2010_us_050_00_5m.json', 'rb') as f:\n",
" county_json = json.load(f, encoding='latin-1')\n",
"\n",
"geo_id_stream = (feature['properties']['GEO_ID'] for feature in county_json['features'])"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 454
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"def to_county_id(geo_id):\n",
" return int(geo_id[-5:])"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 455
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"counties = [to_county_id(geo_id) for geo_id in geo_id_stream]\n",
"denials = [query(key) for key in counties]"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 456
},
{
"cell_type": "heading",
"level": 2,
"metadata": {},
"source": [
"Visualizing the Query"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"normalized_denials = mpl.colors.Normalize()(denials)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 457
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"colormap=mpl.cm.Blues\n",
"denial_colors = [mpl.colors.rgb2hex(d[0:3]) for d in colormap(normalized_denials)]\n",
"\n",
"for feature, color in zip(county_json['features'],\n",
" denial_colors):\n",
" feature['properties']['style'] = {'color': color, 'weight': 1, 'fillColor': color, 'fillOpacity': 0.5}"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 458
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"m = lw.Map(zoom=4, center=[37.996162679728116, -97.294921875], default_tiles=None)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 459
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"m"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 460
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"g = lw.GeoJSON(data=county_json)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 461
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"m.add_layer(g)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 462
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": []
}
],
"metadata": {}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment