Skip to content

Instantly share code, notes, and snippets.

@egy1st

egy1st/TornotoNeighborhood.ipynb

Created January 14, 2019 23:13
Show Gist options
  • Save egy1st/903f21a7db48424daa5027e00a33687e to your computer and use it in GitHub Desktop.
Save egy1st/903f21a7db48424daa5027e00a33687e to your computer and use it in GitHub Desktop.
Download ZIP
Created on Cognitive Class Labs
Display the source blob
Display the rendered blob
Raw
TornotoNeighborhood.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Tornoto Neighborhood Scrape"
]
},
{
"cell_type": "code",
"execution_count": 58,
"metadata": {},
"outputs": [],
"source": [
"# html_doc = \"https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M\"\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 44,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: BeautifulSoup4 in /home/jupyterlab/conda/lib/python3.6/site-packages (4.6.3)\n",
"Requirement already satisfied: lxml in /home/jupyterlab/conda/lib/python3.6/site-packages (4.2.5)\n",
"Requirement already satisfied: html5lib in /home/jupyterlab/conda/lib/python3.6/site-packages (0.9999999)\n"
]
}
],
"source": [
"# installing beautifulsoup4 package for scraping wikipedia page\n",
"# that will be used as an allternative to data provided by foursquare\n",
"# !conda install -c anaconda beautifulsoup4\n",
"!pip install BeautifulSoup4\n",
"!pip install lxml\n",
"!pip install html5lib"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"import bs4\n",
"import requests # library to handle requests\n",
"import json # library to handle JSON files\n",
"from bs4 import BeautifulSoup"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"url = \"https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M\"\n",
"html_doc = requests.get(url).text"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"soup = BeautifulSoup(html_doc, 'html.parser')"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>PostalCode</th>\n",
" <th>Borough</th>\n",
" <th>Neighborhood</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>M3A</td>\n",
" <td>North York</td>\n",
" <td>Parkwoods, Parkwoods</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>M4A</td>\n",
" <td>North York</td>\n",
" <td>Victoria Village, Victoria Village</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>M5A</td>\n",
" <td>Downtown Toronto</td>\n",
" <td>Harbourfront, Harbourfront</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>M5A</td>\n",
" <td>Downtown Toronto</td>\n",
" <td>Regent Park</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>M6A</td>\n",
" <td>North York</td>\n",
" <td>Lawrence Heights, Lawrence Heights</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>M6A</td>\n",
" <td>North York</td>\n",
" <td>Lawrence Manor</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>M7A</td>\n",
" <td>Queen's Park</td>\n",
" <td>Queen's Park, Queen's Park</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>M9A</td>\n",
" <td>Etobicoke</td>\n",
" <td>Islington Avenue, Islington Avenue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>M1B</td>\n",
" <td>Scarborough</td>\n",
" <td>Rouge, Rouge</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>M1B</td>\n",
" <td>Scarborough</td>\n",
" <td>Malvern</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>M3B</td>\n",
" <td>North York</td>\n",
" <td>Don Mills North, Don Mills North</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>M4B</td>\n",
" <td>East York</td>\n",
" <td>Woodbine Gardens, Woodbine Gardens</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>M4B</td>\n",
" <td>East York</td>\n",
" <td>Parkview Hill</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>M5B</td>\n",
" <td>Downtown Toronto</td>\n",
" <td>Ryerson, Ryerson</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>M5B</td>\n",
" <td>Downtown Toronto</td>\n",
" <td>Garden District</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>M6B</td>\n",
" <td>North York</td>\n",
" <td>Glencairn, Glencairn</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>M9B</td>\n",
" <td>Etobicoke</td>\n",
" <td>Cloverdale, Cloverdale</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24</th>\n",
" <td>M9B</td>\n",
" <td>Etobicoke</td>\n",
" <td>Islington</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25</th>\n",
" <td>M9B</td>\n",
" <td>Etobicoke</td>\n",
" <td>Martin Grove</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26</th>\n",
" <td>M9B</td>\n",
" <td>Etobicoke</td>\n",
" <td>Princess Gardens</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27</th>\n",
" <td>M9B</td>\n",
" <td>Etobicoke</td>\n",
" <td>West Deane Park</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>M1C</td>\n",
" <td>Scarborough</td>\n",
" <td>Highland Creek, Highland Creek</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>M1C</td>\n",
" <td>Scarborough</td>\n",
" <td>Rouge Hill</td>\n",
" </tr>\n",
" <tr>\n",
" <th>30</th>\n",
" <td>M1C</td>\n",
" <td>Scarborough</td>\n",
" <td>Port Union</td>\n",
" </tr>\n",
" <tr>\n",
" <th>32</th>\n",
" <td>M3C</td>\n",
" <td>North York</td>\n",
" <td>Flemingdon Park, Flemingdon Park</td>\n",
" </tr>\n",
" <tr>\n",
" <th>33</th>\n",
" <td>M3C</td>\n",
" <td>North York</td>\n",
" <td>Don Mills South</td>\n",
" </tr>\n",
" <tr>\n",
" <th>34</th>\n",
" <td>M4C</td>\n",
" <td>East York</td>\n",
" <td>Woodbine Heights, Woodbine Heights</td>\n",
" </tr>\n",
" <tr>\n",
" <th>35</th>\n",
" <td>M5C</td>\n",
" <td>Downtown Toronto</td>\n",
" <td>St. James Town, St. James Town</td>\n",
" </tr>\n",
" <tr>\n",
" <th>36</th>\n",
" <td>M6C</td>\n",
" <td>York</td>\n",
" <td>Humewood-Cedarvale, Humewood-Cedarvale</td>\n",
" </tr>\n",
" <tr>\n",
" <th>39</th>\n",
" <td>M9C</td>\n",
" <td>Etobicoke</td>\n",
" <td>Bloordale Gardens, Bloordale Gardens</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>237</th>\n",
" <td>M1W</td>\n",
" <td>Scarborough</td>\n",
" <td>L'Amoreaux West, L'Amoreaux West</td>\n",
" </tr>\n",
" <tr>\n",
" <th>238</th>\n",
" <td>M1W</td>\n",
" <td>Scarborough</td>\n",
" <td>Steeles West</td>\n",
" </tr>\n",
" <tr>\n",
" <th>241</th>\n",
" <td>M4W</td>\n",
" <td>Downtown Toronto</td>\n",
" <td>Rosedale, Rosedale</td>\n",
" </tr>\n",
" <tr>\n",
" <th>242</th>\n",
" <td>M5W</td>\n",
" <td>Downtown Toronto</td>\n",
" <td>Stn A PO Boxes 25 The Esplanade, Stn A PO Boxe...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>245</th>\n",
" <td>M8W</td>\n",
" <td>Etobicoke</td>\n",
" <td>Alderwood, Alderwood</td>\n",
" </tr>\n",
" <tr>\n",
" <th>246</th>\n",
" <td>M8W</td>\n",
" <td>Etobicoke</td>\n",
" <td>Long Branch</td>\n",
" </tr>\n",
" <tr>\n",
" <th>247</th>\n",
" <td>M9W</td>\n",
" <td>Etobicoke</td>\n",
" <td>Northwest, Northwest</td>\n",
" </tr>\n",
" <tr>\n",
" <th>248</th>\n",
" <td>M1X</td>\n",
" <td>Scarborough</td>\n",
" <td>Upper Rouge, Upper Rouge</td>\n",
" </tr>\n",
" <tr>\n",
" <th>251</th>\n",
" <td>M4X</td>\n",
" <td>Downtown Toronto</td>\n",
" <td>Cabbagetown, Cabbagetown</td>\n",
" </tr>\n",
" <tr>\n",
" <th>252</th>\n",
" <td>M4X</td>\n",
" <td>Downtown Toronto</td>\n",
" <td>St. James Town</td>\n",
" </tr>\n",
" <tr>\n",
" <th>253</th>\n",
" <td>M5X</td>\n",
" <td>Downtown Toronto</td>\n",
" <td>First Canadian Place, First Canadian Place</td>\n",
" </tr>\n",
" <tr>\n",
" <th>254</th>\n",
" <td>M5X</td>\n",
" <td>Downtown Toronto</td>\n",
" <td>Underground city</td>\n",
" </tr>\n",
" <tr>\n",
" <th>257</th>\n",
" <td>M8X</td>\n",
" <td>Etobicoke</td>\n",
" <td>The Kingsway, The Kingsway</td>\n",
" </tr>\n",
" <tr>\n",
" <th>258</th>\n",
" <td>M8X</td>\n",
" <td>Etobicoke</td>\n",
" <td>Montgomery Road</td>\n",
" </tr>\n",
" <tr>\n",
" <th>259</th>\n",
" <td>M8X</td>\n",
" <td>Etobicoke</td>\n",
" <td>Old Mill North</td>\n",
" </tr>\n",
" <tr>\n",
" <th>264</th>\n",
" <td>M4Y</td>\n",
" <td>Downtown Toronto</td>\n",
" <td>Church and Wellesley, Church and Wellesley</td>\n",
" </tr>\n",
" <tr>\n",
" <th>267</th>\n",
" <td>M7Y</td>\n",
" <td>East Toronto</td>\n",
" <td>Business Reply Mail Processing Centre 969 East...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>268</th>\n",
" <td>M8Y</td>\n",
" <td>Etobicoke</td>\n",
" <td>Humber Bay, Humber Bay</td>\n",
" </tr>\n",
" <tr>\n",
" <th>269</th>\n",
" <td>M8Y</td>\n",
" <td>Etobicoke</td>\n",
" <td>King's Mill Park</td>\n",
" </tr>\n",
" <tr>\n",
" <th>270</th>\n",
" <td>M8Y</td>\n",
" <td>Etobicoke</td>\n",
" <td>Kingsway Park South East</td>\n",
" </tr>\n",
" <tr>\n",
" <th>271</th>\n",
" <td>M8Y</td>\n",
" <td>Etobicoke</td>\n",
" <td>Mimico NE</td>\n",
" </tr>\n",
" <tr>\n",
" <th>272</th>\n",
" <td>M8Y</td>\n",
" <td>Etobicoke</td>\n",
" <td>Old Mill South</td>\n",
" </tr>\n",
" <tr>\n",
" <th>273</th>\n",
" <td>M8Y</td>\n",
" <td>Etobicoke</td>\n",
" <td>The Queensway East</td>\n",
" </tr>\n",
" <tr>\n",
" <th>274</th>\n",
" <td>M8Y</td>\n",
" <td>Etobicoke</td>\n",
" <td>Royal York South East</td>\n",
" </tr>\n",
" <tr>\n",
" <th>275</th>\n",
" <td>M8Y</td>\n",
" <td>Etobicoke</td>\n",
" <td>Sunnylea</td>\n",
" </tr>\n",
" <tr>\n",
" <th>284</th>\n",
" <td>M8Z</td>\n",
" <td>Etobicoke</td>\n",
" <td>Kingsway Park South West, Kingsway Park South ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>285</th>\n",
" <td>M8Z</td>\n",
" <td>Etobicoke</td>\n",
" <td>Mimico NW</td>\n",
" </tr>\n",
" <tr>\n",
" <th>286</th>\n",
" <td>M8Z</td>\n",
" <td>Etobicoke</td>\n",
" <td>The Queensway West</td>\n",
" </tr>\n",
" <tr>\n",
" <th>287</th>\n",
" <td>M8Z</td>\n",
" <td>Etobicoke</td>\n",
" <td>Royal York South West</td>\n",
" </tr>\n",
" <tr>\n",
" <th>288</th>\n",
" <td>M8Z</td>\n",
" <td>Etobicoke</td>\n",
" <td>South of Bloor</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>212 rows × 3 columns</p>\n",
"</div>"
],
"text/plain": [
" PostalCode Borough \\\n",
"3 M3A North York \n",
"4 M4A North York \n",
"5 M5A Downtown Toronto \n",
"6 M5A Downtown Toronto \n",
"7 M6A North York \n",
"8 M6A North York \n",
"9 M7A Queen's Park \n",
"11 M9A Etobicoke \n",
"12 M1B Scarborough \n",
"13 M1B Scarborough \n",
"15 M3B North York \n",
"16 M4B East York \n",
"17 M4B East York \n",
"18 M5B Downtown Toronto \n",
"19 M5B Downtown Toronto \n",
"20 M6B North York \n",
"23 M9B Etobicoke \n",
"24 M9B Etobicoke \n",
"25 M9B Etobicoke \n",
"26 M9B Etobicoke \n",
"27 M9B Etobicoke \n",
"28 M1C Scarborough \n",
"29 M1C Scarborough \n",
"30 M1C Scarborough \n",
"32 M3C North York \n",
"33 M3C North York \n",
"34 M4C East York \n",
"35 M5C Downtown Toronto \n",
"36 M6C York \n",
"39 M9C Etobicoke \n",
".. ... ... \n",
"237 M1W Scarborough \n",
"238 M1W Scarborough \n",
"241 M4W Downtown Toronto \n",
"242 M5W Downtown Toronto \n",
"245 M8W Etobicoke \n",
"246 M8W Etobicoke \n",
"247 M9W Etobicoke \n",
"248 M1X Scarborough \n",
"251 M4X Downtown Toronto \n",
"252 M4X Downtown Toronto \n",
"253 M5X Downtown Toronto \n",
"254 M5X Downtown Toronto \n",
"257 M8X Etobicoke \n",
"258 M8X Etobicoke \n",
"259 M8X Etobicoke \n",
"264 M4Y Downtown Toronto \n",
"267 M7Y East Toronto \n",
"268 M8Y Etobicoke \n",
"269 M8Y Etobicoke \n",
"270 M8Y Etobicoke \n",
"271 M8Y Etobicoke \n",
"272 M8Y Etobicoke \n",
"273 M8Y Etobicoke \n",
"274 M8Y Etobicoke \n",
"275 M8Y Etobicoke \n",
"284 M8Z Etobicoke \n",
"285 M8Z Etobicoke \n",
"286 M8Z Etobicoke \n",
"287 M8Z Etobicoke \n",
"288 M8Z Etobicoke \n",
"\n",
" Neighborhood \n",
"3 Parkwoods, Parkwoods \n",
"4 Victoria Village, Victoria Village \n",
"5 Harbourfront, Harbourfront \n",
"6 Regent Park \n",
"7 Lawrence Heights, Lawrence Heights \n",
"8 Lawrence Manor \n",
"9 Queen's Park, Queen's Park \n",
"11 Islington Avenue, Islington Avenue \n",
"12 Rouge, Rouge \n",
"13 Malvern \n",
"15 Don Mills North, Don Mills North \n",
"16 Woodbine Gardens, Woodbine Gardens \n",
"17 Parkview Hill \n",
"18 Ryerson, Ryerson \n",
"19 Garden District \n",
"20 Glencairn, Glencairn \n",
"23 Cloverdale, Cloverdale \n",
"24 Islington \n",
"25 Martin Grove \n",
"26 Princess Gardens \n",
"27 West Deane Park \n",
"28 Highland Creek, Highland Creek \n",
"29 Rouge Hill \n",
"30 Port Union \n",
"32 Flemingdon Park, Flemingdon Park \n",
"33 Don Mills South \n",
"34 Woodbine Heights, Woodbine Heights \n",
"35 St. James Town, St. James Town \n",
"36 Humewood-Cedarvale, Humewood-Cedarvale \n",
"39 Bloordale Gardens, Bloordale Gardens \n",
".. ... \n",
"237 L'Amoreaux West, L'Amoreaux West \n",
"238 Steeles West \n",
"241 Rosedale, Rosedale \n",
"242 Stn A PO Boxes 25 The Esplanade, Stn A PO Boxe... \n",
"245 Alderwood, Alderwood \n",
"246 Long Branch \n",
"247 Northwest, Northwest \n",
"248 Upper Rouge, Upper Rouge \n",
"251 Cabbagetown, Cabbagetown \n",
"252 St. James Town \n",
"253 First Canadian Place, First Canadian Place \n",
"254 Underground city \n",
"257 The Kingsway, The Kingsway \n",
"258 Montgomery Road \n",
"259 Old Mill North \n",
"264 Church and Wellesley, Church and Wellesley \n",
"267 Business Reply Mail Processing Centre 969 East... \n",
"268 Humber Bay, Humber Bay \n",
"269 King's Mill Park \n",
"270 Kingsway Park South East \n",
"271 Mimico NE \n",
"272 Old Mill South \n",
"273 The Queensway East \n",
"274 Royal York South East \n",
"275 Sunnylea \n",
"284 Kingsway Park South West, Kingsway Park South ... \n",
"285 Mimico NW \n",
"286 The Queensway West \n",
"287 Royal York South West \n",
"288 South of Bloor \n",
"\n",
"[212 rows x 3 columns]"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# (soup.prettify())\n",
"postal_data = []\n",
"table = soup.find('table', attrs={'class':'wikitable sortable'})\n",
"table_body = table.find('tbody')\n",
"\n",
"rows = table_body.find_all('tr')\n",
"for row in rows:\n",
" cols = row.find_all('td')\n",
" cols = [ele.text.strip() for ele in cols]\n",
" postal_data.append([ele for ele in cols if ele]) # Get rid of empty values\n",
"len(postal_data)\n",
"df = pd.DataFrame(data=postal_data)\n",
"df.columns = ['PostalCode', 'Borough', 'Neighborhood']\n",
"df_new = df [df.Borough != \"Not assigned\"]\n",
"#map ( df['Neighborhood'] = df['Neighborhood'].apply(lambda x: df['Borough'] if x == 'Not assigned' else)\n",
"#df_new\n",
"for index, row in df_new.iterrows():\n",
" if row[\"Neighborhood\"] == \"Not assigned\" :\n",
" row[\"Neighborhood\"] = row[\"Borough\"]\n",
"df_new.sort_values(by=['PostalCode'])\n",
"\n",
"old_postal_code = ''\n",
"old_neighborhood = ''\n",
"\n",
"df_new = df_new.iloc[1:]\n",
"\n",
"for index, row in df_new.iterrows():\n",
" \n",
" if old_postal_code == row[\"PostalCode\"]:\n",
" old_postal_code = old_postal_code\n",
" elif old_postal_code != row[\"PostalCode\"]:\n",
" row[\"Neighborhood\"] = old_neighborhood + ', ' + row[\"Neighborhood\"]\n",
" old_postal_code = row[\"PostalCode\"]\n",
"df_new \n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.6"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment