Skip to content

Instantly share code, notes, and snippets.

@marks

marks/scraping-because-there-is-no-API-or-export.ipynb

Last active February 12, 2016 17:41
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save marks/0a082fb53475d8fe51aa to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save marks/0a082fb53475d8fe51aa to your computer and use it in GitHub Desktop.
Download ZIP
Web scraping solution of CDC Vessel Sanitation Program Inspections -- DDOD issue #11 (https://github.com/demand-driven-open-data/ddod-intake/issues/11)
Display the source blob
Display the rendered blob
Raw
scraping-because-there-is-no-API-or-export.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"# Import dependencies\n",
"import pandas\n",
"import mechanize\n",
"from bs4 import BeautifulSoup\n",
"from dateutil import rrule\n",
"from datetime import datetime, timedelta\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# CONFIGURATION\n",
"\n",
"# Date range for search.\n",
"# This program searches \n",
"from_date = datetime(2015,10,01) # October 2015\n",
"to_date = datetime.today()"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"# HELPERS\n",
"def row_values_to_dictionary(row_values):\n",
" dictionary = {}\n",
" dictionary['cruise_ship'] = row_values[0].get_text()\n",
" dictionary['cruise_line'] = row_values[1].get_text()\n",
" dictionary['date'] = row_values[2].get_text()\n",
" dictionary['score'] = row_values[3].get_text()\n",
" return dictionary\n",
"\n",
"def soup_to_list_of_dictionaries(soup):\n",
" rows_of_data = []\n",
" table = soup.find('table',id='ctl00_ContentPlaceHolder1_dgMasterList')\n",
" table_rows = table.find_all('tr')\n",
"\n",
"\n",
" for row in table_rows:\n",
" values = row.find_all('td')\n",
" # If the row looks like a row of data, add it to the list\n",
" if(len(values) == 7 and values[0].get_text() != \"Cruise Ship\"):\n",
" rows_of_data.append(row_values_to_dictionary(values))\n",
" \n",
" return rows_of_data\n",
"\n",
"def get_inspections_between(start_date, end_date):\n",
" start_date_str = start_date.strftime('%m/%d/%Y')\n",
" end_date_str = end_date.strftime('%m/%d/%Y')\n",
" \n",
" print \"INFO :: {} => {} :: About to scrape inspections\".format(\n",
" start_date_str, end_date_str)\n",
" \n",
" # create mechanize browser and go to search form page\n",
" browser = mechanize.Browser()\n",
" browser.open('https://wwwn.cdc.gov/InspectionQueryTool/InspectionSearch.aspx')\n",
" browser.select_form(name='aspnetForm')\n",
" \n",
" # set form input values and submit form\n",
" browser['ctl00$ContentPlaceHolder1$rb_InspectionDateCriteria'] = ['rb_BetweenDate']\n",
" browser['ctl00$ContentPlaceHolder1$txtFromSearchDate'] = start_date_str\n",
" browser['ctl00$ContentPlaceHolder1$txtToSearchDate'] = end_date_str\n",
" browser.submit()\n",
"\n",
" # read form response (resulting HTML page) and parse the data\n",
" b = browser.response()\n",
" soup = BeautifulSoup(b.read())\n",
" data_for_range = soup_to_list_of_dictionaries(soup)\n",
" print \"INFO :: {} => {} :: {} inspections retrieved\".format(\n",
" start_date_str, end_date_str, len(data_for_range))\n",
" return data_for_range"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"INFO :: 10/01/2015 => 10/31/2015 :: About to scrape inspections\n",
"INFO :: 10/01/2015 => 10/31/2015 :: 20 inspections retrieved\n",
"INFO :: 11/01/2015 => 11/30/2015 :: About to scrape inspections\n",
"INFO :: 11/01/2015 => 11/30/2015 :: 18 inspections retrieved\n",
"INFO :: 12/01/2015 => 12/31/2015 :: About to scrape inspections\n",
"INFO :: 12/01/2015 => 12/31/2015 :: 10 inspections retrieved\n",
"INFO :: 01/01/2016 => 01/31/2016 :: About to scrape inspections\n",
"INFO :: 01/01/2016 => 01/31/2016 :: 1 inspections retrieved\n",
"49\n"
]
}
],
"source": [
"# GET THE DATA! ( by month :-\\ )\n",
"\n",
"data = []\n",
"\n",
"# Month-looping code adapted from StackOverflow answer\n",
"months_within_range = list(rrule.rrule(rrule.MONTHLY, dtstart=from_date).between(from_date, to_date, inc=True))\n",
"\n",
"i = 0\n",
"while i < len(months_within_range) - 1:\n",
" month_start = months_within_range[i]\n",
" month_end = months_within_range[i+1] - timedelta(days=1)\n",
" data.extend(get_inspections_between(month_start,month_end))\n",
" i += 1\n",
"\n",
"print len(data)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" cruise_line cruise_ship date score\n",
"0 Royal Caribbean International Adventure Of The Seas 10/17/2015 84\n",
"1 Phoenix Reissen Cruise Line Albatross 10/16/2015 91\n",
"2 Holland America Line Amsterdam 10/11/2015 99\n",
"3 America Cruise Ferries Caribbean Fantasy 10/16/2015 86\n",
"4 Carnival Cruise Lines, Inc. Carnival Conquest 10/25/2015 92\n",
"5 Carnival Cruise Lines, Inc. Carnival Ecstasy 10/5/2015 92\n",
"6 Carnival Cruise Lines, Inc. Carnival Freedom 10/29/2015 100\n",
"7 Costa Crociere S.P.A. Costa Luminosa 10/14/2015 100\n",
"8 Disney Cruise Lines Disney Fantasy 10/3/2015 100\n",
"9 Disney Cruise Lines Disney Magic 10/4/2015 96\n",
"10 Princess Cruises Emerald Princess 10/24/2015 93\n",
"11 Holland America Line Eurodam 10/25/2015 100\n",
"12 Royal Caribbean International Liberty of the Seas 10/22/2015 99\n",
"13 Royal Caribbean International Majesty Of The Seas 10/26/2015 91\n",
"14 Holland America Line Noordam 10/10/2015 99\n",
"15 Norwegian Cruise Lines Norwegian Star 10/19/2015 94\n",
"16 Princess Cruises Ruby Princess 10/9/2015 96\n",
"17 Silversea Cruises Ltd Silver Whisper 10/6/2015 99\n",
"18 Holland America Line Westerdam 10/24/2015 96\n",
"19 Holland America Line Zaandam 10/7/2015 97\n",
"20 Aida Cruises AIDAdiva 11/3/2015 100\n",
"21 Royal Caribbean International Brilliance Of The Seas 11/12/2015 92\n",
"22 Carnival Cruise Lines, Inc. Carnival Inspiration 11/9/2015 94\n",
"23 Celebrity Cruises Celebrity Constellation 11/23/2015 98\n",
"24 Celebrity Cruises Celebrity Eclipse 11/15/2015 99\n",
"25 Disney Cruise Lines Disney Dream 11/13/2015 94\n",
"26 Princess Cruises Grand Princess 11/16/2015 100\n",
"27 Royal Caribbean International Jewel Of The Seas 11/23/2015 95\n",
"28 Holland America Line Nieuw Amsterdam 11/22/2015 98\n",
"29 Norwegian Cruise Lines Norwegian Breakaway 11/8/2015 95\n",
"30 Norwegian Cruise Lines Norwegian Jewel 11/8/2015 100\n",
"31 Norwegian Cruise Lines Norwegian Sky 11/9/2015 97\n",
"32 Norwegian Cruise Lines Norwegian Sun 11/4/2015 91\n",
"33 V. Ships Leisure U.S.A. Pearl Mist 11/8/2015 88\n",
"34 Holland America Line Prinsendam 11/24/2015 96\n",
"35 Sea Dream Yacht Club Seadream II 11/22/2015 98\n",
"36 Princess Cruises Star Princess 11/3/2015 100\n",
"37 Holland America Line Zuiderdam 11/30/2015 100\n",
"38 Azamara Cruises Azamara Journey 12/11/2015 91\n",
"39 Celebrity Cruises Celebrity Infinity 12/5/2015 99\n",
"40 Celebrity Cruises Celebrity Silhouette 12/6/2015 97\n",
"41 Celebrity Cruises Celebrity Summit 12/12/2015 95\n",
"42 Club Med Marine Club Med 2 12/14/2015 88\n",
"43 Norwegian Cruise Lines Norwegian Pearl 12/10/2015 95\n",
"44 Norwegian Cruise Lines Norwegian Spirit 12/5/2015 100\n",
"45 Oceania Cruises Oceania Riviera 12/22/2015 94\n",
"46 Regent Seven Seas Seven Seas Navigator 12/17/2015 94\n",
"47 Silversea Cruises Ltd Silver Wind 12/11/2015 95\n",
"48 Silversea Cruises Ltd Silver Whisper 1/5/2016 97\n"
]
}
],
"source": [
"# DO STUFF WITH THE DATA \n",
"\n",
"# First things first, convert the dictionary to a pandas dataframe\n",
"df = pandas.DataFrame.from_dict(data)\n",
"df = df.dropna(how='all') # remove empty rows\n",
"print df"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.11"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment