babynames.py

#!/usr/bin/env python
"""
This script downloads data on baby names from the Social Security
Administration's web site and saves it to a database.

Run the script from the command line with no arguments to download national data
from 1880 to 2008.

Run the script with --states as an argument to download state data from 
1960 to 2008.
"""
import re
import sys
import urllib2

from newsdatadb import cursor # A MySQLdb cursor instance


def get_page(year, state=None):
    """Get the content of the page listing the top 1,000 baby names 
    for the given year.
    """
    if state:
        body = 'year=%s&state=%s' % (str(year), state)
        uri = 'http://www.ssa.gov/cgi-bin/namesbystate.cgi'
    else:
        body = 'year=%s&top=1000&number=n' % str(year)
        uri = 'http://www.ssa.gov/cgi-bin/popularnames.cgi'

    req = urllib2.Request(uri, body)
    response = urllib2.urlopen(req)

    if response.msg == 'OK':
        return response.read()

    return None


def parse_page(page, state=None):
    """Get the relevant baby-name data from the given HTML page.
    """
    rows = re.findall(r'<tr align="right">.*?<\/tr>', page, re.S)

    fields = ['rank', 'male', 'male_count', 'female', 'female_count', ]


    if state:
        regex = re.compile(r'<td(?: align="center")?>(.*?)<\/td>')
    else:
        regex = re.compile(r'<td>(?P<cell_content>.*?)<\/td>')

    for row in rows:
        row_data = regex.findall(row)
        yield dict(zip(fields, row_data))


def separate_sex_data(data):
    male_data = data.copy()
    male_data['name'] = data['male']
    male_data['sex'] = 'M'
    try:
        male_data['number'] = int(data['male_count'].replace(',', ''))
    except ValueError:
        male_data['number'] = 0

    female_data = data.copy()
    female_data['name'] = data['female']
    female_data['sex'] = 'F'
    try:
        female_data['number'] = int(data['female_count'].replace(',', ''))
    except ValueError:
        female_data['number'] = 0

    return (male_data, female_data)


def save_data(data):
    """Save a row of data to the database.
    """
    male_data, female_data = separate_sex_data(data)

    for sex_data in male_data, female_data:
        query = """
                INSERT INTO baby_names
                (`year`, `name`, `sex`, `number`, `rank`)
                VALUES
                (%(year)s, %(name)s, %(sex)s, %(number)s, %(rank)s)
                """
        cursor.execute(query, sex_data)

    return True


def save_state_data(data):
    """Save a row of state data to the database.
    """
    male_data, female_data = separate_sex_data(data)

    for sex_data in male_data, female_data:
        query = """
                INSERT INTO baby_names_by_state
                (`year`, `name`, `sex`, `number`, `rank`, `state`)
                VALUES
                (%(year)s, %(name)s, %(sex)s, %(number)s, %(rank)s, %(state)s)
                """
        cursor.execute(query, sex_data)

    return True


STATES = ['AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DE', 'DC', 'FL', 'GA',
          'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD', 'MA',
          'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ', 'NM', 'NY',
          'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', 'TN', 'TX',
          'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY', ]


def _main():
    if sys.argv[-1] == '--states':
        for year in range(1960, 2009):
            for state in STATES:
                page = get_page(year, state)
                for data in parse_page(page, state):
                    data['year'] = year
                    data['state'] = state
                    saved = save_state_data(data)

    else:
        for year in range(1880, 2009):
            page = get_page(year)
            for data in parse_page(page):
                data['year'] = year
                saved = save_data(data)


if __name__ == '__main__':
    _main()

LICENSE

Copyright 2010 Aaron Bycoffe. All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are
permitted provided that the following conditions are met:

   1. Redistributions of source code must retain the above copyright notice, this list of
      conditions and the following disclaimer.

   2. Redistributions in binary form must reproduce the above copyright notice, this list
      of conditions and the following disclaimer in the documentation and/or other materials
      provided with the distribution.

THIS SOFTWARE IS PROVIDED BY Aaron Bycoffe ``AS IS'' AND ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Aaron Bycoffe OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

The views and conclusions contained in the software and documentation are those of the
authors and should not be interpreted as representing official policies, either expressed
or implied, of Aaron Bycoffe.