horoscope_scraper.py

# Scrape two months worth of horoscopes from shine.yahoo.com

from BeautifulSoup import BeautifulSoup as Soup
import urllib, datetime

URL = "http://shine.yahoo.com/astrology/%s/daily-extended/%s/"

signs = (
    'aries',
    'leo',
    'sagittarius',
    'taurus',
    'virgo',
    'capricorn',
    'gemini',
    'libra',
    'aquarius',
    'cancer',
    'scorpio',
    'pisces',
)

def horoscope_for_date_and_sign(date, sign):
    url = URL % (sign, date.strftime('%Y%m%d'))
    soup = Soup(urllib.urlopen(url))
    return soup.find('div', {
        'class': 'content'
    }).find('p').contents[0].strip()

def days_in_month(month):
    days = []
    current = month.replace(day = 1)
    while current.month == month.month:
        days.append(current)
        current = current + datetime.timedelta(days = 1)
    return days

def go():
    import csv
    fp = open('results.csv', 'w')
    writer = csv.writer(fp, dialect='excel-tab')

    for month in (datetime.date(2009, 11, 1), datetime.date(2009, 12, 1)):
        for day in days_in_month(month):
            for sign in signs:
                horoscope = horoscope_for_date_and_sign(day, sign)
                writer.writerow([day.strftime('%Y-%m-%d'), sign, horoscope])
                print "Got %s for %s" % (sign, day)