ipsacrawler.py

#!/usr/bin/env python

'''
Autor: Cristian Carpenter G.
'''

from BeautifulSoup import BeautifulSoup
import urllib
import MySQLdb
from datetime import datetime

IPSA = {}
IPSA[1]  = 'ALMENDRAL'
IPSA[2]  = 'ANDINA-B'
IPSA[3]  = 'ANTARCHILE'
IPSA[4]  = 'BCI'
IPSA[5]  = 'BSANTANDER'
IPSA[6]  = 'CALICHERAA'
IPSA[7]  = 'CAP'
IPSA[8]  = 'CCU'
IPSA[9]  = 'CENCOSUD'
IPSA[10] = 'CGE'
IPSA[11] = 'CHILE'
IPSA[12] = 'CMPC'
IPSA[13] = 'COLBUN'
IPSA[14] = 'CONCHATORO'
IPSA[15] = 'COPEC'
IPSA[16] = 'CORPBANCA'
IPSA[17] = 'ECL'
IPSA[18] = 'ENDESA'
IPSA[19] = 'ENERSIS'
IPSA[20] = 'ENTEL'
IPSA[21] = 'FALABELLA'
IPSA[22] = 'GENER'
IPSA[23] = 'IAM'
IPSA[24] = 'LA POLAR'
IPSA[25] = 'LAN'
IPSA[26] = 'MADECO'
IPSA[27] = 'MASISA'
IPSA[28] = 'MULTIFOODS'
IPSA[29] = 'NORTEGRAN'
IPSA[30] = 'ORO BLANCO'
IPSA[31] = 'PARAUCO'
IPSA[32] = 'PROVIDA'
IPSA[33] = 'RIPLEY'
IPSA[34] = 'SALFACORP'
IPSA[35] = 'SK'
IPSA[36] = 'SM-CHILE B'
IPSA[37] = 'SOCOVESA'
IPSA[38] = 'SONDA'
IPSA[39] = 'SQM-B'
IPSA[40] = 'VAPORES'

db = MySQLdb.Connect(user='root', passwd='******', db='ipsa')
stockdata = db.cursor()

for i in range(1, 41):
    params = {'chartType': 'Lineas', 'zoomType': '1a', 'intervalType': '1d', 'fechaInicio': '12/05/2010', 'symbolType': 'A'}
    params['symbol'] = IPSA[i]
    params = urllib.urlencode(params)
    xml = urllib.urlopen("http://www.bolsadesantiago.com/financialServlet?%s" % params).read()
    soup = BeautifulSoup(xml)
    data = soup('serie')[0]('entry')

    for entry in data:
        price = entry['close']
        volume = entry['volume']
        date = datetime.strptime(entry['date'], "%d/%m/%Y")
        date = date.date()
        symbol = IPSA[i]
        stockdata.execute("""INSERT INTO acciones (symbol, price, volume, date) VALUES (%s, %s, %s, %s)""", (symbol, price, volume, date))
        db.commit()