fetch.py

import urllib
import xlrd
import csv
import json

partis = ["GOUVERNEMENT","LCR-CAP 21","DEBOUT LA FRANCE","EUROPE ECOLOGIE LES VERTS ","EN MARCHE","FRONT DEMOCRATE","FRONT NATIONAL","JEANNE AU SECOURS ","LA FRANCE QUI OSE ","LES CENTRISTES","LES REPUBLICAINS","MODEM","NOUVEAU PARTI ANTICAPITALISTE","PARTI COMMUNISTE FRANCAIS","PARTI DE GAUCHE","PARTI DE LA FRANCE","PARTI ECOLOGISTE","PARTI RADICAL DE GAUCHE","PARTI SOCIALISTE","UNION DES DEMOCRATES ET INDEPENDANTS","UNION POPULAIRE REPUBLICAINE","UDI NOUVEAU CENTRE","UDI PARTI RADICAL ","DIVERS"]

stats = ['JT%20-%201er%20au%2031%20janvier%202017', 'Cha%C3%AEnes%20Info%20MAG%201er%20au%2031%20janvier', 'Mag%20-%201er%20au%2031%20janvier%202017', 'Cha%C3%AEnes%20Info%20MAG%201er%20au%2031%20janvier', 'Cha%C3%AEnes%20Info%20Autres%20%C3%A9missions%201%20au%2031%20janvier%20', 'Cha%C3%AEnes%20Info%20JT%201er%20au%2031%20janvier%2017', '1-JOURNAUX-1er-31%20janvier%202017', '2-MAGAZINES-1er-31%20janvier%202017', '3-EMPROG-1er-31%20janvier%202017']

def xls2csv(ExcelFile, CSVFile):
    workbook = xlrd.open_workbook(ExcelFile)
    worksheet = workbook.sheet_by_index(0)
    csvfile = open(CSVFile, 'wb')
    wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
    print ExcelFile

    for rownum in xrange(worksheet.nrows):
        wr.writerow(
            list(x.encode('utf-8') if type(x) == type(u'') else x
                for x in worksheet.row_values(rownum)))

    csvfile.close()

def download_xls(url, fname):
    f = urllib.URLopener()
    f.retrieve(url, fname)

def download_stats(list):
    base_url = "http://www.csa.fr/var/ezflow_site/storage/csa/pdf/pluralisme/"
    for l in list:
        download_xls(base_url + l + ".xls", l + ".xls")
        xls2csv(l + ".xls", l + ".csv")

def compile_stats(list):
    times = {}
    for p in partis:
        times[p] = 0

    for l in list:
        f = open(l + ".csv", 'rt')
        try:
            reader = csv.reader(f, delimiter=',')
            count = 0
            for row in reader:
                if row[0] in partis:
                    for i,col in enumerate(row):
                        if i > 0:
                            try:
                                times[row[0]] = times[row[0]] + float(col)
                            except ValueError:
                                pass
        finally:
            f.close()

    for p in partis:
        times[p] = times[p] * 24

    with open('stats.json', 'w') as outfile:
        json.dump(times, outfile, indent=4)

download_stats(stats)
compile_stats(stats)