Extract hi-res cosm data

fetch_cosm.py

#!/usr/bin/env python
#-----------------------------------------------
# Name:  fetch_cosm.py
#
# Purpose: retrieve historical hi-res data from cosm
# Usage:  ./fetch_cosm.py >>cosm_data.csv
# History:
#   Date        Author  Remarks
#   15Oct2012   RW     Created.
#-----------------------------------------------
import sys
import urllib2
from datetime import *
from time import *
COSM_API_KEY="YOUR API KEY"
FEED_ID="YOUR FEED ID NUMBER"

def main():
    api_key= { 'X-ApiKey' : COSM_API_KEY }
    # change this interval depending on how frequent your data points are- Cosm API limits you to 1000 points per fetch
    interval=timedelta(hours=4)
    # work backwards until this date
    earlist=datetime(2012,05,24)

    # start from now (or change to another date using above syntax)
    end=datetime.utcnow()
    begin=end-interval
    retry_count=0
    while begin > earlist :
        try:
            req=urllib2.Request('http://api.cosm.com/v2/feeds/%d.csv?start=%s&end=%s&interval=0' % ( FEED_ID, begin.strftime('%Y-%m-%dT%H:%M:%SZ'),end.strftime('%Y-%m-%dT%H:%M:%SZ')) , headers=api_key)
            resp=urllib2.urlopen(req)
            if resp.code == 200:
                csv=resp.read()
                print csv
                end=begin
                begin=end-interval
                retry_count=0

        except urllib2.HTTPError:
            # sometimes requests fail (usually 500-internal server error) so try again
            ++retry_count
            print >> sys.stderr,"Error %s %s (%d)" %  (begin.strftime('%Y-%m-%dT%H:M:%SZ'),end.strftime('%Y-%m-%dT%H:%M:%SZ'),retry_count)

# wait 20 seconds between fetchs to avoid API rate limit
        sleep(20)


if __name__ == '__main__':
    main()