markethistory.py

import time
import requests
from requests_futures.sessions import FuturesSession
import requests_futures
from concurrent.futures import as_completed
import datetime
import csv
import time
from tqdm import tqdm
import sys
import glob
import pprint
from math import floor,log10,fabs

import logging
logging.basicConfig(filename='c:/app/gethistory.log',level=logging.INFO,format='%(asctime)s %(levelname)s %(message)s')



def RateLimited(maxPerSecond):
    minInterval = 1.0 / float(maxPerSecond)
    def decorate(func):
        lastTimeCalled = [0.0]
        def rateLimitedFunction(*args,**kargs):
            elapsed = time.clock() - lastTimeCalled[0]
            leftToWait = minInterval - elapsed
            if leftToWait>0:
                time.sleep(leftToWait)
            ret = func(*args,**kargs)
            lastTimeCalled[0] = time.clock()
            return ret
        return rateLimitedFunction
    return decorate


    
    
def processData(result,csvwriter):
    
    try:
        resp=result.result()
        if resp.status_code==200:
            orders=resp.json()
            for order in orders['items']:
                csvwriter.writerow([result.typeid,
                    result.regionid,
                    order['date'],
                    order['lowPrice'],
                    order['highPrice'],
                    order['avgPrice'],
                    order['volume'],
                    order['orderCount']
                    ])
            return True;
        else:
            logging.warn("None 200 status. {} Returned: {}".format(result.typeid,resp.status_code))
            return False
    except requests.exceptions.ConnectionError as e:
        logging.warn(e)
        return False;
    
    
    

    


@RateLimited(150)
def getData(requestsConnection,typeid,regionid):
    url="https://public-crest.eveonline.com/market/{}/types/{}/history/".format(regionid,typeid)
    future=requestsConnection.get(url)
    future.typeid=typeid
    future.regionid=regionid
    return future


if __name__ == "__main__":
    # build the basic list
    baseitemids=[]
    
    
    
    files=glob.glob('*.csv')
    i=0
    print "Pick a file"
    for file in files:
        print "{}: {}".format(i,file)
        i+=1
    
    file=raw_input("File Number:")
    
    with open(files[int(file)], 'rb') as csvfile:
        typereader = csv.reader(csvfile, delimiter=',', quotechar='"')
        for row in typereader:
            baseitemids.append(row[0])
    reqs_num_workers = 10
    session = FuturesSession(max_workers=reqs_num_workers)
    session.headers.update({'UserAgent':'Fuzzwork Market Monitor'});
       
    region=10000002
    
    futures=[]
    retryfutures=[]
    

    for typeid in tqdm(baseitemids,desc="downloading",leave=True):
        futures.append(getData(session,typeid,region))
    with open("history.csv", 'wb') as csvfile:
        csvwriter = csv.writer(csvfile, dialect='excel')    
        for result in tqdm(as_completed(futures),desc='Processing',total=len(baseitemids),leave=True):
            status=processData(result,csvwriter)
            if not status:
                retryfutures.append(getData(session,result.typeid,result.regionid))
                logging.warn("adding {} to retry in {}".format(result.typeid,result.regionid))
   
       
        for result in tqdm(as_completed(retryfutures),desc='Processing Failures',total=len(retryfutures),leave=True):
            status=processData(result,csvwriter)
            if not status:
                logging.warn("{} failed in {}".format(result.typeid,result.regionid))