gistfile1.py

import random, time, operator, math

from deap import base
from deap import creator
from deap import tools
from deap import gp
import cloud

import jsonrpclib



config={"SERVER":"http://aqueous-fjord-3740.herokuapp.com/evospace","SAMPLE_SIZE":5,"WORKER_GENERATIONS":10, "CXPB":.9,"MUTPB":.05,"MAX_SAMPLES":10,"POPULATION_SIZE":200,"WORKER_TYPE":"c1","NUMBER_OF_WORKERS":1,"RETURN_RATE":1.0}

def safeDiv(left, right):
    try:
        return left / right
    except ZeroDivisionError:
        return 0

pset = gp.PrimitiveSet("MAIN", 1)
pset.addPrimitive(operator.add, 2)
pset.addPrimitive(operator.sub, 2)
pset.addPrimitive(operator.mul, 2)
pset.addPrimitive(safeDiv, 2)
pset.addPrimitive(operator.neg, 1)
pset.addPrimitive(math.cos, 1)
pset.addPrimitive(math.sin, 1)
pset.addTerminal(1)
#pset.addEphemeralConstant("rand101", lambda: random.randint(-1,1))
pset.renameArguments(ARG0='x')


#creator.create("FitnessMin", base.Fitness, weights=(-1.0,))
#creator.create("Individual", gp.PrimitiveTree, fitness=creator.FitnessMin)

def staticLimitCrossover(ind1, ind2, heightLimit):
    # Store a backup of the original individuals
    keepInd1, keepInd2 = getToolBox().clone(ind1), getToolBox().clone(ind2)

    # Mate the two individuals
    # The crossover is done in place (see the documentation)
    # If using STGP (like spambase), replace this line by gp.cxTypedOnePoint(ind1, ind2)

    gp.cxOnePoint(ind1, ind2)

    # If a child is higher than the maximum allowed, then
    # it is replaced by one of its parent
    if ind1.height > heightLimit:
        ind1[:] = keepInd1
    if ind2.height > heightLimit:
        ind2[:] = keepInd2

def staticLimitMutation(individual, expr, pset, heightLimit):
    # Store a backup of the original individual
    keepInd = getToolBox().clone(individual)

    # Mutate the individual
    # The mutation is done in place (see the documentation)
    # If using STGP (like spambase), replace this line by gp.mutTypedUniform(individual,expr)

    gp.mutUniform(individual, expr,pset)

    # If the mutation sets the individual higher than the maximum allowed,

    # replaced it by the original individual
    if individual.height > heightLimit:
        individual[:] = keepInd
def getToolBox():
    creator.create("FitnessMin", base.Fitness, weights=(-1.0,))
    creator.create("Individual", gp.PrimitiveTree, fitness=creator.FitnessMin)
    
    toolbox = base.Toolbox()
    # Attribute generator
    toolbox.register("expr", gp.genHalfAndHalf, pset=pset, min_=1, max_=6)
    toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.expr)
    toolbox.register("population", tools.initRepeat, list, toolbox.individual)
    toolbox.register("compile", gp.compile, pset=pset)
    toolbox.register("select", tools.selTournament, tournsize=4)
    toolbox.register("keepbest", tools.selBest)
    toolbox.register("mate", staticLimitCrossover,heightLimit=20)
    toolbox.register("expr_mut", gp.genFull, min_=0, max_=2)
    toolbox.register("mutate", staticLimitMutation, expr=toolbox.expr_mut, pset=pset,heightLimit=17)
    return toolbox
def choosepoints(start,stop,n):
    p=[0]*n
    for x in range(n):
        p[x]=random.uniform(start,stop)
    return p

def evalSymbReg(individual,points):
#def evalSymbReg(individual, points):
    



    # Transform the tree expression in a callable function
    func = getToolBox().compile(expr=individual)
    print func
    print gp.PrimitiveTree.__str__(individual)


    # Evaluate the mean squared error between the expression
    # and the real function : x**4 + x**3 + x**2 + x
    #sqerrors = ((func(x,y) - (x+y))**2 for x,y in zip(points,points2))
    sqerrors = [(func(x) - (math.log10(x+1)))**2 for x in points]

    print sqerrors



    


    
    print math.fsum(sqerrors) / len(points)
    
    return math.fsum(sqerrors) / len(points),

#bloat=gp.staticLimit(operator.attrgetter('height'), 15)



def initialize(pset,config):
    pop = getToolBox().population(config["POPULATION_SIZE"])
    server = jsonrpclib.Server(config["SERVER"])
    server.initialize(None)
    #print pop
    #print str(pop[0])
    sample = [{"chromosome":[str(ind)], "id":None, "fitness":{"DefaultContext":0.0}} for ind in pop]
    init_pop = {'sample_id': 'None' , 'sample':   sample}
    server.putSample(init_pop)



def get_sample(config):
    for attempts in range(3):
        try:
            server = jsonrpclib.Server(config["SERVER"])
            sample = server.getSample(config["SAMPLE_SIZE"])
            return sample
        except jsonrpclib.ProtocolError as err:
            print "Error %s" % err




def put_sample(config,sample):
    for attempts in range(3):
        try:
            server = jsonrpclib.Server(config["SERVER"])
            server.putSample(sample)
            break
        except jsonrpclib.ProtocolError as err:
            print "Error %s" % err


def evolve(sample_num,config):
    #random.seed(64)

    toolbox = getToolBox()

    trainpoints=choosepoints(0,2,20)
    testpoints=choosepoints(0,2,20)

    toolbox.register("evaluate", evalSymbReg, points=trainpoints)
    toolbox.register("evaluateferreals", evalSymbReg, points=testpoints)

    start= time.time()
    evospace_sample = get_sample(config)
    tGetSample= time.time()-start

    startEvol = time.time()

    pop = [ creator.Individual( gp.PrimitiveTree.from_string(cs['chromosome'][0],pset)) for cs in evospace_sample['sample']]

    # Evaluate the entire population
    fitnesses = map(toolbox.evaluate, pop)
    print fitnesses
    for ind, fit in zip(pop, fitnesses):
        ind.fitness.values = fit


    total_evals = len(pop)
    best_first   = None
    CXPB=config["CXPB"]
    MUTPB=config["MUTPB"]
    # Begin the evolution

    for g in range(config["WORKER_GENERATIONS"]):
        # Select the next generation individuals


        offspring = toolbox.select(pop,len(pop))



    # Clone the selected individuals
        offspring = map(toolbox.clone, offspring)

    # Apply crossover on the offspring
        for child1, child2 in zip(offspring[::2], offspring[1::2]):


            if random.random() < CXPB:
                toolbox.mate(child1, child2)
                del child1.fitness.values
                del child2.fitness.values

    # Apply mutation on the offspring
        for mutant in offspring:
            if random.random() < MUTPB:
                toolbox.mutate(mutant)
                del mutant.fitness.values

        offspring2= toolbox.keepbest(pop,1)
        offspring[0]=offspring2[0]

        # Evaluate the individuals with an invalid fitness
        invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
        fitnesses = map(toolbox.evaluate, invalid_ind)
        
        for ind, fit in zip(invalid_ind, fitnesses):
            ind.fitness.values = fit

        total_evals+=len(invalid_ind)
        

       
        
            #EXTRA evaluate fitnesses for test values
        evaluations=0
        fitnesses2 = toolbox.map(toolbox.evaluateferreals, invalid_ind)
        

         # The population is entirely replaced by the offspring
        pop[:] = offspring
        


        # Gather all the fitnesses in one list and print the stats
        print "ok"

       

        fits = [ind.fitness.values[0] for ind in pop]
        sizes= [ind.height for ind in pop]
        
        

        best = max(fits)
        treesiz_best= min(sizes)
        
        if not best_first:
            best_first = best

        if best <= 0.0:
            print tools.selBest(pop, 1)[0]
            break





        

    ending=toolbox.map(gp.PrimitiveTree.__str__,pop)
    print ending

    sample = [ {"chromosome":str(ind),"id":None,
                "fitness":{"DefaultContext":ind.fitness.values[0]} }
               for ind in pop]
    evospace_sample['sample'] = sample
    tEvol = time.time()-startEvol


    startPutback =  time.time()
    if random.random() < 1.0:
        put_sample(config,evospace_sample)
        was_returned= "RETURNED"
    else:
         was_returned= "LOST"
    tPutBack = time.time() - startPutback

    return best <= 0.0, \
           [treesiz_best,best, sample_num, round(time.time() - start, 2), round(tGetSample,2) , round( tEvol,2), round(tPutBack, 2), total_evals, best_first,was_returned]

       


def work(params):
    worker_id = params[0]
    config = params[1]
    results = []
    global trainpoints
    global testpoints
    


    for sample_num in range(config["MAX_SAMPLES"]):
        server = jsonrpclib.Server(config["SERVER"]) #Create every time to prevent timeouts
        if int(server.found(None)):
            break
        else:
            gen_data = evolve(sample_num,config)
            if gen_data[0]:
                server.found_it(None)
            results.append([worker_id] + gen_data[1])
    
    return results


if __name__ == '__main__':
   
   #initialize(pset,config)
   #evolve(1,config)
   for i in range(1):
    start = time.time()
    init_job = cloud.call(initialize, config=config, pset=pset,  _type=config["WORKER_TYPE"],_env="deap_1")
    
    tInitialize = time.time()-start
    print i, tInitialize
    params = [(w, config) for w in range(config["NUMBER_OF_WORKERS"])]
    print "ok"
    jids = cloud.map(work, params, _type=config["WORKER_TYPE"], _depends_on= init_job )
    print "ok"
    results_list = cloud.result(jids)

    tTotal = time.time()-start
    totals = "%d,%0.2f,%0.2f" % (i, round(tTotal,2), round(tInitialize,2))
    #print totals
    #datafile.write(totals + '\n')
    #for worker_list in results_list:
        #for data_list in worker_list:
           # datafile.write(str(i) +"," + ",".join(map(str,data_list)) + '\n')