Example of onemax_short.py, but this includes a large init pop with selection before starting the evolution via local eaSimple for mutability going forward. At end we mess with dataframes and select via log to give options for analysis.

one_max_short_large_init_and_dataframes.py

# Derek Tishler
# Aug 3 2020
# Based On:
# https://github.com/DEAP/deap/blob/master/examples/ga/onemax_short.py
# V2

import array
import random

import numpy as np
import pandas as pd

import operator

from deap import algorithms
from deap import base
from deap import creator
from deap import tools

creator.create("FitnessMax", base.Fitness, weights=(1.0,))
creator.create("Individual", array.array, typecode='b', fitness=creator.FitnessMax)

toolbox = base.Toolbox()

# Attribute generator
toolbox.register("attr_bool", random.randint, 0, 1)

# Structure initializers
toolbox.register("individual", tools.initRepeat, creator.Individual, toolbox.attr_bool, 100)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)

def evalOneMax(individual):
    return sum(individual),

toolbox.register("evaluate", evalOneMax)
toolbox.register("mate", tools.cxTwoPoint)
toolbox.register("mutate", tools.mutFlipBit, indpb=0.05)
toolbox.register("select", tools.selTournament, tournsize=3)


def main():
    random.seed(64)
    
    # OLD WAY OF POP SETUP
    #pop = toolbox.population(n=300)

    ##### New way to setup pop so its of a higher quality ######
    init_pop_size = 1000
    pop_size = 100 # note, only 91 are actually unique!!!

    print("Forming extra large init_pop of %d, and performing selection to pear down to %d\n" % (init_pop_size, pop_size))
    init_pop = toolbox.population(n=init_pop_size)

    fitnesses = toolbox.map(toolbox.evaluate, init_pop)
    for ind, fit in zip(init_pop, fitnesses):
        ind.fitness.values = fit

    # two example of pearing down the larger pop
    if False:
        # Take top best items, then shuffle pop
        sorted_pop = sorted(init_pop, key=operator.attrgetter("fitness"), reverse=True) #check reverse to match min/max
        pop = sorted_pop[:pop_size]
        random.shuffle(pop) # I do not like having a sorted pop...I don't know why
    else:
        # perform selction to keep in a real of random distributions vs skewed
        pop = toolbox.select(init_pop, pop_size) #can use this vs the sort, slice, and shuffle above
    
    # save each gens pop to list
    individuals_by_gen_dict = {}

    print("Unique in Init Pop: %d / %d" % ( len(list(set([str(ind) for ind in init_pop]))), len(init_pop) ))
    print("Unique in Starting Pop: %d / %d\n" % ( len(list(set([str(ind) for ind in pop]))), len(pop) ))
    ##############################################################


    hof = tools.HallOfFame(1)
    stats = tools.Statistics(lambda ind: ind.fitness.values)
    stats.register("avg", np.mean)
    stats.register("std", np.std)
    stats.register("min", np.min)
    stats.register("max", np.max)
    
    pop, log, hof, individuals_by_gen_dict = eaSimple(pop, toolbox, individuals_by_gen_dict, cxpb=0.5, mutpb=0.2, ngen=3, 
                                   stats=stats, halloffame=hof, verbose=True)
    
    ################################
    print('\n######################\n')
    # Simple way to export dataframe to file via:
    # https://groups.google.com/d/msg/deap-users/EdmXhFvA_0c/wGiw4WuOBAAJ
    log_df = pd.DataFrame(log)
    print(log_df.head(3)) # print top 3 items
    print(log_df['min']) # print min columns with index

    # a dataframe is handy for doing other things like I mentioned, 
    #  but if you just need to select items from logbook that is also possible
    gen_from_log = log.select('gen')
    mins_from_log = log.select('min')
    # not in a fancy dataframe/series, but now you have the two lists of same info
    print(gen_from_log)
    print(mins_from_log)
    ################################

    return pop, log, hof



def eaSimple(population, toolbox, individuals_by_gen_dict, cxpb, mutpb, ngen, stats=None,
             halloffame=None, verbose=__debug__):

    logbook = tools.Logbook()
    logbook.header = ['gen', 'nevals'] + (stats.fields if stats else [])

    # Evaluate the individuals with an invalid fitness
    invalid_ind = [ind for ind in population if not ind.fitness.valid]
    fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
    for ind, fit in zip(invalid_ind, fitnesses):
        ind.fitness.values = fit

    if halloffame is not None:
        halloffame.update(population)

    record = stats.compile(population) if stats else {}
    logbook.record(gen=0, nevals=len(invalid_ind), **record)
    if verbose:
        print (logbook.stream)

    ###########
    individuals_by_gen_dict[0] = [toolbox.clone(ind) for ind in population]
    print('\n')
    for ind in population[:3]: # for pop of 100 this gets a big big...
        print("%0.5f\t%s" % (ind.fitness.values[0], str(ind)))
    ###########

    # Begin the generational process
    for gen in range(1, ngen + 1):
        # Select the next generation individuals
        offspring = toolbox.select(population, len(population))

        # Vary the pool of individuals. NOTE USE OF algorithms. to fix ##########################
        offspring = algorithms.varAnd(offspring, toolbox, cxpb, mutpb)

        # Evaluate the individuals with an invalid fitness
        invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
        fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
        for ind, fit in zip(invalid_ind, fitnesses):
            ind.fitness.values = fit

        # Update the hall of fame with the generated individuals
        if halloffame is not None:
            halloffame.update(offspring)

        # Replace the current population by the offspring
        population[:] = offspring

        ###########
        individuals_by_gen_dict[gen] = [toolbox.clone(ind) for ind in population]
        print('\n')
        print("Unique in Pop: %d / %d" % ( len(list(set([str(ind) for ind in population]))), len(population) ))
        for ind in population[:3]: # for pop of 100 this gets a big big...
            print("%0.5f\t%s" % (ind.fitness.values[0], str(ind)))
        ###########

        # Append the current generation statistics to the logbook
        record = stats.compile(population) if stats else {}
        logbook.record(gen=gen, nevals=len(invalid_ind), **record)
        if verbose:
            print (logbook.stream)

    return population, logbook, halloffame, individuals_by_gen_dict




if __name__ == "__main__":
    main()


# EXPECTED OUTPUT
'''
Forming extra large init_pop of 1000, and performing selection to pear down to 100

Unique in Init Pop: 1000 / 1000
Unique in Starting Pop: 91 / 100 # !!!! note this is becuase of selection, wow already 10% lost in 1 step!

gen nevals  avg     std     min max
0   0       54.75   3.47958 47  64 


59.00000    Individual('b', [1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1])
54.00000    Individual('b', [0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1])
56.00000    Individual('b', [1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1])


Unique in Pop: 90 / 100
55.00000    Individual('b', [1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0])
60.00000    Individual('b', [0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0])
64.00000    Individual('b', [0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1])
1   68      58.07   3.29319 47  66 


Unique in Pop: 78 / 100
60.00000    Individual('b', [1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1])
60.00000    Individual('b', [0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1])
60.00000    Individual('b', [0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1])
2   49      60.68   3.02946 49  69 


Unique in Pop: 93 / 100
62.00000    Individual('b', [1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1])
65.00000    Individual('b', [0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1])
61.00000    Individual('b', [0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1])
3   66      63.14   3.50149 55  73 

######################

   gen  nevals    avg       std   min   max
0    0       0  54.75  3.479583  47.0  64.0
1    1      68  58.07  3.293190  47.0  66.0
2    2      49  60.68  3.029455  49.0  69.0
0    47.0
1    47.0
2    49.0
3    55.0
Name: min, dtype: float64
[0, 1, 2, 3]
[47.0, 47.0, 49.0, 55.0]

'''