Benchmark on openml's CC18 using TPOT and AutoSklearn

bench.py

# /home/jhoof/python/python36/bin/python3 bench.py

import openml

from arbok.bench import Benchmark


# We create a benchmark setup where we specify the headers, the interpreter we
# want to use, the directory to where we store the jobs (.sh-files), and we give
# it the config-file we created earlier.
bench = Benchmark(
    headers="#PBS -lnodes=1:cpu3\n#PBS -lwalltime=1:30:00",
    python_interpreter="/home/jhoof/python/python36/bin/python3",  # Path to interpreter
    root="/home/jhoof/benchmark/",
    jobs_dir="jobs",
    config_file="config.json",
    log_file="log.json"
)

# Config file
config_file = bench.create_config_file(

    # Wrapper parameters
    wrapper={"refit": True, "verbose": False, "retry_on_error": True},

    # TPOT parameters
    tpot={
        "max_time_mins": 6,  # Max total time in minutes
        "max_eval_time_mins": 1  # Max time per candidate in minutes
    },

    # Autosklearn parameters
    autosklearn={
        "time_left_for_this_task": 360,  # Max total time in seconds
        "per_run_time_limit": 60  # Max time per candidate in seconds
    }
)


# Next, we load the tasks we want to benchmark on from OpenML.
# In this case, we load a list of task id's from study 99.
tasks = openml.study.get_study(99).tasks

# Next, we create jobs for both tpot and autosklearn.
bench.create_jobs(tasks, classifiers=["tpot", "autosklearn"])

# And finally, we submit the jobs using qsub
bench.submit_jobs()