Benchmarks for https://github.com/optuna/optuna/pull/974.

optuna_pr_974.py

from collections import OrderedDict
import time

from matplotlib import pyplot as plt
import optuna
from optuna import samplers
from optuna.storages import RDBStorage
from optuna.storages import RedisStorage


# Objective function from  `examples/quadratic_simple.py`.
def objective(trial):
    x = trial.suggest_uniform('x', -100, 100)
    y = trial.suggest_categorical('y', [-1, 0, 1])
    return x**2 + y


if __name__ == '__main__':
    storages = [
        None,  # `InMemoryStorage`.
        # MySQL 
        # 1. Start a server (e.g. for macOS as follows).
        #    $ brew services start mysql 
        # 2. Create a database.
        #    $ mysql -u root -e "CREATE DATABASE IF NOT EXISTS optuna974"
        RDBStorage(url='mysql://root@localhost/optuna974'), 
        # RDBStorage(url='sqlite:///974.db'),
        # Redis
        # 1. Start a server.
        #    $ redis-server
        RedisStorage(url='redis://localhost:6379/0'),
    ]
    study_name = '974'
    n_trials_configs = [1, 10, 100, 300, 500, 1000]

    storage_benchmarks = OrderedDict()

    for n_trials in n_trials_configs:
        for storage in storages:
            study = optuna.create_study(
                study_name=study_name, storage=storage,
                sampler=samplers.TPESampler(), load_if_exists=False)

            if storage is None:
                storage_name = 'InMemoryStorage'
            elif isinstance(storage, (RDBStorage, RedisStorage)):
                storage_name = storage.__class__.__name__
            else:
                assert False

            start = time.time()
            study.optimize(objective, n_trials=n_trials)
            end = time.time()

            if storage_name not in storage_benchmarks:
                storage_benchmarks[storage_name] = []
            storage_benchmarks[storage_name].append((end - start) / n_trials)

            # Tidy up.
            if isinstance(storage, (RDBStorage, RedisStorage)):
                optuna.delete_study(study_name, storage)

    for storage_name, t in storage_benchmarks.items():
        plt.plot(n_trials_configs, storage_benchmarks[storage_name], label=storage_name)

    plt.ylabel('s / trial')
    plt.xlabel('# trials')
    plt.legend()
    plt.savefig('974.png')

Environment

• maxOS Mojave
• 2.8 GHz Intel Core i7
• 16 GB 2133 MHz LPDDR3

Storage

• redis 5.0.7
• mysql 8.0.19
• sqlite3 3.24.0

Results

optuna.get_all_study_summaries with 1000 trials

• RDBStorage (MySQL) 0.7059779167175293 s.
• RedisStorage 0.0002129077911376953 s.

RDBStorage (MySQL)

RDBStorage (SQLite)