benchmark_for_optuna_pr_4631.py

import math
import time
import optuna
import numpy as np


def objective(trial: optuna.Trial, n_params: int) -> float:
    return sum([
        math.sin(trial.suggest_float('param-{}'.format(i), 0, math.pi * 2))
        for i in range(n_params)
    ])


def main():
    storage_url = "mysql+pymysql://optuna:[email protected]:3306/optuna"
    sampler = optuna.samplers.TPESampler(seed=1)
    optuna.logging.set_verbosity(optuna.logging.ERROR)

    for n_params in [5, 30]:
        elapsed = []
        for i in range(5):
            study = optuna.create_study(storage=storage_url, sampler=sampler)

            start = time.time()
            study.optimize(lambda t: objective(t, n_params=n_params), n_trials=100)
            elapsed.append(time.time() - start)

            optuna.delete_study(study_name=study.study_name, storage=storage_url)
        print(f"Elapsed {n_params=}: {np.mean(elapsed):.4f} (+/- {np.std(elapsed):.4f})s", elapsed)


if __name__ == '__main__':
    main()

PR 4672

$ python tmp/benchmark_mysql.py
Elapsed n_params=5: 10.6553 (+/- 0.1029)s [10.456530094146729, 10.68004822731018, 10.673655033111572, 10.745635986328125, 10.720831871032715]
Elapsed n_params=30: 38.3893 (+/- 1.0469)s [40.44225525856018, 38.048758029937744, 38.034186124801636, 37.936782121658325, 37.484426975250244]

master ee15fcc95

$ python tmp/benchmark_mysql.py
Elapsed n_params=5: 10.7897 (+/- 0.0556)s [10.832643032073975, 10.73856782913208, 10.873303890228271, 10.728083848953247, 10.77579402923584]
Elapsed n_params=30: 38.0671 (+/- 0.5129)s [37.734352111816406, 39.08030891418457, 37.885014057159424, 37.919699907302856, 37.71600008010864

Branch	n_params=5 (Average Time ± Std Deviation) (s)	n_params=30 (Average Time ± Std Deviation) (s)
master ee15fcc95	10.7897 (+/- 0.0556)	38.0671 (+/- 0.5129)
PR 4672	10.6553 (+/- 0.1029)	38.3893 (+/- 1.0469)