Comparison between pickle, _pickle and joblib

pickle_vs_joblib_vs__pickle.py

#!/usr/bin/env python3

from sklearn.externals import joblib
import numpy
import pickle
import _pickle
import timeit


def wrapper(func, *args, **kwargs):
    def wrapped():
        return func(*args, **kwargs)
    return wrapped

N = 100
big_array = [numpy.ones((5000, 5000)), numpy.random.random((5000, 5000))]

with open("big_array1.pkl", "wb") as f:
    wrapped = wrapper(joblib.dump, big_array, f)
    print("JOBLIB Dump:", timeit.timeit(wrapped, setup="from sklearn.externals import joblib", number=N))

with open("big_array2.pkl", "wb") as f:
    wrapped = wrapper(pickle.dump, big_array, f)
    print("PICKLE Dump:", timeit.timeit(wrapped, setup="import pickle", number=N))

with open("big_array3.pkl", "wb") as f:
    wrapped = wrapper(_pickle.dump, big_array, f)
    print("_PICKLE Dump:", timeit.timeit(wrapped, number=N))

with open("big_array1.pkl", "rb") as f:
    wrapped = wrapper(joblib.load, f)
    print("JOBLIB Load:", timeit.timeit(wrapped, setup="from sklearn.externals import joblib", number=N))

with open("big_array2.pkl", "rb") as f:
    wrapped = wrapper(pickle.load, f)
    print("PICKLE Load:", timeit.timeit(wrapped, setup="from sklearn.externals import joblib", number=N))

with open("big_array3.pkl", "rb") as f:
    wrapped = wrapper(_pickle.load, f)
    print("_PICKLE Load:", timeit.timeit(wrapped, setup="from sklearn.externals import joblib", number=N))