aabadie

"""Example of usage of Joblib with Amazon S3."""

import s3io
import joblib
import numpy as np

big_obj = [np.ones((500, 500)), np.random.random((1000, 1000))]

# Customize the following values with yours
bucket = "my-bucket"

aabadie

import os
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt


# General configuration variables
# Script configuration variables:
CSV_FILE = '/tmp/comparison_results.csv'
PNG_FILE = '/tmp/comparison_results.png'

aabadie

import numpy as np
import joblib

obj = [np.ones((5000, 5000)), np.random.random((5000, 5000))]
joblib.dump(obj, '/tmp/test.pkl', compress=True)
joblib.load('/tmp/test.pkl')

aabadie

"""Persistence strategies comparison script.

This script compute the speed, memory used and disk space used when dumping and
loading arbitrary data. The data are taken among:
- scikit-learn Labeled Faces in the Wild dataset (LFW)
- a fully random numpy array with 10000x10000 shape
- a dictionary with 1M random keys/values
- a list containing 10M random value
The compared persistence strategies are:

aabadie

"""Script comparing different pickling strategies."""

from joblib.numpy_pickle import NumpyPickler, NumpyUnpickler
from joblib.numpy_pickle_utils import JoblibZFile
from joblib.numpy_pickle_utils import BinaryZlibFile, BinaryGzipFile
from pickle import _Pickler, _Unpickler, Pickler, Unpickler
import numpy as np
import bz2
import lzma
import time

aabadie

"""
Benching I/O with joblib and other libraries. Comment and
un-comment what you are interested in.

Warning: this is slow, and the benchs are easily offset by other disk
activity.
"""
import os
import time
import shutil