Skip to content

Instantly share code, notes, and snippets.

@yassineAlouini
Last active November 6, 2017 08:53
Show Gist options
  • Save yassineAlouini/cf95a93ec32e9bbe55f59da5b1924e12 to your computer and use it in GitHub Desktop.
Save yassineAlouini/cf95a93ec32e9bbe55f59da5b1924e12 to your computer and use it in GitHub Desktop.
Get information about the pickling process
# Inspired from: https://airflow.incubator.apache.org/_modules/airflow/models.html#BaseOperator
import pickle
import logging
from datetime import datetime
import traceback
def pickle_info(obj, session=None):
d = {}
d['is_picklable'] = True
try:
dttm = datetime.now()
pickled = pickle.dumps(obj)
d['pickle_len'] = len(pickled)
d['pickling_duration'] = "{}".format(datetime.now() - dttm)
except Exception as e:
logging.debug(e)
d['is_picklable'] = False
d['stacktrace'] = traceback.format_exc()
return d
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment