Skip to content

Instantly share code, notes, and snippets.

@richpsharp
Last active November 18, 2020 21:28
Show Gist options
  • Select an option

  • Save richpsharp/04dae344eb7f165534f51cafba6f6608 to your computer and use it in GitHub Desktop.

Select an option

Save richpsharp/04dae344eb7f165534f51cafba6f6608 to your computer and use it in GitHub Desktop.
Example of pickleless numpy serialization
# coding=UTF-8
import io
import multiprocessing
import numpy
def numpy_dumps(numpy_array):
"""Safely pickle numpy array to string.
Args:
numpy_array (numpy.ndarray): arbitrary numpy array.
Returns:
A string representation of the array that can be loaded using
`numpy_loads.
"""
with io.BytesIO() as file_stream:
numpy.save(file_stream, numpy_array, allow_pickle=False)
return file_stream.getvalue()
def numpy_loads(binary_numpy_string):
"""Safely unpickle binary string to numpy array.
Args:
binary_numpy_string (str): binary string representing a pickled
numpy array.
Returns:
A numpy representation of ``binary_numpy_string``.
"""
with io.BytesIO(binary_numpy_string) as file_stream:
return numpy.load(file_stream)
array_in = numpy.empty(1, dtype='datetime64,f4')
print(array_in.dtype['f0'].metadata) # prints None
q = multiprocessing.Queue()
# safely pickle the array
q.put(numpy_dumps(array_in))
# unpickle the result
array_out = numpy_loads(q.get())
print(array_out.dtype['f0'].metadata) # prints None
numpy.save('out', array_out) # totally fine
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment