Last active
November 18, 2020 21:28
-
-
Save richpsharp/04dae344eb7f165534f51cafba6f6608 to your computer and use it in GitHub Desktop.
Example of pickleless numpy serialization
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # coding=UTF-8 | |
| import io | |
| import multiprocessing | |
| import numpy | |
| def numpy_dumps(numpy_array): | |
| """Safely pickle numpy array to string. | |
| Args: | |
| numpy_array (numpy.ndarray): arbitrary numpy array. | |
| Returns: | |
| A string representation of the array that can be loaded using | |
| `numpy_loads. | |
| """ | |
| with io.BytesIO() as file_stream: | |
| numpy.save(file_stream, numpy_array, allow_pickle=False) | |
| return file_stream.getvalue() | |
| def numpy_loads(binary_numpy_string): | |
| """Safely unpickle binary string to numpy array. | |
| Args: | |
| binary_numpy_string (str): binary string representing a pickled | |
| numpy array. | |
| Returns: | |
| A numpy representation of ``binary_numpy_string``. | |
| """ | |
| with io.BytesIO(binary_numpy_string) as file_stream: | |
| return numpy.load(file_stream) | |
| array_in = numpy.empty(1, dtype='datetime64,f4') | |
| print(array_in.dtype['f0'].metadata) # prints None | |
| q = multiprocessing.Queue() | |
| # safely pickle the array | |
| q.put(numpy_dumps(array_in)) | |
| # unpickle the result | |
| array_out = numpy_loads(q.get()) | |
| print(array_out.dtype['f0'].metadata) # prints None | |
| numpy.save('out', array_out) # totally fine |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment