From 149c05db53c2368fb41b1132e220620448273de5 Mon Sep 17 00:00:00 2001 From: 1313e Date: Thu, 12 Mar 2020 18:06:43 +1100 Subject: [PATCH 1/4] Added six requirement, and added minimum versions for all requirements. --- requirements.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index b3846b87..9dce5b33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ -h5py -numpy -dill +dill>=0.3.0 +h5py>=2.8.0 +numpy>=1.8 +six>=1.11.0 \ No newline at end of file From c514e1959672e1be83da84ca8e383bd2295bd26d Mon Sep 17 00:00:00 2001 From: 1313e Date: Thu, 12 Mar 2020 18:29:38 +1100 Subject: [PATCH 2/4] Now 'dill' is always used for dumping and loading serialized objects. Added a test for dumping and loading a local session function. --- hickle/hickle.py | 11 ++--------- hickle/loaders/load_python3.py | 5 +---- hickle/tests/test_hickle.py | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/hickle/hickle.py b/hickle/hickle.py index 6a986eff..ed0c1ea8 100644 --- a/hickle/hickle.py +++ b/hickle/hickle.py @@ -51,15 +51,8 @@ if PY3: file = io.TextIOWrapper -# Import a default 'pickler' -# Not the nicest import code, but should work on Py2/Py3 -try: - import dill as pickle -except ImportError: - try: - import cPickle as pickle - except ImportError: - import pickle +# Import dill as pickle +import dill as pickle try: from pathlib import Path diff --git a/hickle/loaders/load_python3.py b/hickle/loaders/load_python3.py index c6b173fd..f532f324 100644 --- a/hickle/loaders/load_python3.py +++ b/hickle/loaders/load_python3.py @@ -153,10 +153,7 @@ def load_none_dataset(h_node): def load_pickled_data(h_node): py_type, data = get_type_and_data(h_node) - try: - import cPickle as pickle - except ModuleNotFoundError: - import pickle + import dill as pickle return pickle.loads(data[0]) diff --git a/hickle/tests/test_hickle.py b/hickle/tests/test_hickle.py index 54910542..53172872 100644 --- a/hickle/tests/test_hickle.py +++ b/hickle/tests/test_hickle.py @@ -14,6 +14,7 @@ import six import time from pprint import pprint +import pytest from py.path import local @@ -42,9 +43,24 @@ } } +# Define a test function that must be serialized and unpacked again +def func(a, b, c=0): + return(a, b, c) + + DUMP_CACHE = [] # Used in test_track_times() +def test_local_func(): + """ Dumping and loading a local function """ + filename, mode = 'test.h5', 'w' + with pytest.warns(SerializedWarning): + dump(func, filename, mode) + func_hkl = load(filename) + assert type(func) == type(func_hkl) + assert func(1, 2) == func_hkl(1, 2) + + def test_string(): """ Dumping and loading a string """ if six.PY2: From 37d953471a2b21c4d76e1d80a7d871c6cffa2c10 Mon Sep 17 00:00:00 2001 From: 1313e Date: Thu, 12 Mar 2020 18:47:32 +1100 Subject: [PATCH 3/4] Add support for actually loading serialized data in Python 2. --- hickle/loaders/load_python.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hickle/loaders/load_python.py b/hickle/loaders/load_python.py index 58de921e..f53aee0f 100644 --- a/hickle/loaders/load_python.py +++ b/hickle/loaders/load_python.py @@ -102,6 +102,11 @@ def load_unicode_dataset(h_node): def load_none_dataset(h_node): return None +def load_pickled_data(h_node): + py_type, data = get_type_and_data(h_node) + import dill as pickle + return pickle.loads(data[0]) + def load_python_dtype_dataset(h_node): py_type, data = get_type_and_data(h_node) subtype = h_node.attrs["python_subdtype"] @@ -136,6 +141,7 @@ def load_python_dtype_dataset(h_node): "python_dtype" : load_python_dtype_dataset, "string" : load_string_dataset, "unicode" : load_unicode_dataset, + "pickle" : load_pickled_data, "none" : load_none_dataset } From 8f813b63936a97f59a215fbc8ce98926e6da3a7b Mon Sep 17 00:00:00 2001 From: 1313e Date: Thu, 12 Mar 2020 18:52:45 +1100 Subject: [PATCH 4/4] Add new test to main as well. --- hickle/tests/test_hickle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hickle/tests/test_hickle.py b/hickle/tests/test_hickle.py index 53172872..57686306 100644 --- a/hickle/tests/test_hickle.py +++ b/hickle/tests/test_hickle.py @@ -837,6 +837,7 @@ def test_np_scalar(): test_complex_dict() test_multi_hickle() test_dict_int_key() + test_local_func() # Cleanup print("ALL TESTS PASSED!") \ No newline at end of file