Created
May 15, 2019 15:06
-
-
Save rbdixon/3d749a6f99dd7f3a47a7f320d0867aa8 to your computer and use it in GitHub Desktop.
POC doit modifications to handle non-serializable python-action results
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
''' | |
To use: | |
touch testfile | |
doit | |
Serialized .doit.db database (JSON format) will contain: | |
{ | |
"create_object": { | |
"_values_:": { | |
"obj": null | |
}, | |
"result:": { | |
"obj": null | |
}, | |
"checker:": "MD5Checker", | |
"deps:": [ | |
"testfile" | |
], | |
"testfile": [ | |
1557929656.729679, | |
0, | |
"d41d8cd98f00b204e9800998ecf8427e" | |
] | |
}, | |
"null": { | |
"_values_:": { | |
"null": null | |
}, | |
"result:": { | |
"null": null | |
}, | |
"checker:": "MD5Checker", | |
"deps:": [] | |
}, | |
"use_object": { | |
"_values_:": { | |
"_result:null": { | |
"null": null | |
}, | |
"_result:create_object": { | |
"obj": null | |
} | |
}, | |
"checker:": "MD5Checker", | |
"deps:": [] | |
} | |
} | |
''' | |
import random | |
def task_create_object(): | |
'Create an instance of Test()' | |
class Test: | |
a = random.randint(0, 100) | |
def instantiate(): | |
return {'obj': Test()} | |
# uptodate: this task can never be uptodate since the result is not serializable | |
# and the cached value will always be null | |
return {'actions': [instantiate], 'file_dep': ['testfile'], 'uptodate': [False]} | |
def task_use_object(): | |
'Display Test().a' | |
def use_object(obj): | |
assert obj is not None, 'Did not receive expected Test() instance' | |
print(f'Value: {obj.a}') | |
return { | |
'actions': [use_object], | |
'getargs': {'obj': ('create_object', 'obj')}, | |
'verbosity': 2, | |
} | |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/doit/dependency.py b/doit/dependency.py | |
index 8497314..0597137 100644 | |
--- a/doit/dependency.py | |
+++ b/doit/dependency.py | |
@@ -18,6 +18,13 @@ import dbm as ddbm | |
import json | |
+class SkipEncoder(json.JSONEncoder): | |
+ def default(self, obj): | |
+ if type(obj) not in [str, int, float, bool, None]: | |
+ print(f'Serializing {type(obj)} as None') | |
+ return None | |
+ else: | |
+ return obj | |
class DatabaseException(Exception): | |
"""Exception class for whatever backend exception""" | |
@@ -80,7 +87,7 @@ class JsonDB(object): | |
"""save DB content in file""" | |
try: | |
db_file = open(self.name, 'w') | |
- json.dump(self._db, db_file) | |
+ json.dump(self._db, db_file, cls=SkipEncoder) | |
finally: | |
db_file.close() | |
@@ -159,7 +166,8 @@ class DbmDB(object): | |
def dump(self): | |
"""save/close DBM file""" | |
for task_id in self.dirty: | |
- self._dbm[task_id] = json.dumps(self._db[task_id]) | |
+ self._dbm[task_id] = json.dumps(self._db[task_id], cls=SkipEncoder) | |
+ pass | |
self._dbm.close() | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment