Created
July 11, 2018 07:02
-
-
Save ixaxaar/7ea4ecab9503a92050cc5b24c2ea0fde to your computer and use it in GitHub Desktop.
Backoff by blocks when batch inserting data somewhere, and a few records tend to fail
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from collections import Iterable | |
def flatten(l): | |
if not isinstance(l, Iterable): | |
yield l | |
else: | |
for el in l: | |
if isinstance(el, Iterable) and not isinstance(el, (str, bytes)): | |
yield from flatten(el) | |
else: | |
yield el | |
def recursive_backoff(fn, data, **args): | |
res = [] | |
try: | |
res = fn(data, **args) | |
except Exception as e: | |
if len(data) == 1: return [None] | |
l = len(data) | |
res.append(recursive_backoff(fn, data[ :l//2], **args)) | |
res.append(recursive_backoff(fn, data[l//2: ], **args)) | |
return list(flatten(res)) | |
def f(d, args=None): | |
return (len(d) if (len(d) % 2 == 0) else 1/0) | |
recursive_backoff(f, [1,2,3,4,5,6,7,8,9,10,11], ) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment