Hyperband traceback with dask.compute

dask-compute.pytb

distributed.core - ERROR - "('array-ce35610051c66c361205cb57db0700ff', 89, 7)"
Traceback (most recent call last):
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/core.py", line 375, in handle_stream
    handler(**merge(extra, msg))
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/scheduler.py", line 1355, in update_graph
    ts = self.tasks[key]
KeyError: "('array-ce35610051c66c361205cb57db0700ff', 89, 7)"
distributed.core - ERROR - "('array-ce35610051c66c361205cb57db0700ff', 89, 7)"
Traceback (most recent call last):
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/core.py", line 375, in handle_stream
    handler(**merge(extra, msg))
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/scheduler.py", line 1355, in update_graph
    ts = self.tasks[key]
KeyError: "('array-ce35610051c66c361205cb57db0700ff', 89, 7)"
distributed.core - ERROR - "('array-ce35610051c66c361205cb57db0700ff', 89, 7)"
Traceback (most recent call last):
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/core.py", line 321, in handle_comm
    result = yield result
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py", line 1099, in run
    value = future.result()
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py", line 1113, in run
    yielded = self.gen.send(value)
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/scheduler.py", line 1922, in add_client
    yield self.handle_stream(comm=comm, extra={'client': client})
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py", line 1099, in run
    value = future.result()
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py", line 1113, in run
    yielded = self.gen.send(value)
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/core.py", line 375, in handle_stream
    handler(**merge(extra, msg))
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/scheduler.py", line 1355, in update_graph
    ts = self.tasks[key]
KeyError: "('array-ce35610051c66c361205cb57db0700ff', 89, 7)"
distributed.core - ERROR - "('array-ce35610051c66c361205cb57db0700ff', 89, 7)"
Traceback (most recent call last):
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/core.py", line 321, in handle_comm
    result = yield result
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py", line 1099, in run
    value = future.result()
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py", line 1113, in run
    yielded = self.gen.send(value)
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/scheduler.py", line 1922, in add_client
    yield self.handle_stream(comm=comm, extra={'client': client})
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py", line 1099, in run
    value = future.result()
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py", line 1113, in run
    yielded = self.gen.send(value)
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/core.py", line 375, in handle_stream
    handler(**merge(extra, msg))
  File "/Users/ssievert/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/scheduler.py", line 1355, in update_graph
    ts = self.tasks[key]
KeyError: "('array-ce35610051c66c361205cb57db0700ff', 89, 7)"


---------------------------------------------------------------------------
CancelledError                            Traceback (most recent call last)
<ipython-input-6-1d884ce5f66b> in <module>()
      1 alg = Hyperband(model, params, max_iter=9)
      2 
----> 3 alg.fit(X, y, classes=classes)

~/Developer/stsievert/dask-searchcv/dask_searchcv/adaptive.py in fit(self, X, y, dry_run, **fit_kwargs)
    401                                                                  **fit_kwargs)
    402                                for kwargs in all_kwargs]
--> 403             results = [r.compute() for r in delayed_results]
    404             #  results = dask.compute(delayed_results)
    405 

~/Developer/stsievert/dask-searchcv/dask_searchcv/adaptive.py in <listcomp>(.0)
    401                                                                  **fit_kwargs)
    402                                for kwargs in all_kwargs]
--> 403             results = [r.compute() for r in delayed_results]
    404             #  results = dask.compute(delayed_results)
    405 

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/dask/base.py in compute(self, **kwargs)
    154         dask.base.compute
    155         """
--> 156         (result,) = compute(self, traverse=False, **kwargs)
    157         return result
    158 

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/dask/base.py in compute(*args, **kwargs)
    398     keys = [x.__dask_keys__() for x in collections]
    399     postcomputes = [x.__dask_postcompute__() for x in collections]
--> 400     results = schedule(dsk, keys, **kwargs)
    401     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    402 

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/client.py in get(self, dsk, keys, restrictions, loose_restrictions, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, **kwargs)
   2157             try:
   2158                 results = self.gather(packed, asynchronous=asynchronous,
-> 2159                                       direct=direct)
   2160             finally:
   2161                 for f in futures.values():

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/client.py in gather(self, futures, errors, maxsize, direct, asynchronous)
   1560             return self.sync(self._gather, futures, errors=errors,
   1561                              direct=direct, local_worker=local_worker,
-> 1562                              asynchronous=asynchronous)
   1563 
   1564     @gen.coroutine

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/client.py in sync(self, func, *args, **kwargs)
    650             return future
    651         else:
--> 652             return sync(self.loop, func, *args, **kwargs)
    653 
    654     def __repr__(self):

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/utils.py in sync(loop, func, *args, **kwargs)
    273             e.wait(10)
    274     if error[0]:
--> 275         six.reraise(*error[0])
    276     else:
    277         return result[0]

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/six.py in reraise(tp, value, tb)
    691             if value.__traceback__ is not tb:
    692                 raise value.with_traceback(tb)
--> 693             raise value
    694         finally:
    695             value = None

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/utils.py in f()
    258             yield gen.moment
    259             thread_state.asynchronous = True
--> 260             result[0] = yield make_coro()
    261         except Exception as exc:
    262             error[0] = sys.exc_info()

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py in run(self)
   1097 
   1098                     try:
-> 1099                         value = future.result()
   1100                     except Exception:
   1101                         self.had_exception = True

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py in run(self)
   1105                     if exc_info is not None:
   1106                         try:
-> 1107                             yielded = self.gen.throw(*exc_info)
   1108                         finally:
   1109                             # Break up a reference to itself

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/client.py in _gather(self, futures, errors, direct, local_worker)
   1437                             six.reraise(type(exception),
   1438                                         exception,
-> 1439                                         traceback)
   1440                     if errors == 'skip':
   1441                         bad_keys.add(key)

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/six.py in reraise(tp, value, tb)
    690                 value = tp()
    691             if value.__traceback__ is not tb:
--> 692                 raise value.with_traceback(tb)
    693             raise value
    694         finally:

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/dask/compatibility.py in apply()
     47     def apply(func, args, kwargs=None):
     48         if kwargs:
---> 49             return func(*args, **kwargs)
     50         else:
     51             return func(*args)

~/Developer/stsievert/dask-searchcv/dask_searchcv/adaptive.py in _successive_halving()
    188                                for k, model in models.items()}
    189             #  results = {k: v.compute() for k, v in delayed_results.items()}
--> 190             results = dask.compute(delayed_results)[0]
    191 
    192         val_scores = {k: r[0] for k, r in results.items()}

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/dask/base.py in compute()
    398     keys = [x.__dask_keys__() for x in collections]
    399     postcomputes = [x.__dask_postcompute__() for x in collections]
--> 400     results = schedule(dsk, keys, **kwargs)
    401     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    402 

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/client.py in get()
   2157             try:
   2158                 results = self.gather(packed, asynchronous=asynchronous,
-> 2159                                       direct=direct)
   2160             finally:
   2161                 for f in futures.values():

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/client.py in gather()
   1560             return self.sync(self._gather, futures, errors=errors,
   1561                              direct=direct, local_worker=local_worker,
-> 1562                              asynchronous=asynchronous)
   1563 
   1564     @gen.coroutine

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/client.py in sync()
    650             return future
    651         else:
--> 652             return sync(self.loop, func, *args, **kwargs)
    653 
    654     def __repr__(self):

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/utils.py in sync()
    273             e.wait(10)
    274     if error[0]:
--> 275         six.reraise(*error[0])
    276     else:
    277         return result[0]

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/six.py in reraise()
    691             if value.__traceback__ is not tb:
    692                 raise value.with_traceback(tb)
--> 693             raise value
    694         finally:
    695             value = None

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/utils.py in f()
    258             yield gen.moment
    259             thread_state.asynchronous = True
--> 260             result[0] = yield make_coro()
    261         except Exception as exc:
    262             error[0] = sys.exc_info()

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py in run()
   1097 
   1098                     try:
-> 1099                         value = future.result()
   1100                     except Exception:
   1101                         self.had_exception = True

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py in run()
   1105                     if exc_info is not None:
   1106                         try:
-> 1107                             yielded = self.gen.throw(*exc_info)
   1108                         finally:
   1109                             # Break up a reference to itself

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/client.py in _gather()
   1437                             six.reraise(type(exception),
   1438                                         exception,
-> 1439                                         traceback)
   1440                     if errors == 'skip':
   1441                         bad_keys.add(key)

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/six.py in reraise()
    690                 value = tp()
    691             if value.__traceback__ is not tb:
--> 692                 raise value.with_traceback(tb)
    693             raise value
    694         finally:

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/dask/compatibility.py in apply()
     47     def apply(func, args, kwargs=None):
     48         if kwargs:
---> 49             return func(*args, **kwargs)
     50         else:
     51             return func(*args)

~/Developer/stsievert/dask-searchcv/dask_searchcv/adaptive.py in _train()
     61         logger.info(msg, k, s, i, iter * 100.0 / max_iter)
     62         if not dry_run:
---> 63             model.partial_fit(X, y, **fit_kwargs)
     64     fit_time = default_timer() - start_time
     65     msg = ("Training model %s for %d partial_fit calls took %0.2f seconds")

~/Developer/dask/dask-ml/dask_ml/wrappers.py in partial_fit()
    332         self : object
    333         """
--> 334         return self.fit(X, y=y, **fit_kwargs)
    335 
    336 

~/Developer/dask/dask-ml/dask_ml/wrappers.py in fit()
    287     def fit(self, X, y=None, **fit_kwargs):
    288         check_scoring(self.estimator, self.scoring)
--> 289         result = fit(self.estimator, X, y, **fit_kwargs)
    290 
    291         # Copy the learned attributes over to self

~/Developer/dask/dask-ml/dask_ml/_partial.py in fit()
    184 
    185     if compute:
--> 186         return value.compute()
    187     else:
    188         return value

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/dask/base.py in compute()
    154         dask.base.compute
    155         """
--> 156         (result,) = compute(self, traverse=False, **kwargs)
    157         return result
    158 

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/dask/base.py in compute()
    398     keys = [x.__dask_keys__() for x in collections]
    399     postcomputes = [x.__dask_postcompute__() for x in collections]
--> 400     results = schedule(dsk, keys, **kwargs)
    401     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    402 

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/client.py in get()
   2157             try:
   2158                 results = self.gather(packed, asynchronous=asynchronous,
-> 2159                                       direct=direct)
   2160             finally:
   2161                 for f in futures.values():

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/client.py in gather()
   1560             return self.sync(self._gather, futures, errors=errors,
   1561                              direct=direct, local_worker=local_worker,
-> 1562                              asynchronous=asynchronous)
   1563 
   1564     @gen.coroutine

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/client.py in sync()
    650             return future
    651         else:
--> 652             return sync(self.loop, func, *args, **kwargs)
    653 
    654     def __repr__(self):

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/utils.py in sync()
    273             e.wait(10)
    274     if error[0]:
--> 275         six.reraise(*error[0])
    276     else:
    277         return result[0]

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/six.py in reraise()
    691             if value.__traceback__ is not tb:
    692                 raise value.with_traceback(tb)
--> 693             raise value
    694         finally:
    695             value = None

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/utils.py in f()
    258             yield gen.moment
    259             thread_state.asynchronous = True
--> 260             result[0] = yield make_coro()
    261         except Exception as exc:
    262             error[0] = sys.exc_info()

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py in run()
   1097 
   1098                     try:
-> 1099                         value = future.result()
   1100                     except Exception:
   1101                         self.had_exception = True

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/tornado/gen.py in run()
   1105                     if exc_info is not None:
   1106                         try:
-> 1107                             yielded = self.gen.throw(*exc_info)
   1108                         finally:
   1109                             # Break up a reference to itself

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/distributed/client.py in _gather()
   1433                             six.reraise(CancelledError,
   1434                                         CancelledError(key),
-> 1435                                         None)
   1436                         else:
   1437                             six.reraise(type(exception),

~/anaconda3/envs/dask-master/lib/python3.6/site-packages/six.py in reraise()
    691             if value.__traceback__ is not tb:
    692                 raise value.with_traceback(tb)
--> 693             raise value
    694         finally:
    695             value = None

CancelledError: ('fit-1d7332fedef241aa80083dfad0c3bda3', 89)