ipython E2E.py

shutting down current dask cluster if it exists...
... cluster shut down


initializing dask scheduler...
... scheduler started


starting 2 worker(s)...
... starting gpu worker 1
... starting gpu worker 2


... 2 worker(s) successfully started


printing status ...


There are screens on:
        194.gpu_worker_2        (12/13/2018 09:01:50 PM)        (Detached)
        189.gpu_worker_1        (12/13/2018 09:01:50 PM)        (Detached)
        175.dask_scheduler      (12/13/2018 09:01:44 PM)        (Detached)
3 Sockets in /var/run/screen/S-jovyan.



---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
~/work/E2E.py in <module>
    121
    122
--> 123 client.run(initialize_rmm_pool)
    124
    125

/conda/envs/rapids/lib/python3.5/site-packages/distributed/client.py in run(self, function, *args, **kwargs)
   2040          '192.168.0.101:9000': 'running}
   2041         """
-> 2042         return self.sync(self._run, function, *args, **kwargs)
   2043
   2044     @gen.coroutine

/conda/envs/rapids/lib/python3.5/site-packages/distributed/client.py in sync(self, func, *args, **kwargs)
    645             return future
    646         else:
--> 647             return sync(self.loop, func, *args, **kwargs)
    648
    649     def __repr__(self):

/conda/envs/rapids/lib/python3.5/site-packages/distributed/utils.py in sync(loop, func, *args, **kwargs)
    275             e.wait(10)
    276     if error[0]:
--> 277         six.reraise(*error[0])
    278     else:
    279         return result[0]

/conda/envs/rapids/lib/python3.5/site-packages/six.py in reraise(tp, value, tb)
    691             if value.__traceback__ is not tb:
    692                 raise value.with_traceback(tb)
--> 693             raise value
    694         finally:
    695             value = None

/conda/envs/rapids/lib/python3.5/site-packages/distributed/utils.py in f()
    260             if timeout is not None:
    261                 future = gen.with_timeout(timedelta(seconds=timeout), future)
--> 262             result[0] = yield future
    263         except Exception as exc:
    264             error[0] = sys.exc_info()

/conda/envs/rapids/lib/python3.5/site-packages/tornado/gen.py in run(self)
   1131
   1132                     try:
-> 1133                         value = future.result()
   1134                     except Exception:
   1135                         self.had_exception = True

/conda/envs/rapids/lib/python3.5/asyncio/futures.py in result(self)
    292             self._tb_logger = None
    293         if self._exception is not None:
--> 294             raise self._exception
    295         return self._result
    296

/conda/envs/rapids/lib/python3.5/site-packages/tornado/gen.py in run(self)
   1145                             exc_info = None
   1146                     else:
-> 1147                         yielded = self.gen.send(value)
   1148
   1149                     if stack_context._state.contexts is not orig_stack_contexts:

/conda/envs/rapids/lib/python3.5/site-packages/distributed/client.py in _run(self, function, *args, **kwargs)
   1994                 results[key] = resp['result']
   1995             elif resp['status'] == 'error':
-> 1996                 six.reraise(*clean_exception(**resp))
   1997         raise gen.Return(results)
   1998

/conda/envs/rapids/lib/python3.5/site-packages/six.py in reraise(tp, value, tb)
    690                 value = tp()
    691             if value.__traceback__ is not tb:
--> 692                 raise value.with_traceback(tb)
    693             raise value
    694         finally:

/conda/envs/rapids/bin/dask-worker in initialize_rmm_pool()
    103
    104 def initialize_rmm_pool():
--> 105     from librmm_cffi import librmm_config as rmm_cfg
    106
    107     rmm_cfg.use_pool_allocator = True

/conda/envs/rapids/lib/python3.5/site-packages/librmm_cffi/__init__.py in <module>
     43             return path
     44
---> 45     librmm_api = ffi.dlopen(_get_lib_name())
     46     librmm = _RMMWrapper(ffi, librmm_api)
     47

OSError: cannot load library 'librmm.so': libcuda.so.1: cannot open shared object file: No such file or directory