gistfile1.txt

root@cbb77e9ace8b:/app# cat tftest2.py
import tensorflow as tf

xs = []
for i in range(16):
  with tf.device('/gpu:1'):
    xs.append(tf.ones(()))

from functools import reduce
from operator import add
sum = reduce(add, xs)
with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:
  print(sess.run([sum]))
root@cbb77e9ace8b:/app# BF_FORCE_DEVICE_BUS_ID="0000:00:00.0" LD_LIBRARY_PATH=/usr/local/cuda/lib64 bfboost client -l "172.16.2.26;172.16.1.2" python3 tftest2.py
I tensorflow/stream_executor/dso_loader.cc:111] successfully opened CUDA library libcublas.so locally
I tensorflow/stream_executor/dso_loader.cc:111] successfully opened CUDA library libcudnn.so locally
I tensorflow/stream_executor/dso_loader.cc:111] successfully opened CUDA library libcufft.so locally
I tensorflow/stream_executor/dso_loader.cc:111] successfully opened CUDA library libcuda.so.1 locally
I tensorflow/stream_executor/dso_loader.cc:111] successfully opened CUDA library libcurand.so locally
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 0 with properties:
name: GeForce GTX TITAN X
major: 5 minor: 2 memoryClockRate (GHz) 1.076
pciBusID 0000:00:00.0
Total memory: 12.00GiB
Free memory: 11.87GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7f5be8051f20
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 1 with properties:
name: GeForce GTX TITAN X
major: 5 minor: 2 memoryClockRate (GHz) 1.076
pciBusID 0000:00:00.0
Total memory: 12.00GiB
Free memory: 11.87GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7f5be8466dd0
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 2 with properties:
name: GeForce GTX TITAN X
major: 5 minor: 2 memoryClockRate (GHz) 1.076
pciBusID 0000:00:00.0
Total memory: 12.00GiB
Free memory: 11.87GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7f5be88e1310
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 3 with properties:
name: GeForce GTX TITAN X
major: 5 minor: 2 memoryClockRate (GHz) 1.076
pciBusID 0000:00:00.0
Total memory: 12.00GiB
Free memory: 11.87GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7f5be8d5fb70
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 4 with properties:
name: GeForce GTX TITAN X
major: 5 minor: 2 memoryClockRate (GHz) 1.076
pciBusID 0000:00:00.0
Total memory: 12.00GiB
Free memory: 11.87GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7f5be91e1e00
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 5 with properties:
name: GeForce GTX TITAN X
major: 5 minor: 2 memoryClockRate (GHz) 1.076
pciBusID 0000:00:00.0
Total memory: 12.00GiB
Free memory: 11.87GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7f5be9667ca0
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 6 with properties:
name: GeForce GTX TITAN X
major: 5 minor: 2 memoryClockRate (GHz) 1.076
pciBusID 0000:00:00.0
Total memory: 12.00GiB
Free memory: 11.87GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7f5be9af1e20
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 7 with properties:
name: GeForce GTX TITAN X
major: 5 minor: 2 memoryClockRate (GHz) 1.076
pciBusID 0000:00:00.0
Total memory: 12.00GiB
Free memory: 11.87GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7f5be9f7f160
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 8 with properties:
name: TITAN X (Pascal)
major: 6 minor: 1 memoryClockRate (GHz) 1.531
pciBusID 0000:00:00.0
Total memory: 11.90GiB
Free memory: 11.76GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7fdc840310d0
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 9 with properties:
name: TITAN X (Pascal)
major: 6 minor: 1 memoryClockRate (GHz) 1.531
pciBusID 0000:00:00.0
Total memory: 11.90GiB
Free memory: 11.76GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7fdc8442e8f0
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 10 with properties:
name: TITAN X (Pascal)
major: 6 minor: 1 memoryClockRate (GHz) 1.531
pciBusID 0000:00:00.0
Total memory: 11.90GiB
Free memory: 11.76GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7fdc84a53750
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 11 with properties:
name: TITAN X (Pascal)
major: 6 minor: 1 memoryClockRate (GHz) 1.531
pciBusID 0000:00:00.0
Total memory: 11.90GiB
Free memory: 11.76GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7fdc8507c950
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 12 with properties:
name: TITAN X (Pascal)
major: 6 minor: 1 memoryClockRate (GHz) 1.531
pciBusID 0000:00:00.0
Total memory: 11.90GiB
Free memory: 11.76GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7fdc856a9690
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 13 with properties:
name: TITAN X (Pascal)
major: 6 minor: 1 memoryClockRate (GHz) 1.531
pciBusID 0000:00:00.0
Total memory: 11.90GiB
Free memory: 11.76GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7fdc85cd9f00
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 14 with properties:
name: TITAN X (Pascal)
major: 6 minor: 1 memoryClockRate (GHz) 1.531
pciBusID 0000:00:00.0
Total memory: 11.90GiB
Free memory: 11.76GiB
W tensorflow/stream_executor/cuda/cuda_driver.cc:572] creating context when one is currently active; existing: 0x7fdc8630e680
I tensorflow/core/common_runtime/gpu/gpu_device.cc:951] Found device 15 with properties:
name: TITAN X (Pascal)
major: 6 minor: 1 memoryClockRate (GHz) 1.531
pciBusID 0000:00:00.0
Total memory: 11.90GiB
Free memory: 11.76GiB
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 0 to device ordinal 8
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 0 to device ordinal 9
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 0 to device ordinal 10
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 0 to device ordinal 11
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 0 to device ordinal 12
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 0 to device ordinal 13
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 0 to device ordinal 14
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 0 to device ordinal 15
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 1 to device ordinal 8
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 1 to device ordinal 9
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 1 to device ordinal 10
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 1 to device ordinal 11
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 1 to device ordinal 12
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 1 to device ordinal 13
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 1 to device ordinal 14
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 1 to device ordinal 15
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 2 to device ordinal 8
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 2 to device ordinal 9
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 2 to device ordinal 10
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 2 to device ordinal 11
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 2 to device ordinal 12
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 2 to device ordinal 13
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 2 to device ordinal 14
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 2 to device ordinal 15
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 3 to device ordinal 8
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 3 to device ordinal 9
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 3 to device ordinal 10
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 3 to device ordinal 11
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 3 to device ordinal 12
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 3 to device ordinal 13
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 3 to device ordinal 14
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 3 to device ordinal 15
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 4 to device ordinal 8
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 4 to device ordinal 9
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 4 to device ordinal 10
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 4 to device ordinal 11
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 4 to device ordinal 12
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 4 to device ordinal 13
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 4 to device ordinal 14
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 4 to device ordinal 15
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 5 to device ordinal 8
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 5 to device ordinal 9
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 5 to device ordinal 10
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 5 to device ordinal 11
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 5 to device ordinal 12
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 5 to device ordinal 13
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 5 to device ordinal 14
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 5 to device ordinal 15
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 6 to device ordinal 8
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 6 to device ordinal 9
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 6 to device ordinal 10
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 6 to device ordinal 11
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 6 to device ordinal 12
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 6 to device ordinal 13
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 6 to device ordinal 14
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 6 to device ordinal 15
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 7 to device ordinal 8
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 7 to device ordinal 9
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 7 to device ordinal 10
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 7 to device ordinal 11
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 7 to device ordinal 12
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 7 to device ordinal 13
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 7 to device ordinal 14
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 7 to device ordinal 15
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 8 to device ordinal 0
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 8 to device ordinal 1
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 8 to device ordinal 2
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 8 to device ordinal 3
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 8 to device ordinal 4
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 8 to device ordinal 5
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 8 to device ordinal 6
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 8 to device ordinal 7
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 9 to device ordinal 0
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 9 to device ordinal 1
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 9 to device ordinal 2
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 9 to device ordinal 3
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 9 to device ordinal 4
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 9 to device ordinal 5
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 9 to device ordinal 6
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 9 to device ordinal 7
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 10 to device ordinal 0
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 10 to device ordinal 1
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 10 to device ordinal 2
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 10 to device ordinal 3
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 10 to device ordinal 4
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 10 to device ordinal 5
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 10 to device ordinal 6
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 10 to device ordinal 7
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 11 to device ordinal 0
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 11 to device ordinal 1
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 11 to device ordinal 2
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 11 to device ordinal 3
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 11 to device ordinal 4
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 11 to device ordinal 5
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 11 to device ordinal 6
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 11 to device ordinal 7
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 12 to device ordinal 0
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 12 to device ordinal 1
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 12 to device ordinal 2
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 12 to device ordinal 3
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 12 to device ordinal 4
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 12 to device ordinal 5
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 12 to device ordinal 6
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 12 to device ordinal 7
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 13 to device ordinal 0
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 13 to device ordinal 1
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 13 to device ordinal 2
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 13 to device ordinal 3
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 13 to device ordinal 4
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 13 to device ordinal 5
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 13 to device ordinal 6
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 13 to device ordinal 7
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 14 to device ordinal 0
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 14 to device ordinal 1
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 14 to device ordinal 2
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 14 to device ordinal 3
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 14 to device ordinal 4
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 14 to device ordinal 5
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 14 to device ordinal 6
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 14 to device ordinal 7
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 15 to device ordinal 0
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 15 to device ordinal 1
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 15 to device ordinal 2
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 15 to device ordinal 3
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 15 to device ordinal 4
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 15 to device ordinal 5
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 15 to device ordinal 6
I tensorflow/core/common_runtime/gpu/gpu_device.cc:855] cannot enable peer access from device ordinal 15 to device ordinal 7
I tensorflow/core/common_runtime/gpu/gpu_device.cc:972] DMA: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 0:   Y Y Y Y Y Y Y Y N N N N N N N N
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 1:   Y Y Y Y Y Y Y Y N N N N N N N N
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 2:   Y Y Y Y Y Y Y Y N N N N N N N N
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 3:   Y Y Y Y Y Y Y Y N N N N N N N N
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 4:   Y Y Y Y Y Y Y Y N N N N N N N N
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 5:   Y Y Y Y Y Y Y Y N N N N N N N N
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 6:   Y Y Y Y Y Y Y Y N N N N N N N N
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 7:   Y Y Y Y Y Y Y Y N N N N N N N N
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 8:   N N N N N N N N Y Y Y Y Y Y Y Y
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 9:   N N N N N N N N Y Y Y Y Y Y Y Y
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 10:   N N N N N N N N Y Y Y Y Y Y Y Y
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 11:   N N N N N N N N Y Y Y Y Y Y Y Y
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 12:   N N N N N N N N Y Y Y Y Y Y Y Y
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 13:   N N N N N N N N Y Y Y Y Y Y Y Y
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 14:   N N N N N N N N Y Y Y Y Y Y Y Y
I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] 15:   N N N N N N N N Y Y Y Y Y Y Y Y
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:0) -> (device: 0, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:1) -> (device: 1, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:2) -> (device: 2, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:3) -> (device: 3, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:4) -> (device: 4, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:5) -> (device: 5, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:6) -> (device: 6, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:7) -> (device: 7, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:8) -> (device: 8, name: TITAN X (Pascal), pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:9) -> (device: 9, name: TITAN X (Pascal), pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:10) -> (device: 10, name: TITAN X (Pascal), pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:11) -> (device: 11, name: TITAN X (Pascal), pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:12) -> (device: 12, name: TITAN X (Pascal), pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:13) -> (device: 13, name: TITAN X (Pascal), pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:14) -> (device: 14, name: TITAN X (Pascal), pci bus id: 0000:00:00.0)
I tensorflow/core/common_runtime/gpu/gpu_device.cc:1041] Creating TensorFlow device (/gpu:15) -> (device: 15, name: TITAN X (Pascal), pci bus id: 0000:00:00.0)
Device mapping:
/job:localhost/replica:0/task:0/gpu:0 -> device: 0, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:1 -> device: 1, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:2 -> device: 2, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:3 -> device: 3, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:4 -> device: 4, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:5 -> device: 5, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:6 -> device: 6, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:7 -> device: 7, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:8 -> device: 8, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:9 -> device: 9, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:10 -> device: 10, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:11 -> device: 11, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:12 -> device: 12, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:13 -> device: 13, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:14 -> device: 14, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:15 -> device: 15, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
I tensorflow/core/common_runtime/direct_session.cc:252] Device mapping:
/job:localhost/replica:0/task:0/gpu:0 -> device: 0, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:1 -> device: 1, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:2 -> device: 2, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:3 -> device: 3, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:4 -> device: 4, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:5 -> device: 5, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:6 -> device: 6, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:7 -> device: 7, name: GeForce GTX TITAN X, pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:8 -> device: 8, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:9 -> device: 9, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:10 -> device: 10, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:11 -> device: 11, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:12 -> device: 12, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:13 -> device: 13, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:14 -> device: 14, name: TITAN X (Pascal), pci bus id: 0000:00:00.0
/job:localhost/replica:0/task:0/gpu:15 -> device: 15, name: TITAN X (Pascal), pci bus id: 0000:00:00.0

add: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add: /job:localhost/replica:0/task:0/gpu:0
add_1: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_1: /job:localhost/replica:0/task:0/gpu:0
add_2: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_2: /job:localhost/replica:0/task:0/gpu:0
add_3: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_3: /job:localhost/replica:0/task:0/gpu:0
add_4: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_4: /job:localhost/replica:0/task:0/gpu:0
add_5: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_5: /job:localhost/replica:0/task:0/gpu:0
add_6: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_6: /job:localhost/replica:0/task:0/gpu:0
add_7: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_7: /job:localhost/replica:0/task:0/gpu:0
add_8: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_8: /job:localhost/replica:0/task:0/gpu:0
add_9: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_9: /job:localhost/replica:0/task:0/gpu:0
add_10: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_10: /job:localhost/replica:0/task:0/gpu:0
add_11: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_11: /job:localhost/replica:0/task:0/gpu:0
add_12: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_12: /job:localhost/replica:0/task:0/gpu:0
add_13: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_13: /job:localhost/replica:0/task:0/gpu:0
add_14: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] add_14: /job:localhost/replica:0/task:0/gpu:0
ones_15: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_15: /job:localhost/replica:0/task:0/gpu:0
ones_14: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_14: /job:localhost/replica:0/task:0/gpu:0
ones_13: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_13: /job:localhost/replica:0/task:0/gpu:0
ones_12: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_12: /job:localhost/replica:0/task:0/gpu:0
ones_11: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_11: /job:localhost/replica:0/task:0/gpu:0
ones_10: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_10: /job:localhost/replica:0/task:0/gpu:0
ones_9: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_9: /job:localhost/replica:0/task:0/gpu:0
ones_8: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_8: /job:localhost/replica:0/task:0/gpu:0
ones_7: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_7: /job:localhost/replica:0/task:0/gpu:0
ones_6: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_6: /job:localhost/replica:0/task:0/gpu:0
ones_5: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_5: /job:localhost/replica:0/task:0/gpu:0
ones_4: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_4: /job:localhost/replica:0/task:0/gpu:0
ones_3: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_3: /job:localhost/replica:0/task:0/gpu:0
ones_2: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_2: /job:localhost/replica:0/task:0/gpu:0
ones_1: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones_1: /job:localhost/replica:0/task:0/gpu:0
ones: /job:localhost/replica:0/task:0/gpu:0
I tensorflow/core/common_runtime/simple_placer.cc:819] ones: /job:localhost/replica:0/task:0/gpu:0
Failed to get the number of CUDA devices: CUDA driver version is insufficient for CUDA runtime version
[16.0]
root@cbb77e9ace8b:/app#