Using Tensorflow 2.1.0 with CUDA 10.0

tf_2_10_cuda10_0.md

Use cuda:10.0-cudnn7-devel-centos7 container

docker run -it -u 0 docker.io/nvidia/cuda:10.0-cudnn7-devel-centos7 /bin/bash

Setup LD_LIBRARY_PATH and Links

yum install -y centos-release-scl
yum install -y devtoolset-7 rh-python36 mlocate
source scl_source enable devtoolset-7 rh-python36
pip install --upgrade pip
pip install tensorflow-gpu

export PATH=/usr/local/cuda-10.0/bin${PATH:+:${PATH}}
export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64/${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
export LD_LIBRARY_PATH=/usr/local/cuda-10.0/extras/CUPTI/lib64:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/usr/local/cuda-10.0/targets/x86_64-linux/lib/:$LD_LIBRARY_PATH

ln -s /usr/local/cuda-10.0/targets/x86_64-linux/lib/libcudart.so.10.0.130 /usr/local/cuda-10.0/lib64/libcudart.so.10.1
ln -s /usr/local/cuda-10.0/targets/x86_64-linux/lib/libcudart.so.10.0.130 /usr/local/cuda-10.0/lib64/libcudart.so.10
ln -s /usr/local/cuda-10.0/targets/x86_64-linux/lib/libcufft.so.10.0.145 /usr/local/cuda-10.0/lib64/libcufft.so.10
ln -s /usr/local/cuda-10.0/targets/x86_64-linux/lib/libcurand.so.10.0.130 /usr/local/cuda-10.0/lib64/libcurand.so.10
ln -s /usr/local/cuda-10.0/targets/x86_64-linux/lib/libcusolver.so.10.0.130 /usr/local/cuda-10.0/lib64/libcusolver.so.10
ln -s /usr/local/cuda-10.0/targets/x86_64-linux/lib/libcusparse.so.10.0.130 /usr/local/cuda-10.0/lib64/libcusparse.so.10
ln -s /usr/local/cuda-10.0/targets/x86_64-linux/lib/libcublas.so.10.0.130 /usr/local/cuda-10.0/lib64/libcublas.so.10

Logs

[root@62c8308150b4 idna]# cat /etc/redhat-release
CentOS Linux release 7.7.1908 (Core)
[root@62c8308150b4 idna]# gcc -v
Using built-in specs.
COLLECT_GCC=gcc
COLLECT_LTO_WRAPPER=/opt/rh/devtoolset-7/root/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper
Target: x86_64-redhat-linux
Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,fortran,lto --prefix=/opt/rh/devtoolset-7/root/usr --mandir=/opt/rh/devtoolset-7/root/usr/share/man --infodir=/opt/rh/devtoolset-7/root/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --enable-plugin --with-linker-hash-style=gnu --enable-initfini-array --with-default-libstdcxx-abi=gcc4-compatible --with-isl=/builddir/build/BUILD/gcc-7.3.1-20180303/obj-x86_64-redhat-linux/isl-install --enable-libmpx --enable-gnu-indirect-function --with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux
Thread model: posix
gcc version 7.3.1 20180303 (Red Hat 7.3.1-5) (GCC)
[root@62c8308150b4 idna]# python -V
Python 3.6.9
[root@62c8308150b4 idna]# python
Python 3.6.9 (default, Nov 11 2019, 11:24:16)
[GCC 4.8.5 20150623 (Red Hat 4.8.5-39)] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import tensorflow as tf
2020-03-23 20:49:45.380993: W tensorflow/stream_executor/platform/default/dso_loader.cc:55] Could not load dynamic library 'libnvinfer.so.6'; dlerror: libnvinfer.so.6: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.0/targets/x86_64-linux/lib/:/usr/local/cuda-10.0/extras/CUPTI/lib64:/usr/local/cuda-10.0/lib64/:/usr/local/cuda-10.0/targets/x86_64-linux/lib/:/usr/local/cuda-10.0/extras/CUPTI/lib64:/usr/local/cuda-10.0/lib64:/usr/local/cuda-10.0/lib64:/opt/rh/rh-python36/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/opt/rh/devtoolset-7/root/usr/lib64/dyninst:/opt/rh/devtoolset-7/root/usr/lib/dyninst:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
2020-03-23 20:49:45.381071: W tensorflow/stream_executor/platform/default/dso_loader.cc:55] Could not load dynamic library 'libnvinfer_plugin.so.6'; dlerror: libnvinfer_plugin.so.6: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.0/targets/x86_64-linux/lib/:/usr/local/cuda-10.0/extras/CUPTI/lib64:/usr/local/cuda-10.0/lib64/:/usr/local/cuda-10.0/targets/x86_64-linux/lib/:/usr/local/cuda-10.0/extras/CUPTI/lib64:/usr/local/cuda-10.0/lib64:/usr/local/cuda-10.0/lib64:/opt/rh/rh-python36/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/opt/rh/devtoolset-7/root/usr/lib64/dyninst:/opt/rh/devtoolset-7/root/usr/lib/dyninst:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
2020-03-23 20:49:45.381081: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:30] Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
>>> tf.__version__
'2.1.0'
>>>

>>> import tensorflow as tf
>>> from tensorflow.python.client import device_lib
>>>
>>> def get_available_devices():
...     local_device_protos = device_lib.list_local_devices()
...     return [x.name for x in local_device_protos]
...
>>>
>>> print("TF version: ", tf.__version__)
TF version:  2.1.0
>>> print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
2020-03-23 20:53:59.572505: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1
2020-03-23 20:53:59.597525: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties:
pciBusID: 0000:65:00.0 name: GeForce RTX 2080 Ti computeCapability: 7.5
coreClock: 1.545GHz coreCount: 68 deviceMemorySize: 10.76GiB deviceMemoryBandwidth: 573.69GiB/s
2020-03-23 20:53:59.597686: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-03-23 20:53:59.598561: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-03-23 20:53:59.599340: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-03-23 20:53:59.599543: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-03-23 20:53:59.600534: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-03-23 20:53:59.601297: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-03-23 20:53:59.603745: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-03-23 20:53:59.604956: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0
Num GPUs Available:  1
>>> print("Devices Available: ", get_available_devices())
2020-03-23 20:54:00.384887: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 AVX512F FMA
2020-03-23 20:54:00.402239: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 3500000000 Hz
2020-03-23 20:54:00.403379: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x421e100 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2020-03-23 20:54:00.403403: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version
2020-03-23 20:54:00.503658: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x42840a0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
2020-03-23 20:54:00.503691: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): GeForce RTX 2080 Ti, Compute Capability 7.5
2020-03-23 20:54:00.504176: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties:
pciBusID: 0000:65:00.0 name: GeForce RTX 2080 Ti computeCapability: 7.5
coreClock: 1.545GHz coreCount: 68 deviceMemorySize: 10.76GiB deviceMemoryBandwidth: 573.69GiB/s
2020-03-23 20:54:00.504214: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-03-23 20:54:00.504224: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10
2020-03-23 20:54:00.504232: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10
2020-03-23 20:54:00.504240: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10
2020-03-23 20:54:00.504248: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10
2020-03-23 20:54:00.504256: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10
2020-03-23 20:54:00.504265: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-03-23 20:54:00.504913: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0
2020-03-23 20:54:00.504935: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
2020-03-23 20:54:00.505546: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-03-23 20:54:00.505555: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102]      0
2020-03-23 20:54:00.505561: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0:   N
2020-03-23 20:54:00.506264: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/device:GPU:0 with 10128 MB memory) -> physical GPU (device: 0, name: GeForce RTX 2080 Ti, pci bus id: 0000:65:00.0, compute capability: 7.5)
Devices Available:  ['/device:CPU:0', '/device:XLA_CPU:0', '/device:XLA_GPU:0', '/device:GPU:0']