Last active
July 13, 2018 17:39
-
-
Save sub-mod/036dfc3bbff4ce3eab5ba79ede9126d9 to your computer and use it in GitHub Desktop.
tensorflow build notes
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
need to install bazel | |
https://copr.fedorainfracloud.org/coprs/vbatts/bazel/ | |
oc new-app --template=tensorflow-build-image --param=APPLICATION_NAME=tf-fedora27-builder-image-36 --param=S2I_IMAGE=registry.fedoraproject.org/f27/s2i-core --param=DOCKER_FILE_PATH=Dockerfile.fedora27 --param=NB_PYTHON_VER=3.6 --param=VERSION=4 | |
oc new-app --template=tensorflow-build-job --param=APPLICATION_NAME=tf-build-fc27-363 --param=BUILDER_IMAGESTREAM=docker-registry.default.svc:5000/dh-prod-analytics-factory/tf-fedora27-builder-image-36:3 --param=NB_PYTHON_VER=3.6 --param=GIT_TOKEN= --param=CUSTOM_BUILD="gcc -march=native -Q --help=target|grep march && gcc -march=native -E -v - </dev/null 2>&1 | grep cc1 && grep flags -m1 /proc/cpuinfo | cut -d ":" -f 2 | tr '[:upper:]' '[:lower:]' && lscpu && bazel build -c opt --copt='-march=core-avx2' --cxxopt='-D_GLIBCXX_USE_CXX11_ABI=0' --local_resources 2048,2.0,1.0 --verbose_failures //tensorflow/tools/pip_package:build_pip_package" | |
gcc -march=native -Q --help=target|grep march && lscpu | |
echo "============================" | |
grep flags -m1 /proc/cpuinfo | cut -d ":" -f 2 | tr '[:upper:]' '[:lower:]' | |
echo "============================" | |
#To be compatible with as wide a range of machines as possible, | |
#TensorFlow defaults to only using SSE4.1 SIMD | |
# which flags -march=native will activate? | |
# native means the architecture of the compiling machine which is not very applicable when you cross-compile | |
echo "============================" | |
gcc -march=native -E -v - </dev/null 2>&1 | grep cc1 | |
gcc -### -E - -march=native 2>&1 | sed -r '/cc1/!d;s/(")|(^.* - )|( -mno-[^\ ]+)//g' | |
echo "============================" | |
gcc -march=native -Q --help=target | grep enable | |
echo "============================" | |
#gcc with march=native and ask it what flags it would use : in short | |
echo "============================" | |
gcc -march=native -Q --help=target|grep march | cut -d " " -f 2 | |
echo "============================" | |
#skylake-avx512 | |
A=$(gcc -march=native -Q --help=target|grep march) | |
B=$(echo "${A##* }" | tr -s [:space:] | sed -e 's/^\s*//' -e '/^$/d') | |
echo $B | |
tensorflow/tools/ci_build/builds/configured CPU bazel build -c opt --copt=-mavx --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" tensorflow/tools/pip_package:build_pip_package | |
bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg | |
git clone --branch=r1.9 --depth=1 https://github.com/tensorflow/tensorflow.git . | |
# du -h tensorflow-1.9.0rc0-cp27-cp27mu-linux_x86_64.whl | |
45M tensorflow-1.9.0rc0-cp27-cp27mu-linux_x86_64.whl | |
root@39f9c07a383c:/tmp/tensorflow_pkg# pip install tensorflow-1.9.0rc0-cp27-cp27mu-linux_x86_64.whl | |
Processing ./tensorflow-1.9.0rc0-cp27-cp27mu-linux_x86_64.whl | |
Collecting protobuf>=3.4.0 (from tensorflow==1.9.0rc0) | |
Downloading https://files.pythonhosted.org/packages/9d/61/54c3a9cfde6ffe0ca6a1786ddb8874263f4ca32e7693ad383bd8cf935015/protobuf-3.5.2.post1-cp27-cp27mu-manylinux1_x86_64.whl (6.4MB) | |
100% |████████████████████████████████| 6.4MB 5.7MB/s | |
Collecting astor>=0.6.0 (from tensorflow==1.9.0rc0) | |
Downloading https://files.pythonhosted.org/packages/b2/91/cc9805f1ff7b49f620136b3a7ca26f6a1be2ed424606804b0fbcf499f712/astor-0.6.2-py2.py3-none-any.whl | |
Collecting backports.weakref>=1.0rc1 (from tensorflow==1.9.0rc0) | |
Downloading https://files.pythonhosted.org/packages/88/ec/f598b633c3d5ffe267aaada57d961c94fdfa183c5c3ebda2b6d151943db6/backports.weakref-1.0.post1-py2.py3-none-any.whl | |
Collecting setuptools<=39.1.0 (from tensorflow==1.9.0rc0) | |
Downloading https://files.pythonhosted.org/packages/8c/10/79282747f9169f21c053c562a0baa21815a8c7879be97abd930dbcf862e8/setuptools-39.1.0-py2.py3-none-any.whl (566kB) | |
100% |████████████████████████████████| 573kB 14.5MB/s | |
Requirement already satisfied: wheel in /usr/lib/python2.7/site-packages (from tensorflow==1.9.0rc0) (0.31.1) | |
Requirement already satisfied: mock>=2.0.0 in /usr/lib/python2.7/site-packages (from tensorflow==1.9.0rc0) (2.0.0) | |
Requirement already satisfied: enum34>=1.1.6 in /usr/lib/python2.7/site-packages (from tensorflow==1.9.0rc0) (1.1.6) | |
Collecting gast>=0.2.0 (from tensorflow==1.9.0rc0) | |
Downloading https://files.pythonhosted.org/packages/5c/78/ff794fcae2ce8aa6323e789d1f8b3b7765f601e7702726f430e814822b96/gast-0.2.0.tar.gz | |
Collecting termcolor>=1.1.0 (from tensorflow==1.9.0rc0) | |
Downloading https://files.pythonhosted.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz | |
Collecting absl-py>=0.1.6 (from tensorflow==1.9.0rc0) | |
Downloading https://files.pythonhosted.org/packages/57/8d/6664518f9b6ced0aa41cf50b989740909261d4c212557400c48e5cda0804/absl-py-0.2.2.tar.gz (82kB) | |
100% |████████████████████████████████| 92kB 17.3MB/s | |
Requirement already satisfied: six>=1.10.0 in /usr/lib/python2.7/site-packages (from tensorflow==1.9.0rc0) (1.11.0) | |
Collecting tensorboard<1.9.0,>=1.8.0 (from tensorflow==1.9.0rc0) | |
Downloading https://files.pythonhosted.org/packages/4d/1e/3bfb48ff165e331c0c5fdca5de79d497fa5d71f3cb2eee2733ff22e898df/tensorboard-1.8.0-py2-none-any.whl (3.1MB) | |
100% |████████████████████████████████| 3.1MB 5.3MB/s | |
Collecting grpcio>=1.8.6 (from tensorflow==1.9.0rc0) | |
Downloading https://files.pythonhosted.org/packages/1d/0d/146582f71161a0074dda2378617ae5f7e2c3d6cf62d4588eb586c1d6b675/grpcio-1.12.1-cp27-cp27mu-manylinux1_x86_64.whl (8.9MB) | |
100% |████████████████████████████████| 8.9MB 3.0MB/s | |
Requirement already satisfied: numpy>=1.13.3 in /usr/lib64/python2.7/site-packages (from tensorflow==1.9.0rc0) (1.14.4) | |
Requirement already satisfied: funcsigs>=1; python_version < "3.3" in /usr/lib/python2.7/site-packages (from mock>=2.0.0->tensorflow==1.9.0rc0) (1.0.2) | |
Requirement already satisfied: pbr>=0.11 in /usr/lib/python2.7/site-packages (from mock>=2.0.0->tensorflow==1.9.0rc0) (4.0.4) | |
Collecting bleach==1.5.0 (from tensorboard<1.9.0,>=1.8.0->tensorflow==1.9.0rc0) | |
Downloading https://files.pythonhosted.org/packages/33/70/86c5fec937ea4964184d4d6c4f0b9551564f821e1c3575907639036d9b90/bleach-1.5.0-py2.py3-none-any.whl | |
Requirement already satisfied: futures>=3.1.1; python_version < "3" in /usr/lib/python2.7/site-packages (from tensorboard<1.9.0,>=1.8.0->tensorflow==1.9.0rc0) (3.2.0) | |
Requirement already satisfied: markdown>=2.6.8 in /usr/lib64/python2.7/site-packages (from tensorboard<1.9.0,>=1.8.0->tensorflow==1.9.0rc0) (2.6.11) | |
Collecting werkzeug>=0.11.10 (from tensorboard<1.9.0,>=1.8.0->tensorflow==1.9.0rc0) | |
Downloading https://files.pythonhosted.org/packages/20/c4/12e3e56473e52375aa29c4764e70d1b8f3efa6682bef8d0aae04fe335243/Werkzeug-0.14.1-py2.py3-none-any.whl (322kB) | |
100% |████████████████████████████████| 327kB 13.4MB/s | |
Collecting html5lib==0.9999999 (from tensorboard<1.9.0,>=1.8.0->tensorflow==1.9.0rc0) | |
Downloading https://files.pythonhosted.org/packages/ae/ae/bcb60402c60932b32dfaf19bb53870b29eda2cd17551ba5639219fb5ebf9/html5lib-0.9999999.tar.gz (889kB) | |
100% |████████████████████████████████| 890kB 12.1MB/s | |
Building wheels for collected packages: gast, termcolor, absl-py, html5lib | |
Running setup.py bdist_wheel for gast ... done | |
Stored in directory: /home/default/.cache/pip/wheels/9a/1f/0e/3cde98113222b853e98fc0a8e9924480a3e25f1b4008cedb4f | |
Running setup.py bdist_wheel for termcolor ... done | |
Stored in directory: /home/default/.cache/pip/wheels/7c/06/54/bc84598ba1daf8f970247f550b175aaaee85f68b4b0c5ab2c6 | |
Running setup.py bdist_wheel for absl-py ... done | |
Stored in directory: /home/default/.cache/pip/wheels/a0/f8/e9/1933dbb3447ea6ef57062fd5461cb118deb8c2ed074e8344bf | |
Running setup.py bdist_wheel for html5lib ... done | |
Stored in directory: /home/default/.cache/pip/wheels/50/ae/f9/d2b189788efcf61d1ee0e36045476735c838898eef1cad6e29 | |
Successfully built gast termcolor absl-py html5lib | |
Installing collected packages: setuptools, protobuf, astor, backports.weakref, gast, termcolor, absl-py, html5lib, bleach, werkzeug, tensorboard, grpcio, tensorflow | |
Found existing installation: setuptools 39.2.0 | |
Uninstalling setuptools-39.2.0: | |
Successfully uninstalled setuptools-39.2.0 | |
Successfully installed absl-py-0.2.2 astor-0.6.2 backports.weakref-1.0.post1 bleach-1.5.0 gast-0.2.0 grpcio-1.12.1 html5lib-0.9999999 protobuf-3.5.2.post1 setuptools-39.1.0 tensorboard-1.8.0 tensorflow-1.9.0rc0 termcolor-1.1.0 werkzeug-0.14.1 | |
test | |
=========== | |
import tensorflow as tf | |
a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a') | |
b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b') | |
c = tf.matmul(a, b) | |
sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) | |
print(sess.run(c)) | |
yum install -y which python gcc gcc-c++ | |
yum install -y findutils x86info cpuid dmidecode | |
which python | |
curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py" | |
python get-pip.py | |
pip install tensorflow --user | |
python -c "import tensorflow as tf;a=tf.constant([1.0,2.0,3.0,4.0,5.0,6.0],shape=[2,3],name='a');b=tf.constant([1.0,2.0,3.0,4.0,5.0,6.0],shape=[3,2],name='b');c=tf.matmul(a,b);sess=tf.Session(config=tf.ConfigProto(log_device_placement=True));print(sess.run(c))" | |
{ | |
"apiVersion": "batch/v1", | |
"kind": "Job", | |
"metadata": { | |
"labels": { | |
"app": "pyspark-ceph-connection" | |
}, | |
"name": "pyspark-ceph-connection" | |
}, | |
"spec": { | |
"backoffLimit": 6, | |
"completions": 1, | |
"parallelism": 1, | |
"template": { | |
"metadata": { | |
"labels": { | |
"job-name": "pyspark-ceph-connection" | |
} | |
}, | |
"spec": { | |
"containers": [ | |
{ | |
"env": [ | |
{ | |
"name": "POD_NAME", | |
"valueFrom": { | |
"fieldRef": { | |
"apiVersion": "v1", | |
"fieldPath": "metadata.name" | |
} | |
} | |
} | |
], | |
"image": "docker-registry.default.svc:5000/dh-prod-analytics-factory/pyspark-ceph-connection", | |
"imagePullPolicy": "Always", | |
"name": "spark-sample", | |
"resources": {}, | |
"terminationMessagePath": "/dev/termination-log", | |
"terminationMessagePolicy": "File" | |
} | |
], | |
"restartPolicy": "OnFailure", | |
} | |
} | |
} | |
} | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment