yaroslavvb

# XLA compilation controlled by "compile_ops" option
# compile_ops=False: 4.39 sec
# compile_ops=True: 0.90 sec


import os
os.environ['CUDA_VISIBLE_DEVICES']=''
import tensorflow as tf

yaroslavvb

# make things wide
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))

from IPython.display import clear_output, Image, display, HTML

def strip_consts(graph_def, max_const_size=32):
    """Strip large constant values from graph_def."""
    strip_def = tf.GraphDef()
    for n0 in graph_def.node:

yaroslavvb

# Try to copy "a" value to "c" while simultaneously adding vector of 1's to a.
# If the copy is started before the first assign_add, the copied value will be inconsistent.
#
# Running it on macbook my "c" ends up with a mix of values between 1 and 6
#
#
# 16.017478 copy 1 (0) starting
# 17.006894 write 1 (0) starting
# 28.431654 write 1 ending (11.4247 sec)
# 29.436692 write 1 (1) starting

yaroslavvb

"""Example of barrier implementation using TensorFlow shared variables.

All workers synchronize on barrier, copy global parameters to local versions
and increment global parameter variable asynchronously. Should see something
like this:

bash> killall python
bash> python simple_barrier.py --num_workers=4
worker  0, local_param  4 global_param  5
worker  2, local_param  4 global_param  7

yaroslavvb

# Dependencies:
# portpicker (pip install portpicker)
# tcmalloc4 (sudo apt-get install google-perftools)
# TF 0.12
#
#
# Benchmarks on Xeon E5-2630 v3 @ 2.40GHz
#
# export LD_PRELOAD=/usr/lib/libtcmalloc.so.4
# python benchmark_grpc_recv.py --data_mb=128

yaroslavvb

from google.protobuf import text_format
from tensorflow.core.framework import op_def_pb2
ops = op_def_pb2.OpList()

ops_text = open("/local_home/yaroslav/tensorflow.git/tensorflow/tensorflow/core/ops/ops.pbtxt").read()
text_format.Merge(ops_text, ops)
print(len(ops.op))

yaroslavvb

# Benchmark transferring data from TF into Python runtime
#
## Dependencies:
# portpicker (pip install portpicker)
# tcmalloc4 (sudo apt-get install google-perftools)
# TF 0.12  (for var.read_value(), ones_initializer())
#
# On Linux default malloc is slow
# sudo apt-get install google-perftools
# export LD_PRELOAD="/usr/lib/libtcmalloc.so.4"

yaroslavvb

# export LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH"

import tensorflow as tf
a = tf.ones((10, 10))
b = tf.ones((10, 10))
c = tf.matmul(a, b)
ss = tf.contrib.stat_summarizer.NewStatSummarizer(tf.get_default_graph().as_graph_def().SerializeToString())

sess = tf.Session()
for i in range(10):

yaroslavvb

#!/usr/bin/env python
# Benchmark transferring data, part of troubleshooting https://github.com/tensorflow/tensorflow/issues/6116
#
# Take a independent workers communicating with b parameter shards
# Each worker tries to add to variables stored on parameter server as fast as
# possible.
# 
# macbook
# ps=1: 1.6 GB/s
# ps=2: 2.6 GB/s

yaroslavvb

import tensorflow as tf
a = tf.ones((10, 10))
b = tf.ones((10, 10))
c = tf.matmul(a, b)
ss = tf.contrib.stat_summarizer.NewStatSummarizer(tf.get_default_graph().as_graph_def().SerializeToString())

sess = tf.Session()
for i in range(10):
  run_metadata = tf.RunMetadata()
  run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)