Performance decrease in TensorFlow 1.9 for large graphs #23847

flowers-nasnet.sh

#!/usr/bin/env bash
[ ! -d "models" ] && git clone https://github.com/tensorflow/models.git
cd models/research/slim/
MODEL_NAME="nasnet_large"
TRAIN_DIR=/tmp/flowers-models/${MODEL_NAME}
DATASET_DIR=/tmp/flowers
export TF_CUDNN_WORKSPACE_LIMIT_IN_MB=500
export CUDA_DEVICE_ORDER="PCI_BUS_ID"
export CUDA_VISIBLE_DEVICES="0,1"

python download_and_convert_data.py \
    --dataset_name=flowers \
    --dataset_dir=${DATASET_DIR}

python train_image_classifier.py \
    --train_dir=${TRAIN_DIR}/all \
    --dataset_name=flowers \
    --dataset_split_name=train \
    --dataset_dir=${DATASET_DIR} \
    --model_name=${MODEL_NAME} \
    --max_number_of_steps=500 \
    --batch_size=16 \
    --learning_rate=0.015 \
    --save_interval_secs=60000 \
    --save_summaries_secs=60 \
    --log_every_n_steps=10 \
    --optimizer=rmsprop \
    --num_preprocessing_threads 4 \
    --num_readers 8 \
    --moving_average_decay=0.9999 \
    --weight_decay=0.00005 \
    --learning_rate_decay_type=exponential \
    --learning_rate_decay_factor=0.97 \
    --label_smoothing=0.1

git-bisect-layout_optimizer-on-off.log

devel commit: d4976f754009d084514f4308d3bfc7dc3a106e29 (layout_optimizer=ON (default))
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global step 10: loss = 77.8448 (1.096 sec/step)
INFO:tensorflow:global_step/sec: 0.222987
INFO:tensorflow:global step 20: loss = 41.6071 (1.079 sec/step)
INFO:tensorflow:global step 30: loss = 55.1761 (1.083 sec/step)
INFO:tensorflow:global step 40: loss = 25.6517 (1.078 sec/step)
INFO:tensorflow:global step 50: loss = 22.6773 (1.084 sec/step)
INFO:tensorflow:global step 60: loss = 28.2089 (1.078 sec/step)
INFO:tensorflow:global_step/sec: 0.933322
INFO:tensorflow:global step 70: loss = 48.3094 (1.082 sec/step)
INFO:tensorflow:global step 80: loss = 24.6723 (1.079 sec/step)
INFO:tensorflow:global step 90: loss = 46.9032 (1.065 sec/step)
INFO:tensorflow:global step 100: loss = 30.3739 (1.062 sec/step)
INFO:tensorflow:global step 110: loss = 33.5094 (1.073 sec/step)
INFO:tensorflow:global step 120: loss = 31.5804 (1.063 sec/step)
INFO:tensorflow:global_step/sec: 0.91668
INFO:tensorflow:global step 130: loss = 19.0180 (1.077 sec/step)
INFO:tensorflow:global step 140: loss = 29.7252 (1.071 sec/step)
INFO:tensorflow:global step 150: loss = 18.1041 (1.069 sec/step)
INFO:tensorflow:global step 160: loss = 17.3742 (1.076 sec/step)
INFO:tensorflow:global step 170: loss = 31.1170 (1.050 sec/step)
INFO:tensorflow:global_step/sec: 0.933334
INFO:tensorflow:global step 180: loss = 25.0502 (1.065 sec/step)
INFO:tensorflow:global step 190: loss = 22.6136 (1.073 sec/step)
INFO:tensorflow:global step 200: loss = 21.5477 (1.067 sec/step)


devel commit: d4976f754009d084514f4308d3bfc7dc3a106e29 (layout_optimizer=OFF)
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global step 10: loss = 86.5514 (0.777 sec/step)
INFO:tensorflow:global_step/sec: 0.366127
INFO:tensorflow:global step 20: loss = 40.5931 (0.764 sec/step)
INFO:tensorflow:global step 30: loss = 82.1632 (0.772 sec/step)
INFO:tensorflow:global step 40: loss = 57.8050 (0.755 sec/step)
INFO:tensorflow:global step 50: loss = 38.0561 (0.752 sec/step)
INFO:tensorflow:global step 60: loss = 31.1644 (0.763 sec/step)
INFO:tensorflow:global step 70: loss = 41.3120 (0.759 sec/step)
INFO:tensorflow:global step 80: loss = 57.0420 (0.771 sec/step)
INFO:tensorflow:global step 90: loss = 22.8730 (0.754 sec/step)
INFO:tensorflow:global_step/sec: 1.29997
INFO:tensorflow:global step 100: loss = 19.0714 (0.758 sec/step)
INFO:tensorflow:global step 110: loss = 24.2251 (0.768 sec/step)
INFO:tensorflow:global step 120: loss = 18.9120 (0.767 sec/step)
INFO:tensorflow:global step 130: loss = 18.2566 (0.767 sec/step)
INFO:tensorflow:global step 140: loss = 17.1418 (0.771 sec/step)
INFO:tensorflow:global step 150: loss = 24.1894 (0.756 sec/step)
INFO:tensorflow:global step 160: loss = 32.5523 (0.757 sec/step)
INFO:tensorflow:global step 170: loss = 17.5002 (0.769 sec/step)
INFO:tensorflow:global_step/sec: 1.30003
INFO:tensorflow:global step 180: loss = 31.7285 (0.750 sec/step)
INFO:tensorflow:global step 190: loss = 22.0445 (0.773 sec/step)
INFO:tensorflow:global step 200: loss = 19.4750 (0.763 sec/step)

tensorflow/tensorflow:1.12.0-devel-gpu-py3
1.12.0
devel commit: a6d8ffae097d0132989ae4688d224121ec6d8f35 (layout_optimizer=OFF)
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global step 10: loss = 51.9313 (0.788 sec/step)
INFO:tensorflow:global_step/sec: 0.219973
INFO:tensorflow:global step 20: loss = 63.9743 (0.776 sec/step)
INFO:tensorflow:global step 30: loss = 48.4911 (0.790 sec/step)
INFO:tensorflow:global step 40: loss = 86.4540 (0.805 sec/step)
INFO:tensorflow:global step 50: loss = 27.5662 (0.804 sec/step)
INFO:tensorflow:global step 60: loss = 28.5045 (0.788 sec/step)
INFO:tensorflow:global step 70: loss = 27.8477 (0.783 sec/step)
INFO:tensorflow:global step 80: loss = 26.0450 (1.097 sec/step)
INFO:tensorflow:global_step/sec: 1.24996
INFO:tensorflow:global step 90: loss = 22.9502 (0.811 sec/step)
INFO:tensorflow:global step 100: loss = 34.4691 (0.799 sec/step)
INFO:tensorflow:global step 110: loss = 18.2673 (0.796 sec/step)
INFO:tensorflow:global step 120: loss = 27.0335 (0.808 sec/step)
INFO:tensorflow:global step 130: loss = 21.4064 (0.800 sec/step)
INFO:tensorflow:global step 140: loss = 49.4867 (0.795 sec/step)
INFO:tensorflow:global step 150: loss = 29.2122 (0.809 sec/step)
INFO:tensorflow:global step 160: loss = 18.0968 (0.802 sec/step)
INFO:tensorflow:global_step/sec: 1.23337
INFO:tensorflow:global step 170: loss = 25.6444 (0.796 sec/step)
INFO:tensorflow:global step 180: loss = 25.8588 (0.802 sec/step)
INFO:tensorflow:global step 190: loss = 18.6244 (0.809 sec/step)
INFO:tensorflow:global step 200: loss = 21.5316 (0.804 sec/step)

tensorflow/tensorflow:1.12.0-devel-gpu-py3
1.12.0
devel commit: a6d8ffae097d0132989ae4688d224121ec6d8f35 (layout_optimizer=ON (default))
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global_step/sec: 0.0787004
INFO:tensorflow:global step 10: loss = 35.8201 (1.097 sec/step)
INFO:tensorflow:global step 20: loss = 41.5376 (1.102 sec/step)
INFO:tensorflow:global step 30: loss = 36.2800 (1.094 sec/step)
INFO:tensorflow:global step 40: loss = 45.9087 (1.108 sec/step)
INFO:tensorflow:global step 50: loss = 47.8143 (1.102 sec/step)
INFO:tensorflow:global_step/sec: 0.900003
INFO:tensorflow:global step 60: loss = 22.7945 (1.099 sec/step)
INFO:tensorflow:global step 70: loss = 34.6515 (1.127 sec/step)
INFO:tensorflow:global step 80: loss = 44.4557 (1.115 sec/step)
INFO:tensorflow:global step 90: loss = 35.1953 (1.108 sec/step)
INFO:tensorflow:global step 100: loss = 19.1494 (1.102 sec/step)
INFO:tensorflow:global step 110: loss = 23.8404 (1.107 sec/step)
INFO:tensorflow:global_step/sec: 0.9
INFO:tensorflow:global step 120: loss = 18.3944 (1.079 sec/step)
INFO:tensorflow:global step 130: loss = 18.0790 (1.078 sec/step)
INFO:tensorflow:global step 140: loss = 30.3031 (1.098 sec/step)
INFO:tensorflow:global step 150: loss = 27.0260 (1.107 sec/step)
INFO:tensorflow:global step 160: loss = 18.9154 (1.092 sec/step)
INFO:tensorflow:global_step/sec: 0.899997
INFO:tensorflow:global step 170: loss = 19.1641 (1.105 sec/step)
INFO:tensorflow:global step 180: loss = 20.8932 (1.130 sec/step)
INFO:tensorflow:global step 190: loss = 33.3864 (1.091 sec/step)
INFO:tensorflow:global step 200: loss = 25.4152 (1.088 sec/step)

Mon Nov 19 08:21:01 Europe 2018
tensorflow/tensorflow:1.9.0-devel-gpu-py3
1.9.0
devel commit: 25c197e02393bd44f50079945409009dd4d434f8 (layout_optimizer=OFF)
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global_step/sec: 0.177561
INFO:tensorflow:global step 10: loss = 67.7237 (0.833 sec/step)
INFO:tensorflow:global step 20: loss = 36.5267 (0.847 sec/step)
INFO:tensorflow:global step 30: loss = 49.9012 (0.844 sec/step)
INFO:tensorflow:global step 40: loss = 25.3782 (0.819 sec/step)
INFO:tensorflow:global step 50: loss = 21.8855 (0.822 sec/step)
INFO:tensorflow:global step 60: loss = 25.6266 (0.839 sec/step)
INFO:tensorflow:global step 70: loss = 25.5132 (0.832 sec/step)
INFO:tensorflow:global step 80: loss = 27.3603 (0.826 sec/step)
INFO:tensorflow:global_step/sec: 1.2
INFO:tensorflow:global step 90: loss = 19.6148 (0.845 sec/step)
INFO:tensorflow:global step 100: loss = 19.4294 (0.829 sec/step)
INFO:tensorflow:global step 110: loss = 21.8289 (0.821 sec/step)
INFO:tensorflow:global step 120: loss = 19.5455 (0.838 sec/step)
INFO:tensorflow:global step 130: loss = 23.7997 (0.827 sec/step)
INFO:tensorflow:global step 140: loss = 17.4499 (0.824 sec/step)
INFO:tensorflow:global step 150: loss = 16.9958 (0.825 sec/step)
INFO:tensorflow:global_step/sec: 1.19995
INFO:tensorflow:global step 160: loss = 29.1579 (0.836 sec/step)
INFO:tensorflow:global step 170: loss = 16.7707 (0.828 sec/step)
INFO:tensorflow:global step 180: loss = 17.3891 (0.836 sec/step)
INFO:tensorflow:global step 190: loss = 18.0500 (0.827 sec/step)
INFO:tensorflow:global step 200: loss = 17.5038 (0.819 sec/step)

tensorflow/tensorflow:1.9.0-devel-gpu-py3
1.9.0
devel commit: 25c197e02393bd44f50079945409009dd4d434f8 (layout_optimizer=ON (default))
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global_step/sec: 0.0801179
INFO:tensorflow:global step 10: loss = 53.2845 (1.157 sec/step)
INFO:tensorflow:global step 20: loss = 46.9081 (1.133 sec/step)
INFO:tensorflow:global step 30: loss = 28.1337 (1.161 sec/step)
INFO:tensorflow:global step 40: loss = 66.8712 (1.150 sec/step)
INFO:tensorflow:global step 50: loss = 41.4689 (1.144 sec/step)
INFO:tensorflow:global_step/sec: 0.866785
INFO:tensorflow:global step 60: loss = 29.3436 (1.133 sec/step)
INFO:tensorflow:global step 70: loss = 27.5818 (1.144 sec/step)
INFO:tensorflow:global step 80: loss = 42.5981 (1.122 sec/step)
INFO:tensorflow:global step 90: loss = 21.0384 (1.132 sec/step)
INFO:tensorflow:global step 100: loss = 24.9422 (1.135 sec/step)
INFO:tensorflow:global_step/sec: 0.866668
INFO:tensorflow:global step 110: loss = 32.5086 (1.127 sec/step)
INFO:tensorflow:global step 120: loss = 28.8373 (1.142 sec/step)
INFO:tensorflow:global step 130: loss = 18.0380 (1.121 sec/step)
INFO:tensorflow:global step 140: loss = 24.8458 (1.152 sec/step)
INFO:tensorflow:global step 150: loss = 19.4204 (1.138 sec/step)
INFO:tensorflow:global step 160: loss = 18.0976 (1.136 sec/step)
INFO:tensorflow:global_step/sec: 0.866667
INFO:tensorflow:global step 170: loss = 17.8945 (1.139 sec/step)
INFO:tensorflow:global step 180: loss = 17.8198 (1.139 sec/step)
INFO:tensorflow:global step 190: loss = 22.4533 (1.128 sec/step)
INFO:tensorflow:global step 200: loss = 17.1611 (1.143 sec/step)

git-bisect.log

tensorflow/tensorflow:1.9.0-devel-gpu-py3
1.9.0
devel commit: 25c197e02393bd44f50079945409009dd4d434f8
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global_step/sec: 0.0394692
INFO:tensorflow:global step 10: loss = 41.9093 (1.146 sec/step)
INFO:tensorflow:global step 20: loss = 53.8440 (1.156 sec/step)
INFO:tensorflow:global step 30: loss = 30.2608 (1.161 sec/step)
INFO:tensorflow:global step 40: loss = 49.8703 (1.160 sec/step)
INFO:tensorflow:global step 50: loss = 31.5418 (1.143 sec/step)
INFO:tensorflow:global_step/sec: 0.85
INFO:tensorflow:global step 60: loss = 34.6669 (1.165 sec/step)
INFO:tensorflow:global step 70: loss = 42.2785 (1.146 sec/step)
INFO:tensorflow:global step 80: loss = 28.3640 (1.135 sec/step)
INFO:tensorflow:global step 90: loss = 22.8678 (1.178 sec/step)
INFO:tensorflow:global step 100: loss = 21.3473 (1.368 sec/step)
INFO:tensorflow:global_step/sec: 0.86667
INFO:tensorflow:global step 110: loss = 24.0175 (1.125 sec/step)
INFO:tensorflow:global step 120: loss = 31.7661 (1.158 sec/step)
INFO:tensorflow:global step 130: loss = 20.2401 (1.155 sec/step)
INFO:tensorflow:global step 140: loss = 17.8373 (1.141 sec/step)
INFO:tensorflow:global step 150: loss = 17.6278 (1.149 sec/step)
INFO:tensorflow:global_step/sec: 0.849998
INFO:tensorflow:global step 160: loss = 30.6285 (1.139 sec/step)
INFO:tensorflow:global step 170: loss = 17.8899 (1.168 sec/step)
INFO:tensorflow:global step 180: loss = 16.9702 (1.180 sec/step)
INFO:tensorflow:global step 190: loss = 21.8870 (1.172 sec/step)
INFO:tensorflow:global step 200: loss = 19.6049 (1.138 sec/step)
INFO:tensorflow:global_step/sec: 0.866658
INFO:tensorflow:global step 210: loss = 20.4830 (1.159 sec/step)
INFO:tensorflow:global step 220: loss = 17.1234 (1.164 sec/step)
INFO:tensorflow:global step 230: loss = 17.1645 (1.188 sec/step)
INFO:tensorflow:global step 240: loss = 29.6436 (1.157 sec/step)
INFO:tensorflow:global step 250: loss = 29.5057 (1.155 sec/step)
INFO:tensorflow:global_step/sec: 0.850008
INFO:tensorflow:global step 260: loss = 21.4966 (1.170 sec/step)
INFO:tensorflow:global step 270: loss = 17.4952 (1.133 sec/step)
INFO:tensorflow:global step 280: loss = 17.1351 (1.154 sec/step)
INFO:tensorflow:global step 290: loss = 18.0270 (1.188 sec/step)
INFO:tensorflow:global step 300: loss = 16.5770 (1.148 sec/step)
INFO:tensorflow:global step 310: loss = 16.9056 (1.133 sec/step)
INFO:tensorflow:global_step/sec: 0.866668
INFO:tensorflow:global step 320: loss = 16.3095 (1.154 sec/step)
INFO:tensorflow:global step 330: loss = 16.8750 (1.146 sec/step)
INFO:tensorflow:global step 340: loss = 21.6861 (1.132 sec/step)
INFO:tensorflow:global step 350: loss = 20.0684 (1.168 sec/step)
INFO:tensorflow:global step 360: loss = 19.3319 (1.169 sec/step)
INFO:tensorflow:global_step/sec: 0.866662
INFO:tensorflow:global step 370: loss = 17.6751 (1.153 sec/step)
INFO:tensorflow:global step 380: loss = 20.9411 (1.162 sec/step)
INFO:tensorflow:global step 390: loss = 17.4878 (1.142 sec/step)
INFO:tensorflow:global step 400: loss = 17.6261 (1.166 sec/step)
INFO:tensorflow:global step 410: loss = 17.5331 (1.102 sec/step)
INFO:tensorflow:global_step/sec: 0.850002
INFO:tensorflow:global step 420: loss = 21.1279 (1.154 sec/step)
INFO:tensorflow:global step 430: loss = 16.6489 (1.172 sec/step)
INFO:tensorflow:global step 440: loss = 17.3688 (1.143 sec/step)
INFO:tensorflow:global step 450: loss = 23.8969 (1.129 sec/step)
INFO:tensorflow:global step 460: loss = 19.7581 (1.181 sec/step)
INFO:tensorflow:global step 470: loss = 26.7453 (1.161 sec/step)
INFO:tensorflow:global step 480: loss = 21.8421 (1.168 sec/step)
INFO:tensorflow:global step 490: loss = 19.3461 (1.147 sec/step)
INFO:tensorflow:global step 500: loss = 16.8896 (1.162 sec/step)

tensorflow/tensorflow:1.9.0-rc0-devel-gpu-py3
1.9.0-rc0
devel commit: e1436b2952c7600c8ac88114210381db0398be16
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global step 10: loss = 55.4198 (1.116 sec/step)
INFO:tensorflow:global step 20: loss = 33.3220 (1.118 sec/step)
INFO:tensorflow:global step 30: loss = 44.3401 (1.126 sec/step)
INFO:tensorflow:global step 40: loss = 32.6873 (1.126 sec/step)
INFO:tensorflow:global_step/sec: 0.823548
INFO:tensorflow:global step 50: loss = 66.1069 (1.417 sec/step)
INFO:tensorflow:global step 60: loss = 39.4708 (1.143 sec/step)
INFO:tensorflow:global step 70: loss = 45.2507 (1.121 sec/step)
INFO:tensorflow:global step 80: loss = 29.5897 (1.137 sec/step)
INFO:tensorflow:global step 90: loss = 27.3361 (1.127 sec/step)
INFO:tensorflow:global step 100: loss = 32.6940 (1.158 sec/step)
INFO:tensorflow:global_step/sec: 0.883326
INFO:tensorflow:global step 110: loss = 19.0330 (1.128 sec/step)
INFO:tensorflow:global step 120: loss = 31.1948 (1.114 sec/step)
INFO:tensorflow:global step 130: loss = 23.3815 (1.122 sec/step)
INFO:tensorflow:global step 140: loss = 20.0340 (1.146 sec/step)
INFO:tensorflow:global step 150: loss = 29.5272 (1.128 sec/step)
INFO:tensorflow:global_step/sec: 0.866676
INFO:tensorflow:global step 160: loss = 22.8941 (1.137 sec/step)
INFO:tensorflow:global step 170: loss = 18.1229 (1.129 sec/step)
INFO:tensorflow:global step 180: loss = 26.4751 (1.130 sec/step)
INFO:tensorflow:global step 190: loss = 18.9285 (1.137 sec/step)
INFO:tensorflow:global step 200: loss = 21.4825 (1.140 sec/step)
INFO:tensorflow:global_step/sec: 0.883334
INFO:tensorflow:global step 210: loss = 22.7128 (1.119 sec/step)
INFO:tensorflow:global step 220: loss = 17.4327 (1.148 sec/step)
INFO:tensorflow:global step 230: loss = 20.7170 (1.142 sec/step)
INFO:tensorflow:global step 240: loss = 22.6502 (1.144 sec/step)
INFO:tensorflow:global step 250: loss = 27.8108 (1.116 sec/step)
INFO:tensorflow:global step 260: loss = 17.6209 (1.118 sec/step)
INFO:tensorflow:global_step/sec: 0.883333
INFO:tensorflow:global step 270: loss = 19.5009 (1.144 sec/step)
INFO:tensorflow:global step 280: loss = 21.6900 (1.126 sec/step)
INFO:tensorflow:global step 290: loss = 17.1734 (1.144 sec/step)
INFO:tensorflow:global step 300: loss = 21.7089 (1.130 sec/step)
INFO:tensorflow:global step 310: loss = 17.1693 (1.141 sec/step)
INFO:tensorflow:global_step/sec: 0.866664
INFO:tensorflow:global step 320: loss = 17.4280 (1.123 sec/step)
INFO:tensorflow:global step 330: loss = 17.5584 (1.124 sec/step)
INFO:tensorflow:global step 340: loss = 24.8683 (1.139 sec/step)
INFO:tensorflow:global step 350: loss = 17.2827 (1.131 sec/step)
INFO:tensorflow:global step 360: loss = 19.0061 (1.137 sec/step)
INFO:tensorflow:global_step/sec: 0.883338
INFO:tensorflow:global step 370: loss = 18.0653 (1.131 sec/step)
INFO:tensorflow:global step 380: loss = 18.9377 (1.142 sec/step)
INFO:tensorflow:global step 390: loss = 18.5246 (1.132 sec/step)
INFO:tensorflow:global step 400: loss = 19.0135 (1.136 sec/step)
INFO:tensorflow:global step 410: loss = 16.9987 (1.126 sec/step)
INFO:tensorflow:global_step/sec: 0.883327
INFO:tensorflow:global step 420: loss = 18.9109 (1.118 sec/step)
INFO:tensorflow:global step 430: loss = 19.4710 (1.123 sec/step)
INFO:tensorflow:global step 440: loss = 21.3032 (1.154 sec/step)
INFO:tensorflow:global step 450: loss = 17.4385 (1.124 sec/step)
INFO:tensorflow:global step 460: loss = 21.5500 (1.122 sec/step)
INFO:tensorflow:global step 470: loss = 18.0666 (1.165 sec/step)
INFO:tensorflow:global step 480: loss = 22.0548 (1.105 sec/step)
INFO:tensorflow:global step 490: loss = 24.3514 (1.115 sec/step)
INFO:tensorflow:global step 500: loss = 17.5704 (1.142 sec/step)


devel commit: 93bc2e2072e0daccbcff7a90d397b704a9e8f778
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global step 10: loss = 62.7191 (0.796 sec/step)
INFO:tensorflow:global step 20: loss = 42.6149 (0.795 sec/step)
INFO:tensorflow:global_step/sec: 0.481212
INFO:tensorflow:global step 30: loss = 33.2815 (0.782 sec/step)
INFO:tensorflow:global step 40: loss = 53.4033 (0.782 sec/step)
INFO:tensorflow:global step 50: loss = 23.1952 (0.763 sec/step)
INFO:tensorflow:global step 60: loss = 43.4205 (0.779 sec/step)
INFO:tensorflow:global step 70: loss = 35.2502 (0.797 sec/step)
INFO:tensorflow:global step 80: loss = 31.8134 (0.779 sec/step)
INFO:tensorflow:global step 90: loss = 47.6353 (0.762 sec/step)
INFO:tensorflow:global step 100: loss = 31.7606 (0.774 sec/step)
INFO:tensorflow:global_step/sec: 1.26665
INFO:tensorflow:global step 110: loss = 31.1573 (0.772 sec/step)
INFO:tensorflow:global step 120: loss = 23.1848 (0.767 sec/step)
INFO:tensorflow:global step 130: loss = 28.1278 (0.778 sec/step)
INFO:tensorflow:global step 140: loss = 24.6071 (0.786 sec/step)
INFO:tensorflow:global step 150: loss = 18.2558 (0.775 sec/step)
INFO:tensorflow:global step 160: loss = 22.5694 (0.778 sec/step)
INFO:tensorflow:global step 170: loss = 29.8637 (0.776 sec/step)
INFO:tensorflow:global_step/sec: 1.28335
INFO:tensorflow:global step 180: loss = 18.6625 (0.775 sec/step)
INFO:tensorflow:global step 190: loss = 24.0972 (0.769 sec/step)
INFO:tensorflow:global step 200: loss = 22.4856 (0.780 sec/step)


devel commit: bc86da090f2f2e850768bbdfd603c7217aecdb53
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global_step/sec: 0.101649
INFO:tensorflow:global step 10: loss = 30.6683 (1.173 sec/step)
INFO:tensorflow:global step 20: loss = 91.3520 (1.207 sec/step)
INFO:tensorflow:global step 30: loss = 37.6055 (1.188 sec/step)
INFO:tensorflow:global step 40: loss = 44.8454 (1.184 sec/step)
INFO:tensorflow:global step 50: loss = 33.0068 (1.191 sec/step)
INFO:tensorflow:global_step/sec: 0.849999
INFO:tensorflow:global step 60: loss = 43.0847 (1.197 sec/step)
INFO:tensorflow:global step 70: loss = 26.2046 (1.176 sec/step)
INFO:tensorflow:global step 80: loss = 23.9322 (1.190 sec/step)
INFO:tensorflow:global step 90: loss = 29.7891 (1.203 sec/step)
INFO:tensorflow:global step 100: loss = 25.8478 (1.185 sec/step)
INFO:tensorflow:global_step/sec: 0.849998
INFO:tensorflow:global step 110: loss = 19.4219 (1.174 sec/step)
INFO:tensorflow:global step 120: loss = 22.3297 (1.167 sec/step)
INFO:tensorflow:global step 130: loss = 19.0054 (1.156 sec/step)
INFO:tensorflow:global step 140: loss = 36.6793 (1.148 sec/step)
INFO:tensorflow:global step 150: loss = 29.8156 (1.174 sec/step)
INFO:tensorflow:global_step/sec: 0.833337
INFO:tensorflow:global step 160: loss = 21.8821 (1.180 sec/step)
INFO:tensorflow:global step 170: loss = 25.2779 (1.150 sec/step)
INFO:tensorflow:global step 180: loss = 19.1096 (1.179 sec/step)
INFO:tensorflow:global step 190: loss = 18.9300 (1.198 sec/step)
INFO:tensorflow:global step 200: loss = 17.7248 (1.184 sec/step)



devel commit: 04a5547817ad758cc7c32cd580335fd2e8a5e1e6
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global_step/sec: 0.121511
INFO:tensorflow:global step 10: loss = 37.4533 (1.115 sec/step)
INFO:tensorflow:global step 20: loss = 35.7348 (1.097 sec/step)
INFO:tensorflow:global step 30: loss = 60.8260 (1.117 sec/step)
INFO:tensorflow:global step 40: loss = 54.5362 (1.102 sec/step)
INFO:tensorflow:global step 50: loss = 34.5627 (1.103 sec/step)
INFO:tensorflow:global step 60: loss = 59.0764 (1.108 sec/step)
INFO:tensorflow:global_step/sec: 0.899999
INFO:tensorflow:global step 70: loss = 49.0319 (1.098 sec/step)
INFO:tensorflow:global step 80: loss = 35.7102 (1.125 sec/step)
INFO:tensorflow:global step 90: loss = 33.2336 (1.099 sec/step)
INFO:tensorflow:global step 100: loss = 31.9586 (1.102 sec/step)
INFO:tensorflow:global step 110: loss = 19.0082 (1.105 sec/step)
INFO:tensorflow:global_step/sec: 0.9
INFO:tensorflow:global step 120: loss = 24.0201 (1.079 sec/step)
INFO:tensorflow:global step 130: loss = 25.3416 (1.110 sec/step)
INFO:tensorflow:global step 140: loss = 23.3222 (1.088 sec/step)
INFO:tensorflow:global step 150: loss = 31.6109 (1.085 sec/step)
INFO:tensorflow:global step 160: loss = 20.4977 (1.089 sec/step)
INFO:tensorflow:global_step/sec: 0.916548
INFO:tensorflow:global step 170: loss = 17.6889 (1.086 sec/step)
INFO:tensorflow:global step 180: loss = 27.2714 (1.104 sec/step)
INFO:tensorflow:global step 190: loss = 26.6638 (1.115 sec/step)
INFO:tensorflow:global step 200: loss = 21.7155 (1.116 sec/step)


devel commit: 603aad77e69ea856b39566769361c022b6af933a
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global step 10: loss = 44.9329 (0.805 sec/step)
INFO:tensorflow:global_step/sec: 0.374356
INFO:tensorflow:global step 20: loss = 51.3308 (0.777 sec/step)
INFO:tensorflow:global step 30: loss = 31.2073 (0.787 sec/step)
INFO:tensorflow:global step 40: loss = 44.6608 (0.781 sec/step)
INFO:tensorflow:global step 50: loss = 34.2890 (0.768 sec/step)
INFO:tensorflow:global step 60: loss = 26.2657 (0.775 sec/step)
INFO:tensorflow:global step 70: loss = 21.7431 (0.773 sec/step)
INFO:tensorflow:global step 80: loss = 39.8482 (0.772 sec/step)
INFO:tensorflow:global step 90: loss = 37.8229 (0.747 sec/step)
INFO:tensorflow:global_step/sec: 1.28333
INFO:tensorflow:global step 100: loss = 19.2881 (0.771 sec/step)
INFO:tensorflow:global step 110: loss = 19.6185 (0.781 sec/step)
INFO:tensorflow:global step 120: loss = 24.0603 (0.776 sec/step)
INFO:tensorflow:global step 130: loss = 42.3605 (0.773 sec/step)
INFO:tensorflow:global step 140: loss = 21.2523 (0.788 sec/step)
INFO:tensorflow:global step 150: loss = 23.3521 (0.786 sec/step)
INFO:tensorflow:global step 160: loss = 34.7682 (0.783 sec/step)
INFO:tensorflow:global step 170: loss = 28.5749 (0.776 sec/step)
INFO:tensorflow:global_step/sec: 1.28333
INFO:tensorflow:global step 180: loss = 24.9694 (0.784 sec/step)
INFO:tensorflow:global step 190: loss = 18.2603 (0.773 sec/step)
INFO:tensorflow:global step 200: loss = 17.2641 (0.767 sec/step)


devel commit: 89ff74a7b25c01a511e84a805d3b2edf780142a6
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global step 10: loss = 68.2446 (1.082 sec/step)
INFO:tensorflow:global_step/sec: 0.201259
INFO:tensorflow:global step 20: loss = 48.5203 (1.089 sec/step)
INFO:tensorflow:global step 30: loss = 33.6766 (1.080 sec/step)
INFO:tensorflow:global step 40: loss = 49.8812 (1.077 sec/step)
INFO:tensorflow:global step 50: loss = 35.7311 (1.101 sec/step)
INFO:tensorflow:global step 60: loss = 36.6608 (1.369 sec/step)
INFO:tensorflow:global_step/sec: 0.899997
INFO:tensorflow:global step 70: loss = 33.0950 (1.096 sec/step)
INFO:tensorflow:global step 80: loss = 26.2097 (1.101 sec/step)
INFO:tensorflow:global step 90: loss = 53.1002 (1.097 sec/step)
INFO:tensorflow:global step 100: loss = 34.8386 (1.093 sec/step)
INFO:tensorflow:global step 110: loss = 20.2384 (1.103 sec/step)
INFO:tensorflow:global_step/sec: 0.916657
INFO:tensorflow:global step 120: loss = 19.5256 (1.104 sec/step)
INFO:tensorflow:global step 130: loss = 18.2252 (1.096 sec/step)
INFO:tensorflow:global step 140: loss = 25.2375 (1.100 sec/step)
INFO:tensorflow:global step 150: loss = 19.5416 (1.094 sec/step)
INFO:tensorflow:global step 160: loss = 24.6995 (1.097 sec/step)
INFO:tensorflow:global step 170: loss = 17.8404 (1.069 sec/step)
INFO:tensorflow:global_step/sec: 0.900013
INFO:tensorflow:global step 180: loss = 19.3932 (1.091 sec/step)
INFO:tensorflow:global step 190: loss = 28.2675 (1.082 sec/step)
INFO:tensorflow:global step 200: loss = 17.8288 (1.085 sec/step)


devel commit: c93a883fcea141dc0f63fe63afcd9490e39e3eaf
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global_step/sec: 0.160342
INFO:tensorflow:global step 10: loss = 77.0541 (1.073 sec/step)
INFO:tensorflow:global step 20: loss = 48.5982 (1.073 sec/step)
INFO:tensorflow:global step 30: loss = 31.3864 (1.104 sec/step)
INFO:tensorflow:global step 40: loss = 46.9027 (1.100 sec/step)
INFO:tensorflow:global step 50: loss = 32.1115 (1.070 sec/step)
INFO:tensorflow:global step 60: loss = 25.8082 (1.064 sec/step)
INFO:tensorflow:global_step/sec: 0.91668
INFO:tensorflow:global step 70: loss = 29.2531 (1.086 sec/step)
INFO:tensorflow:global step 80: loss = 23.6930 (1.072 sec/step)
INFO:tensorflow:global step 90: loss = 29.8911 (1.078 sec/step)
INFO:tensorflow:global step 100: loss = 21.4892 (1.081 sec/step)
INFO:tensorflow:global step 110: loss = 19.1849 (1.100 sec/step)
INFO:tensorflow:global_step/sec: 0.916667
INFO:tensorflow:global step 120: loss = 25.6123 (1.075 sec/step)
INFO:tensorflow:global step 130: loss = 17.9063 (1.095 sec/step)
INFO:tensorflow:global step 140: loss = 33.5016 (1.095 sec/step)
INFO:tensorflow:global step 150: loss = 19.2937 (1.093 sec/step)
INFO:tensorflow:global step 160: loss = 17.4196 (1.068 sec/step)
INFO:tensorflow:global step 170: loss = 21.3618 (1.064 sec/step)
INFO:tensorflow:global_step/sec: 0.916669
INFO:tensorflow:global step 180: loss = 25.2028 (1.070 sec/step)
INFO:tensorflow:global step 190: loss = 30.7240 (1.094 sec/step)
INFO:tensorflow:global step 200: loss = 20.8082 (1.117 sec/step)


devel commit: bf86d3a46b4e2ef4dabcba211c1ce36cb81ac315
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global_step/sec: 0.142282
INFO:tensorflow:global step 10: loss = 40.2229 (1.097 sec/step)
INFO:tensorflow:global step 20: loss = 51.7422 (1.063 sec/step)
INFO:tensorflow:global step 30: loss = 37.5562 (1.090 sec/step)
INFO:tensorflow:global step 40: loss = 43.2558 (1.074 sec/step)
INFO:tensorflow:global step 50: loss = 59.6657 (1.083 sec/step)
INFO:tensorflow:global step 60: loss = 25.7144 (1.064 sec/step)
INFO:tensorflow:global_step/sec: 0.933337
INFO:tensorflow:global step 70: loss = 28.0388 (1.070 sec/step)
INFO:tensorflow:global step 80: loss = 35.4460 (1.085 sec/step)
INFO:tensorflow:global step 90: loss = 27.2240 (1.079 sec/step)
INFO:tensorflow:global step 100: loss = 22.7069 (1.085 sec/step)
INFO:tensorflow:global step 110: loss = 26.4674 (1.080 sec/step)
INFO:tensorflow:global_step/sec: 0.916668
INFO:tensorflow:global step 120: loss = 20.6134 (1.085 sec/step)
INFO:tensorflow:global step 130: loss = 18.3419 (1.076 sec/step)
INFO:tensorflow:global step 140: loss = 18.5712 (1.071 sec/step)
INFO:tensorflow:global step 150: loss = 21.2448 (1.071 sec/step)
INFO:tensorflow:global step 160: loss = 20.4065 (1.080 sec/step)
INFO:tensorflow:global step 170: loss = 18.0304 (1.294 sec/step)
INFO:tensorflow:global_step/sec: 0.933331
INFO:tensorflow:global step 180: loss = 38.4310 (1.061 sec/step)
INFO:tensorflow:global step 190: loss = 19.2659 (1.075 sec/step)
INFO:tensorflow:global step 200: loss = 19.7380 (1.072 sec/step)


devel commit: d961d8ffae1500aca0c6191e4b1e37a2a44bf527
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global step 10: loss = 67.5196 (0.770 sec/step)
INFO:tensorflow:global_step/sec: 0.263487
INFO:tensorflow:global step 20: loss = 79.2548 (0.780 sec/step)
INFO:tensorflow:global step 30: loss = 29.7414 (0.760 sec/step)
INFO:tensorflow:global step 40: loss = 50.1389 (0.784 sec/step)
INFO:tensorflow:global step 50: loss = 34.0233 (0.766 sec/step)
INFO:tensorflow:global step 60: loss = 31.5582 (0.774 sec/step)
INFO:tensorflow:global step 70: loss = 26.7938 (0.761 sec/step)
INFO:tensorflow:global step 80: loss = 34.2674 (0.766 sec/step)
INFO:tensorflow:global step 90: loss = 21.8281 (0.776 sec/step)
INFO:tensorflow:global_step/sec: 1.29999
INFO:tensorflow:global step 100: loss = 31.8484 (0.793 sec/step)
INFO:tensorflow:global step 110: loss = 20.1656 (0.757 sec/step)
INFO:tensorflow:global step 120: loss = 37.9373 (0.753 sec/step)
INFO:tensorflow:global step 130: loss = 18.8595 (0.762 sec/step)
INFO:tensorflow:global step 140: loss = 19.1332 (0.759 sec/step)
INFO:tensorflow:global step 150: loss = 18.6528 (0.773 sec/step)
INFO:tensorflow:global step 160: loss = 17.5451 (0.772 sec/step)
INFO:tensorflow:global_step/sec: 1.30001
INFO:tensorflow:global step 170: loss = 20.1951 (0.748 sec/step)
INFO:tensorflow:global step 180: loss = 18.7759 (0.757 sec/step)
INFO:tensorflow:global step 190: loss = 27.1669 (0.787 sec/step)
INFO:tensorflow:global step 200: loss = 23.1967 (0.771 sec/step)


devel commit: 81cabadc78811a216381fbf30715b1313684e32f
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global_step/sec: 0.1416
INFO:tensorflow:global step 10: loss = 73.0054 (1.078 sec/step)
INFO:tensorflow:global step 20: loss = 53.6742 (1.085 sec/step)
INFO:tensorflow:global step 30: loss = 36.5160 (1.096 sec/step)
INFO:tensorflow:global step 40: loss = 35.7159 (1.097 sec/step)
INFO:tensorflow:global step 50: loss = 25.8626 (1.084 sec/step)
INFO:tensorflow:global step 60: loss = 24.9598 (1.087 sec/step)
INFO:tensorflow:global_step/sec: 0.916669
INFO:tensorflow:global step 70: loss = 28.7914 (1.073 sec/step)
INFO:tensorflow:global step 80: loss = 26.4148 (1.072 sec/step)
INFO:tensorflow:global step 90: loss = 40.5804 (1.068 sec/step)
INFO:tensorflow:global step 100: loss = 21.3673 (1.090 sec/step)
INFO:tensorflow:global step 110: loss = 44.3514 (1.066 sec/step)
INFO:tensorflow:global_step/sec: 0.916669
INFO:tensorflow:global step 120: loss = 17.9662 (1.075 sec/step)
INFO:tensorflow:global step 130: loss = 20.8716 (1.073 sec/step)
INFO:tensorflow:global step 140: loss = 28.7359 (1.099 sec/step)
INFO:tensorflow:global step 150: loss = 41.6168 (1.092 sec/step)
INFO:tensorflow:global step 160: loss = 27.5819 (1.082 sec/step)
INFO:tensorflow:global step 170: loss = 20.6188 (1.077 sec/step)
INFO:tensorflow:global_step/sec: 0.916665
INFO:tensorflow:global step 180: loss = 16.7592 (1.055 sec/step)
INFO:tensorflow:global step 190: loss = 23.8085 (1.062 sec/step)
INFO:tensorflow:global step 200: loss = 17.9391 (1.064 sec/step)


devel commit: a699d69c621fde118d4c89ba94658a9d7f91faac
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global_step/sec: 0.139098
INFO:tensorflow:global step 10: loss = 114.9229 (1.084 sec/step)
INFO:tensorflow:global step 20: loss = 27.4050 (1.107 sec/step)
INFO:tensorflow:global step 30: loss = 52.5609 (1.105 sec/step)
INFO:tensorflow:global step 40: loss = 37.2694 (1.084 sec/step)
INFO:tensorflow:global step 50: loss = 20.9535 (1.085 sec/step)
INFO:tensorflow:global step 60: loss = 28.3000 (1.063 sec/step)
INFO:tensorflow:global_step/sec: 0.933333
INFO:tensorflow:global step 70: loss = 26.2898 (1.064 sec/step)
INFO:tensorflow:global step 80: loss = 26.4547 (1.085 sec/step)
INFO:tensorflow:global step 90: loss = 45.9689 (1.075 sec/step)
INFO:tensorflow:global step 100: loss = 23.6273 (1.062 sec/step)
INFO:tensorflow:global step 110: loss = 30.1636 (1.090 sec/step)
INFO:tensorflow:global_step/sec: 0.916667
INFO:tensorflow:global step 120: loss = 30.8477 (1.066 sec/step)
INFO:tensorflow:global step 130: loss = 22.8348 (1.092 sec/step)
INFO:tensorflow:global step 140: loss = 24.0032 (1.090 sec/step)
INFO:tensorflow:global step 150: loss = 24.4059 (1.074 sec/step)
INFO:tensorflow:global step 160: loss = 26.2792 (1.074 sec/step)
INFO:tensorflow:global step 170: loss = 23.9606 (1.213 sec/step)
INFO:tensorflow:global_step/sec: 0.933332
INFO:tensorflow:global step 180: loss = 27.7266 (1.080 sec/step)
INFO:tensorflow:global step 190: loss = 27.3166 (1.079 sec/step)
INFO:tensorflow:global step 200: loss = 17.5864 (1.066 sec/step)


devel commit: dc0f44a98284e1bd8f9d44ef7a8122b27f9f0f15
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global step 10: loss = 48.5255 (0.774 sec/step)
INFO:tensorflow:global step 20: loss = 29.2131 (0.774 sec/step)
INFO:tensorflow:global_step/sec: 0.424724
INFO:tensorflow:global step 30: loss = 55.2396 (0.757 sec/step)
INFO:tensorflow:global step 40: loss = 24.3604 (0.760 sec/step)
INFO:tensorflow:global step 50: loss = 71.5361 (0.766 sec/step)
INFO:tensorflow:global step 60: loss = 51.0882 (0.753 sec/step)
INFO:tensorflow:global step 70: loss = 23.5861 (0.765 sec/step)
INFO:tensorflow:global step 80: loss = 23.3129 (0.774 sec/step)
INFO:tensorflow:global step 90: loss = 19.9461 (0.759 sec/step)
INFO:tensorflow:global_step/sec: 1.3
INFO:tensorflow:global step 100: loss = 19.9001 (0.764 sec/step)
INFO:tensorflow:global step 110: loss = 29.5116 (0.752 sec/step)
INFO:tensorflow:global step 120: loss = 18.3177 (0.747 sec/step)
INFO:tensorflow:global step 130: loss = 29.4836 (0.751 sec/step)
INFO:tensorflow:global step 140: loss = 33.5815 (0.756 sec/step)
INFO:tensorflow:global step 150: loss = 17.5979 (0.763 sec/step)
INFO:tensorflow:global step 160: loss = 18.7652 (0.745 sec/step)
INFO:tensorflow:global step 170: loss = 24.6465 (0.754 sec/step)
INFO:tensorflow:global_step/sec: 1.31667
INFO:tensorflow:global step 180: loss = 31.0412 (0.754 sec/step)
INFO:tensorflow:global step 190: loss = 17.5818 (0.770 sec/step)
INFO:tensorflow:global step 200: loss = 36.6495 (0.760 sec/step)


devel commit: f1fb08bbb70047af0c86cc440ccc0581e64fd85f
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global_step/sec: 0.145286
INFO:tensorflow:global step 10: loss = 38.6893 (1.085 sec/step)
INFO:tensorflow:global step 20: loss = 63.5405 (1.082 sec/step)
INFO:tensorflow:global step 30: loss = 32.6803 (1.072 sec/step)
INFO:tensorflow:global step 40: loss = 39.0896 (1.074 sec/step)
INFO:tensorflow:global step 50: loss = 46.2860 (1.069 sec/step)
INFO:tensorflow:global step 60: loss = 30.3566 (1.077 sec/step)
INFO:tensorflow:global_step/sec: 0.916667
INFO:tensorflow:global step 70: loss = 26.4089 (1.088 sec/step)
INFO:tensorflow:global step 80: loss = 22.1819 (1.072 sec/step)
INFO:tensorflow:global step 90: loss = 34.5988 (1.088 sec/step)
INFO:tensorflow:global step 100: loss = 21.1399 (1.055 sec/step)
INFO:tensorflow:global step 110: loss = 25.3149 (1.090 sec/step)
INFO:tensorflow:global_step/sec: 0.933334
INFO:tensorflow:global step 120: loss = 21.7527 (1.078 sec/step)
INFO:tensorflow:global step 130: loss = 24.1046 (1.079 sec/step)
INFO:tensorflow:global step 140: loss = 18.8834 (1.061 sec/step)
INFO:tensorflow:global step 150: loss = 27.8211 (1.077 sec/step)
INFO:tensorflow:global step 160: loss = 22.4091 (1.077 sec/step)
INFO:tensorflow:global step 170: loss = 19.9642 (1.056 sec/step)
INFO:tensorflow:global_step/sec: 0.916666
INFO:tensorflow:global step 180: loss = 32.2118 (1.063 sec/step)
INFO:tensorflow:global step 190: loss = 19.8473 (1.072 sec/step)
INFO:tensorflow:global step 200: loss = 21.3244 (1.078 sec/step)


devel commit: d4976f754009d084514f4308d3bfc7dc3a106e29
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global step 10: loss = 77.8448 (1.096 sec/step)
INFO:tensorflow:global_step/sec: 0.222987
INFO:tensorflow:global step 20: loss = 41.6071 (1.079 sec/step)
INFO:tensorflow:global step 30: loss = 55.1761 (1.083 sec/step)
INFO:tensorflow:global step 40: loss = 25.6517 (1.078 sec/step)
INFO:tensorflow:global step 50: loss = 22.6773 (1.084 sec/step)
INFO:tensorflow:global step 60: loss = 28.2089 (1.078 sec/step)
INFO:tensorflow:global_step/sec: 0.933322
INFO:tensorflow:global step 70: loss = 48.3094 (1.082 sec/step)
INFO:tensorflow:global step 80: loss = 24.6723 (1.079 sec/step)
INFO:tensorflow:global step 90: loss = 46.9032 (1.065 sec/step)
INFO:tensorflow:global step 100: loss = 30.3739 (1.062 sec/step)
INFO:tensorflow:global step 110: loss = 33.5094 (1.073 sec/step)
INFO:tensorflow:global step 120: loss = 31.5804 (1.063 sec/step)
INFO:tensorflow:global_step/sec: 0.91668
INFO:tensorflow:global step 130: loss = 19.0180 (1.077 sec/step)
INFO:tensorflow:global step 140: loss = 29.7252 (1.071 sec/step)
INFO:tensorflow:global step 150: loss = 18.1041 (1.069 sec/step)
INFO:tensorflow:global step 160: loss = 17.3742 (1.076 sec/step)
INFO:tensorflow:global step 170: loss = 31.1170 (1.050 sec/step)
INFO:tensorflow:global_step/sec: 0.933334
INFO:tensorflow:global step 180: loss = 25.0502 (1.065 sec/step)
INFO:tensorflow:global step 190: loss = 22.6136 (1.073 sec/step)
INFO:tensorflow:global step 200: loss = 21.5477 (1.067 sec/step)