Skip to content

Instantly share code, notes, and snippets.

@innerlee

innerlee/activation distribution.py

Last active October 23, 2016 10:00
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save innerlee/07436c632e4424a7c7bc83f4c2e5cd90 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save innerlee/07436c632e4424a7c7bc83f4c2e5cd90 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
activation distribution.py
#!/usr/bin/python
# -*- coding: utf-8 -*-
# activation value
import sys
sys.path.append('/home/lizz/parrots/parrots/python')
from pyparrots import dnn
import numpy as np
from numpy import linalg as alg
import seaborn as sb
import matplotlib.pyplot as plt
conf = {
"run" : "20",
"iter" : "640"
}
if len(sys.argv) == 1: raise Exception('pls specify run [iter]')
if len(sys.argv) >= 2: conf["run"] = sys.argv[1]
if len(sys.argv) >= 3: conf["iter"] = sys.argv[2]
if len(sys.argv) >= 4: raise Exception('too much arguments')
## conf end
model_file = '/mnt/gv4/16fall/run{run}/model.yaml'.format(**conf)
with open(model_file) as f: model = dnn.Model.from_yaml_text(f.read())
with open('session.yaml') as f: s = dnn.Session.from_yaml_text(model, f.read()); s.setup()
with s.flow('train') as f: f.load_param('/mnt/gv4/16fall/run{run}/snapshots/iter.00{iter}000.parrots'.format(**conf))
conf['model'] = model.name
with s.flow('train') as f:
f.feed()
f.forward()
f.backward()
lst = [
[
'reduction_b_concat',
'inception_resnet_c_1_elt',
'inception_resnet_c_2_elt',
'inception_resnet_c_3_elt',
],
[
'reduction_a_concat',
'inception_resnet_b_1_elt',
'inception_resnet_b_2_elt',
'inception_resnet_b_3_elt',
'inception_resnet_b_4_elt',
'inception_resnet_b_5_elt',
'inception_resnet_b_6_elt',
],
[
'stem_concat',
'inception_resnet_a_1_elt',
'inception_resnet_a_2_elt',
'inception_resnet_a_3_elt',
],
]
sb.set(style="white")
ii, jj = (len(lst), max(map(len, lst)))
plt.figure(figsize=(jj * 5, ii * 5))
with s.flow('train') as f:
for i, row in enumerate(lst):
for j, col in enumerate(row):
v = f.data(col).value()
g = f.data(col).grad()
nv = np.round(alg.norm(v), 4)
ng = np.round(alg.norm(g), 4)
rho = np.round(1 - np.count_nonzero(v) / (1.0 * v.size), 4)
print col, nv, '\t', ng, '\t', rho, '\t', np.round(ng/(1-rho), 4)
subax = plt.subplot2grid((ii, jj), (i, j))
subax.set_title(col)
vv = v.flatten()
vv[vv <= 0] = -0.5
sb.distplot(vv, bins=np.linspace(-1, 10, 45), norm_hist=True, kde=False, ax=subax)
subax.set_xlim([-1, 10])
subax.set_ylim([0, 2.6])
subax.set_yticks((0, 0.4, 0.8, 1.2, 1.6, 2, 2.4))
subax.set_yticklabels(('0', '0.1', '0.2', '0.3', '0.4', '0.5', '0.6'))
plt.suptitle('{model} (run {run}) at iter {iter}k, distribution of activation values'.format(**conf))
plt.savefig('run{run}.iter{iter}k.png'.format(**conf), dpi=300)
Raw
grad_dist.py
#!/usr/bin/python
# -*- coding: utf-8 -*-
# activation value
import sys
sys.path.append('/home/lizz/parrots/parrots/python')
from pyparrots import dnn
import numpy as np
from numpy import linalg as alg
import seaborn as sb
import matplotlib.pyplot as plt
conf = {
"run" : "20",
"iter" : "640"
}
if len(sys.argv) == 1: raise Exception('pls specify run [iter]')
if len(sys.argv) >= 2: conf["run"] = sys.argv[1]
if len(sys.argv) >= 3: conf["iter"] = sys.argv[2]
if len(sys.argv) >= 4: raise Exception('too much arguments')
## conf end
model_file = '/mnt/gv4/16fall/run{run}/model.yaml'.format(**conf)
with open(model_file) as f: model = dnn.Model.from_yaml_text(f.read())
with open('session.yaml') as f: s = dnn.Session.from_yaml_text(model, f.read()); s.setup()
with s.flow('train') as f: f.load_param('/mnt/gv4/16fall/run{run}/snapshots/iter.00{iter}000.parrots'.format(**conf))
conf['model'] = model.name
with s.flow('train') as f:
f.feed()
f.forward()
f.backward()
lst = [
[
'reduction_b_concat',
'inception_resnet_c_1_elt',
'inception_resnet_c_2_elt',
'inception_resnet_c_3_elt',
],
[
'reduction_a_concat',
'inception_resnet_b_1_elt',
'inception_resnet_b_2_elt',
'inception_resnet_b_3_elt',
'inception_resnet_b_4_elt',
'inception_resnet_b_5_elt',
'inception_resnet_b_6_elt',
],
[
'stem_concat',
'inception_resnet_a_1_elt',
'inception_resnet_a_2_elt',
'inception_resnet_a_3_elt',
],
]
sb.set(style="white")
ii, jj = (len(lst), max(map(len, lst)))
plt.figure(figsize=(jj * 5, ii * 5))
with s.flow('train') as f:
for i, row in enumerate(lst):
for j, col in enumerate(row):
v = f.data(col).value()
g = f.data(col).grad()
nv = np.round(alg.norm(v), 4)
ng = np.round(alg.norm(g), 4)
rho = np.round(1 - np.count_nonzero(v) / (1.0 * v.size), 4)
print col, nv, '\t', ng, '\t', rho, '\t', np.round(ng/(1-rho), 4)
subax = plt.subplot2grid((ii, jj), (i, j))
subax.set_title(col)
vv = g.flatten()
# vv[vv <= 0] = -0.5
sb.distplot(vv, norm_hist=True, kde=False, ax=subax)
# subax.set_xlim([-1, 10])
# subax.set_ylim([0, 2.6])
# subax.set_yticks((0, 0.4, 0.8, 1.2, 1.6, 2, 2.4))
# subax.set_yticklabels(('0', '0.1', '0.2', '0.3', '0.4', '0.5', '0.6'))
plt.suptitle('{model} (run {run}) at iter {iter}k, distribution of gradients'.format(**conf))
plt.savefig('run{run}.iter{iter}k.grad.png'.format(**conf), dpi=300)
Raw
value_norm.py
#!/usr/bin/python
# -*- coding: utf-8 -*-
# value norm
import sys
sys.path.append('/home/lizz/parrots/parrots/python')
from pyparrots import dnn
import numpy as np
from numpy import linalg as alg
conf = {
"run" : "20",
"iter" : "640",
}
if len(sys.argv) == 1: raise Exception('pls specify run [iter]')
if len(sys.argv) >= 2: conf["run"] = sys.argv[1]
if len(sys.argv) >= 3: conf["iter"] = sys.argv[2]
if len(sys.argv) >= 4: raise Exception('too much arguments')
## conf end
model_file = '/mnt/gv4/16fall/run{run}/model.yaml'.format(**conf)
with open(model_file) as f: model = dnn.Model.from_yaml_text(f.read())
with open('session.yaml') as f: s = dnn.Session.from_yaml_text(model, f.read()); s.setup()
with s.flow('train') as f: f.load_param('/mnt/gv4/16fall/run{run}/snapshots/iter.00{iter}000.parrots'.format(**conf))
conf['model'] = model.name
with s.flow('train') as f:
f.feed()
f.forward()
f.backward()
lists = {
"20": [
[
[
(1.0, 'stem_concat'),
],
[
(1.0, 'stem_concat'),
(0.3, 'inception_resnet_a_1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_a_1_elt'),
(0.3, 'inception_resnet_a_2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_a_2_elt'),
(0.3, 'inception_resnet_a_3_conv_1x1_bn'),
],
],
[
[
(1.0, 'reduction_a_concat'),
],
[
(1.0, 'reduction_a_concat'),
(0.3, 'inception_resnet_b_1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_1_elt'),
(0.3, 'inception_resnet_b_2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_2_elt'),
(0.3, 'inception_resnet_b_3_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_3_elt'),
(0.3, 'inception_resnet_b_4_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_4_elt'),
(0.3, 'inception_resnet_b_5_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_5_elt'),
(0.3, 'inception_resnet_b_6_conv_1x1_bn'),
],
],
[
[
(1.0, 'reduction_b_concat'),
],
[
(1.0, 'reduction_b_concat'),
(0.3, 'inception_resnet_c_1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_c_1_elt'),
(0.3, 'inception_resnet_c_2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_c_2_elt'),
(0.3, 'inception_resnet_c_3_conv_1x1_bn'),
],
],
],
"2": [
[
[
(1.0, 'stem_concat'),
],
[
(1.0, 'stem_concat'),
(0.3, 'inception_resnet_a_1_conv_1x1_bn'),
(0.3, 'inception_resnet_a_1_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_a_1_elt'),
(0.3, 'inception_resnet_a_2_conv_1x1_bn'),
(0.3, 'inception_resnet_a_2_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_a_2_elt'),
(0.3, 'inception_resnet_a_3_conv_1x1_bn'),
(0.3, 'inception_resnet_a_3_side_conv_1x1_bn'),
],
],
[
[
(1.0, 'reduction_a_concat'),
],
[
(1.0, 'reduction_a_concat'),
(0.3, 'inception_resnet_b_1_conv_1x1_bn'),
(0.3, 'inception_resnet_b_1_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_1_elt'),
(0.3, 'inception_resnet_b_2_conv_1x1_bn'),
(0.3, 'inception_resnet_b_2_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_2_elt'),
(0.3, 'inception_resnet_b_3_conv_1x1_bn'),
(0.3, 'inception_resnet_b_3_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_3_elt'),
(0.3, 'inception_resnet_b_4_conv_1x1_bn'),
(0.3, 'inception_resnet_b_4_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_4_elt'),
(0.3, 'inception_resnet_b_5_conv_1x1_bn'),
(0.3, 'inception_resnet_b_5_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_5_elt'),
(0.3, 'inception_resnet_b_6_conv_1x1_bn'),
(0.3, 'inception_resnet_b_6_side_conv_1x1_bn'),
],
],
[
[
(1.0, 'reduction_b_concat'),
],
[
(1.0, 'reduction_b_concat'),
(0.3, 'inception_resnet_c_1_conv_1x1_bn'),
(0.3, 'inception_resnet_c_1_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_c_1_elt'),
(0.3, 'inception_resnet_c_2_conv_1x1_bn'),
(0.3, 'inception_resnet_c_2_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_c_2_elt'),
(0.3, 'inception_resnet_c_3_conv_1x1_bn'),
(0.3, 'inception_resnet_c_3_side_conv_1x1_bn'),
],
],
],
"35": [
[
[
(1.0, 'stem_concat'),
],
[
(1.0, 'stem_concat'),
(0.3, 'inception_resnet_a_1_conv_1x1_bn'),
(0.3, 'inception_resnet_a_1_mid_conv_1x1_bn'),
(0.3, 'inception_resnet_a_1_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_a_1_elt'),
(0.3, 'inception_resnet_a_2_conv_1x1_bn'),
(0.3, 'inception_resnet_a_2_mid_conv_1x1_bn'),
(0.3, 'inception_resnet_a_2_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_a_2_elt'),
(0.3, 'inception_resnet_a_3_conv_1x1_bn'),
(0.3, 'inception_resnet_a_3_mid_conv_1x1_bn'),
(0.3, 'inception_resnet_a_3_side_conv_1x1_bn'),
],
],
[
[
(1.0, 'reduction_a_concat'),
],
[
(1.0, 'reduction_a_concat'),
(0.3, 'inception_resnet_b_1_conv_1x1_bn'),
(0.3, 'inception_resnet_b_1_mid_conv_1x1_bn'),
(0.3, 'inception_resnet_b_1_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_1_elt'),
(0.3, 'inception_resnet_b_2_conv_1x1_bn'),
(0.3, 'inception_resnet_b_2_mid_conv_1x1_bn'),
(0.3, 'inception_resnet_b_2_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_2_elt'),
(0.3, 'inception_resnet_b_3_conv_1x1_bn'),
(0.3, 'inception_resnet_b_3_mid_conv_1x1_bn'),
(0.3, 'inception_resnet_b_3_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_3_elt'),
(0.3, 'inception_resnet_b_4_conv_1x1_bn'),
(0.3, 'inception_resnet_b_4_mid_conv_1x1_bn'),
(0.3, 'inception_resnet_b_4_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_4_elt'),
(0.3, 'inception_resnet_b_5_conv_1x1_bn'),
(0.3, 'inception_resnet_b_5_mid_conv_1x1_bn'),
(0.3, 'inception_resnet_b_5_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_5_elt'),
(0.3, 'inception_resnet_b_6_conv_1x1_bn'),
(0.3, 'inception_resnet_b_6_mid_conv_1x1_bn'),
(0.3, 'inception_resnet_b_6_side_conv_1x1_bn'),
],
],
[
[
(1.0, 'reduction_b_concat'),
],
[
(1.0, 'reduction_b_concat'),
(0.3, 'inception_resnet_c_1_conv_1x1_bn'),
(0.3, 'inception_resnet_c_1_mid_conv_1x1_bn'),
(0.3, 'inception_resnet_c_1_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_c_1_elt'),
(0.3, 'inception_resnet_c_2_conv_1x1_bn'),
(0.3, 'inception_resnet_c_2_mid_conv_1x1_bn'),
(0.3, 'inception_resnet_c_2_side_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_c_2_elt'),
(0.3, 'inception_resnet_c_3_conv_1x1_bn'),
(0.3, 'inception_resnet_c_3_mid_conv_1x1_bn'),
(0.3, 'inception_resnet_c_3_side_conv_1x1_bn'),
],
],
],
"4": [
[
[
(1.0, 'stem_concat'),
],
[
(1.0, 'stem_concat'),
(0.3, 'inception_resnet_a_1_conv_1x1_bn'),
(0.15, 'inception_resnet_a_1_poly2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_a_1_elt'),
(0.3, 'inception_resnet_a_2_conv_1x1_bn'),
(0.15, 'inception_resnet_a_2_poly2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_a_2_elt'),
(0.3, 'inception_resnet_a_3_conv_1x1_bn'),
(0.15, 'inception_resnet_a_3_poly2_conv_1x1_bn'),
],
],
[
[
(1.0, 'reduction_a_concat'),
],
[
(1.0, 'reduction_a_concat'),
(0.3, 'inception_resnet_b_1_conv_1x1_bn'),
(0.15, 'inception_resnet_b_1_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_1_elt'),
(0.3, 'inception_resnet_b_2_conv_1x1_bn'),
(0.15, 'inception_resnet_b_2_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_2_elt'),
(0.3, 'inception_resnet_b_3_conv_1x1_bn'),
(0.15, 'inception_resnet_b_3_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_3_elt'),
(0.3, 'inception_resnet_b_4_conv_1x1_bn'),
(0.15, 'inception_resnet_b_4_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_4_elt'),
(0.3, 'inception_resnet_b_5_conv_1x1_bn'),
(0.15, 'inception_resnet_b_5_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_5_elt'),
(0.3, 'inception_resnet_b_6_conv_1x1_bn'),
(0.15, 'inception_resnet_b_6_poly2_block1_conv_1x1_bn'),
],
],
[
[
(1.0, 'reduction_b_concat'),
],
[
(1.0, 'reduction_b_concat'),
(0.3, 'inception_resnet_c_1_conv_1x1_bn'),
(0.15, 'inception_resnet_c_1_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_c_1_elt'),
(0.3, 'inception_resnet_c_2_conv_1x1_bn'),
(0.15, 'inception_resnet_c_2_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_c_2_elt'),
(0.3, 'inception_resnet_c_3_conv_1x1_bn'),
(0.15, 'inception_resnet_c_3_poly2_block1_conv_1x1_bn'),
],
],
],
"36": [
[
[
(1.0, 'stem_concat'),
],
[
(1.0, 'stem_concat'),
(0.3, 'inception_resnet_a_1_conv_1x1_bn'),
(0.15, 'inception_resnet_a_1_poly2_conv_1x1_bn'),
(0.05, 'inception_resnet_a_1_poly3_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_a_1_elt'),
(0.3, 'inception_resnet_a_2_conv_1x1_bn'),
(0.15, 'inception_resnet_a_2_poly2_conv_1x1_bn'),
(0.05, 'inception_resnet_a_2_poly3_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_a_2_elt'),
(0.3, 'inception_resnet_a_3_conv_1x1_bn'),
(0.15, 'inception_resnet_a_3_poly2_conv_1x1_bn'),
(0.05, 'inception_resnet_a_3_poly3_conv_1x1_bn'),
],
],
[
[
(1.0, 'reduction_a_concat'),
],
[
(1.0, 'reduction_a_concat'),
(0.3, 'inception_resnet_b_1_conv_1x1_bn'),
(0.15, 'inception_resnet_b_1_poly2_block1_conv_1x1_bn'),
(0.05, 'inception_resnet_b_1_poly2_block2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_1_elt'),
(0.3, 'inception_resnet_b_2_conv_1x1_bn'),
(0.15, 'inception_resnet_b_2_poly2_block1_conv_1x1_bn'),
(0.05, 'inception_resnet_b_2_poly2_block2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_2_elt'),
(0.3, 'inception_resnet_b_3_conv_1x1_bn'),
(0.15, 'inception_resnet_b_3_poly2_block1_conv_1x1_bn'),
(0.05, 'inception_resnet_b_3_poly2_block2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_3_elt'),
(0.3, 'inception_resnet_b_4_conv_1x1_bn'),
(0.15, 'inception_resnet_b_4_poly2_block1_conv_1x1_bn'),
(0.05, 'inception_resnet_b_4_poly2_block2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_4_elt'),
(0.3, 'inception_resnet_b_5_conv_1x1_bn'),
(0.15, 'inception_resnet_b_5_poly2_block1_conv_1x1_bn'),
(0.05, 'inception_resnet_b_5_poly2_block2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_5_elt'),
(0.3, 'inception_resnet_b_6_conv_1x1_bn'),
(0.15, 'inception_resnet_b_6_poly2_block1_conv_1x1_bn'),
(0.05, 'inception_resnet_b_6_poly2_block2_conv_1x1_bn'),
],
],
[
[
(1.0, 'reduction_b_concat'),
],
[
(1.0, 'reduction_b_concat'),
(0.3, 'inception_resnet_c_1_conv_1x1_bn'),
(0.15, 'inception_resnet_c_1_poly2_block1_conv_1x1_bn'),
(0.05, 'inception_resnet_c_1_poly2_block2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_c_1_elt'),
(0.3, 'inception_resnet_c_2_conv_1x1_bn'),
(0.15, 'inception_resnet_c_2_poly2_block1_conv_1x1_bn'),
(0.05, 'inception_resnet_c_2_poly2_block2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_c_2_elt'),
(0.3, 'inception_resnet_c_3_conv_1x1_bn'),
(0.15, 'inception_resnet_c_3_poly2_block1_conv_1x1_bn'),
(0.05, 'inception_resnet_c_3_poly2_block2_conv_1x1_bn'),
],
],
],
"6": [
[
[
(1.0, 'stem_concat'),
],
[
(1.0, 'stem_concat'),
(0.3, 'inception_resnet_a_1_conv_1x1_bn'),
(0.15, 'inception_resnet_a_1_poly2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_a_1_elt'),
(0.3, 'inception_resnet_a_2_conv_1x1_bn'),
(0.15, 'inception_resnet_a_2_poly2_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_a_2_elt'),
(0.3, 'inception_resnet_a_3_conv_1x1_bn'),
(0.15, 'inception_resnet_a_3_poly2_conv_1x1_bn'),
],
],
[
[
(1.0, 'reduction_a_concat'),
],
[
(1.0, 'reduction_a_concat'),
(0.3, 'inception_resnet_b_1_conv_1x1_bn'),
(0.15, 'inception_resnet_b_1_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_1_elt'),
(0.3, 'inception_resnet_b_2_conv_1x1_bn'),
(0.15, 'inception_resnet_b_2_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_2_elt'),
(0.3, 'inception_resnet_b_3_conv_1x1_bn'),
(0.15, 'inception_resnet_b_3_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_3_elt'),
(0.3, 'inception_resnet_b_4_conv_1x1_bn'),
(0.15, 'inception_resnet_b_4_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_4_elt'),
(0.3, 'inception_resnet_b_5_conv_1x1_bn'),
(0.15, 'inception_resnet_b_5_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_b_5_elt'),
(0.3, 'inception_resnet_b_6_conv_1x1_bn'),
(0.15, 'inception_resnet_b_6_poly2_block1_conv_1x1_bn'),
],
],
[
[
(1.0, 'reduction_b_concat'),
],
[
(1.0, 'reduction_b_concat'),
(0.3, 'inception_resnet_c_1_conv_1x1_bn'),
(0.15, 'inception_resnet_c_1_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_c_1_elt'),
(0.3, 'inception_resnet_c_2_conv_1x1_bn'),
(0.15, 'inception_resnet_c_2_poly2_block1_conv_1x1_bn'),
],
[
(1.0, 'inception_resnet_c_2_elt'),
(0.3, 'inception_resnet_c_3_conv_1x1_bn'),
(0.15, 'inception_resnet_c_3_poly2_block1_conv_1x1_bn'),
],
],
],
}
lst = lists[conf["run"]]
print '{model} (run {run}) at iter {iter}k, the value norm, value mean, grad norm, grad mean, grad max, grad min, zero ratio, negative ratio'.format(**conf)
def meaning(i): return 'path {}:'.format(i)
def blockname(i): return chr(65 + i)
def beauty(v, g, layer, note):
# val
nv = alg.norm(v, axis=0).mean()
mv = v.mean()
# grad
ng = alg.norm(g, axis=0).mean() if g is not None else None
mg = g.mean() if g is not None else None
# ratios
rho = 1 - np.count_nonzero(v) / (1.0 * v.size)
neg = (v <= 0).sum() / (1.0 * v.size)
#
print " >>> {layer} {note} {nv} {mv} {ng} {mg} {maxg} {ming} {zr} {nr}".format(
layer=layer.ljust(32),
note=note.ljust(8),
nv='%10.2f' % nv,
mv='%10.4f' % mv,
ng='%10.4f' % ng if g is not None else " "*10,
mg='%10.2e' % mg if g is not None else " "*10,
maxg='%10.2e' % g.max() if g is not None else " "*10,
ming='%10.2e' % g.min() if g is not None else " "*10,
zr='%10.4f' % rho,
nr='%10.4f' % neg
)
return nv
normstr = ''
with s.flow('train') as f:
for i, block in enumerate(lst):
print "> block ", blockname(i)
normstr += '\n'
for j, subblock in enumerate(block):
print " >> sub-block ", j
normstr += ', '
overallv = 0
subnormstr = ''
for k, (coef, layer) in enumerate(subblock):
# val
v = f.data(layer).value()
v = v.reshape(np.prod(v.shape[0:-1]), v.shape[-1]) * coef
overallv = (v if isinstance(overallv, int) else (overallv + v))
# grad
g = f.data(layer).grad()
g = g.reshape(np.prod(g.shape[0:-1]), g.shape[-1])
# pretty print
nv = beauty(v, g, layer.replace('inception_resnet_', ''), meaning(k))
subnormstr += '%1.0f' % nv + ', '
beauty(overallv, None, '', "overall:")
normstr += '%1.0f' % nv + ', ' + subnormstr
print normstr
# love
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment