Multi-GPU Training Logs

multigpu_mnist_train_log.txt

I0812 15:00:42.248405  8934 caffe.cpp:160] Using GPUs 0, 1, 2
I0812 15:00:42.422600  8934 solver.cpp:38] Initializing solver from parameters: 
test_iter: 100
test_interval: 500
base_lr: 0.01
display: 100
max_iter: 10000
lr_policy: "inv"
gamma: 0.0001
power: 0.75
momentum: 0.9
weight_decay: 0.0005
snapshot: 5000
snapshot_prefix: "examples/mnist/lenet"
solver_mode: GPU
device_id: 0
net: "examples/mnist/lenet_train_test.prototxt"
I0812 15:00:42.422646  8934 solver.cpp:80] Creating training net from net file: examples/mnist/lenet_train_test.prototxt
I0812 15:00:42.422924  8934 net.cpp:339] The NetState phase (0) differed from the phase (1) specified by a rule in layer mnist
I0812 15:00:42.422946  8934 net.cpp:339] The NetState phase (0) differed from the phase (1) specified by a rule in layer accuracy
I0812 15:00:42.423032  8934 net.cpp:50] Initializing net from parameters: 
name: "LeNet"
state {
  phase: TRAIN
}
layer {
  name: "mnist"
  type: "Data"
  top: "data"
  top: "label"
  include {
    phase: TRAIN
  }
  transform_param {
    scale: 0.00390625
  }
  data_param {
    source: "examples/mnist/mnist_train_lmdb"
    batch_size: 64
    backend: LMDB
  }
}
layer {
  name: "conv1"
  type: "Convolution"
  bottom: "data"
  top: "conv1"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 20
    kernel_size: 5
    stride: 1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "pool1"
  type: "Pooling"
  bottom: "conv1"
  top: "pool1"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer {
  name: "conv2"
  type: "Convolution"
  bottom: "pool1"
  top: "conv2"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 50
    kernel_size: 5
    stride: 1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "pool2"
  type: "Pooling"
  bottom: "conv2"
  top: "pool2"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer {
  name: "ip1"
  type: "InnerProduct"
  bottom: "pool2"
  top: "ip1"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  inner_product_param {
    num_output: 500
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu1"
  type: "ReLU"
  bottom: "ip1"
  top: "ip1"
}
layer {
  name: "ip2"
  type: "InnerProduct"
  bottom: "ip1"
  top: "ip2"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  inner_product_param {
    num_output: 10
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "loss"
  type: "SoftmaxWithLoss"
  bottom: "ip2"
  bottom: "label"
  top: "loss"
}
I0812 15:00:42.423115  8934 layer_factory.hpp:75] Creating layer mnist
I0812 15:00:42.423537  8934 net.cpp:110] Creating Layer mnist
I0812 15:00:42.423578  8934 net.cpp:432] mnist -> data
I0812 15:00:42.423631  8934 net.cpp:432] mnist -> label
I0812 15:00:42.424365  8939 db_lmdb.cpp:22] Opened lmdb examples/mnist/mnist_train_lmdb
I0812 15:00:42.424435  8934 data_layer.cpp:44] output data size: 64,1,28,28
I0812 15:00:42.429911  8934 net.cpp:155] Setting up mnist
I0812 15:00:42.429991  8934 net.cpp:163] Top shape: 64 1 28 28 (50176)
I0812 15:00:42.430022  8934 net.cpp:163] Top shape: 64 (64)
I0812 15:00:42.430053  8934 layer_factory.hpp:75] Creating layer conv1
I0812 15:00:42.430095  8934 net.cpp:110] Creating Layer conv1
I0812 15:00:42.430129  8934 net.cpp:476] conv1 <- data
I0812 15:00:42.430166  8934 net.cpp:432] conv1 -> conv1
I0812 15:00:42.430785  8934 net.cpp:155] Setting up conv1
I0812 15:00:42.430821  8934 net.cpp:163] Top shape: 64 20 24 24 (737280)
I0812 15:00:42.430860  8934 layer_factory.hpp:75] Creating layer pool1
I0812 15:00:42.430891  8934 net.cpp:110] Creating Layer pool1
I0812 15:00:42.430914  8934 net.cpp:476] pool1 <- conv1
I0812 15:00:42.430941  8934 net.cpp:432] pool1 -> pool1
I0812 15:00:42.430985  8934 net.cpp:155] Setting up pool1
I0812 15:00:42.431013  8934 net.cpp:163] Top shape: 64 20 12 12 (184320)
I0812 15:00:42.431107  8940 blocking_queue.cpp:50] Waiting for data
I0812 15:00:42.431123  8934 layer_factory.hpp:75] Creating layer conv2
I0812 15:00:42.431198  8934 net.cpp:110] Creating Layer conv2
I0812 15:00:42.431206  8934 net.cpp:476] conv2 <- pool1
I0812 15:00:42.431219  8934 net.cpp:432] conv2 -> conv2
I0812 15:00:42.431434  8934 net.cpp:155] Setting up conv2
I0812 15:00:42.431447  8934 net.cpp:163] Top shape: 64 50 8 8 (204800)
I0812 15:00:42.431463  8934 layer_factory.hpp:75] Creating layer pool2
I0812 15:00:42.431473  8934 net.cpp:110] Creating Layer pool2
I0812 15:00:42.431480  8934 net.cpp:476] pool2 <- conv2
I0812 15:00:42.431489  8934 net.cpp:432] pool2 -> pool2
I0812 15:00:42.431501  8934 net.cpp:155] Setting up pool2
I0812 15:00:42.431510  8934 net.cpp:163] Top shape: 64 50 4 4 (51200)
I0812 15:00:42.431517  8934 layer_factory.hpp:75] Creating layer ip1
I0812 15:00:42.431534  8934 net.cpp:110] Creating Layer ip1
I0812 15:00:42.431540  8934 net.cpp:476] ip1 <- pool2
I0812 15:00:42.431553  8934 net.cpp:432] ip1 -> ip1
I0812 15:00:42.434183  8934 net.cpp:155] Setting up ip1
I0812 15:00:42.434196  8934 net.cpp:163] Top shape: 64 500 (32000)
I0812 15:00:42.434211  8934 layer_factory.hpp:75] Creating layer relu1
I0812 15:00:42.434221  8934 net.cpp:110] Creating Layer relu1
I0812 15:00:42.434228  8934 net.cpp:476] relu1 <- ip1
I0812 15:00:42.434237  8934 net.cpp:419] relu1 -> ip1 (in-place)
I0812 15:00:42.434253  8934 net.cpp:155] Setting up relu1
I0812 15:00:42.434262  8934 net.cpp:163] Top shape: 64 500 (32000)
I0812 15:00:42.434268  8934 layer_factory.hpp:75] Creating layer ip2
I0812 15:00:42.434278  8934 net.cpp:110] Creating Layer ip2
I0812 15:00:42.434285  8934 net.cpp:476] ip2 <- ip1
I0812 15:00:42.434298  8934 net.cpp:432] ip2 -> ip2
I0812 15:00:42.434734  8934 net.cpp:155] Setting up ip2
I0812 15:00:42.434746  8934 net.cpp:163] Top shape: 64 10 (640)
I0812 15:00:42.434757  8934 layer_factory.hpp:75] Creating layer loss
I0812 15:00:42.434772  8934 net.cpp:110] Creating Layer loss
I0812 15:00:42.434778  8934 net.cpp:476] loss <- ip2
I0812 15:00:42.434787  8934 net.cpp:476] loss <- label
I0812 15:00:42.434800  8934 net.cpp:432] loss -> loss
I0812 15:00:42.434823  8934 layer_factory.hpp:75] Creating layer loss
I0812 15:00:42.434881  8934 net.cpp:155] Setting up loss
I0812 15:00:42.434891  8934 net.cpp:163] Top shape: (1)
I0812 15:00:42.434898  8934 net.cpp:168]     with loss weight 1
I0812 15:00:42.434924  8934 net.cpp:236] loss needs backward computation.
I0812 15:00:42.434932  8934 net.cpp:236] ip2 needs backward computation.
I0812 15:00:42.434939  8934 net.cpp:236] relu1 needs backward computation.
I0812 15:00:42.434945  8934 net.cpp:236] ip1 needs backward computation.
I0812 15:00:42.434952  8934 net.cpp:236] pool2 needs backward computation.
I0812 15:00:42.434958  8934 net.cpp:236] conv2 needs backward computation.
I0812 15:00:42.434965  8934 net.cpp:236] pool1 needs backward computation.
I0812 15:00:42.434973  8934 net.cpp:236] conv1 needs backward computation.
I0812 15:00:42.434978  8934 net.cpp:240] mnist does not need backward computation.
I0812 15:00:42.434984  8934 net.cpp:283] This network produces output loss
I0812 15:00:42.435000  8934 net.cpp:297] Network initialization done.
I0812 15:00:42.435006  8934 net.cpp:298] Memory required for data: 5169924
I0812 15:00:42.435240  8934 solver.cpp:170] Creating test net (#0) specified by net file: examples/mnist/lenet_train_test.prototxt
I0812 15:00:42.435277  8934 net.cpp:339] The NetState phase (1) differed from the phase (0) specified by a rule in layer mnist
I0812 15:00:42.435355  8934 net.cpp:50] Initializing net from parameters: 
name: "LeNet"
state {
  phase: TEST
}
layer {
  name: "mnist"
  type: "Data"
  top: "data"
  top: "label"
  include {
    phase: TEST
  }
  transform_param {
    scale: 0.00390625
  }
  data_param {
    source: "examples/mnist/mnist_test_lmdb"
    batch_size: 100
    backend: LMDB
  }
}
layer {
  name: "conv1"
  type: "Convolution"
  bottom: "data"
  top: "conv1"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 20
    kernel_size: 5
    stride: 1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "pool1"
  type: "Pooling"
  bottom: "conv1"
  top: "pool1"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer {
  name: "conv2"
  type: "Convolution"
  bottom: "pool1"
  top: "conv2"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 50
    kernel_size: 5
    stride: 1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "pool2"
  type: "Pooling"
  bottom: "conv2"
  top: "pool2"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer {
  name: "ip1"
  type: "InnerProduct"
  bottom: "pool2"
  top: "ip1"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  inner_product_param {
    num_output: 500
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu1"
  type: "ReLU"
  bottom: "ip1"
  top: "ip1"
}
layer {
  name: "ip2"
  type: "InnerProduct"
  bottom: "ip1"
  top: "ip2"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  inner_product_param {
    num_output: 10
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "accuracy"
  type: "Accuracy"
  bottom: "ip2"
  bottom: "label"
  top: "accuracy"
  include {
    phase: TEST
  }
}
layer {
  name: "loss"
  type: "SoftmaxWithLoss"
  bottom: "ip2"
  bottom: "label"
  top: "loss"
}
I0812 15:00:42.435459  8934 layer_factory.hpp:75] Creating layer mnist
I0812 15:00:42.435576  8934 net.cpp:110] Creating Layer mnist
I0812 15:00:42.435588  8934 net.cpp:432] mnist -> data
I0812 15:00:42.435600  8934 net.cpp:432] mnist -> label
I0812 15:00:42.436486  8941 db_lmdb.cpp:22] Opened lmdb examples/mnist/mnist_test_lmdb
I0812 15:00:42.436527  8934 data_layer.cpp:44] output data size: 100,1,28,28
I0812 15:00:42.437571  8934 net.cpp:155] Setting up mnist
I0812 15:00:42.437583  8934 net.cpp:163] Top shape: 100 1 28 28 (78400)
I0812 15:00:42.437592  8934 net.cpp:163] Top shape: 100 (100)
I0812 15:00:42.437599  8934 layer_factory.hpp:75] Creating layer label_mnist_1_split
I0812 15:00:42.437614  8934 net.cpp:110] Creating Layer label_mnist_1_split
I0812 15:00:42.437623  8934 net.cpp:476] label_mnist_1_split <- label
I0812 15:00:42.437631  8934 net.cpp:432] label_mnist_1_split -> label_mnist_1_split_0
I0812 15:00:42.437643  8934 net.cpp:432] label_mnist_1_split -> label_mnist_1_split_1
I0812 15:00:42.437655  8934 net.cpp:155] Setting up label_mnist_1_split
I0812 15:00:42.437664  8934 net.cpp:163] Top shape: 100 (100)
I0812 15:00:42.437672  8934 net.cpp:163] Top shape: 100 (100)
I0812 15:00:42.437682  8934 layer_factory.hpp:75] Creating layer conv1
I0812 15:00:42.437696  8934 net.cpp:110] Creating Layer conv1
I0812 15:00:42.437702  8934 net.cpp:476] conv1 <- data
I0812 15:00:42.437711  8934 net.cpp:432] conv1 -> conv1
I0812 15:00:42.437780  8934 net.cpp:155] Setting up conv1
I0812 15:00:42.437790  8934 net.cpp:163] Top shape: 100 20 24 24 (1152000)
I0812 15:00:42.437804  8934 layer_factory.hpp:75] Creating layer pool1
I0812 15:00:42.437813  8934 net.cpp:110] Creating Layer pool1
I0812 15:00:42.437820  8934 net.cpp:476] pool1 <- conv1
I0812 15:00:42.437830  8934 net.cpp:432] pool1 -> pool1
I0812 15:00:42.437842  8934 net.cpp:155] Setting up pool1
I0812 15:00:42.437851  8934 net.cpp:163] Top shape: 100 20 12 12 (288000)
I0812 15:00:42.437857  8934 layer_factory.hpp:75] Creating layer conv2
I0812 15:00:42.437870  8934 net.cpp:110] Creating Layer conv2
I0812 15:00:42.437875  8934 net.cpp:476] conv2 <- pool1
I0812 15:00:42.437887  8934 net.cpp:432] conv2 -> conv2
I0812 15:00:42.438129  8934 net.cpp:155] Setting up conv2
I0812 15:00:42.438139  8934 net.cpp:163] Top shape: 100 50 8 8 (320000)
I0812 15:00:42.438151  8934 layer_factory.hpp:75] Creating layer pool2
I0812 15:00:42.438161  8934 net.cpp:110] Creating Layer pool2
I0812 15:00:42.438169  8934 net.cpp:476] pool2 <- conv2
I0812 15:00:42.438189  8934 net.cpp:432] pool2 -> pool2
I0812 15:00:42.438199  8934 net.cpp:155] Setting up pool2
I0812 15:00:42.438208  8934 net.cpp:163] Top shape: 100 50 4 4 (80000)
I0812 15:00:42.438213  8934 layer_factory.hpp:75] Creating layer ip1
I0812 15:00:42.438222  8934 net.cpp:110] Creating Layer ip1
I0812 15:00:42.438228  8934 net.cpp:476] ip1 <- pool2
I0812 15:00:42.438241  8934 net.cpp:432] ip1 -> ip1
I0812 15:00:42.442422  8934 net.cpp:155] Setting up ip1
I0812 15:00:42.442476  8934 net.cpp:163] Top shape: 100 500 (50000)
I0812 15:00:42.442512  8934 layer_factory.hpp:75] Creating layer relu1
I0812 15:00:42.442589  8934 net.cpp:110] Creating Layer relu1
I0812 15:00:42.442616  8934 net.cpp:476] relu1 <- ip1
I0812 15:00:42.442647  8934 net.cpp:419] relu1 -> ip1 (in-place)
I0812 15:00:42.442708  8934 net.cpp:155] Setting up relu1
I0812 15:00:42.442756  8934 net.cpp:163] Top shape: 100 500 (50000)
I0812 15:00:42.442824  8934 layer_factory.hpp:75] Creating layer ip2
I0812 15:00:42.442878  8934 net.cpp:110] Creating Layer ip2
I0812 15:00:42.442901  8934 net.cpp:476] ip2 <- ip1
I0812 15:00:42.442931  8934 net.cpp:432] ip2 -> ip2
I0812 15:00:42.443135  8934 net.cpp:155] Setting up ip2
I0812 15:00:42.443166  8934 net.cpp:163] Top shape: 100 10 (1000)
I0812 15:00:42.443194  8934 layer_factory.hpp:75] Creating layer ip2_ip2_0_split
I0812 15:00:42.443305  8934 net.cpp:110] Creating Layer ip2_ip2_0_split
I0812 15:00:42.443356  8934 net.cpp:476] ip2_ip2_0_split <- ip2
I0812 15:00:42.443382  8934 net.cpp:432] ip2_ip2_0_split -> ip2_ip2_0_split_0
I0812 15:00:42.443433  8934 net.cpp:432] ip2_ip2_0_split -> ip2_ip2_0_split_1
I0812 15:00:42.443462  8934 net.cpp:155] Setting up ip2_ip2_0_split
I0812 15:00:42.443511  8934 net.cpp:163] Top shape: 100 10 (1000)
I0812 15:00:42.443538  8934 net.cpp:163] Top shape: 100 10 (1000)
I0812 15:00:42.443562  8934 layer_factory.hpp:75] Creating layer accuracy
I0812 15:00:42.443616  8934 net.cpp:110] Creating Layer accuracy
I0812 15:00:42.443640  8934 net.cpp:476] accuracy <- ip2_ip2_0_split_0
I0812 15:00:42.443689  8934 net.cpp:476] accuracy <- label_mnist_1_split_0
I0812 15:00:42.443716  8934 net.cpp:432] accuracy -> accuracy
I0812 15:00:42.443770  8934 net.cpp:155] Setting up accuracy
I0812 15:00:42.443799  8934 net.cpp:163] Top shape: (1)
I0812 15:00:42.443821  8934 layer_factory.hpp:75] Creating layer loss
I0812 15:00:42.443850  8934 net.cpp:110] Creating Layer loss
I0812 15:00:42.443874  8934 net.cpp:476] loss <- ip2_ip2_0_split_1
I0812 15:00:42.443900  8934 net.cpp:476] loss <- label_mnist_1_split_1
I0812 15:00:42.443928  8934 net.cpp:432] loss -> loss
I0812 15:00:42.443994  8934 layer_factory.hpp:75] Creating layer loss
I0812 15:00:42.444056  8934 net.cpp:155] Setting up loss
I0812 15:00:42.444066  8934 net.cpp:163] Top shape: (1)
I0812 15:00:42.444072  8934 net.cpp:168]     with loss weight 1
I0812 15:00:42.444085  8934 net.cpp:236] loss needs backward computation.
I0812 15:00:42.444093  8934 net.cpp:240] accuracy does not need backward computation.
I0812 15:00:42.444100  8934 net.cpp:236] ip2_ip2_0_split needs backward computation.
I0812 15:00:42.444108  8934 net.cpp:236] ip2 needs backward computation.
I0812 15:00:42.444113  8934 net.cpp:236] relu1 needs backward computation.
I0812 15:00:42.444119  8934 net.cpp:236] ip1 needs backward computation.
I0812 15:00:42.444125  8934 net.cpp:236] pool2 needs backward computation.
I0812 15:00:42.444133  8934 net.cpp:236] conv2 needs backward computation.
I0812 15:00:42.444139  8934 net.cpp:236] pool1 needs backward computation.
I0812 15:00:42.444145  8934 net.cpp:236] conv1 needs backward computation.
I0812 15:00:42.444152  8934 net.cpp:240] label_mnist_1_split does not need backward computation.
I0812 15:00:42.444162  8934 net.cpp:240] mnist does not need backward computation.
I0812 15:00:42.444169  8934 net.cpp:283] This network produces output accuracy
I0812 15:00:42.444175  8934 net.cpp:283] This network produces output loss
I0812 15:00:42.444226  8934 net.cpp:297] Network initialization done.
I0812 15:00:42.444244  8934 net.cpp:298] Memory required for data: 8086808
I0812 15:00:42.444296  8934 solver.cpp:49] Solver scaffolding done.
I0812 15:00:42.449823  8934 parallel.cpp:392] GPUs pairs 1:2, 0:1
I0812 15:00:42.683681  8934 data_layer.cpp:44] output data size: 64,1,28,28
I0812 15:00:42.713794  8934 parallel.cpp:235] GPU 1 does not have p2p access to GPU 0
I0812 15:00:42.901604  8934 data_layer.cpp:44] output data size: 64,1,28,28
I0812 15:00:42.927168  8934 parallel.cpp:420] Starting Optimization
I0812 15:00:42.927347  8934 solver.cpp:265] Solving LeNet
I0812 15:00:42.927376  8934 solver.cpp:266] Learning Rate Policy: inv
I0812 15:00:42.927517  8934 solver.cpp:310] Iteration 0, Testing net (#0)
I0812 15:00:44.810916  8934 solver.cpp:359]     Test net output #0: accuracy = 0.1171
I0812 15:00:44.810952  8934 solver.cpp:359]     Test net output #1: loss = 2.39372 (* 1 = 2.39372 loss)
I0812 15:00:44.841032  8934 solver.cpp:222] Iteration 0, loss = 2.34509
I0812 15:00:44.841334  8934 solver.cpp:238]     Train net output #0: loss = 2.34509 (* 1 = 2.34509 loss)
I0812 15:00:44.935735  8934 solver.cpp:517] Iteration 0, lr = 0.01
I0812 15:00:53.956465  8934 solver.cpp:222] Iteration 100, loss = 0.259395
I0812 15:00:53.956553  8934 solver.cpp:238]     Train net output #0: loss = 0.259395 (* 1 = 0.259395 loss)
I0812 15:00:54.048570  8934 solver.cpp:517] Iteration 100, lr = 0.00992565
I0812 15:01:02.986958  8934 solver.cpp:222] Iteration 200, loss = 0.220586
I0812 15:01:02.996296  8934 solver.cpp:238]     Train net output #0: loss = 0.220586 (* 1 = 0.220586 loss)
I0812 15:01:03.059922  8934 solver.cpp:517] Iteration 200, lr = 0.00985258
I0812 15:01:11.830909  8934 solver.cpp:222] Iteration 300, loss = 0.113429
I0812 15:01:11.830996  8934 solver.cpp:238]     Train net output #0: loss = 0.113429 (* 1 = 0.113429 loss)
I0812 15:01:11.901401  8934 solver.cpp:517] Iteration 300, lr = 0.00978075
I0812 15:01:20.998740  8934 solver.cpp:222] Iteration 400, loss = 0.120896
I0812 15:01:20.998869  8934 solver.cpp:238]     Train net output #0: loss = 0.120896 (* 1 = 0.120896 loss)
I0812 15:01:21.048179  8934 solver.cpp:517] Iteration 400, lr = 0.00971013
I0812 15:01:30.206492  8934 solver.cpp:310] Iteration 500, Testing net (#0)
I0812 15:01:32.418656  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9734
I0812 15:01:32.418691  8934 solver.cpp:359]     Test net output #1: loss = 0.0829207 (* 1 = 0.0829207 loss)
I0812 15:01:32.430511  8934 solver.cpp:222] Iteration 500, loss = 0.0747729
I0812 15:01:32.430588  8934 solver.cpp:238]     Train net output #0: loss = 0.074773 (* 1 = 0.074773 loss)
I0812 15:01:32.521658  8934 solver.cpp:517] Iteration 500, lr = 0.00964069
I0812 15:01:41.482954  8934 solver.cpp:222] Iteration 600, loss = 0.0415598
I0812 15:01:41.482987  8934 solver.cpp:238]     Train net output #0: loss = 0.0415598 (* 1 = 0.0415598 loss)
I0812 15:01:41.521524  8934 solver.cpp:517] Iteration 600, lr = 0.0095724
I0812 15:01:50.894820  8934 solver.cpp:222] Iteration 700, loss = 0.0332583
I0812 15:01:50.895135  8934 solver.cpp:238]     Train net output #0: loss = 0.0332583 (* 1 = 0.0332583 loss)
I0812 15:01:50.970991  8934 solver.cpp:517] Iteration 700, lr = 0.00950522
I0812 15:01:59.864426  8934 solver.cpp:222] Iteration 800, loss = 0.155629
I0812 15:01:59.864567  8934 solver.cpp:238]     Train net output #0: loss = 0.155629 (* 1 = 0.155629 loss)
I0812 15:01:59.948451  8934 solver.cpp:517] Iteration 800, lr = 0.00943913
I0812 15:02:08.614575  8934 solver.cpp:222] Iteration 900, loss = 0.319555
I0812 15:02:08.614662  8934 solver.cpp:238]     Train net output #0: loss = 0.319555 (* 1 = 0.319555 loss)
I0812 15:02:08.665554  8934 solver.cpp:517] Iteration 900, lr = 0.00937411
I0812 15:02:17.566509  8934 solver.cpp:310] Iteration 1000, Testing net (#0)
I0812 15:02:19.273252  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9828
I0812 15:02:19.273288  8934 solver.cpp:359]     Test net output #1: loss = 0.0536248 (* 1 = 0.0536248 loss)
I0812 15:02:19.303689  8934 solver.cpp:222] Iteration 1000, loss = 0.0737256
I0812 15:02:19.303783  8934 solver.cpp:238]     Train net output #0: loss = 0.0737256 (* 1 = 0.0737256 loss)
I0812 15:02:19.373178  8934 solver.cpp:517] Iteration 1000, lr = 0.00931012
I0812 15:02:28.194737  8934 solver.cpp:222] Iteration 1100, loss = 0.0279671
I0812 15:02:28.194826  8934 solver.cpp:238]     Train net output #0: loss = 0.0279672 (* 1 = 0.0279672 loss)
I0812 15:02:28.288879  8934 solver.cpp:517] Iteration 1100, lr = 0.00924715
I0812 15:02:36.924510  8934 solver.cpp:222] Iteration 1200, loss = 0.0784045
I0812 15:02:36.924669  8934 solver.cpp:238]     Train net output #0: loss = 0.0784045 (* 1 = 0.0784045 loss)
I0812 15:02:36.992279  8934 solver.cpp:517] Iteration 1200, lr = 0.00918515
I0812 15:02:45.303750  8934 solver.cpp:222] Iteration 1300, loss = 0.0156678
I0812 15:02:45.303838  8934 solver.cpp:238]     Train net output #0: loss = 0.0156679 (* 1 = 0.0156679 loss)
I0812 15:02:45.361748  8934 solver.cpp:517] Iteration 1300, lr = 0.00912412
I0812 15:02:54.148757  8934 solver.cpp:222] Iteration 1400, loss = 0.0158745
I0812 15:02:54.149096  8934 solver.cpp:238]     Train net output #0: loss = 0.0158746 (* 1 = 0.0158746 loss)
I0812 15:02:54.246583  8934 solver.cpp:517] Iteration 1400, lr = 0.00906403
I0812 15:03:03.047662  8934 solver.cpp:310] Iteration 1500, Testing net (#0)
I0812 15:03:04.570204  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9851
I0812 15:03:04.570343  8934 solver.cpp:359]     Test net output #1: loss = 0.0448866 (* 1 = 0.0448866 loss)
I0812 15:03:04.599284  8934 solver.cpp:222] Iteration 1500, loss = 0.022271
I0812 15:03:04.599364  8934 solver.cpp:238]     Train net output #0: loss = 0.0222711 (* 1 = 0.0222711 loss)
I0812 15:03:04.652730  8934 solver.cpp:517] Iteration 1500, lr = 0.00900485
I0812 15:03:13.586694  8934 solver.cpp:222] Iteration 1600, loss = 0.0805327
I0812 15:03:13.586817  8934 solver.cpp:238]     Train net output #0: loss = 0.0805328 (* 1 = 0.0805328 loss)
I0812 15:03:13.669781  8934 solver.cpp:517] Iteration 1600, lr = 0.00894657
I0812 15:03:22.720105  8934 solver.cpp:222] Iteration 1700, loss = 0.0096161
I0812 15:03:22.720190  8934 solver.cpp:238]     Train net output #0: loss = 0.00961622 (* 1 = 0.00961622 loss)
I0812 15:03:22.766216  8934 solver.cpp:517] Iteration 1700, lr = 0.00888916
I0812 15:03:31.639554  8934 solver.cpp:222] Iteration 1800, loss = 0.0161298
I0812 15:03:31.639643  8934 solver.cpp:238]     Train net output #0: loss = 0.0161299 (* 1 = 0.0161299 loss)
I0812 15:03:31.682180  8934 solver.cpp:517] Iteration 1800, lr = 0.0088326
I0812 15:03:40.716598  8934 solver.cpp:222] Iteration 1900, loss = 0.0170069
I0812 15:03:40.716635  8934 solver.cpp:238]     Train net output #0: loss = 0.017007 (* 1 = 0.017007 loss)
I0812 15:03:40.811696  8934 solver.cpp:517] Iteration 1900, lr = 0.00877687
I0812 15:03:49.782884  8934 solver.cpp:310] Iteration 2000, Testing net (#0)
I0812 15:03:51.654955  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9884
I0812 15:03:51.654990  8934 solver.cpp:359]     Test net output #1: loss = 0.0382701 (* 1 = 0.0382701 loss)
I0812 15:03:51.666371  8934 solver.cpp:222] Iteration 2000, loss = 0.0166309
I0812 15:03:51.666401  8934 solver.cpp:238]     Train net output #0: loss = 0.016631 (* 1 = 0.016631 loss)
I0812 15:03:51.728466  8934 solver.cpp:517] Iteration 2000, lr = 0.00872196
I0812 15:04:00.866591  8934 solver.cpp:222] Iteration 2100, loss = 0.0416569
I0812 15:04:00.866627  8934 solver.cpp:238]     Train net output #0: loss = 0.041657 (* 1 = 0.041657 loss)
I0812 15:04:00.943202  8934 solver.cpp:517] Iteration 2100, lr = 0.00866784
I0812 15:04:09.844360  8934 solver.cpp:222] Iteration 2200, loss = 0.00415214
I0812 15:04:09.844445  8934 solver.cpp:238]     Train net output #0: loss = 0.00415225 (* 1 = 0.00415225 loss)
I0812 15:04:09.907941  8934 solver.cpp:517] Iteration 2200, lr = 0.0086145
I0812 15:04:18.904716  8934 solver.cpp:222] Iteration 2300, loss = 0.00498838
I0812 15:04:18.904753  8934 solver.cpp:238]     Train net output #0: loss = 0.00498849 (* 1 = 0.00498849 loss)
I0812 15:04:18.929219  8934 solver.cpp:517] Iteration 2300, lr = 0.00856192
I0812 15:04:28.018985  8934 solver.cpp:222] Iteration 2400, loss = 0.0357902
I0812 15:04:28.019296  8934 solver.cpp:238]     Train net output #0: loss = 0.0357903 (* 1 = 0.0357903 loss)
I0812 15:04:28.066203  8934 solver.cpp:517] Iteration 2400, lr = 0.00851008
I0812 15:04:37.121809  8934 solver.cpp:310] Iteration 2500, Testing net (#0)
I0812 15:04:39.005237  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9871
I0812 15:04:39.005322  8934 solver.cpp:359]     Test net output #1: loss = 0.0386158 (* 1 = 0.0386158 loss)
I0812 15:04:39.036412  8934 solver.cpp:222] Iteration 2500, loss = 0.0279232
I0812 15:04:39.036489  8934 solver.cpp:238]     Train net output #0: loss = 0.0279233 (* 1 = 0.0279233 loss)
I0812 15:04:39.127079  8934 solver.cpp:517] Iteration 2500, lr = 0.00845897
I0812 15:04:48.024590  8934 solver.cpp:222] Iteration 2600, loss = 0.0129841
I0812 15:04:48.024677  8934 solver.cpp:238]     Train net output #0: loss = 0.0129842 (* 1 = 0.0129842 loss)
I0812 15:04:48.087580  8934 solver.cpp:517] Iteration 2600, lr = 0.00840857
I0812 15:04:57.082221  8934 solver.cpp:222] Iteration 2700, loss = 0.0233948
I0812 15:04:57.082298  8934 solver.cpp:238]     Train net output #0: loss = 0.0233949 (* 1 = 0.0233949 loss)
I0812 15:04:57.195091  8934 solver.cpp:517] Iteration 2700, lr = 0.00835886
I0812 15:05:06.519223  8934 solver.cpp:222] Iteration 2800, loss = 0.0220501
I0812 15:05:06.519353  8934 solver.cpp:238]     Train net output #0: loss = 0.0220502 (* 1 = 0.0220502 loss)
I0812 15:05:06.542099  8934 solver.cpp:517] Iteration 2800, lr = 0.00830984
I0812 15:05:15.537714  8934 solver.cpp:222] Iteration 2900, loss = 0.020445
I0812 15:05:15.537787  8934 solver.cpp:238]     Train net output #0: loss = 0.0204451 (* 1 = 0.0204451 loss)
I0812 15:05:15.594329  8934 solver.cpp:517] Iteration 2900, lr = 0.00826148
I0812 15:05:24.883646  8934 solver.cpp:310] Iteration 3000, Testing net (#0)
I0812 15:05:26.701876  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9886
I0812 15:05:26.701910  8934 solver.cpp:359]     Test net output #1: loss = 0.032208 (* 1 = 0.032208 loss)
I0812 15:05:26.730984  8934 solver.cpp:222] Iteration 3000, loss = 0.0241014
I0812 15:05:26.731063  8934 solver.cpp:238]     Train net output #0: loss = 0.0241016 (* 1 = 0.0241016 loss)
I0812 15:05:26.783557  8934 solver.cpp:517] Iteration 3000, lr = 0.00821377
I0812 15:05:35.732266  8934 solver.cpp:222] Iteration 3100, loss = 0.0201092
I0812 15:05:35.732301  8934 solver.cpp:238]     Train net output #0: loss = 0.0201094 (* 1 = 0.0201094 loss)
I0812 15:05:35.808251  8934 solver.cpp:517] Iteration 3100, lr = 0.0081667
I0812 15:05:44.867094  8934 solver.cpp:222] Iteration 3200, loss = 0.00797207
I0812 15:05:44.867220  8934 solver.cpp:238]     Train net output #0: loss = 0.00797222 (* 1 = 0.00797222 loss)
I0812 15:05:44.948299  8934 solver.cpp:517] Iteration 3200, lr = 0.00812025
I0812 15:05:53.949679  8934 solver.cpp:222] Iteration 3300, loss = 0.012678
I0812 15:05:53.949769  8934 solver.cpp:238]     Train net output #0: loss = 0.0126782 (* 1 = 0.0126782 loss)
I0812 15:05:54.064254  8934 solver.cpp:517] Iteration 3300, lr = 0.00807442
I0812 15:06:03.066782  8934 solver.cpp:222] Iteration 3400, loss = 0.0928055
I0812 15:06:03.066819  8934 solver.cpp:238]     Train net output #0: loss = 0.0928057 (* 1 = 0.0928057 loss)
I0812 15:06:03.136878  8934 solver.cpp:517] Iteration 3400, lr = 0.00802918
I0812 15:06:12.386310  8934 solver.cpp:310] Iteration 3500, Testing net (#0)
I0812 15:06:14.325238  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9902
I0812 15:06:14.325274  8934 solver.cpp:359]     Test net output #1: loss = 0.029976 (* 1 = 0.029976 loss)
I0812 15:06:14.362886  8934 solver.cpp:222] Iteration 3500, loss = 0.0376475
I0812 15:06:14.362972  8934 solver.cpp:238]     Train net output #0: loss = 0.0376476 (* 1 = 0.0376476 loss)
I0812 15:06:14.381422  8934 solver.cpp:517] Iteration 3500, lr = 0.00798454
I0812 15:06:23.279611  8934 solver.cpp:222] Iteration 3600, loss = 0.00911815
I0812 15:06:23.279723  8934 solver.cpp:238]     Train net output #0: loss = 0.00911829 (* 1 = 0.00911829 loss)
I0812 15:06:23.314371  8934 solver.cpp:517] Iteration 3600, lr = 0.00794046
I0812 15:06:32.182525  8934 solver.cpp:222] Iteration 3700, loss = 0.0182947
I0812 15:06:32.182616  8934 solver.cpp:238]     Train net output #0: loss = 0.0182949 (* 1 = 0.0182949 loss)
I0812 15:06:32.245113  8934 solver.cpp:517] Iteration 3700, lr = 0.00789695
I0812 15:06:41.339375  8934 solver.cpp:222] Iteration 3800, loss = 0.00605706
I0812 15:06:41.339462  8934 solver.cpp:238]     Train net output #0: loss = 0.00605719 (* 1 = 0.00605719 loss)
I0812 15:06:41.423751  8934 solver.cpp:517] Iteration 3800, lr = 0.007854
I0812 15:06:50.360689  8934 solver.cpp:222] Iteration 3900, loss = 0.00607884
I0812 15:06:50.360779  8934 solver.cpp:238]     Train net output #0: loss = 0.00607896 (* 1 = 0.00607896 loss)
I0812 15:06:50.407685  8934 solver.cpp:517] Iteration 3900, lr = 0.00781158
I0812 15:06:59.592584  8934 solver.cpp:310] Iteration 4000, Testing net (#0)
I0812 15:07:01.409979  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9899
I0812 15:07:01.410017  8934 solver.cpp:359]     Test net output #1: loss = 0.0315973 (* 1 = 0.0315973 loss)
I0812 15:07:01.440788  8934 solver.cpp:222] Iteration 4000, loss = 0.00715784
I0812 15:07:01.440866  8934 solver.cpp:238]     Train net output #0: loss = 0.00715796 (* 1 = 0.00715796 loss)
I0812 15:07:01.498203  8934 solver.cpp:517] Iteration 4000, lr = 0.0077697
I0812 15:07:10.408756  8934 solver.cpp:222] Iteration 4100, loss = 0.0181035
I0812 15:07:10.408840  8934 solver.cpp:238]     Train net output #0: loss = 0.0181036 (* 1 = 0.0181036 loss)
I0812 15:07:10.496220  8934 solver.cpp:517] Iteration 4100, lr = 0.00772833
I0812 15:07:19.468960  8934 solver.cpp:222] Iteration 4200, loss = 0.00506859
I0812 15:07:19.469070  8934 solver.cpp:238]     Train net output #0: loss = 0.0050687 (* 1 = 0.0050687 loss)
I0812 15:07:19.563653  8934 solver.cpp:517] Iteration 4200, lr = 0.00768748
I0812 15:07:28.463631  8934 solver.cpp:222] Iteration 4300, loss = 0.00685792
I0812 15:07:28.463716  8934 solver.cpp:238]     Train net output #0: loss = 0.00685804 (* 1 = 0.00685804 loss)
I0812 15:07:28.573341  8934 solver.cpp:517] Iteration 4300, lr = 0.00764712
I0812 15:07:37.770794  8934 solver.cpp:222] Iteration 4400, loss = 0.00494956
I0812 15:07:37.772317  8934 solver.cpp:238]     Train net output #0: loss = 0.00494967 (* 1 = 0.00494967 loss)
I0812 15:07:37.867321  8934 solver.cpp:517] Iteration 4400, lr = 0.00760726
I0812 15:07:47.014811  8934 solver.cpp:310] Iteration 4500, Testing net (#0)
I0812 15:07:49.268638  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9906
I0812 15:07:49.268673  8934 solver.cpp:359]     Test net output #1: loss = 0.02887 (* 1 = 0.02887 loss)
I0812 15:07:49.280167  8934 solver.cpp:222] Iteration 4500, loss = 0.00429833
I0812 15:07:49.280246  8934 solver.cpp:238]     Train net output #0: loss = 0.00429844 (* 1 = 0.00429844 loss)
I0812 15:07:49.304939  8934 solver.cpp:517] Iteration 4500, lr = 0.00756788
I0812 15:07:58.014494  8934 solver.cpp:222] Iteration 4600, loss = 0.00628843
I0812 15:07:58.014528  8934 solver.cpp:238]     Train net output #0: loss = 0.00628854 (* 1 = 0.00628854 loss)
I0812 15:07:58.130872  8934 solver.cpp:517] Iteration 4600, lr = 0.00752897
I0812 15:08:07.087723  8934 solver.cpp:222] Iteration 4700, loss = 0.000914029
I0812 15:08:07.091270  8934 solver.cpp:238]     Train net output #0: loss = 0.000914142 (* 1 = 0.000914142 loss)
I0812 15:08:07.159850  8934 solver.cpp:517] Iteration 4700, lr = 0.00749052
I0812 15:08:16.295464  8934 solver.cpp:222] Iteration 4800, loss = 0.000818103
I0812 15:08:16.295585  8934 solver.cpp:238]     Train net output #0: loss = 0.000818218 (* 1 = 0.000818218 loss)
I0812 15:08:16.340133  8934 solver.cpp:517] Iteration 4800, lr = 0.00745253
I0812 15:08:25.389153  8934 solver.cpp:222] Iteration 4900, loss = 0.0150521
I0812 15:08:25.389377  8934 solver.cpp:238]     Train net output #0: loss = 0.0150522 (* 1 = 0.0150522 loss)
I0812 15:08:25.444224  8934 solver.cpp:517] Iteration 4900, lr = 0.00741498
I0812 15:08:34.499964  8934 solver.cpp:395] Snapshotting to binary proto file examples/mnist/lenet_iter_5000.caffemodel
I0812 15:08:34.511214  8934 solver.cpp:680] Snapshotting solver state to binary proto fileexamples/mnist/lenet_iter_5000.solverstate
I0812 15:08:34.515492  8934 solver.cpp:310] Iteration 5000, Testing net (#0)
I0812 15:08:36.037077  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9896
I0812 15:08:36.037163  8934 solver.cpp:359]     Test net output #1: loss = 0.0328394 (* 1 = 0.0328394 loss)
I0812 15:08:36.069629  8934 solver.cpp:222] Iteration 5000, loss = 0.0221282
I0812 15:08:36.069816  8934 solver.cpp:238]     Train net output #0: loss = 0.0221283 (* 1 = 0.0221283 loss)
I0812 15:08:36.117702  8934 solver.cpp:517] Iteration 5000, lr = 0.00737788
I0812 15:08:45.425815  8934 solver.cpp:222] Iteration 5100, loss = 0.0028408
I0812 15:08:45.425850  8934 solver.cpp:238]     Train net output #0: loss = 0.00284091 (* 1 = 0.00284091 loss)
I0812 15:08:45.514688  8934 solver.cpp:517] Iteration 5100, lr = 0.0073412
I0812 15:08:54.635145  8934 solver.cpp:222] Iteration 5200, loss = 0.0174867
I0812 15:08:54.635305  8934 solver.cpp:238]     Train net output #0: loss = 0.0174868 (* 1 = 0.0174868 loss)
I0812 15:08:54.706445  8934 solver.cpp:517] Iteration 5200, lr = 0.00730495
I0812 15:09:03.833523  8934 solver.cpp:222] Iteration 5300, loss = 0.0100477
I0812 15:09:03.833560  8934 solver.cpp:238]     Train net output #0: loss = 0.0100478 (* 1 = 0.0100478 loss)
I0812 15:09:03.929847  8934 solver.cpp:517] Iteration 5300, lr = 0.00726911
I0812 15:09:12.852756  8934 solver.cpp:222] Iteration 5400, loss = 0.0106576
I0812 15:09:12.852849  8934 solver.cpp:238]     Train net output #0: loss = 0.0106577 (* 1 = 0.0106577 loss)
I0812 15:09:12.963768  8934 solver.cpp:517] Iteration 5400, lr = 0.00723368
I0812 15:09:22.129829  8934 solver.cpp:310] Iteration 5500, Testing net (#0)
I0812 15:09:24.004926  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9903
I0812 15:09:24.004957  8934 solver.cpp:359]     Test net output #1: loss = 0.0280474 (* 1 = 0.0280474 loss)
I0812 15:09:24.016491  8934 solver.cpp:222] Iteration 5500, loss = 0.0179802
I0812 15:09:24.016526  8934 solver.cpp:238]     Train net output #0: loss = 0.0179803 (* 1 = 0.0179803 loss)
I0812 15:09:24.041682  8934 solver.cpp:517] Iteration 5500, lr = 0.00719865
I0812 15:09:32.959800  8934 solver.cpp:222] Iteration 5600, loss = 0.0115019
I0812 15:09:32.959924  8934 solver.cpp:238]     Train net output #0: loss = 0.0115021 (* 1 = 0.0115021 loss)
I0812 15:09:33.076342  8934 solver.cpp:517] Iteration 5600, lr = 0.00716402
I0812 15:09:41.954908  8934 solver.cpp:222] Iteration 5700, loss = 0.0016562
I0812 15:09:41.954982  8934 solver.cpp:238]     Train net output #0: loss = 0.0016563 (* 1 = 0.0016563 loss)
I0812 15:09:42.058688  8934 solver.cpp:517] Iteration 5700, lr = 0.00712977
I0812 15:09:51.092350  8934 solver.cpp:222] Iteration 5800, loss = 0.00438052
I0812 15:09:51.092428  8934 solver.cpp:238]     Train net output #0: loss = 0.00438063 (* 1 = 0.00438063 loss)
I0812 15:09:51.111524  8934 solver.cpp:517] Iteration 5800, lr = 0.0070959
I0812 15:10:00.229512  8934 solver.cpp:222] Iteration 5900, loss = 0.0425145
I0812 15:10:00.229599  8934 solver.cpp:238]     Train net output #0: loss = 0.0425146 (* 1 = 0.0425146 loss)
I0812 15:10:00.343152  8934 solver.cpp:517] Iteration 5900, lr = 0.0070624
I0812 15:10:09.415284  8934 solver.cpp:310] Iteration 6000, Testing net (#0)
I0812 15:10:11.496309  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9896
I0812 15:10:11.496342  8934 solver.cpp:359]     Test net output #1: loss = 0.0310988 (* 1 = 0.0310988 loss)
I0812 15:10:11.506505  8934 solver.cpp:222] Iteration 6000, loss = 0.024175
I0812 15:10:11.506533  8934 solver.cpp:238]     Train net output #0: loss = 0.0241751 (* 1 = 0.0241751 loss)
I0812 15:10:11.570771  8934 solver.cpp:517] Iteration 6000, lr = 0.00702927
I0812 15:10:20.583513  8934 solver.cpp:222] Iteration 6100, loss = 0.00508203
I0812 15:10:20.583612  8934 solver.cpp:238]     Train net output #0: loss = 0.00508214 (* 1 = 0.00508214 loss)
I0812 15:10:20.703438  8934 solver.cpp:517] Iteration 6100, lr = 0.0069965
I0812 15:10:29.704063  8934 solver.cpp:222] Iteration 6200, loss = 0.00718511
I0812 15:10:29.704149  8934 solver.cpp:238]     Train net output #0: loss = 0.00718522 (* 1 = 0.00718522 loss)
I0812 15:10:29.820086  8934 solver.cpp:517] Iteration 6200, lr = 0.00696408
I0812 15:10:38.826932  8934 solver.cpp:222] Iteration 6300, loss = 0.00189943
I0812 15:10:38.826966  8934 solver.cpp:238]     Train net output #0: loss = 0.00189954 (* 1 = 0.00189954 loss)
I0812 15:10:38.860199  8934 solver.cpp:517] Iteration 6300, lr = 0.00693201
I0812 15:10:47.882072  8934 solver.cpp:222] Iteration 6400, loss = 0.00205551
I0812 15:10:47.882189  8934 solver.cpp:238]     Train net output #0: loss = 0.00205561 (* 1 = 0.00205561 loss)
I0812 15:10:47.961988  8934 solver.cpp:517] Iteration 6400, lr = 0.00690029
I0812 15:10:56.863453  8934 solver.cpp:310] Iteration 6500, Testing net (#0)
I0812 15:10:59.112794  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9892
I0812 15:10:59.112825  8934 solver.cpp:359]     Test net output #1: loss = 0.0309178 (* 1 = 0.0309178 loss)
I0812 15:10:59.124754  8934 solver.cpp:222] Iteration 6500, loss = 0.00429155
I0812 15:10:59.124832  8934 solver.cpp:238]     Train net output #0: loss = 0.00429165 (* 1 = 0.00429165 loss)
I0812 15:10:59.205926  8934 solver.cpp:517] Iteration 6500, lr = 0.0068689
I0812 15:11:08.017499  8934 solver.cpp:222] Iteration 6600, loss = 0.0103716
I0812 15:11:08.017586  8934 solver.cpp:238]     Train net output #0: loss = 0.0103717 (* 1 = 0.0103717 loss)
I0812 15:11:08.069288  8934 solver.cpp:517] Iteration 6600, lr = 0.00683784
I0812 15:11:17.202719  8934 solver.cpp:222] Iteration 6700, loss = 0.00595986
I0812 15:11:17.202792  8934 solver.cpp:238]     Train net output #0: loss = 0.00595996 (* 1 = 0.00595996 loss)
I0812 15:11:17.305418  8934 solver.cpp:517] Iteration 6700, lr = 0.00680711
I0812 15:11:26.384793  8934 solver.cpp:222] Iteration 6800, loss = 0.00682748
I0812 15:11:26.385243  8934 solver.cpp:238]     Train net output #0: loss = 0.00682758 (* 1 = 0.00682758 loss)
I0812 15:11:26.455240  8934 solver.cpp:517] Iteration 6800, lr = 0.0067767
I0812 15:11:35.483453  8934 solver.cpp:222] Iteration 6900, loss = 0.00190708
I0812 15:11:35.483487  8934 solver.cpp:238]     Train net output #0: loss = 0.00190719 (* 1 = 0.00190719 loss)
I0812 15:11:35.534250  8934 solver.cpp:517] Iteration 6900, lr = 0.0067466
I0812 15:11:44.375579  8934 solver.cpp:310] Iteration 7000, Testing net (#0)
I0812 15:11:46.136260  8934 solver.cpp:359]     Test net output #0: accuracy = 0.991
I0812 15:11:46.136289  8934 solver.cpp:359]     Test net output #1: loss = 0.0276416 (* 1 = 0.0276416 loss)
I0812 15:11:46.148154  8934 solver.cpp:222] Iteration 7000, loss = 0.00189898
I0812 15:11:46.148188  8934 solver.cpp:238]     Train net output #0: loss = 0.00189909 (* 1 = 0.00189909 loss)
I0812 15:11:46.176205  8934 solver.cpp:517] Iteration 7000, lr = 0.00671681
I0812 15:11:54.379050  8934 solver.cpp:222] Iteration 7100, loss = 0.00385309
I0812 15:11:54.379124  8934 solver.cpp:238]     Train net output #0: loss = 0.0038532 (* 1 = 0.0038532 loss)
I0812 15:11:54.433017  8934 solver.cpp:517] Iteration 7100, lr = 0.00668733
I0812 15:12:02.811902  8934 solver.cpp:222] Iteration 7200, loss = 0.000648332
I0812 15:12:02.812017  8934 solver.cpp:238]     Train net output #0: loss = 0.000648438 (* 1 = 0.000648438 loss)
I0812 15:12:02.856523  8934 solver.cpp:517] Iteration 7200, lr = 0.00665815
I0812 15:12:11.316092  8934 solver.cpp:222] Iteration 7300, loss = 0.000467351
I0812 15:12:11.316174  8934 solver.cpp:238]     Train net output #0: loss = 0.000467456 (* 1 = 0.000467456 loss)
I0812 15:12:11.387338  8934 solver.cpp:517] Iteration 7300, lr = 0.00662927
I0812 15:12:19.648627  8934 solver.cpp:222] Iteration 7400, loss = 0.0100936
I0812 15:12:19.648660  8934 solver.cpp:238]     Train net output #0: loss = 0.0100937 (* 1 = 0.0100937 loss)
I0812 15:12:19.726174  8934 solver.cpp:517] Iteration 7400, lr = 0.00660067
I0812 15:12:27.860529  8934 solver.cpp:310] Iteration 7500, Testing net (#0)
I0812 15:12:29.803882  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9905
I0812 15:12:29.803916  8934 solver.cpp:359]     Test net output #1: loss = 0.0296367 (* 1 = 0.0296367 loss)
I0812 15:12:29.835386  8934 solver.cpp:222] Iteration 7500, loss = 0.0135883
I0812 15:12:29.835468  8934 solver.cpp:238]     Train net output #0: loss = 0.0135884 (* 1 = 0.0135884 loss)
I0812 15:12:29.851951  8934 solver.cpp:517] Iteration 7500, lr = 0.00657236
I0812 15:12:38.653424  8934 solver.cpp:222] Iteration 7600, loss = 0.00192597
I0812 15:12:38.653540  8934 solver.cpp:238]     Train net output #0: loss = 0.00192608 (* 1 = 0.00192608 loss)
I0812 15:12:38.666893  8934 solver.cpp:517] Iteration 7600, lr = 0.00654433
I0812 15:12:47.777663  8934 solver.cpp:222] Iteration 7700, loss = 0.0122421
I0812 15:12:47.777700  8934 solver.cpp:238]     Train net output #0: loss = 0.0122422 (* 1 = 0.0122422 loss)
I0812 15:12:47.847755  8934 solver.cpp:517] Iteration 7700, lr = 0.00651658
I0812 15:12:56.987732  8934 solver.cpp:222] Iteration 7800, loss = 0.00705013
I0812 15:12:56.987818  8934 solver.cpp:238]     Train net output #0: loss = 0.00705024 (* 1 = 0.00705024 loss)
I0812 15:12:57.055039  8934 solver.cpp:517] Iteration 7800, lr = 0.00648911
I0812 15:13:05.894214  8934 solver.cpp:222] Iteration 7900, loss = 0.00460331
I0812 15:13:05.894297  8934 solver.cpp:238]     Train net output #0: loss = 0.00460342 (* 1 = 0.00460342 loss)
I0812 15:13:05.998929  8934 solver.cpp:517] Iteration 7900, lr = 0.0064619
I0812 15:13:15.322681  8934 solver.cpp:310] Iteration 8000, Testing net (#0)
I0812 15:13:17.185226  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9906
I0812 15:13:17.185304  8934 solver.cpp:359]     Test net output #1: loss = 0.0265522 (* 1 = 0.0265522 loss)
I0812 15:13:17.216338  8934 solver.cpp:222] Iteration 8000, loss = 0.0135588
I0812 15:13:17.216421  8934 solver.cpp:238]     Train net output #0: loss = 0.013559 (* 1 = 0.013559 loss)
I0812 15:13:17.298643  8934 solver.cpp:517] Iteration 8000, lr = 0.00643496
I0812 15:13:26.446269  8934 solver.cpp:222] Iteration 8100, loss = 0.00705471
I0812 15:13:26.446355  8934 solver.cpp:238]     Train net output #0: loss = 0.00705481 (* 1 = 0.00705481 loss)
I0812 15:13:26.499325  8934 solver.cpp:517] Iteration 8100, lr = 0.00640827
I0812 15:13:35.745419  8934 solver.cpp:222] Iteration 8200, loss = 0.00144347
I0812 15:13:35.745501  8934 solver.cpp:238]     Train net output #0: loss = 0.00144357 (* 1 = 0.00144357 loss)
I0812 15:13:35.834023  8934 solver.cpp:517] Iteration 8200, lr = 0.00638185
I0812 15:13:44.658489  8934 solver.cpp:222] Iteration 8300, loss = 0.00511854
I0812 15:13:44.658574  8934 solver.cpp:238]     Train net output #0: loss = 0.00511864 (* 1 = 0.00511864 loss)
I0812 15:13:44.710475  8934 solver.cpp:517] Iteration 8300, lr = 0.00635567
I0812 15:13:53.553982  8934 solver.cpp:222] Iteration 8400, loss = 0.0271572
I0812 15:13:53.554057  8934 solver.cpp:238]     Train net output #0: loss = 0.0271573 (* 1 = 0.0271573 loss)
I0812 15:13:53.619370  8934 solver.cpp:517] Iteration 8400, lr = 0.00632975
I0812 15:14:02.415545  8934 solver.cpp:310] Iteration 8500, Testing net (#0)
I0812 15:14:04.291296  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9895
I0812 15:14:04.291326  8934 solver.cpp:359]     Test net output #1: loss = 0.0309789 (* 1 = 0.0309789 loss)
I0812 15:14:04.302868  8934 solver.cpp:222] Iteration 8500, loss = 0.015714
I0812 15:14:04.302906  8934 solver.cpp:238]     Train net output #0: loss = 0.0157141 (* 1 = 0.0157141 loss)
I0812 15:14:04.328243  8934 solver.cpp:517] Iteration 8500, lr = 0.00630407
I0812 15:14:12.892047  8934 solver.cpp:222] Iteration 8600, loss = 0.00425323
I0812 15:14:12.892132  8934 solver.cpp:238]     Train net output #0: loss = 0.00425334 (* 1 = 0.00425334 loss)
I0812 15:14:12.919698  8934 solver.cpp:517] Iteration 8600, lr = 0.00627864
I0812 15:14:22.127713  8934 solver.cpp:222] Iteration 8700, loss = 0.00508571
I0812 15:14:22.127817  8934 solver.cpp:238]     Train net output #0: loss = 0.00508582 (* 1 = 0.00508582 loss)
I0812 15:14:22.196290  8934 solver.cpp:517] Iteration 8700, lr = 0.00625344
I0812 15:14:31.163892  8934 solver.cpp:222] Iteration 8800, loss = 0.00125972
I0812 15:14:31.164049  8934 solver.cpp:238]     Train net output #0: loss = 0.00125983 (* 1 = 0.00125983 loss)
I0812 15:14:31.224545  8934 solver.cpp:517] Iteration 8800, lr = 0.00622847
I0812 15:14:40.362598  8934 solver.cpp:222] Iteration 8900, loss = 0.0014759
I0812 15:14:40.362864  8934 solver.cpp:238]     Train net output #0: loss = 0.00147601 (* 1 = 0.00147601 loss)
I0812 15:14:40.454977  8934 solver.cpp:517] Iteration 8900, lr = 0.00620374
I0812 15:14:49.448246  8934 solver.cpp:310] Iteration 9000, Testing net (#0)
I0812 15:14:51.062361  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9906
I0812 15:14:51.062409  8934 solver.cpp:359]     Test net output #1: loss = 0.02796 (* 1 = 0.02796 loss)
I0812 15:14:51.072724  8934 solver.cpp:222] Iteration 9000, loss = 0.00350029
I0812 15:14:51.072757  8934 solver.cpp:238]     Train net output #0: loss = 0.0035004 (* 1 = 0.0035004 loss)
I0812 15:14:51.148672  8934 solver.cpp:517] Iteration 9000, lr = 0.00617924
I0812 15:14:59.800629  8934 solver.cpp:222] Iteration 9100, loss = 0.00817083
I0812 15:14:59.800715  8934 solver.cpp:238]     Train net output #0: loss = 0.00817095 (* 1 = 0.00817095 loss)
I0812 15:14:59.912013  8934 solver.cpp:517] Iteration 9100, lr = 0.00615496
I0812 15:15:08.791244  8934 solver.cpp:222] Iteration 9200, loss = 0.00630721
I0812 15:15:08.791373  8934 solver.cpp:238]     Train net output #0: loss = 0.00630733 (* 1 = 0.00630733 loss)
I0812 15:15:08.904939  8934 solver.cpp:517] Iteration 9200, lr = 0.0061309
I0812 15:15:17.983322  8934 solver.cpp:222] Iteration 9300, loss = 0.00714821
I0812 15:15:17.983409  8934 solver.cpp:238]     Train net output #0: loss = 0.00714833 (* 1 = 0.00714833 loss)
I0812 15:15:18.064748  8934 solver.cpp:517] Iteration 9300, lr = 0.00610706
I0812 15:15:27.277515  8934 solver.cpp:222] Iteration 9400, loss = 0.0016361
I0812 15:15:27.277743  8934 solver.cpp:238]     Train net output #0: loss = 0.00163623 (* 1 = 0.00163623 loss)
I0812 15:15:27.334009  8934 solver.cpp:517] Iteration 9400, lr = 0.00608343
I0812 15:15:36.018651  8934 solver.cpp:310] Iteration 9500, Testing net (#0)
I0812 15:15:38.153204  8934 solver.cpp:359]     Test net output #0: accuracy = 0.991
I0812 15:15:38.153237  8934 solver.cpp:359]     Test net output #1: loss = 0.026864 (* 1 = 0.026864 loss)
I0812 15:15:38.164965  8934 solver.cpp:222] Iteration 9500, loss = 0.00147481
I0812 15:15:38.165042  8934 solver.cpp:238]     Train net output #0: loss = 0.00147493 (* 1 = 0.00147493 loss)
I0812 15:15:38.246574  8934 solver.cpp:517] Iteration 9500, lr = 0.00606002
I0812 15:15:47.346539  8934 solver.cpp:222] Iteration 9600, loss = 0.00334556
I0812 15:15:47.346612  8934 solver.cpp:238]     Train net output #0: loss = 0.00334569 (* 1 = 0.00334569 loss)
I0812 15:15:47.402668  8934 solver.cpp:517] Iteration 9600, lr = 0.00603682
I0812 15:15:56.393163  8934 solver.cpp:222] Iteration 9700, loss = 0.000684135
I0812 15:15:56.393198  8934 solver.cpp:238]     Train net output #0: loss = 0.00068426 (* 1 = 0.00068426 loss)
I0812 15:15:56.459017  8934 solver.cpp:517] Iteration 9700, lr = 0.00601382
I0812 15:16:05.434850  8934 solver.cpp:222] Iteration 9800, loss = 0.000417975
I0812 15:16:05.434936  8934 solver.cpp:238]     Train net output #0: loss = 0.000418103 (* 1 = 0.000418103 loss)
I0812 15:16:05.505854  8934 solver.cpp:517] Iteration 9800, lr = 0.00599102
I0812 15:16:14.228703  8934 solver.cpp:222] Iteration 9900, loss = 0.00772128
I0812 15:16:14.228785  8934 solver.cpp:238]     Train net output #0: loss = 0.00772141 (* 1 = 0.00772141 loss)
I0812 15:16:14.273255  8934 solver.cpp:517] Iteration 9900, lr = 0.00596843
I0812 15:16:23.274432  8934 solver.cpp:395] Snapshotting to binary proto file examples/mnist/lenet_iter_10000.caffemodel
I0812 15:16:23.281019  8934 solver.cpp:680] Snapshotting solver state to binary proto fileexamples/mnist/lenet_iter_10000.solverstate
I0812 15:16:23.291081  8934 solver.cpp:291] Iteration 10000, loss = 0.010073
I0812 15:16:23.291107  8934 solver.cpp:310] Iteration 10000, Testing net (#0)
I0812 15:16:24.803035  8934 solver.cpp:359]     Test net output #0: accuracy = 0.9908
I0812 15:16:24.803066  8934 solver.cpp:359]     Test net output #1: loss = 0.0285205 (* 1 = 0.0285205 loss)
I0812 15:16:24.803072  8934 solver.cpp:296] Optimization Done.
I0812 15:16:24.815353  8934 caffe.cpp:184] Optimization Done.