chrishokamp · August 6, 2015 00:17
diff --git a/failed_memorization_example.ipynb b/failed_memorization_example.ipynb
 {"nbformat_minor": 0, "cells": [{"execution_count": 4, "cell_type": "code", "source": "import numpy\nimport theano\nimport logging\nfrom theano import tensor\nfrom blocks.bricks import Linear, Tanh\nfrom blocks.bricks.cost import SquaredError\nfrom blocks.initialization import IsotropicGaussian, Constant\nfrom blocks.monitoring import aggregation\nfrom blocks.algorithms import (GradientDescent, Scale,\n                               StepClipping, CompositeRule)\nfrom blocks.algorithms import AdaDelta\nfrom blocks.extensions.monitoring import TrainingDataMonitoring\nfrom blocks.main_loop import MainLoop\nfrom blocks.extensions import FinishAfter, Printing\nfrom blocks.bricks.recurrent import SimpleRecurrent\nfrom matplotlib import pyplot\nfrom fuel.datasets import IndexableDataset, IterableDataset\nfrom fuel.transformers import Mapping, Batch, Padding, Filter\nfrom collections import OrderedDict\nfrom blocks.graph import ComputationGraph\nfrom fuel.transformers import Batch\nfrom fuel.schemes import ConstantScheme\nfrom blocks.filter import VariableFilter\nfrom blocks.extensions import SimpleExtension\nfrom blocks.model import Model\nfrom blocks.bricks import WEIGHT", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 5, "cell_type": "code", "source": "\"\"\"\nThis is a simple example of Vanilla RNN applied on a toy example dataset.\nThe task is a \"simple memorization task\". The network input has a shape of\n(15, 2) and the output is another sequence with a shape of (15, 2).\nThus, we have 2 input signals and 2 output signals (each with a length of 15).\nOutput signals are same as input signals but with 2 and 4 time-steps\ndelay respectively.\n\nOutput_0_at_time_step[t] = Input_0_at_time_step[t-2]\nOutput_1_at_time_step[t] = Input_1_at_time_step[t-4]\n\nChris: some updates and modifications made for monitoring\n\n\"\"\"\n\ndef main(dataset, n_h, n_y, batch_size, dev_split, n_epochs):\n    # Building Model\n    orig_u = tensor.tensor3('input_sequence')\n    # split so that we can monitor performance on dev data\n    u = orig_u[:dev_split]\n    \n    input_to_state = Linear(name='input_to_state',\n                            input_dim=seq_u.shape[-1],\n                            output_dim=n_h)\n    u_transform = input_to_state.apply(u)\n    RNN = SimpleRecurrent(activation=Tanh(),\n                          dim=n_h, name=\"RNN\")\n    h = RNN.apply(u_transform)  # h is hidden states in the RNN\n    \n    state_to_output = Linear(name='state_to_output',\n                             input_dim=n_h,\n                             output_dim=seq_y.shape[-1])\n    \n    y_hat = state_to_output.apply(h)\n    y_hat.name = 'yhat_sequence'\n    \n    # Cost\n    orig_y = tensor.tensor3('target_sequence')\n    y = orig_y[:dev_split]\n    cost = SquaredError().apply(y, y_hat)\n    \n    cost_cg = ComputationGraph(cost)\n    weights = VariableFilter(roles=[WEIGHT])(cost_cg.variables)\n\n    # add some regularization\n    # L2 norm sum as regularization\n    # blocks also has l2_norm(tensors): implemented in `theano_expressions`\n    cost = cost + tensor.sum([(0.005 * W**2).sum() for W in weights])\n    cost.name = 'regularized_MSE'\n    \n    # split out some dev data so we can monitor performance\n    dev_y = orig_y[dev_split:]\n    dev_x = orig_u[dev_split:]\n    dev_transform = input_to_state.apply(dev_x)\n    dev_h = RNN.apply(dev_transform)\n    dev_y_hat = state_to_output.apply(dev_h)\n    dev_mse = SquaredError().apply(dev_y, dev_y_hat)\n    \n    dev_mse = aggregation.mean(dev_mse, 5)\n    dev_mse.name = 'dev_MSE'\n    # end dev data monitoring\n\n    # Initialization\n    for brick in (RNN, state_to_output, input_to_state):\n        brick.weights_init = IsotropicGaussian(0.01)\n        brick.biases_init = Constant(0)\n        brick.initialize()\n\n    stream = dataset.get_example_stream()\n    stream = Batch(stream, iteration_scheme=ConstantScheme(batch_size))\n   \n    cg = ComputationGraph(cost)\n\n    test_model = Model(cost)\n\n    # Training\n    algorithm = GradientDescent(cost=cost, parameters=cg.parameters,\n                                step_rule=CompositeRule([StepClipping(10.0), Scale(0.01)])) \n    \n    observables = [cost, dev_mse]\n    monitor = TrainingDataMonitoring(observables,\n                                     prefix=\"train\")\n   \n    main_loop = MainLoop(\n        model=test_model,\n        data_stream=stream, \n        algorithm=algorithm,\n        extensions=[monitor,\n                    TrainingDataMonitoring(observables, after_batch=True),\n                    FinishAfter(after_n_epochs=n_epochs),\n                    Printing(every_n_batches=100, after_epoch=False)])  \n\n    main_loop.run()", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 6, "cell_type": "code", "source": "# THEANO DEBUGGING EXAMPLES\nfrom blocks import config\nconfig.recursion_limit = 100000\nfloatX = theano.config.floatX\nlogger = logging.getLogger(__name__)\n# this is to let the log print in the notebook\nlogger.setLevel(logging.DEBUG)\n\n# theano debugging stuff\ntheano.config.optimizer='fast_compile'\n# theano.config.optimizer='None'\ntheano.config.exception_verbosity='high'", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 8, "cell_type": "code", "source": "logging.basicConfig(level=logging.INFO)\n\nn_examples = 100\nn_u = 2  # input vector size\nn_h = 10  # hidden vector size\nn_y = 2  # output vector size\ntime_steps = 5  # number of time-steps in time\nn_seq = 10  # number of sequences for training\n\nnumpy.random.seed(0)\n\n# generating random sequences\nseq_u = numpy.random.randn(n_examples, time_steps, n_u).astype(theano.config.floatX)\nseq_y = numpy.zeros((n_examples, time_steps, n_y)).astype(theano.config.floatX)\n\nseq_y[:, 1:] = seq_u[:, :-1]  # 2 time-step delay\n# seq_y[:, 4:, 1] = seq_u[:, :-4, 1]  # 4 time-step delay\n\n# some random noise?\nseq_y += 0.01 * numpy.random.standard_normal(seq_y.shape)\n\ndataset = IndexableDataset(indexables=OrderedDict([('input_sequence', seq_u), ('target_sequence', seq_y)]))\n# split each batch 80/20 train/dev (see the main function)s\nbatch_size = 20\ndev_split = 16\nmain(dataset, n_h, n_y, batch_size, dev_split, 1000)\n", "outputs": [{"output_type": "stream", "name": "stdout", "text": "provides_sources\n('input_sequence', 'target_sequence')\n\n-------------------------------------------------------------------------------\nBEFORE FIRST EPOCH\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 0\n\t iterations_done: 0\n\t received_first_batch: False\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 0:\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 19\n\t iterations_done: 100\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 100:\n\t dev_MSE: 0.452650606632\n\t regularized_MSE: 3.82857203484\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 39\n\t iterations_done: 200\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 200:\n\t dev_MSE: 0.453206598759\n\t regularized_MSE: 3.82864284515\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 59\n\t iterations_done: 300\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 300:\n\t dev_MSE: 0.454001277685\n\t regularized_MSE: 3.82835268974\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 79\n\t iterations_done: 400\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 400:\n\t dev_MSE: 0.455393463373\n\t regularized_MSE: 3.82793784142\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 99\n\t iterations_done: 500\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 500:\n\t dev_MSE: 0.457631736994\n\t regularized_MSE: 3.82750415802\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 119\n\t iterations_done: 600\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 600:\n\t dev_MSE: 0.460746377707\n\t regularized_MSE: 3.82734084129\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 139\n\t iterations_done: 700\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 700:\n\t dev_MSE: 0.464316785336\n\t regularized_MSE: 3.82771945\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 159\n\t iterations_done: 800\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 800:\n\t dev_MSE: 0.467621982098\n\t regularized_MSE: 3.82853102684\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 179\n\t iterations_done: 900\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 900:\n\t dev_MSE: 0.470146745443\n\t regularized_MSE: 3.8293735981\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 199\n\t iterations_done: 1000\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 1000:\n\t dev_MSE: 0.471821129322\n\t regularized_MSE: 3.82996439934\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 219\n\t iterations_done: 1100\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 1100:\n\t dev_MSE: 0.47284245491\n\t regularized_MSE: 3.83026337624\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 239\n\t iterations_done: 1200\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 1200:\n\t dev_MSE: 0.47344571352\n\t regularized_MSE: 3.83034706116\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 259\n\t iterations_done: 1300\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 1300:\n\t dev_MSE: 0.4738060534\n\t regularized_MSE: 3.83030128479\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 279\n\t iterations_done: 1400\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 1400:\n\t dev_MSE: 0.474032700062\n\t regularized_MSE: 3.83018708229\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 299\n\t iterations_done: 1500\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 1500:\n\t dev_MSE: 0.474187761545\n\t regularized_MSE: 3.8300409317\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 319\n\t iterations_done: 1600\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 1600:\n\t dev_MSE: 0.474305450916\n\t regularized_MSE: 3.82988405228\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 339\n\t iterations_done: 1700\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 1700:\n\t dev_MSE: 0.474404245615\n\t regularized_MSE: 3.82972764969\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 359\n\t iterations_done: 1800\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 1800:\n\t dev_MSE: 0.474494159222\n\t regularized_MSE: 3.8295788765\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 379\n\t iterations_done: 1900\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 1900:\n\t dev_MSE: 0.474580824375\n\t regularized_MSE: 3.82944250107\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 399\n\t iterations_done: 2000\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 2000:\n\t dev_MSE: 0.474667489529\n\t regularized_MSE: 3.82932257652\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 419\n\t iterations_done: 2100\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 2100:\n\t dev_MSE: 0.474756151438\n\t regularized_MSE: 3.82922339439\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 439\n\t iterations_done: 2200\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 2200:\n\t dev_MSE: 0.474848181009\n\t regularized_MSE: 3.82915043831\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 459\n\t iterations_done: 2300\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 2300:\n\t dev_MSE: 0.474944591522\n\t regularized_MSE: 3.82910990715\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 479\n\t iterations_done: 2400\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 2400:\n\t dev_MSE: 0.475046098232\n\t regularized_MSE: 3.82911133766\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 499\n\t iterations_done: 2500\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 2500:\n\t dev_MSE: 0.475153297186\n\t regularized_MSE: 3.8291656971\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 519\n\t iterations_done: 2600\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 2600:\n\t dev_MSE: 0.475266396999\n\t regularized_MSE: 3.82928848267\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 539\n\t iterations_done: 2700\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 2700:\n\t dev_MSE: 0.47538536787\n\t regularized_MSE: 3.82949972153\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 559\n\t iterations_done: 2800\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 2800:\n\t dev_MSE: 0.475509703159\n\t regularized_MSE: 3.82982587814\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 579\n\t iterations_done: 2900\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 2900:\n\t dev_MSE: 0.475638151169\n\t regularized_MSE: 3.83030104637\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 599\n\t iterations_done: 3000\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 3000:\n\t dev_MSE: 0.475768864155\n\t regularized_MSE: 3.83096933365\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 619\n\t iterations_done: 3100\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 3100:\n\t dev_MSE: 0.475898832083\n\t regularized_MSE: 3.83188652992\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 639\n\t iterations_done: 3200\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 3200:\n\t dev_MSE: 0.476023852825\n\t regularized_MSE: 3.83312320709\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 659\n\t iterations_done: 3300\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 3300:\n\t dev_MSE: 0.476138353348\n\t regularized_MSE: 3.83476424217\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 679\n\t iterations_done: 3400\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 3400:\n\t dev_MSE: 0.476235300303\n\t regularized_MSE: 3.83690953255\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 699\n\t iterations_done: 3500\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 3500:\n\t dev_MSE: 0.476307153702\n\t regularized_MSE: 3.83966612816\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 719\n\t iterations_done: 3600\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 3600:\n\t dev_MSE: 0.476346582174\n\t regularized_MSE: 3.84313368797\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 739\n\t iterations_done: 3700\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 3700:\n\t dev_MSE: 0.476349294186\n\t regularized_MSE: 3.84737420082\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 759\n\t iterations_done: 3800\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 3800:\n\t dev_MSE: 0.476317107677\n\t regularized_MSE: 3.85236954689\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 779\n\t iterations_done: 3900\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 3900:\n\t dev_MSE: 0.476261049509\n\t regularized_MSE: 3.85797309875\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 799\n\t iterations_done: 4000\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 4000:\n\t dev_MSE: 0.476202428341\n\t regularized_MSE: 3.8638894558\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 819\n\t iterations_done: 4100\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 4100:\n\t dev_MSE: 0.476170837879\n\t regularized_MSE: 3.86971592903\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 839\n\t iterations_done: 4200\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 4200:\n\t dev_MSE: 0.476199448109\n\t regularized_MSE: 3.87504243851\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 859\n\t iterations_done: 4300\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 4300:\n\t dev_MSE: 0.476320117712\n\t regularized_MSE: 3.8795633316\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 879\n\t iterations_done: 4400\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 4400:\n\t dev_MSE: 0.47656005621\n\t regularized_MSE: 3.88313579559\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 899\n\t iterations_done: 4500\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 4500:\n\t dev_MSE: 0.476938486099\n\t regularized_MSE: 3.88576054573\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 919\n\t iterations_done: 4600\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 4600:\n\t dev_MSE: 0.477463245392\n\t regularized_MSE: 3.88752746582\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 939\n\t iterations_done: 4700\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 4700:\n\t dev_MSE: 0.478127777576\n\t regularized_MSE: 3.88855886459\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 959\n\t iterations_done: 4800\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 4800:\n\t dev_MSE: 0.4789095819\n\t regularized_MSE: 3.88898181915\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 979\n\t iterations_done: 4900\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 4900:\n\t dev_MSE: 0.479772746563\n\t regularized_MSE: 3.88891649246\n\n\n-------------------------------------------------------------------------------\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: True\n\t epochs_done: 999\n\t iterations_done: 5000\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 5000:\n\t dev_MSE: 0.480674028397\n\t regularized_MSE: 3.88847732544\n\n\n-------------------------------------------------------------------------------\nTRAINING HAS BEEN FINISHED:\n-------------------------------------------------------------------------------\nTraining status:\n\t batch_interrupt_received: False\n\t epoch_interrupt_received: False\n\t epoch_started: False\n\t epochs_done: 1000\n\t iterations_done: 5000\n\t received_first_batch: True\n\t resumed_from: None\n\t training_started: True\nLog records from the iteration 5000:\n\t dev_MSE: 0.480674028397\n\t regularized_MSE: 3.88847732544\n\t training_finish_requested: True\n\t training_finished: True\n\n"}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": null, "cell_type": "code", "source": "", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}], "nbformat": 4, "metadata": {"kernelspec": {"display_name": "Python 2", "name": "python2", "language": "python"}, "language_info": {"mimetype": "text/x-python", "nbconvert_exporter": "python", "version": "2.7.10", "name": "python", "file_extension": ".py", "pygments_lexer": "ipython2", "codemirror_mode": {"version": 2, "name": "ipython"}}}}