Highway network.ipynb

highway-network.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Highway network.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "[View in Colaboratory](https://colab.research.google.com/gist/NTT123/ecdac1c66b04c83fcc309fa2f93bf734/highway-network.ipynb)"
      ]
    },
    {
      "metadata": {
        "id": "YXdzZAsrZEzY",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Training Very Deep Networks\n",
        "Rupesh Kumar Srivastava, Klaus Greff, Jürgen Schmidhuber\n",
        "https://arxiv.org/abs/1507.06228"
      ]
    },
    {
      "metadata": {
        "id": "HjxGMdJy8rRn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "7176bed0-6fd6-4a2f-fd2e-6337c9433cb1"
      },
      "cell_type": "code",
      "source": [
        "# http://pytorch.org/\n",
        "from os import path\n",
        "from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag\n",
        "platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())\n",
        "\n",
        "accelerator = 'cu80' if path.exists('/opt/bin/nvidia-smi') else 'cpu'\n",
        "\n",
        "!pip install -q http://download.pytorch.org/whl/{accelerator}/torch-0.4.1-{platform}-linux_x86_64.whl torchvision\n",
        "import torch"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tcmalloc: large alloc 1073750016 bytes == 0x560ee000 @  0x7ff0c05321c4 0x46d6a4 0x5fcbcc 0x4c494d 0x54f3c4 0x553aaf 0x54e4c8 0x54f4f6 0x553aaf 0x54efc1 0x54f24d 0x553aaf 0x54efc1 0x54f24d 0x553aaf 0x54efc1 0x54f24d 0x551ee0 0x54e4c8 0x54f4f6 0x553aaf 0x54efc1 0x54f24d 0x551ee0 0x54efc1 0x54f24d 0x551ee0 0x54e4c8 0x54f4f6 0x553aaf 0x54e4c8\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "WZ2kb7mI8t75",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import torchvision\n",
        "from torchvision import datasets, transforms\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "OoRDDwXR80Oz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torchvision.transforms as transforms\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2W9yv1LN9kQJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "7b6da2e4-103d-49ff-d5df-493b04cdcb3c"
      },
      "cell_type": "code",
      "source": [
        "transform = transforms.Compose(\n",
        "    [transforms.ToTensor(),\n",
        "     transforms.Normalize((0.1307,), (0.3081,))])\n",
        "\n",
        "trainset = torchvision.datasets.MNIST(root='./data', train=True,\n",
        "                                        download=True, transform=transform)\n",
        "trainloader = torch.utils.data.DataLoader(trainset, batch_size=32,\n",
        "                                          shuffle=True, num_workers=2)\n",
        "\n",
        "testset = torchvision.datasets.MNIST(root='./data', train=False,\n",
        "                                       download=True, transform=transform)\n",
        "testloader = torch.utils.data.DataLoader(testset, batch_size=32,\n",
        "                                         shuffle=False, num_workers=2)\n",
        "\n"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
            "Processing...\n",
            "Done!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "lsYbZg2t9mG0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276
        },
        "outputId": "78194f64-f83d-4ba6-bf98-53f5a44168f8"
      },
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "\n",
        "def imshow(img):\n",
        "    img = img / 2 + 0.5     # unnormalize\n",
        "    npimg = img.numpy()\n",
        "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
        "\n",
        "\n",
        "dataiter = iter(trainloader)\n",
        "images, labels = dataiter.next()\n",
        "\n",
        "imshow(torchvision.utils.make_grid(images))\n"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeQAAAEDCAYAAADk73fDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJztvXmgjHX///+wZqssHWQJRShLKoq0\nUJFSpKwJRYi0KURKKWVplSWiLFmKu0h8+EqkO5QlRcpWSGXLkvUO5/fH/J7XNWfOzFlnueac1+Of\n48zMGe/rmmuu63q+lucrR2JiYiKGYRiGYcSUnLFegGEYhmEYdkE2DMMwDE9gF2TDMAzD8AB2QTYM\nwzAMD2AXZMMwDMPwAHZBNgzDMAwPkDvcbzhkyBDWr19Pjhw56N+/PzVq1Aj3f2EYhmEYWY6wXpC/\n/fZbduzYwcyZM9m2bRv9+/dn5syZ4fwvDMMwDCNLEtaQ9YoVK7jlllsAuOSSSzh8+DBHjx4N539h\nGIZhGFmSsCrk/fv3c/nllzu/Fy1alH379lGoUKGgrx80aBA9evRg9OjR4VyGkQK2v6OL7e/oY/s8\nutj+Tj+DBg0K+niOcFpnDhw4kBtvvNFRyW3btmXIkCFUqFAh6Ov37t1L8eLFw/XfG4ZhGIanGTRo\nUMgLclgVcvHixdm/f7/z+969e0lISAj5+tGjR6e4OCP82P6OLra/o4/t8+hi+zt8hPWCfN111zFy\n5EjatGnDxo0bKV68eMhwdTC+/fbbcC4nptSpUyfJ717atnCsxcvbl1nCvW1e2zf+2+e1tWUWOy5j\ng51TUiZw20IR1gvylVdeyeWXX06bNm3IkSMHzz//fDjf3jAMwzCyLGHvQ37qqafC/ZaGYRiGkeUx\npy7DMAzD8AB2QTYMwzAMDxD2kLVhGEZaefrppwGoUKECs2fPBuCLL76I5ZIMI2aYQjYMwzAMD5At\nFHL79u0BaNiwIQCNGzemdOnSsVxSppk0aRIAp06dAmD16tUAjBs3LmZrihVXX3014HMMAli8eDHT\npk2L5ZKMVGjatCkAR44cASB37tycPXs2lksKymWXXQb4TI4AXnzxRQDeffddAGfNDz/8MJUrVwZg\nw4YN0V6mkUUwhWwYhmEYHiBbKOSaNWsm+X3YsGGUKVMGcHNY8cbLL78MQO/evQF46aWXAEJakfbp\n04ft27dHZ3FRomDBggDkyZMHgNOnTwNQvXp1XnvtNcDdP/HAJ598AsDatWspUKBA0NfUr18fgOuv\nvz5q6won1atXB6Bu3boAKTr5xZqTJ09y2223AVCiRAkARo0aBbjHmhg1ahRvvPFGdBcYRhISEhg6\ndCjgqv9Vq1bFcklhR9+djz76CIChQ4eyceNGAPr37w/Agw8+CMBvv/0W/QViCtkwDMMwPEG2UMhi\n/fr1APzwww+Oal64cCHgyyvHC61ateLiiy9O02t1N3jo0CFmzZrl3AHGM3nz5gXgzJkzADzwwANJ\nnj9+/HgST3WvULJkScBVIMuWLUvy/MGDBwEoW7ZsyPdYt24d4HPFA/eu3+sULlwYgGLFigGuMn7v\nvfcA2LlzJ8eOHYvN4gL48ssvAfj555/ZtWsXgKN+33zzzSSvvfvuu53Hn3jiCcD9nO+6666orDcz\nLF68GIDt27ezYsUKALZu3RrLJYWNSpUqAbB582YAmjRpAkCnTp2c1yhi06ZNG8A3IAnciMiePXui\nslZhCtkwDMMwPEC2UMg1atQAoFq1agCULl2aH3/8EYDdu3cDvhyrngNXTXuR8uXLhxxpGchNN90E\nwKeffuo8ljOn7z7Mi1WtaUEztx9++OGgz2/ZsoUZM2ZEc0kpUrFiRTp27Mj7778PuHfdVapUSfK6\ntHwe+fLlA3Dmz8ZLNfl3330HwIgRIwD466+/AMiRI0eS373AY489BkCuXLkcK+CiRYsCySNp+jze\ne+89unTpArjfNdWp/P7775FfdAZ56623AN85Ye7cuQAcOHAgzX8/YMAAAO68807Aza0vWLAgnMtM\nF8899xzgntv79u2b5r8dPHgwAF9//TUAr7/+OgDTp08HYM6cOSxatChsaw3EFLJhGIZheIBsoZB1\nVzt8+HDAV1mtHOOSJUsAN5dw//33A+5dYrxWYQdDd4rKdakSOd74448/Unz+ueeec/pHY0XFihWd\nfxcsWJASJUrQr1+/oK9duXIlAGvWrAEgMTHReU499MrBxhsPPfQQ4EanRPny5QFvVvL+/PPPgO+8\ncd999wGuYgrk77//BuDSSy/lf//7HwDXXnst4FPYAOPHj4/oejPDBx98AECzZs0YMmQIAJ07d07z\n32usoKr+N23aFN4FppPnnnuOCy+8EICLLroow++j2ozAGo27777bia7mz58fIKzdK9nigiyCXVyn\nTp2a5HeFt1X0VbNmTc+Fr7/99lsuueSSJI+pEKV169ZBHz98+HB0FhdBPvzwQyB0S8KWLVsAXyg7\nsGAq2viHn3v27JnkuRMnTgBuGEwn8mDHmYq34rFlLVeuXHz11VcA1K5dG8ApHPJSSiEQfR7lypUL\neSEOZOrUqbzyyiuAG65VWkFteV5EIdohQ4awY8eODL+PCvKUlok2am8877zzku1vf/MZgH/++Qdw\nC7fSQ548eZx9phs3fdcV3s4MFrI2DMMwDA+QrRRyWrj99tsBtyBg/vz5nrPZVLjPn0BlHPi4v6Xm\nzJkzI7KuSLN3714guZmEHj906BCQvJ0oFsj4IhhSuwpVp4RCpvGEQrVr165lzJgxSZ6T4p8wYULU\n1xVp1JLmpQK1UEjlyUioUaNGzrkvIyil0qBBA8CNzEULFQv6H1dSzQqnFypUCIBt27YB8PHHHzsh\ndqUuRalSpQDXgOnVV19N9n+qKFOGPmpFzUw0yxSyYRiGYXgAU8gB/PDDD7FeQki6du0KkESxhyoU\nSgndxU6ZMiU8C4sCf/zxR0g7SRXWeKl4RqYE4Ct8eeKJJ2jVqhXgGkekhBTHHXfcEZkFRhCt3b9g\nS+0x+/bti8maYkH37t0BmDx5MgD//vtvLJcDuMrvm2++AeCee+4ByHD+eMCAAQwbNswx6VGUKlqo\n6LFq1arJnpMpUqj6mfLlyzvRRhX8BqIWUQ3vqVatmtPqJmQOU69ePcA91jOCKWTDMAzD8ACmkP9/\nVFWtJn5Z/N1+++0xr7LWXZrGDAL8+eefANx7771A6uowd+7cTtO+cnzxgEZmlilTxjGRiAcGDRoE\n+GoQrrjiCr788st05dU0xCCUneQLL7wAJFXiXkF2mP5q+JprrgFg3rx5MVlTLFA1r5fQuUxjJKUw\nzz333HS/V6VKlZy8qbZVefRooS4ZHXPgdjW0aNECCP0dSguqoB47dizgU92qWQlUyhpQkZnzlClk\nwzAMw/AA2V4hK3cgZSylLGOQWKtjcHNx/oYRMoBPax749OnTzt8r3xMPyLBFdpn+yEhD0YKshLYt\n0F5TpgQyb1CfrxdQVOn7778HfEMyypUrB8Ctt94as3XFCvXNe6kPObBqXz3TyoOmxI033ghA8+bN\nAZ8avuGGG8K8wvQhm1N/fvrpJ4BkXg3hoFWrVo4dsaILQh0EmYkSmEI2DMMwDA+QbRWyxi5OnDgR\ncO/uZbPphd7jpk2bAtCtW7ckj//999/O4PSMVEqrWlkVl6lZUcYC5YSC5eHkdKVcjkzxswpDhw51\njsdAZHua1vGb0USj63R8nT171hm0oJx4VkbWkxpxqpxiqAreaKJIi0YQClX+N27c2BkQob5dnWMU\n3bjqqqsAnxUs+CJ3hQoVcs4jsUARTX9nPCnVSLkT6rup0Zwal6rctYaMZIRsdUHWRXj+/PnOhVgo\nRO2FC/Ett9wCuGvRASaKFy+eKcMIvZ/CogrLqzk+lMlINFAbgubPBgtJKeSuhvysggxPihQpEnLy\nk4rFAkOM48aNi+mEnWB88803fPbZZ7FeRlR46KGHHHtQfZ90vvECsnnUBUzfnUDjFnAncwlZ1Q4d\nOhRwWy337NnD+vXradq0acxuOgK/J0OGDIl4Md3s2bMB+PzzzwF45513wvbeFrI2DMMwDA+QpRWy\n7gY1y1SqeP369Z6e5iQF6z8xyJ+rrrrKaeSXtZtMCNTaFIh/25PK8mU00qNHD8AN8fTr148NGzYA\n0W9TkRGGPjt/jh49CriFXvFaKCTjDNnxPfnkkwDOcIKU5iJr+kygwYaGokDsZtFqUpoUxKhRo6hc\nuTLgqv+sagxSv359Tp06BbjDKR555JEkr5GSUkFmLFAboeZoX3HFFQC8++67joGJjp8vvvgCcIe2\nKLX16KOPJntfRexibc5TtGjRTA3JyAzhSMuYQjYMwzAMD5BhhTxs2DDWrFnD6dOn6datG9WrV6dP\nnz6cOXOGhIQEhg8fTt68ecO51nSjsvRgBTJeVMbi22+/BUKX7a9atcoxsFf7i9Svf2uUP/5tT4Gv\nCfzb1atXO8MRoqWQ9VnJAD4YUpDxpowrVqxIx44dHYWkKIAMX958881M/x+XXHJJWN4nM6hASDRr\n1swpBJK60pATGZqo3TBeUORGBVsy2Dh8+LBjFKFcciCaDe1voamolKJVgeg8peK4cKEIof/sYxV+\nqXAwNfLmzesUeMWqsOvxxx8H3O9Q9+7dWbp0KZB8tG640OesnLpYu3YtkDljkAxdkFeuXMmWLVuY\nOXMmBw8e5O6776Zu3bq0a9eOJk2a8PrrrzNr1izatWuX4YUZhmEYRnYiQxfk2rVrOzmr8847jxMn\nTrBq1SrHzq9BgwZMnDgx5hdk3V1KfSnHtXv3budO1wvGH0J3mYFmEMFQ87sGK8gOM5Tph1oZ/P9W\nA7ul1JS73Lp1K4sXL073+jNDr169gJQ/DxnIezkPKcXRrFkz57G6detSokQJZ+RdpFAeL7CDIFbc\ncMMNjimGcpUylwjMr3qVatWqAa6q1ecrZRusC0BGNYH51AceeABw22SmTZuWbGyh8tCKBvkP6PAa\nnTt3diIgI0eOjMkaDh48mOwx7Tvt53CjbhRFB/bs2QO41xkZwmSEDOWQc+XK5fQazpo1ixtuuIET\nJ044IepixYp5+qRpGIZhGF4jR2KopGMaWLx4Me+++y4TJ06kUaNGrFixAvCN8urbty8zZsxI8e/3\n7t3rDMg2DMMwjKzOoEGDHD+BQDJc1LV8+XLGjh3Le++9x7nnnkuBAgU4efIk+fLlY8+ePWm60I4e\nPTrJ4lTMFCnUvF6sWLGIO3LVqVMnye9p2bY5c+YAOAVbgfjvU4U/VUiQGuPGjSMxMZGyZcsyffp0\nIHPzkDOyfSmh0FyokPXAgQOd8JTaSiJFRrZNRgs7d+4EIH/+/IBvSlVaC5c2b94MwFNPPZWh8LZu\niFM7Jvy3L5zfueuuuw6Ajh07Ar6QoZyhVAijwrbjx48D7n5LzySslKhTp07Yzilt27Zl48aNQOoh\n9mXLljlpsNGjRwOuoYaQr7eOjd27dzshfaWeUiLc37nMsmDBAnbv3k2pUqWclFhGpkaJjGyfWkM1\n9a5ixYrOfp81axbgmqKEg1atWnHppZcCromRUhEpTdEL3LZQZOiC/M8//zBs2DA++OADJwdSr149\nFi5cSLNmzVi0aFGSnKVXUN579+7dTp+tF5y5hPKOgU45oao204NXxy/qJKb6g0DUG3n69OmIX4gz\ngr6U6tXU9yEtKN+vz71Dhw4AlCxZMi6tJlu2bJnk95UrVzo2p6owVr3CypUrAXe/eQmdYMuUKePk\ndkOxfPlyADZs2JBq7jBW/bFZGfV0d+rUCYCvv/7a+U7qfHHZZZcBbu1MelBqVgMjLr30Uuf91W+v\nXvtw9Jdn6II8f/58Dh486JScg8/k4Nlnn2XmzJmUKlXKmQhiGIZhGEbqZOiC3Lp166B+x++//36m\nFxQJpML8q0/79OkDeKvKWoRDEQfi34dcq1YtwB0a//DDD4f9/0srKgS88MILgz4vX1r1TXoNRV3S\nooxlRi9VqP2v6vpoV7aHG4V3VZnsH31SSFqhWY3TlGOUl3j77bcBnNROMBQS12fnxfNIpNGYTS+g\nATzr169P1iuuELZU9HPPPQf4KucVpVLqQNFD9dQPGzYMcCMhO3bscFIP+j+3b98etu2IO+vM3bt3\nA0mtFfv27Rv0tTpZBrYmlC5d2skhZ0d0AOkgMzLON998A7gX10DKlCnDvn376NChg5PnUr48taLH\neENtPhoMEozXXnsN8PYMa7XNHDhwIJmpkD5DhbJXr14d3cV5iNWrVzs1AoEzgtNqLhIuli1bBvhq\nEzZt2gRAmzZtADetJFTHMHbsWCd1qfXr8xUySJJxy7nnnusYE4XzQizMOtMwDMMwPEDcKeTbb78d\ncEMJ9913n1PF+sMPPwBJDUDArSZW2DM7quOEhARWrVpF2bJlnVCLih1iie4yFeUItKPTHatXQ4L7\n9+8HYMCAAUGfnz9/PgMGDKBs2bKeNnkIJ9FWR+FGc42XLFnihN01AKRkyZKAd4/HaLJp0yZHISvt\nJYWsqvpo89133zmGHRqCoYiHQtYKS2sgTzDUsaC/UZFypGYsC1PIhmEYhuEB4k4h685UKnf//v28\n9dZbSV4Trp7GrIQG2t9www2eUMYisFgrlMm+YUQbjSoEdxi94bJ7927eeOMNhg8f7hQkqqAvliMm\njx07BsBdd92V5HFZEr/00ktA0oEYGjU5c+ZMwO2Bjnb7rilkwzAMw/AAcaeQDcMwjNhz+PBhJ6ca\nDx0DassNZgZ1yy23AMkHgkQbU8iGYRiG4QHsgmwYhmEYHsAuyIZhGIbhAeyCbBiGYRgewC7IhmEY\nhuEBPFVlndaZkfGIl7YtEmvx0vaFm8xum5f3jZfXFg6y8vZ5advsnBIeTCEbhmEYhgewC7JhGIZh\neABPhaxlV5YVCAy3eGnbwrEWL29fZgn3tnlt3/hvn9fWllnsuIwNdk5JmbSG300hG4ZhGIYHsAuy\nYRiGYXgAuyAbhmEYhgfwVA7ZMAzDiD/27t0LuMMZ2rZtC8DFF18cszXFI6aQDcMwDMMDmEI2DMMw\nMkTu3L5LyKeffgpAzpw+jbd48WIAvvnmGwA++OCD6C8uDjGFbBiGYRgewBRyFqV79+6A2/82duzY\nLNXXZxhGbOnQoQMXXXQR4CrjdevWAXD//fcD0KlTp5isLV4xhWwYhmEYHsAUchZFCnnt2rVJfje8\nQ4ECBQCYPHkyAK+88goATZo0YebMmQBs2bIlNosLQt68eQF46qmnAHj11VcBSEhIAKBatWoA3H33\n3YBv+z788EMAHn30UQBat24NwMmTJ6O0aiPcKG983XXXkStXLgAWLVoEwC+//ALAtddeC8DPP/8c\ngxXGL3ZBzqLoQuxPPF6U69atC8DXX39N8eLFAThw4EC636dDhw4AnHvuuQD06dMHgHLlyoVjmemi\nYcOGAE647++//wagcOHCADz22GO89957AFSuXBmA4cOHAzBo0CAAevfuDUCjRo0AOHz4MBMmTIjo\nuv/66y/AvQDXq1cPgH79+gFQtGhRAAoWLAjAVVdd5Vyc9+3bB0DLli0B+P777wH48ccfI7rmSKEb\njJtuusl5LEeOHAAkJiYGfe3vv/8encVFCIWlR48e7fz++OOP89FHHzkX4vXr18dsfVkBC1kbhmEY\nhgfIlEI+efIkTZs2pUePHtStW5c+ffpw5swZEhISGD58uBPiijRSOffccw8AjzzySIqvHzRokBMm\nzGpIeQRj7NixjsLyOtOnTwfcO+42bdowYsQIIH0KWQq7VKlSgGtU0KtXLwDmzp0bngWngerVqwNw\n5MgRwFWYQsfxmDFj+O233wDo0qVLkp8vvPACAP/88w8AL7/8MgBPPPFEBFfu45xzzgHgyiuvBODG\nG28EoGnTpoC7Pdqnc+fOpUmTJgDMnz8fwFHxy5YtAyBfvnyA90PYzZo1A6B8+fIAlC5dGnCVP4RW\nyDLJ0Dbr8/d6OFfhaEXWatWqleT56dOnO9GQeFTG9913H8ePHwfgjjvuANwogLanUqVKQOrXlHCR\nKYU8ZswYzj//fADefvtt2rVrx7Rp0yhXrhyzZs0KywINwzAMIzuQYYW8bds2tm7d6uRQVq1a5dy9\nN2jQgIkTJ9KuXbuwLDI11HQ+cOBAAJYsWZLi63ft2uXkwwYMGAAQ8fxbtOjatWvI58aNG+d5hax8\n27FjxwBX0TZv3tzJIacFqTj91Pv07dsXgKpVq4ZnwelALSDKYwdy+PBhACfCpH8DzJkzJ8lrla/V\n/urTp0/EbQqlhqSM08KCBQsAV13Pnj0bgOeeew6Azz77DEi+fV6hYsWKAE7E4vbbb8/we+in1Ffh\nwoU5deoU4LYLeQEpY51LApWxzq9r1qzh0KFD0V1cJmjQoAEAq1evBuCaa65xCivF2bNnATeapeN1\n4cKFAJQoUYIrrrgiYmvMkRgYX0kjXbt2ZeDAgXz66aeULl2a4cOHs2LFCgB27txJnz59mDFjRorv\nsXfv3nSdZA3DMAwjnhk0aFBIYZQhhfzpp59yxRVXULZs2aDPp/UaP3r06CSLy6hxxTvvvAO4uUUp\nENm5CVV8+iuUr7/+Ggi/tVu0h20HGoEE48EHH2T+/PkZussPJNzbt3v3bsBtAcqTJw/gHkvPP/+8\nk+9JjYoVKzq5IFVTS+G8/vrrACm+Vzi3zX9/K5+tCvijR48C0L9/f8CtpE4LyjdLxcybN8+5IU4N\n/+2LtlnMVVddBcDDDz8MhF8h16lTJyznFFGiRAkAXnzxxVRfGyqHnBLz5s0D3OMzparzSJ9TpIzH\njBkDuPlU8dVXXwGuwvzpp58APHtOETo3qptBnQlHjhyhdu3aAHTr1g2AYcOGAW5HQSAHDhxwritp\n/b5ByudlfzJ0QV66dCm7du1i6dKl/PXXX+TNm5cCBQpw8uRJ8uXLx549e0z5GoZhGEY6yNAF+c03\n33T+PXLkSEqXLs26detYuHAhzZo1Y9GiRVx//fVhW2RqqAJOxgT58+cH4LvvvkvyuhMnTgAwY8YM\nx/y8fv36ADz++OMAEc0PRBKprlB3YldeeaUnt23SpEmAmz9V1e1DDz0EuKojPbRt2zZZ9Oall17K\nzDLDQuAdtaIA6VHGGzduBHD6lLW/0nO3HgvUe33ppZcC7nfTi5XGqqR+9tln+eOPP1J87b///gv4\nco06/+inqs1VZ6NjXRX+4FaoKy9fpUqVMGxB+smVK5cTbQlUxooiSrF68TMLhqIw6ve/+uqrAbfK\n/cyZM851StXyI0eOBNx++UC++OKLiH7XwtaH3KtXLz799FPatWvHoUOHaN68ebje2jAMwzCyPJl2\n6vK/23v//fcz+3aZYsOGDSk+r7vySy65xMndqL/Qi+oxPcSbC5f6jFVNrWrHZ555BsiYMpZ7VdGi\nRZ27fN35qlLUS9Ws6eHpp58GoEyZMoCrrh944IGYrSktvPHGG4Crqi677DLAjYjI4ckLaN8WK1YM\n8Cko5ZADUXW0KnerV6/OmTNnAFc1//rrrwB8/vnngFvpP2rUKAB69uzpvJ+sJ1VVr97laNG9e/dk\n1dSqs1i5ciUQP8pYjneKiGqf6lxTo0YNAG655RbnbzQm8rHHHgv6njpOq1Wr5nxWkcCcugzDMAzD\nA2RbL+s///wzye833HAD4FYSxgtpVcZeiwAoQqHcmSo2lfv1d0BKDamJXbt2AT7/Z/UT6nOOV2Ws\nCmzd9bdv3x6AkiVLAq4a8xLKwVaoUMHpaFA1q/LlUsheQm5unTt3DvkaKeObb74ZcPtVwT13KLoT\nmH/evn074HOdC0R92spvRgsdT9p2cF3TVJUcL8pY7N27F3Drg/QdUVeDvzJWDYYMrvS9CkTR1aVL\nl4Z/wX5kywty586dk4Vn4pXUirnGjh0bzeWkiXLlynHBBRckeeyuu+4C3OKf9KAQoNzh9u3b51yk\nI22WESlk2aeTom5Y1DakNhkvomNRFy1/FHrXFKsdO3YApOpZEElkeKL0VTBkgKH2Hv8LcSCyyAyF\nDIxULBVLlCJasWKFY8eqn6mZCPXs2dNzxiDFihVL9jnqIqsiL9GkSRMnjK3W2VBowlmkL8gWsjYM\nwzAMD5AtFLIKXwoVKuQ8plCFin+2bt0a/YVlAg2RmDhxYoqvi7bxQ1rYs2dPspBlqOKZlNBnF6iu\nunfvHrfKGHwhX7UHafyiwvFeVsbio48+Any2mWoBkoWpZijLgEEWpi+++KJjpxlttIbAqI0/O3fu\nBFJWxmkl2gVb/kgtyvjCv0BJIV6lF4QsPxX6ffDBBwG48MILOX36NOCaMgVGdqJN5cqVnSEn//vf\n/wCcmdxCEYyWLVs6A4lCof0TyUIuf0whG4ZhGIYHyBIKWXecsp0LzA9fd911If9WxT/33nsv4Jta\nFQ8odxwKL+aOxcmTJ5MVI2kIgVqXnn32WcAdL+iPjN6lGvUZKlpQunRpT47zUzGJcoeBeat+/foB\nvjzxNddcA7j2r6nlJb3IP//845go6KeKvNQiJMqVK+cos0BDn0jTo0cPAEft+aMWmWuvvTbT/4+M\nKWrWrAlkrLUvs9x5550AzvElBg4c6BRYqt1JUUSdTzVKU983wBmx+8knnwCuApfhSbTp27evUxCq\ntem4UnRDa8yVK5dTaBcqoqbIjqI+kcYUsmEYhmF4gLhXyHPnzmXatGmAa3+ZEWQGIJs14d8GpQEU\naqHRnaPyYWkdfhANdGfnxRwyuPkoDZVQu5PGYCo/rIEgqkwFVxnrNaoKVSRERgZeoXz58hw7dszJ\nk8u2VcYR27ZtA9z86r59+9i/fz8Qn8o4LQwZMgRwK1/LlSvnGPxHWyFLGQcbCqFhJJlRSFKjavWT\nLWyw/09V9FLT4UZV1RqhqGr3MmXKOC2COnesWbMG8EWcwFXGspd89913adq0KZMnT3bOffob1Q6k\nZtYUbvr16+dE2YTO7YH07t07pG2tcuzRzoWbQjYMwzAMDxD3Cnno0KFptg9U1V3r1q2d5n/1IMp4\nIS2VnmoiVz5MvZUauhFp0wNVWAdDd7deMwIJRIpDxgQy4m/VqhXg2t0pb6WfkDSHBe5nKDtOL+Bf\nSbtz504mT57sjIJUNav2QWCmFbQyAAAgAElEQVSNw9GjR52oS1Zn5syZgC/akdLowViwbt06J4qR\nEfQZrl+/HoD//Oc/IV87fvx4ABo3bgy4VcvhJrCSXFEJ/xyqzGgCe4w1iEH2p9dee63zmNarSNd9\n990HuIo80uj/3b9/vzMeMhAZnPTt2xfw9VEr6qkBINoPgwcPBlwjmGhhCtkwDMMwPEDcK+Svvvoq\nVYWsoReqtitWrJhTvSqU31M+MhjKWUqRqT9Pd2eqSoy0Qk6twjqemDp1atDHlQeScva3uwtEVcuP\nPvoo4FYrxwKpCzlQAXz55ZdOJADcXk0dP+qbFJMnT470Mj2DcnQHDx6M2RqU0w90Fvvkk0/Ys2dP\nmt9HVcqq9tf5QLa8gezbt8/pJtBgi0gpYxF4blLFd4ECBZzPQsewIjpSyvoOKkJ4xRVXOPllnQMV\nhfzvf/8bsW0IhupKIPUOE0Xbtm/f7kwllDJW9E3KPjW3snAT9xfku+66y7lAKcyp0OWAAQMA9+BI\n6SDRFy/eT4ZeD1WnhgrmFP73P5lpeous8XQyU7FMLD87hZ9l3KGwNLgnaqU21Jahk6FsMVUIk51Q\nwVPt2rVj5jeuGyIVGmqSln8YN9SFWa013bp1c27YdYEPhd5rwIAB5MqVC4Dff/89o8tPFzonqu1M\n07jWrVvnhHQDjUt0oe7YsSPgepX7o31XoUIFIGmKyavkzJnTKaTUuUM397E6l1jI2jAMwzA8QNwr\n5M8//9wJlyiUsnnzZsC9o8sOeNkIJCOoHcbfQELtC7pDDxXujgWTJk0C3GIQFdfNnz+fatWqMXPm\nTCcsKUX81ltvAW5h08aNG6O65nAjxb9p0ybADV+mhMz6H3roIbp27QrAE088EZkFhkDtkjKhUeFT\np06d+Pjjj4HQ1q5SyNWqVXOMPgLbmaR+1WIj0wyp42gi5aeJU5oDXKtWrZADd1Kyod2wYQOlSpVy\n2qfiQRmLGTNmJJuu1ahRI8AUsmEYhmFka+JeIYOb0PdP7Gc3vG4EklbUjia1obzcjh07nNYQLyL1\nLiMZ3WkD3H///RQqVMhRTspTqcBGxTMqKGnZsqVjSBFPFChQAPC1IkLKSleFQ7Ix/Pzzzx1lHStk\nbiEzlhtvvJGWLVtm+P1kLtGrVy8AihcvDrhjQmOJxkhK+c+ePdtRzRqFqnOKCPw8Bw4cyIQJE2jU\nqFFcHa+a/bxy5UqnhUstaplpcwsHppANwzAMwwNkCYUc78joI7UK6e7du4d8LqvkkFWdrNyxKjx/\n/fVXR3F4EbVaSf0q9wi+Kt4WLVqEtHaVIrv//vsB3yAGqU0v2bGmhqIDMsKoXLkyv/zyC+CaScyZ\nMwdwTXWUTy9ZsqQzoCBWKIdct25dAJ5//nlnAIg+j8B8qo7TTZs2OREcVS5LbUkZexFVfNevX9/J\ndY8ePTpNf6vITryxatUqwDfy89ixYwB06dIFiP02mUI2DMMwDA9gCtkDpLV3WCq4Tp06yR7LKijf\npj7kL7/8EnDHM3oVWQjKtF+V/rVr1+b66693eo9TQsYLCxYsoHLlygDcdtttQNoqlmNNYJXyyZMn\nnTGYMsl48sknAXjttdcAN28uJe0FNCqycuXKziAFmbm88sorSV6rCIZeB26HgOEt7rnnHgDnuwXu\nd+7yyy+PyZoCMYVsGIZhGB7AFHIc8uCDD8Z6CRFD0QLZ9cnu1OsKWZaHM2bMyPB7qL964MCBbN26\nFXDv5uX25WVkM9ihQwfA56DWqVMnwK3+b9CgAeDmV6WgvY76bO+9994Yr8TIKKrVUK/0unXraNGi\nBeCdCI0pZMMwDMPwAKaQDU8RmKNT1XJ2Yv/+/Y5rVTwo40D8XY4C+1Oz4+dpxBZVzStyI8qWLesZ\nZSzsgmx4ChVH2YnbMIxwIAMeDSFS6Do9k7yihYWsDcMwDMMDZFghz507l/fee4/cuXPz6KOPUrly\nZfr06cOZM2dISEhg+PDhjvG6YRiGYcQCmc/op5fJkEI+ePAgo0aNYtq0aYwdO5YvvviCt99+m3bt\n2jFt2jTKlSvnCb9WwzAMw4gXMqSQV6xYQd26dSlUqBCFChVi8ODBNGzYkBdeeAHwtTZMnDiRdu3a\npet9/Q0vshpe2rZIrMVL2xduMrttXt43Xl5bOMjK2+elbbNzSnjIkRg4vDMNjBs3ju3bt3Po0CGO\nHDlCr169ePLJJx2Hm507d9KnT59UezL37t3raZ9XwzAMwwgngwYNcnr2A8lwDvnQoUO88847/PHH\nH3To0CHJUO60XuNHjx6d4uKM8GP7O7rY/o4+ts+ji+3v8JGhHHKxYsWoVasWuXPn5qKLLqJgwYIU\nLFjQcd3Zs2ePKV/DMAzDSAcZUsj169enX79+PPTQQxw+fJjjx49Tv359Fi5cSLNmzVi0aJHTT5oe\nZK+XFQjMf3hp28KxFi9vX2YJ97Z5bd/4b5/X1pZZ7LiMDXZOSZm05sMzdEEuUaIEjRs3plWrVgA8\n++yzVK9enb59+zJz5kxKlSpF8+bNM/LWhmEYhpEtyXAOuU2bNrRp0ybJY++//36mF2QYhmEY2RFz\n6jIMwzAMD2Be1tmE3r17O0O4T5w4AUCRIkWA+BmBZxiGkZUxhZxNyJMnD6dPnwZgypQpTJkyhZIl\nS1KyZMkYr8wwDMMAuyAbhmEYhifIUiHrEiVKADjV3xqzdcUVVwDw8ssvA9CwYUO2bNkSgxVGn3r1\n6gHwzz//OI/99ttvgIWqDcMwvIQpZMMwDMPwAFlCId9xxx0AlC9fHoDq1asn+SkWLFgAwJYtW8if\nPz8AL774YpRWGRtef/11AH744QfnsZEjRwJJVXNW4IILLgBg2LBhPPjggzFejZFdqVKlCgBPPfUU\nADlz5kzye6dOnQD3u2kYwhSyYRiGYXiALKGQlSu+++67ATh79iwA06dPB+CXX34BoHPnzgBUrlzZ\nUcuVK1dO8pqsQu/evQFXNQLs3r2bMmXKUKpUKcCb2zx8+HAAatSoAUDjxo3T/LfDhg1z/t2+fXsA\npk6dGsbVGemhZMmSLFy4EID58+cDbp3Hrl27APjf//4HuPUdWYElS5YA8OSTTwLu+ejVV18F4I8/\n/ojNwtLI5MmTAahatSoAPXv2BKBatWqAzzoZ3O0CXxcHwL///hu1daYXHXtFixYF4PHHHydXrlwA\nfP3114BbczNgwAAAPvroIwAeeeQRAJ544gluvPFGAD7//HPnfcKFKWTDMAzD8ABxr5Dr16/vVFUf\nOnQIcMc/rlq1CoDjx48DMH78eOfxTz/9NMn7yPbzgQceALypHtOC+oq3b98OuBEA8OWsZs6c6clt\nk6ItVqwY4FPz4Crk0qVLp/q38c7TTz8N+OoaPvzwQ8BVIc8//zzgmrroWPciefPmBaBQoUL85z//\nAVxV9cUXXwDw7rvvAji98Tlz5kyiuOKZbdu2pfh8xYoVo7SS1GnUqBEAzzzzjPNZ6Rhbt25d0L/R\nZ+bPqFGjAChYsCAA9913X9jXmlGkjLW2J554wnlOx5yUsSI2PXr0AJJHbs6ePcuXX34JuF0qUshv\nvvlmptdqCtkwDMMwPEDcK+Q8efKQL18+wI33Kx8QyF9//QVAuXLlnApH5QqkxNSfrFxmvHHOOecA\n0KRJkySPf/LJJ7FYTpoZOnQo4OsR96dmzZoA7N+/P+Tf6jXBHvNyDvniiy8GYPDgwQAcO3YM8OXw\ndOeuaM8LL7wAwMCBA6O9zDSjfJyiGW3btmXTpk0AzjhWDbLXzzJlygAwYsQIfv755yiuNvwkJCQA\nyY9h0bZtWwDuvffeqK0pFFLGqtUIly/DmjVrwvI+4WTr1q2AW1OUEorupBSRE/v27QPCWxNgCtkw\nDMMwPEDcK+T69es7fX76mRZUKRf4N+l5Dy8hJdWvX78kj2/evBnw5T683HfsXyGd1VFeSrktKWPl\nuMDNwSmXrLye1HTXrl2js9h0oLzwjh07AJ/aUNQqkPPPPz/J79WqVYt7hTxixAgATp06FfT5H3/8\nEYBzzz03amsKRdmyZQG49NJL0/w3qkvp378/AKVKlUrWS12gQIEwrTDzqCZDUcNwMGvWLObOnQvA\nPffcA7jV1uEg7i/Ir732GldffTUAzZs3B+DOO+8E3A8kGDr5BRaS6MLmdVS8pRaLUGH6I0eOADjt\nJ14lWNgZ3AtTnz59QoaR2rVrB3h/G3UhVouIjr26desCyY1sABo0aADAvHnzAPfz9hLlypUD3JSQ\nCiMVkg+GPlddsNevXx/JJUacAgUKOO1CrVu3TvLcypUrATdkrRuWWBJ4Q+SPbhy0XpnsqIjpvPPO\nA6B27dqRXGKmWb58OUCGTIJU3KXCYJm9LF261EmRhvNCLOJTDhqGYRhGFiPuFfLx48ed4RG6c1MI\nUEn3Zs2aAfDNN984f6e7p1q1agHuXWFgO5RXmTlzJhC6PatSpUoA3HzzzVFbU0ZIrWVJyrlmzZoh\nC7uCKWMvFnO98847gFugpYKaa6+9NtlrFQ5T25oiHZpp7SXURlKoUCEAxo0bB7hGGP7o+9a0aVMA\nHn30USB5pCreePvtt3njjTeCPtelSxfAba3xAoq4SPl9//33HDx4EIDVq1cD7npDpRIWLFjgqGih\ncG4skYJVgVZa0LYruqH9M2XKlDCvLmVMIRuGYRiGB4h7hQy+u1NwC2BU5q5Wn59++glw73YqVark\nNH4LtWHImMLLzJ071zHOCET5dOUeva487r//foCQ26OcaUptT8HwYk5Sylh1CjKQUOudcswNGjRw\nlLHys4rcyPLVS1SoUAFwI1Ia7xkMWWZqu5SHC2zTixdk8jFhwgQee+wxwI3QSX3K0tdL404VcVEk\naf/+/c55Mq3IgtcfHcuBefRoIttgHWOhmDp1qlNMqvOQBhTJKjPamEI2DMMwDA+QJRSyUOWmqlaV\nB1CeSrZ9/jZ9EydOBNxGcC/dxQYya9YswFfpp7tvIUWsu3EvVHKmhVDKuE+fPkDKyjhU/tmL6hhc\n+0spZbUyqY1Ev8+bN89pv1OrSahWGi8gy8C77roLcFsKZ8yY4bxGRju5c2epU45TMa/KcnC/i1L9\nXjynqCJeP9ODoole/Sy/++47wI1UXHbZZUFfd/LkSWbPng14xwjKFLJhGIZheABv3uJkkhUrViT5\nXeO2VJF84YUXOs+pAlJ2dsrd7dmzJ+LrTCu649b6q1SpwpkzZwA3T667QA0d+H//7/8BrloZNWqU\nMyDdC6RWXa2eXOW4pk2bluw1GukXSEo9y7FEClg9jlJVssNU326ePHmcCI5ycv6mIV5DZvt79+4F\nXOOLevXqOeP4Dhw4AECnTp0AGDt2bJRXGV70WfoPKhAySBkzZkxU1xRpbrnlFsA1TypevLjznCI5\noYxgosmiRYsA99wRqnOmS5cuTi5dAyJ03ty4cWOklxkUU8iGYRiG4QGypEIORFV3yumUKFEiWfWx\n7uxUse0lhSxl7N/zp7vvH374AXBtGGVw/+uvvyZ7H/8+7FgTyplLqOpRP1N6TSBeVMfgRi9UwSnV\nK/tBVbl+/fXXTuRDfccpVS57BfV9XnLJJYDPnlEV5YsXLwZwKpGLFCkSgxVmHg1lkPWkP+ozP3r0\naFTXFGlUTa3aDI27BXdbFdEK1nseK1Rno9GQPXv2TPYaeVcowqhhJ7oeKMITahRluMkWF2SF1BT+\n27Fjh2M9qXChDiQVgMn/tGPHjlFdqz+FCxcG4Kabbkry+JIlSzh8+HCSx7Q9wQ46ITMRL6BwkrYt\ntQt0Woh2E3+4kf1pYmKiE9L30lzZ1NBJ6++//wZ8IUEVPekxtaJ49aYpNWRoou/SNddcA/guxrrR\n8qLPeEbQhVgXpcDZwACbNm2iSpUqnjxOdcypBeuzzz5L8nzTpk3JkSNHksd0gVYqU0JH5lKRxkLW\nhmEYhuEBMqSQjx07Rt++fTl8+DD//vsvPXv2JCEhwSmHr1y5stPaEUsUvg1cS/Xq1Z2wi6aTqEhK\nM111t3/xxRc7BQvRRsVoUpEqQKhUqVKSlhKAhx56KMnvMmCQ7d2tt97qhNm07bEswFD4K1SLUmAb\nQo0aNZy78FCToTTpSrOt4wUNH/BvdRoyZEgsl5Qp1HIXbHazrDPjDSklzb4NjJzlyJEjyyhjRdvU\nzqZUSyDTp093jFG8TKjiuptuusk53y9btgxIblCjfRAtMnRB/uSTT6hQoQK9e/dmz549dOzYkYSE\nBPr370+NGjXo3bs3y5Ytcyp8DcMwDMNImQxdkIsUKeIUGh05coTChQuze/duxyy/QYMGrFixIuYX\nZOVT/ducwM0lg284BbjqSjkv5ZTXrVsXM4UspSFFKOMF/7FfMrQPNFKfM2cO4Cr+W2+91WlHSWks\nnlcIHJ3ZuHHjVO0z400ZN2zYEHA/XxUaetFIIruiIS0qQgv87txxxx2Ab8SfVw1p0oqUsQrxQilj\nFXLVrFmTN9980zEjijdU7AuuEVEgOueqDinS5EjM4ADgzp07s3PnTo4cOcKYMWN48cUXnX6vFStW\nMGvWLF577bUU32Pv3r1JetkMwzAMIyszaNAgJ70bSIYU8pw5cyhVqhQTJkzg559/pmfPno76AvcO\nKzVGjx6dZHHffvttRpYTEimNwFF8qqALhipdN2/eDMCkSZOc/EJ6qFOnTpLfM7NtqnbUftq+fbvT\nwC4jk8BK7GDs2rWLunXr8vDDDwOZs9cM5/alF9mdBiKFnNkK3nBu2/z587n99tuDPuff5gRutGPT\npk0Z/v/Sgv/2RfNzA3espHJ1+qzCNVyiTp06YT2naF2KCCqSJlQ1Ho3oTKS/czqHhxJSUsZqa1Pl\ncUrHeHqI5TlFSlhV9EL2m8EMYNJD4LaFIkMX5LVr11K/fn3A1+t16tQpTp8+7Ty/Z88eU76GYRiG\nkQ4ydEEuV64c69evp3HjxuzevZuCBQtSunRpVq9ezdVXX82iRYtSNHSIFsoDKzcnJeKPzBleeukl\nwK229tLYQo2V1F3/xRdfTLdu3dL9PgMHDmTx4sVxM3gikNQM4GWvGZh/9hrq51QXgCJKsmM0vMHD\nDz/sjGMNzJOqgvzOO+8EXDvUeEPb16NHj1SHRUyYMAGIna1kelEdjaKIgQN5AEdYhhrVqHNvtMjQ\nBbl169b079+f9u3bc/r0aQYNGkRCQgLPPfccZ8+epWbNmtSrVy/cazUMwzCMLEuGLsgFCxbkrbfe\nSvZ4sAEAXmLLli2Az+nqq6++AlyrTC+b96v/Vn3Jy5cvdywVU0MD7QcOHMj1118fmQVGiXbt2gGw\ncOHCoM97XRnr85PF4uDBgwF3XGhaP1MjOmzfvt1x7BMaV6g6lGDuVfGAjkXVY6RkFXzzzTcD0XOr\nyizqrtm2bRvg2rbKOlg2nx07dnQUcqjOE0VCzKnLMAzDMLIRWdrLWmO4mjdvDrjj7jp37uzkiEOZ\noet5LwxkkDuQRkRmhCuvvNKpGIxXQo1sjJf+T9UwLF++HPBFagCuu+46IHTfZ1ZCSkyuc16kfPny\nAJw+fTrZcHvlXFPyjI8H5FAYTBmrO0XjFuXst2HDhiitLnP8888/gFsVrv7qwE6UefPmRXVdaSFL\nX5B1MdUcYM1JVpLfn169egGuachzzz0XjSUa6SBUoaAmtHiZhg0bOuHBrVu3Am74MztciIVuSrwc\nnj///PMBdygBuDdPwQpD4wmd51L6zshiVzcj8XIhFpMnTwbgvffeAzJWoKswd7RNoSxkbRiGYRge\nIEsrZNGiRYs0v9aUsXdRaFqjGvX70qVLY7WkNHPOOec4vfrqQIjnARKZRa0z77//PuArPlSoMdYE\nDmoBN90Vb/asgah4NXAe9ZkzZxxzD818jnf0/brggguAlA2hxPfffw+4pkmrV6+O0OqCYwrZMAzD\nMDxAtlDIRtYgWKtdPLBgwQLGjRvn2PLJCjU7IwtK5fnOP/98zyjkRx55JNlj8a6MhVo/J02aBLij\nac877zzHHEM2ofGO8r8qUguMrPmj2gBFr0aPHh2NJSbDFLJhGIZheABTyIYRYZYvX86SJUucu+/f\nfvsttgvyAIEm/kZ0mD17dpLfu3btGqOVRA+1jV5zzTVAytusCu1YYQrZMAzDMDyAKWTDiDDZuZra\nMIy0YwrZMAzDMDyApxRyWoc4xyNe2rZIrMVL2xduMrttXt43Xl5bOMjK2+elbbNzSngwhWwYhmEY\nHsAuyIZhGIbhATwVsv72229jvYSwERhu8dK2hWMtXt6+zBLubfPavvHfPq+tLbPYcRkb7JySMmkN\nv5tCNgzDMAwPYBdkwzAMw/AAdkE2DMMwDA9gF2TDMAzD8AB2QTYMwzAMD+CpKmsj+7J48WLAN4YP\nYNGiRQD8+OOPgG8gw8iRI4P+7aOPPgrAihUrIr1MwzCMiGEK2TAMwzA8gClkI6a8+OKLgDuoPpAS\nJUo4/+7SpQsAOXP67iPPnj0LwL///gtAw4YNAXj55Zcjs1jDMLI0ZcqUAdxzTe/evQH4+OOPAViz\nZg1ff/01AOvXrw/7/28K2TAMwzA8gClkI6Z89NFHADz++ONJHu/cuTMAe/bsSfY3I0aMAKBKlSoA\n5MmTB4CDBw8CUL9+fecu1oge06dPB6BDhw4AjBkzJtlrAqMbIleuXDzwwAMRXmH2Q/v7mWeeAaBT\np04AlCtXDoBSpUoBkC9fvmSfiahatSoA9913H5D0s1uwYEGS9wn8fGfOnAnATTfdBMCZM2cyuUXh\n4/zzz+fyyy8HYNKkSYB7bjnvvPMA95g+ffo0ABUqVGD79u2AKWTDMAzDyLKYQo5DcuXKxT333APA\nO++8A8D48eMBmDdvHgBz5swB4IYbbgDg559/jvYy08SuXbsAOHz4MABvv/02AHnz5gWgZMmSyf5m\n6tSpgHu3X7FiRQCuvPJKAMqXL+9phVywYEHAVfQffPABAKtXrwZg3LhxaX6vhIQEwHccaB8WLVoU\ngHvvvTcs602Nv/76C4AjR44AMGrUKCC5Cg71GECOHDkitLrwMHHixCS/jx07FoDu3bsne608mPWa\nWHL33XcDriL+8ssvg74uZ86cIT8boef9X9e4cWMABg0a5LyP/2v69u0LwPLlywGYNWtWurchXOTL\nlw9wuzKWLl3qnEP0M5ATJ04A8NlnnwG+75SiApEgTRfkzZs306NHDzp16kT79u35888/6dOnD2fO\nnCEhIYHhw4eTN29e5s6dy6RJk8iZMyetWrWiZcuWEVt4pFGBUPv27SlcuDAALVq0iOWSqFy5MgCT\nJ092woG33HJLktco1Kvwny7cJUuWdIqfvIQuIiqeSAu6ACicrQuyWLJkSZhWF150Idb6p02bBsBP\nP/0EpO+mSSdYFaEcPnyYd999F4D9+/eHZ8GpcNdddwFQs2ZNIPlF9emnn2bTpk0AXHrppQC8/vrr\nUVlbOOjevTtr164N+VwoNEhAf6sbrCuuuCLMK0ydH374AXALH3XT161bNwAuvPBCwHdj/Mgjjzj/\nBjccrXZCXdy/+uorAPLnz8+ff/5Jjx49mDt3LpD8gvzPP/8AsGzZsohsX3rQDarOF4HnDXA/M237\n9ddfD7htmfoZKVINWR8/fpzBgwdTt25d57G3336bdu3aMW3aNMqVK8esWbM4fvw4o0aN4oMPPmDK\nlClMmjSJQ4cORXTxhmEYhpFVSFUh582bl/HjxzshUYBVq1bxwgsvANCgQQMmTpxIhQoVqF69Ouee\ney7gCx+uXbvWUZqxpH379oCrFsENe77//vuAG2YTuXLlAnzJfN1VxgrdyVWoUAFIXzHB7NmzAZ86\nU1FGVkdhXK+hELWUsbjmmmsAePPNN1N9DyljtYOpgOrIkSNOkduHH34YngWnglSRbtYD281OnTrl\nfP/Lly8flTWFkzp16qQ6Ni+l0LUe00+9NppjBbds2QK46YQbb7wRcM9volixYs73Rj8HDBiQ5DVK\nhwXSo0ePkM95AX0fNm7cmOw5PaZobteuXQHYsWNHkp/RIkdiYmJiWl44cuRIihQpQvv27albt64T\nxti5cyd9+vThvvvu48cff6R///6A7+Ry4YUX0rp165DvuXfvXooXLx6GzTAMwzAM7zNo0CAn5x5I\npou6Ql3P03KdHz16dJLFZfbOUQVABQoUAKB58+YAye78zp49y759+5K8RvmvatWqAfDwww8DvvyY\nckDpUcrhHLatth6p+MTERKc4Q3kfNbKLSy65BIBmzZoBcPLkST766CP69+/Phg0bALekf926dele\nkxeGiStPft111yV5fNmyZUyZMiXD7xvObZs/fz633347U6dO5fjx44D73ahduzbgKuSUcvw6dpWL\nlTIWgwcP5vfff0/Tmvy3LxKfW/Xq1QGYMWOGc4xddNFFQV/7/PPPA77imXCkuOrUqZPpc4r2T0p5\n4sDCre7duzuvD5V3Fio+TG9OORzH5ffffx/0b8eMGZOh8wC4x3hmidQ55bvvvgOSn+fuvvtu5zv4\n22+/heX/CkVqkRaRobanAgUKcPLkScBXWFO8eHGKFy+epJjE1K9hGIZhpJ0MKeR69eqxcOFCmjVr\nxqJFi7j++uupWbMmzz77LEeOHCFXrlysXbvWCV+Hk1q1agHunWm5cuWcCl0pY931hOKyyy5z8iN6\nrdSLqgN1wwE42xHtKmvle9S8LlatWsWMGTMAnwkGJK/QVQO+hjNUr17dUVWqGlczf0bvjGPFxRdf\nDLhKJFAht2jRIlMKOdw88MAD7Nq1y8n7ymTgtddeA9KW85XqbNOmTZLHDxw4AJBmdRxJVIGqVpde\nvXqFfO2qVasA+N///gfgqQLQlJRxqDzw2LFjHcX74IMPAqGVto5bvS4WqBpa57l4OwekB3WiqG5I\nNUWzZs1i4cKFgNudoohNai1gkSLVC/KGDRsYOnQou3fvJnfu3CxcuJARI0bQr18/Zs6cSalSpWje\nvDl58uShd+/edO7cmeG64uwAABFSSURBVBw5ctCzZ0+nwMswDMMwjJRJ9YJcrVq1oGpD1cn+3Hbb\nbdx2223hWVkIihUrBmTMtkwqo0GDBqn2xQUzpIg26k9VdbTurLdu3Zrq36q6UspD2x7PSBnrRi9Q\ngb366qtA0uhGrPCv9D5y5AhFixZ1lLGOvbSYl1x22WWA228cWJtx9dVXA3DrrbdmftEZRH3fqljN\nnz9/qn+jz0gK30ukVDmdnp5ivTbwfTKaQw4HipJJAWaHQSzyOvjjjz8AN19cvnz5ZLlvmfQ88cQT\nQPSPT7PONAzDMAwPEDfWmerBvfPOO1N9rWzOAis701LN2qBBAyBtVeKR5umnn07yu3IgR48eTfVv\nVZm9e/fukK9R/vzzzz/P6BKjxsUXXxxSGasHsl69eoDPySzW+CuPRo0aAW4vaKh+YTlfqS4AoF+/\nfoA7fEPHpRyYVF0f7X5JcPtU33rrLSBtylhccMEFgLtdaenBjhb++eFAG9NYqNpwEtgPLne3UONP\n/VHfebyiyKJUcO7cuR0bTQ2TUORGDnNXXXUV4LqXRRpTyIZhGIbhAeJGIctBRXfhqtI8ceKEk0+W\nCcnOnTsBVymnBRnyq1K0bdu2gM+AfNu2bZldfsTRna8U/n//+18AnnrqqZB/I8XtZeSutnXr1mTK\nWBWSa9asAaJ3F5sSyvkqd6/cL7hdAFKzgdujqvfzzz/feSzYCENwnZZioYyFtlHqQt9NHVe33nqr\nU82rz0oVruockHn/c889x4svvhillaeNb7/9Nu4VsShbtizgHp9i5cqVgC86k1plsY7lv//+G/BF\nb+T1HE8olwzucRh47vj0008BN9qmiKNGL0aKuLkg68uudh6FWWXUD74vdUZR65SKcdRGtG3bNufi\nH2t69uwJQPHixZ1ChZdeeglw542qBcjfezweadq0KeAW1zVp0sR5TqYoClF//PHHUV5daPTFVWsZ\nuKYzmmAVqsBu7969gM+QXxc5oQvbG2+8AQS3AYw2MqpRUZdCgVqbhhCAW4ypubMdO3YE3O91y5Yt\ns0ThoT+pGYREE7VQqjBSpMfSVMVounB/++23junGsGHDAOjTp09mlxpVJPS++eYbwG0fbNWqFeAW\nwWlaVZ06dZyW00hgIWvDMAzD8ABxo5BV/BFuNCNTd0iymlQBQCzVsYwkTp06Bbh2mOC2/qQ2c1Vh\ne/9iON3tadiBl1CIVqFMKcU9e/Y4QzakvKI1RCEj+BcF6t9SFvpdRVFKK+hz+euvv+jRo0eS91Or\nhpSyl9AAiZSK6dQ+okJCpZVUeLlkyRLnNV4LXacX2VN6iXbt2gFJw7Xgtq/eeOONDBkyJOjfauyi\njkEVun399ddOoaxSfipI1NjFjLSnRhOFqhXdUWg/8Lyp0PykSZOcqEIk7Da99+02DMMwjGxI3Cjk\nSCGVIjUqq0kvWPk99thjgGuOPmLEiJCvVUuCcsrKjSgX4q+s1AzvJXtJobYDKWMNAQFSnBzmFRRZ\nUT64bNmyTutSoAmBcst6XjUCwSxadceu4zNeUXtT4Gi//PnzJ7OIjVdSyx3HolBM0bDAwi0NKwml\njgE++eSTJL8ryvHFF19w8OBBBgwY4Lxvp06dANf8Rko50sVQYtiwYU5UUxE1UaNGDYAUx9AGFnfp\nO6qalo4dOzr1ITq3hhNTyIZhGIbhAbKtQlbVrvKzgbk5/wrRWCGVXqlSJcBt4k8JmaQL//zl8uXL\nadu2bTLDkVgixaT9r9YgtVbIxOXjjz9ONjTk2LFjgGsx6gW0poEDBwK+0XRSJ6Fy3mrt8m+RElIn\ngbm/eEc1G+XKlXMee/vtt2O1nLCQ0lAKiM2IUvHQQw8BvoEK4EaiMtP6OG/ePKd6XmNr9V1UzlUW\nlKrQjhRFihQBfC2DjRs3BtxtVF2K6hbSgpSyIqf+BNZ3hBNTyIZhGIbhAbKtQlZ1p36qGtDrVYFp\nRXkSDR8At8fOC6P6lDdVb65+il9++QVwLf1efPFFSpcuneQ1yteq73Xz5s2AqzjBVZbKZal3VjlM\n/xx1tFEuSmp69erVgC+aIZtNKYv/+7//i8EKk/LOO+8AULNmTQC6desGhC9CEWhqE09079491SH0\nqXVERBKNlw2sY8gsqoyXJ4CGnMgIRp0DsvBVv3K4yZ3bdylTzhfcPLaioNOnT0/3+yoCEC1MIRuG\nYRiGB8i2CrlgwYKA2zOo/IIXcseZQU5R/fv3B5KqjXvvvTcmawpGmzZtAFcVBiKnsZQcx1RFOWHC\nhJCvkSpR5aty1aoujaVCVt/un3/+CSTtU9bxmZL1abTRPpTxvixLlT/UkHuN/sxOrF27NqRCjmXu\nONoE5lflxKYctuphwo3qYg4cOOCoWuWSlSdXFX9aKr5r1aoFwIUXXpjk8QMHDjjfSUX5wkm2vCDf\ncccdjoWcpvLopBjvKMSp0KxYvnw5GzdudNqiYknBggWdop5QqKhLxXfBPJsVOp05cyaQ1K5ShCq0\nURtDLMKjKmSS5af/dCfwhdP9Q29e4ddffwVcu0VNwhGa4CTb2Q4dOjiewCp207bLAlSe8eAzf4k3\ndBEOdpzFcu5xepk6dapTiHXLLbcAGbuxkqWxCvRkcyvL1NGjRzvf7XAiy87hw4c7YXRdmNXeqQJJ\nnVP8p9zdcccdSd5Px6tC8aJ8+fJJjtlwYyFrwzAMw/AA2Uohq0Do33//dablKIw7cuTImK0rnNxw\nww1BH+/WrVsyJRYrli5dmqyIS0NDVHghlaWBA1K0/sjmTp/r6NGjAZ/6UnGJ7tRlEq99EDjnNhpo\n0o7s+AKN/rXN//77rzM8xEtI7Wj+dGAbl5SgfwrB3w4UXHUSaAxy5swZx4Zx/Pjx4V56xEip1Unp\nklgWc6XGk08+CfhahFSIpc9XhYUZiVzob5SGUfGsrIojxdNPP+0Ujmn9MhmSBaiQCgY32haI3qNQ\noUKA28YVKUwhG4ZhGIYHyFYKWZZqtWvXdozB47HFIhi685TNZmDxRPv27SNihh4ulFdaunRpuv9W\nylJDDj777LNks7A1mlOvieQItWB07drVaf3wHxICbuuI8uReVMf+SGmoRVBFXYo6+fPss8+m+F4y\nrmnbtm3QGgCvklKLUzzljl9//XXAjTKBW2gZ7PNMK48++miS31UAqPbDSKJ8sj6HUFGMwPywPzJQ\nUUQtWsWVppANwzAMwwNkaYWsFqA5c+YAbh5rx44djiLz0hDxzKA7Og01OH36NAATJ04EvGW9WLt2\nbbp06RLrZUSV/v37s2jRIiDpaEZwW+5mz54d9XVlBG2HfqqCtn379oBrVRhYJwDutstMREozXtSx\n2iT1vQpGPChjoRy/PzfddBPgtg2tWLECSFsufNCgQYCbtxXq7khISMjoUtONzu1SyGp7+s9//gPA\nK6+84uS4VdOgwS76TuoYjxamkA3DMAzDA2RphVylShXAtYqUvVrr1q2ditesRuB4NY1s9EqFdXak\nV69eySqqwc37S2HGKx07dgTcnKOqr9944w0eeeQRwLXdlN3pW2+9Fe1lRhwvV1OHQr3HNWvWdEYw\nBg7eUS5W9QDBvAyqVq0KuL3oio4oKimVGosokM6JGl2q+hr/KIcei/V4U1PIhmEYhuEBsrRCnj9/\nPuBTKACvvfYa4Bu4ILP1rIoqYJWrVSWyEX2mTJlCkSJFeOaZZwA316p8WlY7FpVzvPbaa52K3Wuv\nvRZwtz1eCaWCx44dG9cWmbVr13acrPQzPchOVRERI2OYQjYMwzAMD5ClFbL/6EFwq+sqV64ci+VE\nlGBOVoY3OHToEIcOHaJr166AO9gkqynj7EA8uG8Z8UuaLsibN2+mR48edOrUifbt2/Pnn3/yzDPP\ncPr0aXLnzs3w4cNJSEhg7ty5TJo0iZw5c9KqVStatmwZ6fWniOzbdCLMihdiwzAMI2uQasj6+PHj\nDB48OMkYvDfffJNWrVoxdepUbr31Vt5//32OHz/OqFGj+OCDD5gyZQqTJk3i0KFDEV28YRiGYWQV\nUlXIefPmZfz48UkM359//nnOOeccAIoUKcLGjRtZv3491atX59xzzwV8pfJr1651rArTQkp2dPGO\nl7YtEmvx0vaFm8xum5f3jZfXFg6y8vZ5advsnBIeciQG2gaFYOTIkRQpUiRJleSZM2fo2LEjPXv2\nZP/+/fz444/0798f8KnoCy+8kNatW4d8z71791K8ePFMboJhGIZhxAeDBg1yHM0CyXBR15kzZ+jT\npw/XXnstdevWTWbmn5br/OjRo1NcnBF+bH9HF9vf0cf2eXSx/R0+Mtz29Mwzz1CuXDmn76x48eLs\n37/fed7Ur2EYhmGknQyFrOfOncuqVat4+eWXnedPnjzJnXfeyezZs8mVKxctWrRg1qxZTk7ZMAzD\nMIzQpHpB3rBhA0OHDmX37t3kzp2bEiVKcODAAc455xwKFSoE+Oa7Dho0iP/7v/9jwoQJ5MiRg/bt\n23PXXXdFZSMMwzAMI95Js0I2DMMwDCNymHWmYRiGYXgAuyAbhmEYhgewC7JhGIZheAC7IBuGYRiG\nB4jptKchQ4awfv16cuTIQf/+/alRo0Ysl5PlWLVqFY899hiVKlUC4NJLL6VLly706dOHM2fOkJCQ\nwPDhw8mbN2+MVxr/BBvAEmw/e20AS7wSuL/79evHxo0bKVy4MACdO3fmpptusv0dJoYNG8aaNWs4\nffo03bp1o3r16nZ8R4LEGLFq1arErl27JiYmJiZu3bo1sVWrVrFaSpZl5cqVib169UryWL9+/RLn\nz5+fmJiYmPjaa68lfvjhh7FYWpbi2LFjie3bt0989tlnE6dMmZKYmBh8Px87diyxUaNGiUeOHEk8\nceJE4h133JF48ODBWC49Lgm2v/v27Zu4ZMmSZK+z/Z15VqxYkdilS5fExMTExL///jvxxhtvtOM7\nQsQsZL1ixQpuueUWwNfHfPjwYY4ePRqr5WQbVq1axc033wxAgwYNWLFiRYxXFP9oAIu/M12w/ew/\ngCVfvnzOABYjfQTb38Gw/R0eateuzVtvvQXAeeedx4kTJ+z4jhAxuyDv37+fIkWKOL8XLVqUffv2\nxWo5WZatW7fSvXt32rZty3//+19OnDjhhKiLFStm+zwM5M6dm3z58iV5LNh+3r9/P0WLFnVeY8d8\nxgi2vwGmTp1Khw4deOKJJ/j7779tf4eJXLlyUaBAAQBmzZrFDTfcYMd3hIhpDtmfRPMnCTvly5fn\nkUceoUmTJuzatYsOHTpw5swZ53nb59Eh1H62/R8+mjVrRuHChalatSrjxo3jnXfeoVatWkleY/s7\ncyxevJhZs2YxceJEGjVq5Dxux3f4iJlCDjaMIiEhIVbLyZKUKFGC22+/nRw5cnDRRRdxwQUXcPjw\nYU6ePAnAnj17bABIhChQoECy/WwDWCJH3bp1qVq1KgANGzZk8+bNtr/DyPLlyxk7dizjx4/n3HPP\nteM7QsTsgnzdddexcOFCADZu3Ejx4sUdb2wjPMydO5cJEyYAsG/fPg4cOECLFi2c/b5o0SKuv/76\nWC4xy1KvXr1k+7lmzZr8+OOPHDlyhGPHjrF27VquvvrqGK80a9CrVy927doF+PL3lSpVsv0dJv75\n5x+GDRvGu+++61Sx2/EdGWLqZT1ixAhWr15Njhw5eP7556lSpUqslpIlOXr0KE899RRHjhzh33//\n5ZFHHqFq1ar07duXU6dOUapUKV555RXy5MkT66XGNcEGsIwYMYJ+/fol2882gCXzBNvf7du3Z9y4\nceTPn58CBQrwyiuvUKxYMdvfYWDmzJmMHDmSChUqOI+9+uqrPPvss3Z8hxkbLmEYhmEYHsCcugzD\nMAzDA9gF2TAMwzA8gF2QDcMwDMMD2AXZMAzDMDyAXZANwzAMwwPYBdkwDMMwPIBdkA3DMAzDA9gF\n2TAMwzA8wP8HgbU5YlEcKHIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f12b983e898>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "pxYwY0Ej-s0v",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "35c6dc55-e7f7-485c-d389-dee10cba0086"
      },
      "cell_type": "code",
      "source": [
        "images[0].size()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 28, 28])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "metadata": {
        "id": "5YXMcKJP9rt8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import torch.nn as nn"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ercl8jN597GM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1V5O65279Abj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class RP(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        mylist = nn.ModuleList()\n",
        "        for i in range(1):\n",
        "            mylist.append(nn.Linear(28*28, 28*28))\n",
        "            mylist.append(nn.SELU())\n",
        "        self.seq = nn.Sequential(*mylist)\n",
        "\n",
        "        \n",
        "    def forward(self, x):\n",
        "        return self.seq(x.view(x.size(0), -1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "NuSZYC6bKhpx",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class Highway(nn.Module):\n",
        "    def __init__(self, F, input_size):\n",
        "        super().__init__()\n",
        "        self.F = F\n",
        "        self.input_size = input_size\n",
        "        self.T = nn.Linear(input_size, input_size)\n",
        "        self.T.bias.data.fill_(-3)\n",
        "        self.sigmoid = nn.Sigmoid()\n",
        "        \n",
        "    def forward(self, x):\n",
        "        out = self.F(x)\n",
        "        T = self.sigmoid(self.T(x))\n",
        "        return T*out + (1.0 - T)*x\n",
        "        \n",
        "        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "gDyi_5ZdLWJX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class HWN(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        \n",
        "        ml = nn.ModuleList()\n",
        "        for i in range(50):\n",
        "            seq = nn.Sequential(\n",
        "                nn.Linear(28*28, 28*28),\n",
        "                nn.ReLU()\n",
        "            )\n",
        "            \n",
        "            ml.append( Highway( seq, 28*28 ) )\n",
        "        \n",
        "        self.hw = nn.Sequential( *ml )\n",
        "        self.fc = nn.Linear(28*28, 10)\n",
        "        self.softmax = nn.Softmax(dim=1)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        x = x.view(x.size(0), -1)\n",
        "        x = self.hw(x)\n",
        "        x = self.fc(x)\n",
        "        return x\n",
        "        #return self.softmax(x)\n",
        "            "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "M556Gy4cNDE8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "net = HWN()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WZR0I1PrLhWW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "net = net.cuda()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-1xN0SosNlIZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(net.parameters(), lr=0.01)#, momentum=0.9)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "H0XxuS6DWtWM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "device = torch.device(\"cuda\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "SvdljO99WjGg",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def test(model, device, test_loader):\n",
        "    model.eval()\n",
        "    test_loss = 0\n",
        "    correct = 0\n",
        "    with torch.no_grad():\n",
        "        for data, target in test_loader:\n",
        "            data, target = data.to(device), target.to(device)\n",
        "            output = torch.log_softmax(model(data), dim=1)\n",
        "            test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss\n",
        "            pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
        "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
        "\n",
        "    test_loss /= len(test_loader.dataset)\n",
        "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
        "        test_loss, correct, len(test_loader.dataset),\n",
        "        100. * correct / len(test_loader.dataset)))\n",
        "    \n",
        "    model.train()\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "qMpkjYAlJLgV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4wBf-0v5PLR_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1054
        },
        "outputId": "c0e3f291-ad16-487e-d833-0e5e95c15144"
      },
      "cell_type": "code",
      "source": [
        "for epoch in range(100):  # loop over the dataset multiple times\n",
        "    scheduler.step()\n",
        "    running_loss = 0.0\n",
        "    for i, data in enumerate(trainloader, 0):\n",
        "        # get the inputs\n",
        "        inputs, labels = data\n",
        "\n",
        "        # zero the parameter gradients\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        # forward + backward + optimize\n",
        "        outputs = net(inputs.cuda())\n",
        "        loss = criterion(outputs, labels.cuda())\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # print statistics\n",
        "        running_loss += loss.item()\n",
        "        if i % 200 == 199:    # print every 200 mini-batches\n",
        "            print('[%d, %5d] loss: %.7f' %\n",
        "                  (epoch + 1, i + 1, running_loss / 200))\n",
        "            running_loss = 0.0\n",
        "            #test(net, device, testloader)\n",
        "\n",
        "\n",
        "print('Finished Training')\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1,   200] loss: 2.2228167\n",
            "[1,   400] loss: 2.0689299\n",
            "[1,   600] loss: 1.9293880\n",
            "[1,   800] loss: 1.7969398\n",
            "[1,  1000] loss: 1.6691611\n",
            "[1,  1200] loss: 1.5360413\n",
            "[1,  1400] loss: 1.4119227\n",
            "[1,  1600] loss: 1.3066598\n",
            "[1,  1800] loss: 1.2026807\n",
            "[2,   200] loss: 1.0800963\n",
            "[2,   400] loss: 0.9861976\n",
            "[2,   600] loss: 0.9234034\n",
            "[2,   800] loss: 0.8803566\n",
            "[2,  1000] loss: 0.8169722\n",
            "[2,  1200] loss: 0.7670708\n",
            "[2,  1400] loss: 0.7282899\n",
            "[2,  1600] loss: 0.6872202\n",
            "[2,  1800] loss: 0.6547833\n",
            "[3,   200] loss: 0.6240231\n",
            "[3,   400] loss: 0.6151459\n",
            "[3,   600] loss: 0.6009396\n",
            "[3,   800] loss: 0.5700441\n",
            "[3,  1000] loss: 0.5584773\n",
            "[3,  1200] loss: 0.5339702\n",
            "[3,  1400] loss: 0.5293032\n",
            "[3,  1600] loss: 0.5035653\n",
            "[3,  1800] loss: 0.5154778\n",
            "[4,   200] loss: 0.4791781\n",
            "[4,   400] loss: 0.4846514\n",
            "[4,   600] loss: 0.4846456\n",
            "[4,   800] loss: 0.4628088\n",
            "[4,  1000] loss: 0.4526667\n",
            "[4,  1200] loss: 0.4597595\n",
            "[4,  1400] loss: 0.4615633\n",
            "[4,  1600] loss: 0.4491191\n",
            "[4,  1800] loss: 0.4376944\n",
            "[5,   200] loss: 0.4324945\n",
            "[5,   400] loss: 0.4232682\n",
            "[5,   600] loss: 0.4284526\n",
            "[5,   800] loss: 0.4137639\n",
            "[5,  1000] loss: 0.4290935\n",
            "[5,  1200] loss: 0.4027735\n",
            "[5,  1400] loss: 0.3935796\n",
            "[5,  1600] loss: 0.4033876\n",
            "[5,  1800] loss: 0.3871577\n",
            "[6,   200] loss: 0.3852943\n",
            "[6,   400] loss: 0.3714200\n",
            "[6,   600] loss: 0.4025640\n",
            "[6,   800] loss: 0.3840378\n",
            "[6,  1000] loss: 0.3878418\n",
            "[6,  1200] loss: 0.3814455\n",
            "[6,  1400] loss: 0.3746588\n",
            "[6,  1600] loss: 0.3953459\n",
            "[6,  1800] loss: 0.3796620\n",
            "[7,   200] loss: 0.3810881\n",
            "[7,   400] loss: 0.3638740\n",
            "[7,   600] loss: 0.3856409\n",
            "[7,   800] loss: 0.3863000\n",
            "[7,  1000] loss: 0.3616689\n",
            "[7,  1200] loss: 0.3688783\n",
            "[7,  1400] loss: 0.3555660\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "u_5qHM5IhscS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "f8e8d787-70f2-4af6-b9bd-d6590ab70b20"
      },
      "cell_type": "code",
      "source": [
        "test(net, device, testloader)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.3665, Accuracy: 9847/10000 (98%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "U2e3QgwMpNNi",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}