mnist_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST Visualization Example\n",
    "\n",
    "Real-time visualization of MNIST training on a CNN, using TensorFlow and [TensorDebugger](https://github.com/ericjang/tdb)\n",
    "\n",
    "The visualizations in this notebook won't show up on http://nbviewer.ipython.org. To view the widgets and interact with them, you will need to download this notebook and run it with a Jupyter Notebook server."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: Load TDB Notebook Extension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "Jupyter.utils.load_extensions('tdb_ext/main')"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%javascript\n",
    "Jupyter.utils.load_extensions('tdb_ext/main')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#import sys\n",
    "#sys.path.append('/home/evjang/thesis/tensor_debugger')\n",
    "import tdb\n",
    "from tdb.examples import viz\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import urllib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from tdb.examples import mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'module' object has no attribute 'reload'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-4ad06dd4f710>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mimportlib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mimportlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmnist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'module' object has no attribute 'reload'"
     ]
    }
   ],
   "source": [
    "import importlib\n",
    "importlib.reload(mnist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Build TensorFlow Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'module' object has no attribute 'histogram_summary'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-f9617cffa412>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0mlearning_rate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m     summaries) = mnist.build_model()\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tdb/examples/mnist.pyc\u001b[0m in \u001b[0;36mbuild_model\u001b[0;34m()\u001b[0m\n\u001b[1;32m    146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m   \u001b[0;31m# TRAINING LOSS / REGULARIZATION NODES\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 148\u001b[0;31m   \u001b[0mlogits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuild_lenet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data_node\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    149\u001b[0m   \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreduce_mean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax_cross_entropy_with_logits\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_labels_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tdb/examples/mnist.pyc\u001b[0m in \u001b[0;36mbuild_lenet\u001b[0;34m(data, train)\u001b[0m\n\u001b[1;32m    139\u001b[0m       \u001b[0mfc1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfc1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mSEED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m       \u001b[0;31m# append summary ops to train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m       \u001b[0m_activation_summary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconv1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    142\u001b[0m       \u001b[0m_activation_summary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfc1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tdb/examples/mnist.pyc\u001b[0m in \u001b[0;36m_activation_summary\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m     88\u001b[0m   \u001b[0;31m# Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m   \u001b[0;31m# session. This helps the clarity of presentation on tensorboard.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m   \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistogram_summary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'/activations'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     91\u001b[0m   \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscalar_summary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'/sparsity'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_fraction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'module' object has no attribute 'histogram_summary'"
     ]
    }
   ],
   "source": [
    "(train_data_node,\n",
    "    train_labels_node,\n",
    "    validation_data_node,\n",
    "    test_data_node,\n",
    "    # predictions\n",
    "    train_prediction,\n",
    "    validation_prediction,\n",
    "    test_prediction,\n",
    "    # weights\n",
    "    conv1_weights,\n",
    "    conv2_weights,\n",
    "    fc1_weights,\n",
    "    fc2_weights,\n",
    "    # training\n",
    "    optimizer,\n",
    "    loss,\n",
    "    learning_rate,\n",
    "    summaries) = mnist.build_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Attach Plotting Ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def viz_activations(ctx, m):\n",
    "    plt.matshow(m.T,cmap=plt.cm.gray)\n",
    "    plt.title(\"LeNet Predictions\")\n",
    "    plt.xlabel(\"Batch\")\n",
    "    plt.ylabel(\"Digit Activation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# plotting a user-defined function 'viz_activations'\n",
    "p0=tdb.plot_op(viz_activations,inputs=[train_prediction])\n",
    "# weight variables are of type tf.Variable, so we need to find the corresponding tf.Tensor instead\n",
    "g=tf.get_default_graph()\n",
    "p1=tdb.plot_op(viz.viz_conv_weights,inputs=[g.as_graph_element(conv1_weights)])\n",
    "p2=tdb.plot_op(viz.viz_conv_weights,inputs=[g.as_graph_element(conv2_weights)])\n",
    "p3=tdb.plot_op(viz.viz_fc_weights,inputs=[g.as_graph_element(fc1_weights)])\n",
    "p4=tdb.plot_op(viz.viz_fc_weights,inputs=[g.as_graph_element(fc2_weights)])\n",
    "p2=tdb.plot_op(viz.viz_conv_hist,inputs=[g.as_graph_element(conv1_weights)])\n",
    "ploss=tdb.plot_op(viz.watch_loss,inputs=[loss])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: Download the MNIST dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "base_url='http://yann.lecun.com/exdb/mnist/'\n",
    "files=['train-images-idx3-ubyte.gz',\n",
    " 'train-labels-idx1-ubyte.gz',\n",
    " 't10k-images-idx3-ubyte.gz',\n",
    " 't10k-labels-idx1-ubyte.gz']\n",
    "download_dir='/tmp/'\n",
    "for f in files:\n",
    "    print(f)\n",
    "    urllib.urlretrieve(base_url+f, download_dir+f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5: Debug + Visualize!\n",
    "\n",
    "Upon evaluating plot nodes p1,p2,p3,p4,ploss, plots will be generated in the Plot view on the right."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# return the TF nodes corresponding to graph input placeholders\n",
    "(train_data, \n",
    " train_labels, \n",
    " validation_data, \n",
    " validation_labels, \n",
    " test_data, \n",
    " test_labels) = mnist.get_data(download_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# start the TensorFlow session that will be used to evaluate the graph\n",
    "s=tf.InteractiveSession()\n",
    "tf.initialize_all_variables().run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 64\n",
    "NUM_EPOCHS = 5\n",
    "TRAIN_SIZE=10000\n",
    "\n",
    "for step in xrange(NUM_EPOCHS * TRAIN_SIZE // BATCH_SIZE):\n",
    "    offset = (step * BATCH_SIZE) % (TRAIN_SIZE - BATCH_SIZE)\n",
    "    batch_data = train_data[offset:(offset + BATCH_SIZE), :, :, :]\n",
    "    batch_labels = train_labels[offset:(offset + BATCH_SIZE)]\n",
    "    feed_dict = {\n",
    "        train_data_node: batch_data,\n",
    "        train_labels_node: batch_labels\n",
    "    }\n",
    "    # run training node and visualization node\n",
    "    status,result=tdb.debug([optimizer,p0], feed_dict=feed_dict, session=s)\n",
    "    if step % 10 == 0:  \n",
    "        status,result=tdb.debug([loss,p1,p2,p3,p4,ploss], feed_dict=feed_dict, breakpoints=None, break_immediately=False, session=s)\n",
    "        print('loss: %f' % (result[0]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}