TensorFlow+Problem.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/johnb/anaconda/lib/python3.6/importlib/_bootstrap.py:205: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "# Common imports\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"end_to_end_project\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Problem 12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "* f - number of features\n",
    "* m - number of items in batch\n",
    "* X - m-by-f matrix of inputs\n",
    "* theta - 1-by-f matrix of parameters\n",
    "* y - m-by-1 matrix of target values\n",
    "* p_hat - m-by-1 matrix of predictino probabilities -    1/(1+exp(- theta^T * x)\n",
    "* loss = -1/m * (y^T *log(p_hat) + (1-y)^T *log(1-p_hat) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DONE\n",
    "* created logistic evaluation code, loss, and optimization op\n",
    "* create training code, train, and make sure it works by evaluating against some data set\n",
    "* add saves and reloads\n",
    "* view in TensorBoard\n",
    "* track a variable and view in TensorBoard\n",
    "* add scopes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Get training data\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "save_path = '/tmp/my_model.ckpt'\n",
    "logdir = 'tf_logs/run-{}'.format(datetime.utcnow().strftime(\"%Y%m%d%H%M%S\"))\n",
    "\n",
    "# create graph\n",
    "learning_rate = 0.1\n",
    "batch_size = 100\n",
    "num_epochs = 4000\n",
    "f = 784\n",
    "m = batch_size\n",
    "with tf.name_scope('predictor') as scope:\n",
    "    X = tf.placeholder(tf.float32, shape=(None, f), name='X')\n",
    "    y = tf.placeholder(tf.float32, shape=(None, 1), name='y')\n",
    "    theta = tf.Variable(tf.random_uniform([1, f], -1.0, 1.0), name='theta')\n",
    "\n",
    "    with tf.name_scope('p_hat') as scope:\n",
    "        p_hat = 1 / (\n",
    "            1 +  tf.exp(- tf.matmul(\n",
    "                X,\n",
    "                tf.transpose(theta),\n",
    "            ))\n",
    "        )\n",
    "\n",
    "# this loss verified to be a correct function of y and p_hat\n",
    "with tf.name_scope('training') as scope:\n",
    "    with tf.name_scope('loss') as scope:\n",
    "        loss = - 1/m * (\n",
    "            tf.matmul(tf.transpose(y), tf.log(p_hat))\n",
    "            + tf.matmul(tf.transpose(1-y), tf.log(1-p_hat) )\n",
    "        )\n",
    "\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "with tf.name_scope('other_stuff') as scope:    \n",
    "    init = tf.global_variables_initializer()\n",
    "\n",
    "    # writing stuff\n",
    "    saver = tf.train.Saver()\n",
    "    loss_summary = tf.summary.scalar('loss', tf.reduce_mean(loss))\n",
    "    file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 [[ 3.47815657]]\n",
      "100 [[ 0.59215534]]\n",
      "200 [[ 0.53353268]]\n",
      "300 [[ 0.51380879]]\n",
      "400 [[ 0.36483687]]\n",
      "500 [[ 0.43761894]]\n",
      "600 [[ 0.39674804]]\n",
      "700 [[ 0.46109581]]\n",
      "800 [[ 0.3795715]]\n",
      "900 [[ 0.36957067]]\n",
      "1000 [[ 0.45197573]]\n",
      "1100 [[ 0.34083092]]\n",
      "1200 [[ 0.3478775]]\n",
      "1300 [[ 0.36157089]]\n",
      "1400 [[ 0.26671299]]\n",
      "1500 [[ 0.26312533]]\n",
      "1600 [[ 0.26403239]]\n",
      "1700 [[ 0.38385996]]\n",
      "1800 [[ 0.44032452]]\n",
      "1900 [[ 0.26759982]]\n",
      "2000 [[ 0.25534591]]\n",
      "2100 [[ 0.28244549]]\n",
      "2200 [[ 0.41844669]]\n",
      "2300 [[ 0.31011257]]\n",
      "2400 [[ 0.3331832]]\n",
      "2500 [[ 0.24795561]]\n",
      "2600 [[ 0.2687422]]\n",
      "2700 [[ 0.24933818]]\n",
      "2800 [[ 0.26276705]]\n",
      "2900 [[ 0.36464542]]\n",
      "3000 [[ 0.33911484]]\n",
      "3100 [[ 0.29623201]]\n",
      "3200 [[ 0.20152988]]\n",
      "3300 [[ 0.20380083]]\n",
      "3400 [[ 0.42993292]]\n",
      "3500 [[ 0.21006083]]\n",
      "3600 [[ 0.41066816]]\n",
      "3700 [[ 0.22908619]]\n",
      "3800 [[ 0.25793368]]\n",
      "3900 [[ 0.2999942]]\n",
      "900\n"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch_index in range(num_epochs):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        batch_ys = np.any(batch_ys[:,(1,3,5,7,9)], axis=1, keepdims=True).astype(float)\n",
    "        training_op_val, loss_val, loss_summary_val = sess.run(\n",
    "            [training_op, loss, loss_summary], \n",
    "            feed_dict={X: batch_xs, y:batch_ys}\n",
    "        )\n",
    "        if not epoch_index % 10:\n",
    "            file_writer.add_summary(loss_summary_val, epoch_index)\n",
    "        if not epoch_index % 100: \n",
    "            print(epoch_index, loss_val) \n",
    "        if not epoch_index % 1000: \n",
    "            saver.save(sess, save_path)\n",
    "    \n",
    "    saver.save(sess, save_path)\n",
    "            \n",
    "    # Evaluate\n",
    "    batch_xs, batch_ys = mnist.test.next_batch(1000)\n",
    "    batch_ys = np.any(batch_ys[:,(1,3,5,7,9)], axis=1, keepdims=True).astype(float)\n",
    "    p_hat_val = sess.run(p_hat, feed_dict={X: batch_xs, y:batch_ys})\n",
    "    \n",
    "    file_writer.close()\n",
    "\n",
    "result = (p_hat_val>0.5).astype(int) == batch_ys\n",
    "print(np.sum(result))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /tmp/my_model.ckpt\n",
      "887\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, save_path)\n",
    "    batch_xs, batch_ys = mnist.test.next_batch(1000)\n",
    "    batch_ys = np.any(batch_ys[:,(1,3,5,7,9)], axis=1, keepdims=True).astype(float)\n",
    "    p_hat_val = sess.run(p_hat, feed_dict={X: batch_xs, y:batch_ys})\n",
    "    \n",
    "result = (p_hat_val>0.5).astype(int) == batch_ys\n",
    "print(np.sum(result))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}