basic neural net implementation for digit recognition

neuralnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAANnUlEQVR4nO3db6wV9Z3H8c9Hbf1HjbAgIRS3BXmCxtj1BjdZIm5q0fWBUE0UEjeITW9jqmmTmmhYY03UpNls2/jEJoAGurISDLigadaypIo8IV4NVQRblGDKH8GGGCzRsMJ3H9yhucV7fnM5/+X7fiU359z5npn55lw+zJyZM/NzRAjA2e+cXjcAoDsIO5AEYQeSIOxAEoQdSOK8bq7MNof+gQ6LCI82vaUtu+2bbf/B9nu2H2plWQA6y82eZ7d9rqQ/SvqOpH2SXpe0KCJ2FuZhyw50WCe27LMlvRcReyLiuKQ1kua3sDwAHdRK2KdK+tOI3/dV0/6G7UHbQ7aHWlgXgBZ1/ABdRCyTtExiNx7opVa27PslTRvx+9eraQD6UCthf13STNvftP1VSQslbWxPWwDarend+Ij43PZ9kl6WdK6kZyLinbZ1BqCtmj711tTK+MwOdFxHvlQD4MuDsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5BE0+OzS5LtvZI+kXRC0ucRMdCOpgC0X0thr/xzRPy5DcsB0EHsxgNJtBr2kPRb22/YHhztBbYHbQ/ZHmpxXQBa4IhofmZ7akTst32ZpE2S7o+ILYXXN78yAGMSER5tektb9ojYXz0elvSCpNmtLA9A5zQddtsX2/7aqeeS5kna0a7GALRXK0fjJ0t6wfap5fxXRPxPW7oC0HYtfWY/45XxmR3ouI58Zgfw5UHYgSQIO5AEYQeSIOxAEu24EAZ97LrrrivW77rrrmJ97ty5xfqVV155xj2d8sADDxTrBw4cKNbnzJlTrD/77LMNa9u2bSvOezZiyw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSXDV21ngzjvvbFh78skni/NOnDixWK8uYW7olVdeKdYnTZrUsDZr1qzivHXqenv++ecb1hYuXNjSuvsZV70ByRF2IAnCDiRB2IEkCDuQBGEHkiDsQBJcz94Hzjuv/GcYGCgPjrt8+fKGtYsuuqg475YtDQfwkSQ99thjxfrWrVuL9fPPP79hbe3atcV5582bV6zXGRpixLGR2LIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKcZ+8DdfduX7FiRdPL3rRpU7FeuhZeko4ePdr0uuuW3+p59H379hXrq1atamn5Z5vaLbvtZ2wftr1jxLQJtjfZ3l09ju9smwBaNZbd+JWSbj5t2kOSNkfETEmbq98B9LHasEfEFklHTps8X9KpfaRVkha0ty0A7dbsZ/bJEXGwev6hpMmNXmh7UNJgk+sB0CYtH6CLiCjdSDIilklaJnHDSaCXmj31dsj2FEmqHg+3ryUAndBs2DdKWlw9XyxpQ3vaAdAptfeNt/2cpBskTZR0SNJPJf23pLWSLpf0gaQ7IuL0g3ijLSvlbnzdNeFLly4t1uv+Rk899VTD2sMPP1yct9Xz6HV27drVsDZz5syWln377bcX6xs25NwGNbpvfO1n9ohY1KD07ZY6AtBVfF0WSIKwA0kQdiAJwg4kQdiBJLjEtQ0eeeSRYr3u1Nrx48eL9ZdffrlYf/DBBxvWPv300+K8dS644IJive4y1csvv7xhrW7I5ccff7xYz3pqrVls2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgidpLXNu6si/xJa6XXnppw9q7775bnHfixInF+ksvvVSsL1iwoFhvxRVXXFGsr169uli/9tprm173unXrivV77rmnWD927FjT6z6bNbrElS07kARhB5Ig7EAShB1IgrADSRB2IAnCDiTBefYxuuyyyxrWDhw40NKyp0+fXqx/9tlnxfqSJUsa1m699dbivFdddVWxPm7cuGK97t9PqX7bbbcV533xxReLdYyO8+xAcoQdSIKwA0kQdiAJwg4kQdiBJAg7kATn2ceodD17aVhiSZo0aVKxXnf/9E7+jeq+I1DX25QpU4r1jz76qOl50Zymz7Pbfsb2Yds7Rkx71PZ+29urn1va2SyA9hvLbvxKSTePMv2XEXFN9fOb9rYFoN1qwx4RWyQd6UIvADqolQN099l+q9rNH9/oRbYHbQ/ZHmphXQBa1GzYfyVphqRrJB2U9PNGL4yIZRExEBEDTa4LQBs0FfaIOBQRJyLipKTlkma3ty0A7dZU2G2PPGfyXUk7Gr0WQH+oHZ/d9nOSbpA00fY+ST+VdIPtaySFpL2SftC5FvvDxx9/3LBWd1/3uvvCT5gwoVh///33i/XSOOUrV64sznvkSPnY65o1a4r1unPldfOje2rDHhGLRpn8dAd6AdBBfF0WSIKwA0kQdiAJwg4kQdiBJGqPxqPetm3bivW6S1x76frrry/W586dW6yfPHmyWN+zZ88Z94TOYMsOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lwnj25Cy+8sFivO49ed5trLnHtH2zZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJhmxG0YkTJ4r1un8/pVtNl4ZzRvOaHrIZwNmBsANJEHYgCcIOJEHYgSQIO5AEYQeS4Hr25G666aZet4Auqd2y255m+3e2d9p+x/aPqukTbG+yvbt6HN/5dgE0ayy78Z9L+klEzJL0j5J+aHuWpIckbY6ImZI2V78D6FO1YY+IgxHxZvX8E0m7JE2VNF/SquplqyQt6FCPANrgjD6z2/6GpG9J2iZpckQcrEofSprcYJ5BSYMt9AigDcZ8NN72OEnrJP04Io6OrMXw1RCjXhEREcsiYiAiBlrqFEBLxhR221/RcNBXR8T6avIh21Oq+hRJhzvTIoB2qN2Nt21JT0vaFRG/GFHaKGmxpJ9Vjxs60iE6avr06b1uAV0yls/s/yTpXyW9bXt7NW2phkO+1vb3JH0g6Y6OdAigLWrDHhFbJY16Mbykb7e3HQCdwtdlgSQIO5AEYQeSIOxAEoQdSIJLXJN77bXXivVzzilvD+qGdEb/YMsOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lwnj25HTt2FOu7d+8u1uuuh58xY0bDGkM2dxdbdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IwsODuXRpZXb3Voa2uPvuu4v1FStWFOuvvvpqw9r9999fnHfnzp3FOkYXEaPeDZotO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kUXue3fY0Sb+WNFlSSFoWEU/aflTS9yWduih5aUT8pmZZnGf/krnkkkuK9bVr1xbrN954Y8Pa+vXri/MuWbKkWD927FixnlWj8+xjuXnF55J+EhFv2v6apDdsb6pqv4yI/2hXkwA6Zyzjsx+UdLB6/ontXZKmdroxAO11Rp/ZbX9D0rckbasm3Wf7LdvP2B7fYJ5B20O2h1prFUArxhx22+MkrZP044g4KulXkmZIukbDW/6fjzZfRCyLiIGIGGi9XQDNGlPYbX9Fw0FfHRHrJSkiDkXEiYg4KWm5pNmdaxNAq2rDbtuSnpa0KyJ+MWL6lBEv+66k8m1KAfTUWE69zZH0mqS3JZ0an3eppEUa3oUPSXsl/aA6mFdaFqfezjJ1p+aeeOKJhrV77723OO/VV19drHMJ7OiaPvUWEVsljTZz8Zw6gP7CN+iAJAg7kARhB5Ig7EAShB1IgrADSXAraeAsw62kgeQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJsdxdtp3+LOmDEb9PrKb1o37trV/7kuitWe3s7e8bFbr6pZovrNwe6td70/Vrb/3al0RvzepWb+zGA0kQdiCJXod9WY/XX9KvvfVrXxK9NasrvfX0MzuA7un1lh1AlxB2IImehN32zbb/YPs92w/1oodGbO+1/bbt7b0en64aQ++w7R0jpk2wvcn27upx1DH2etTbo7b3V+/ddtu39Ki3abZ/Z3un7Xds/6ia3tP3rtBXV963rn9mt32upD9K+o6kfZJel7QoIvrijv+290oaiIiefwHD9vWS/iLp1xFxVTXt3yUdiYifVf9Rjo+IB/ukt0cl/aXXw3hXoxVNGTnMuKQFku5WD9+7Ql93qAvvWy+27LMlvRcReyLiuKQ1kub3oI++FxFbJB05bfJ8Sauq56s0/I+l6xr01hci4mBEvFk9/0TSqWHGe/reFfrqil6EfaqkP434fZ/6a7z3kPRb22/YHux1M6OYPGKYrQ8lTe5lM6OoHca7m04bZrxv3rtmhj9vFQfovmhORPyDpH+R9MNqd7UvxfBnsH46dzqmYby7ZZRhxv+ql+9ds8Oft6oXYd8vadqI379eTesLEbG/ejws6QX131DUh06NoFs9Hu5xP3/VT8N4jzbMuPrgvevl8Oe9CPvrkmba/qbtr0paKGljD/r4AtsXVwdOZPtiSfPUf0NRb5S0uHq+WNKGHvbyN/plGO9Gw4yrx+9dz4c/j4iu/0i6RcNH5N+X9G+96KFBX9Ml/b76eafXvUl6TsO7df+n4WMb35P0d5I2S9ot6X8lTeij3v5Tw0N7v6XhYE3pUW9zNLyL/pak7dXPLb1+7wp9deV94+uyQBIcoAOSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJP4fBJBcC88tlKgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from functools import reduce\n",
    "from sklearn import datasets\n",
    "import PIL.Image as pil\n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline\n",
    "\n",
    "digits = datasets.load_digits()\n",
    "num = digits.images[28]\n",
    "\n",
    "## 28x28\n",
    "# X, y = datasets.fetch_openml('mnist_784', version=1, return_X_y=True)\n",
    "testnum = X[4]\n",
    "plt.imshow(testnum.reshape((28, 28)), cmap='gray')\n",
    "plt.show()\n",
    "\n",
    "# ## 8x8\n",
    "# pic = pil.fromarray(num)\n",
    "# plt.gray()\n",
    "# plt.matshow(num)\n",
    "# plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(784,)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[4].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 25) (25,) (3,)\n",
      "<function relu at 0x122a259d0> [184.75734833 190.16254672 188.10501189 184.20379922 195.75863049] \n",
      "\n",
      "<function softmax at 0x122a25310> [1.00000000e+000 2.00303272e-095 1.64990330e-078 6.21235649e-058\n",
      " 4.63082926e-103 6.73567466e-149 4.89548781e-045 1.15255295e-075\n",
      " 2.57824440e-106 1.73421184e-045] \n",
      "\n",
      "[1.00000000e+000 2.00303272e-095 1.64990330e-078 6.21235649e-058\n",
      " 4.63082926e-103 6.73567466e-149 4.89548781e-045 1.15255295e-075\n",
      " 2.57824440e-106 1.73421184e-045]\n"
     ]
    }
   ],
   "source": [
    "inputs = np.random.rand(25) # inputs\n",
    "neurons = 3 # layer1 neurons\n",
    "layer1 = np.random.rand(neurons,25) # layer 1 weight matrix\n",
    "res = layer1 @ inputs\n",
    "print(layer1.shape,inputs.shape,res.shape)\n",
    "\n",
    "# print(flat.shape,layer1.shape)\n",
    "# activation = lambda x: x\n",
    "# bias = np.array([2,4,8])\n",
    "# res = activation(flat @ layer1 + bias)\n",
    "\n",
    "# print(res,res.shape,bias.shape)\n",
    "# print(layer1.shape,bias.shape)\n",
    "\n",
    "\n",
    "def relu(vec):\n",
    "    return np.maximum(0,vec)\n",
    "  \n",
    "def softmax(vec):\n",
    "    exps  = np.exp(vec - max(vec))\n",
    "    total = sum(exps)\n",
    "    return exps/total\n",
    "\n",
    "class NeuralNetwork():\n",
    "    def __init__(self,learning_rate=0.001,layers=None,loss_func=None):\n",
    "        self.learning_rate = learning_rate\n",
    "        self.layers = layers\n",
    "        self.loss_func = loss_func\n",
    "  \n",
    "    def init_weights(self):\n",
    "        self.layers[0].init_weights(input_size=1)\n",
    "        self.layers[0].weights = self.layers[0].weights.flatten()\n",
    "        for i,layer in enumerate(self.layers[1:],start=1):\n",
    "            prev_layer = self.layers[i-1]\n",
    "      # every layer will have inputs equal to prev layers neurons\n",
    "      # since each neuron gives one output\n",
    "            layer.init_weights(prev_layer.neurons)\n",
    "\n",
    "    def feedfoward(self):\n",
    "        self.init_weights()\n",
    "        def step(l0,l1):\n",
    "            res = l1.activation(l1.weights @ l0.weights + l1.bias)\n",
    "            print(l1.activation,res,'\\n')\n",
    "            return Layer.FromVec(res)\n",
    "        return reduce(step,self.layers)  \n",
    "\n",
    "    def backprop(self,target):\n",
    "        pass\n",
    "\n",
    "    def train(self,target_output,epochs=0):\n",
    "        for i in range(epochs):\n",
    "            output = self.feedfoward()\n",
    "            loss = self.loss_func(output,target_output)\n",
    "            print('epoch [{}] - loss {}'.format(i,loss))\n",
    "            gradient = self.backprop()\n",
    "            self.weights -= gradient * self.learning_rate\n",
    "\n",
    "class Layer():\n",
    "    def __init__(self,neurons=None,activation=relu):\n",
    "        self.neurons = neurons\n",
    "        self.activation = activation\n",
    "        self.bias = np.random.rand(neurons)\n",
    "\n",
    "    def init_weights(self,input_size):\n",
    "        \"\"\"each neuron has one weight for each input, where the input\n",
    "        is the output from the prior layer\"\"\"\"\"\n",
    "        self.weights = np.random.rand(self.neurons,input_size)\n",
    "\n",
    "    @classmethod\n",
    "    def FromVec(cls,vec):\n",
    "        layer = cls(neurons=len(vec))\n",
    "        layer.weights = vec\n",
    "        return layer\n",
    "\n",
    "net = NeuralNetwork(\n",
    "  learning_rate=0.001,\n",
    "  layers = [\n",
    "    Layer(neurons=28*28), # input layer\n",
    "    Layer(neurons=5,activation=relu),\n",
    "    Layer(neurons=10,activation=softmax)\n",
    "    ]\n",
    "  )\n",
    "\n",
    "net.init_weights()\n",
    "inputlayer = net.layers[0]\n",
    "l1 = net.layers[1]\n",
    "outputlayer = net.layers[2]\n",
    "\n",
    "# l1_out = l1.weights @ inputlayer.weights + l1.bias\n",
    "# print(l1_out)\n",
    "# l2_out = outputlayer.weights @ l1_out + outputlayer.bias\n",
    "# print(l2_out,'\\n\\n',softmax(l2_out),sum(softmax(l2_out)))\n",
    "\n",
    "res = net.feedfoward()\n",
    "print(res.weights)\n",
    "\n",
    "# for l in net.layers:\n",
    "#   print(l.weights.shape)\n",
    "# out = net.feedfoward()\n",
    "# print(out.weights)\n",
    "\n",
    "# a = np.random.rand(2000)\n",
    "# b = np.random.rand(1,2000)\n",
    "# print(b @ a)\n",
    "\n",
    "\n",
    "# ### softmax question ###\n",
    "\n",
    "# ## high values make the max entry consume all others, with prob = 1\n",
    "# print(softmax(np.array([100,200,300,100,400])))\n",
    "# ## single digit values result in more dispersed percentages\n",
    "# print(softmax(np.array([1,2,3,1,4])))\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}