Limiting_Cases_LogLoss.ipynb

{
 "cells": [
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "from sklearn.metrics import log_loss",
   "execution_count": 1,
   "outputs": []
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "import numpy as np",
   "execution_count": 2,
   "outputs": []
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "import sklearn\nprint(sklearn.__version__)",
   "execution_count": 3,
   "outputs": [
    {
     "output_type": "stream",
     "text": "0.19.1\n",
     "name": "stdout"
    }
   ]
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "truths = np.array([1., 0., 1.], dtype=np.int)\n# Perfect Classifier\npredictions = np.zeros(shape=(3,2), dtype=np.float)\npredictions[:, 1]= [1, 0, 1]\npredictions[:, 0] = [0, 1, 0]",
   "execution_count": 4,
   "outputs": []
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "# value from sklearn. Ideally this should be zero, but because of the\n# clipping I expect a small non zero value of epsilon. Works !\nlog_loss(truths, predictions, eps=1.0e-15)",
   "execution_count": 5,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 5,
     "data": {
      "text/plain": "9.992007221626415e-16"
     },
     "metadata": {}
    }
   ]
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "# The probability matrix is still what we put in!\npredictions",
   "execution_count": 6,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 6,
     "data": {
      "text/plain": "array([[0., 1.],\n       [1., 0.],\n       [0., 1.]])"
     },
     "metadata": {}
    }
   ]
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## Proclam"
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "from proclam.metrics import LogLoss",
   "execution_count": 7,
   "outputs": []
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "D = LogLoss()",
   "execution_count": 8,
   "outputs": []
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "# This is equal to epsilon\nD.evaluate(predictions, truths)",
   "execution_count": 9,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 9,
     "data": {
      "text/plain": "2.220446049250313e-16"
     },
     "metadata": {}
    }
   ]
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "# Now prediction\npredictions",
   "execution_count": 10,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 10,
     "data": {
      "text/plain": "array([[2.22044605e-16, 1.00000000e+00],\n       [1.00000000e+00, 2.22044605e-16],\n       [2.22044605e-16, 1.00000000e+00]])"
     },
     "metadata": {}
    }
   ]
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "import proclam\nproclam.__file__",
   "execution_count": 12,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 12,
     "data": {
      "text/plain": "'/Users/rbiswas/astroml_miniconda3/envs/plasticc/lib/python3.6/site-packages/proclam-0.1-py3.6.egg/proclam/__init__.py'"
     },
     "metadata": {}
    }
   ]
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "import sys\neps =sys.float_info.epsilon",
   "execution_count": 37,
   "outputs": []
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "truths = np.array([1, 0, 1, 2, 0], dtype=np.int)\npredictions = np.zeros(shape=(5,3), dtype=np.float)",
   "execution_count": 18,
   "outputs": []
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "predictions[:, 0] = [0, 1, 0, 0, 1]\npredictions[:, 1] = [1, 0, 1, 0, 0]\npredictions[:, 2] = [0, 0, 0, 1, 0]",
   "execution_count": 19,
   "outputs": []
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "D.evaluate(predictions, truths)",
   "execution_count": 21,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 21,
     "data": {
      "text/plain": "4.440892098500626e-16"
     },
     "metadata": {}
    }
   ]
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "# predictions changed\npredictions",
   "execution_count": 28,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 28,
     "data": {
      "text/plain": "array([[2.22044605e-16, 1.00000000e+00, 2.22044605e-16],\n       [1.00000000e+00, 2.22044605e-16, 2.22044605e-16],\n       [2.22044605e-16, 1.00000000e+00, 2.22044605e-16],\n       [2.22044605e-16, 2.22044605e-16, 1.00000000e+00],\n       [1.00000000e+00, 2.22044605e-16, 2.22044605e-16]])"
     },
     "metadata": {}
    }
   ]
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "#probabilities not normalized correctly\nall(predictions.sum(axis=1)>1.)",
   "execution_count": 27,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 27,
     "data": {
      "text/plain": "True"
     },
     "metadata": {}
    }
   ]
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "predictions[3] = (1., 0., 0.)",
   "execution_count": 32,
   "outputs": []
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "predictions.sum(axis=1)",
   "execution_count": 34,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 34,
     "data": {
      "text/plain": "array([1., 1., 1., 1., 1.])"
     },
     "metadata": {}
    }
   ]
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "D.evaluate(predictions, truths)",
   "execution_count": 35,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 35,
     "data": {
      "text/plain": "12.014551129705717"
     },
     "metadata": {}
    }
   ]
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "- np.log(eps)/3.",
   "execution_count": 39,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 39,
     "data": {
      "text/plain": "12.014551129705717"
     },
     "metadata": {}
    }
   ]
  },
  {
   "metadata": {
    "trusted": true
   },
   "cell_type": "code",
   "source": "",
   "execution_count": null,
   "outputs": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  },
  "language_info": {
   "name": "python",
   "version": "3.6.5",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}