On the evaluation of speaker turn segmentation

notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "from pyannote.core import Segment, Timeline, Annotation, SlidingWindow, notebook\n",
    "from pyannote.metrics.segmentation import SegmentationPurity, SegmentationCoverage\n",
    "from pyannote.metrics.segmentation import SegmentationPrecision, SegmentationRecall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# On the evaluation of speaker turn segmentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's consider a 15 seconds audio recording whose \"speaker\" reference is depicted below (three speakers, denoted 1 to 3):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHEAAACsCAYAAAD8BzCjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADJBJREFUeJzt3XGMpOVdB/DvrxymBaqohy0eF69pKkqIhWZtzpI02laL\npTlI/EdTSY39x0RbME1Mq4nIPwZjY22i0TRYwUgwBiglNMoRJDYmpWXvhAMOsbVqufPq0RAsZ6Ol\n5ecfM21PKLndu9l975n9fJLJzjv7zuY7m2dn3v3O8z5T3R0AAAAATm8vmzoAAAAAACemxAEAAAAY\ngBIHAAAAYABKHAAAAIABKHEAAAAABqDEAQAAABiAEgcAAABgAEocAAAAgAEocQAAAAAGoMQBAAAA\nGMBwJU5VfayqjlbVo1NnWaSq2llV91fVwap6rKqumTrTIlTVy6vqs1X18PxxXT91JgAAABhRdffU\nGdalqt6c5FiSv+jui6fOsyhVdX6S87t7f1W9Msm+JFd198GJo52SqqokZ3f3sao6M8k/JLmmux+Y\nOBoAAAAMZbiZON39qSRPT51j0br7SHfvn19/NsnjSXZMm+rU9cyx+eaZ88tYzSEAAACcBoYrcbaC\nqtqV5NIkn5k2yWJU1RlV9VCSo0nu7e6leFwAAACwmbadyp0P79i50BkVOw4/WYv8eSdr93X3LPRx\nPXD929f8uKrqnCS3J7m2u7+yyBx77rxioY/rrqs+uabH1d3fSHJJVZ2b5ONVdXF3L9WaRgAAALDR\nzMQ5jczXjLk9yS3dfcfUeRatu59Jcn+Sy6fOAgAAAKMZbmHj5FunG929ZAsbV5Kbkzzd3ddOnWdR\nquq8JM919zNV9Yoke5P8XnffPXE0AAAAGMpwM3Gq6tYkn05yYVUdqqr3TJ1pQS5LcnWSt1TVQ/PL\nO6YOtQDnJ7m/qg4keTCzNXEUOAAAALBOQ87EAQAAANhqhpuJAwAAALAVKXEAAAAABqDEAQAAABiA\nEgcAAABgAEocAAAAgAFsW8/O27dv7127dm1QFAAAAICtZ9++fV/u7vNOtN+6Spxdu3ZldXX15FMB\nAAAA8P9U1b+vZT+nUwEAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUO\nAAAAwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAl\nDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIAAAAADECJAwAAADAA\nJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAxgXSXOcwcO\nbFQOgFNyeMfOHN6xc9Pvy/IyLmAau6+7J7uvu2fqGEPwu1obz+cwDX97G8NMHAAAAIABKHEAAAAA\nBqDEAQAAABjAtvXewTltwLLy/AZw+rDWC4vmdR5YBmbiAAAAAAxg3TNxdhx+ciNyAJySRby75vmN\n43nHFqb1wPVvnzrCac9spfXxOg+by7HUxjATBwAAAGAAShwAAACAAShxAAAAAAagxAEAAAAYQHX3\nmndeWVnp1dXVDYwDAAAAsLVU1b7uXjnRfmbiAAAAAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIA\nAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1Di\nAAAAAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ\n4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAkD13XpE9d14xdQzYctbzd6fEAQAAABiA\nEgcAAABgAEocAAAAgAEocQAAAAAGsG3qAAAAAJw+LG4Mpy8zcQAAAAAGYCYOAAAA33LXVZ+cOgJs\nKT5iHAAAAGDJKHEAAAAABqDEAQAAABhAdfead15ZWenV1dUNjAMAAACwtVTVvu5eOdF+ZuIAAAAA\nDECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAA\nAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAA\nAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIA\nAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMIDq7rXvXPVskic2Lg5b0PYk\nX546BEvDeGLRjCkWzZhi0YwpFs2YYpGMp7X7oe4+70Q7bVvnD32iu1dOMhC8SFWtGlMsivHEohlT\nLJoxxaIZUyyaMcUiGU+L53QqAAAAgAEocQAAAAAGsN4S56MbkoKtzJhikYwnFs2YYtGMKRbNmGLR\njCkWyXhasHUtbAwAAADANJxOBQAAADCANZU4VXV5VT1RVZ+vqg9sdCiWW1XtrKr7q+pgVT1WVddM\nnYnlUFVnVNU/VtXdU2dhfFV1blXdVlX/VFWPV9VPTJ2JcVXVr89f8x6tqlur6uVTZ2I8VfWxqjpa\nVY8ed9v3VdW9VfW5+dfvnTIj43iJ8fT789e9A1X18ao6d8qMjOU7janjvvf+quqq2j5FtmVywhKn\nqs5I8sdJfjbJRUl+oaou2uhgLLWvJ3l/d1+UZHeSXzWmWJBrkjw+dQiWxkeS/G13/0iS18fY4iRV\n1Y4k70uy0t0XJzkjyc9Pm4pB3ZTk8hfc9oEk93X365LcN9+GtbgpLx5P9ya5uLt/LMk/J/ngZodi\naDflxWMqVbUzyc8k+eJmB1pGa5mJ88Ykn+/uL3T315L8VZIrNzYWy6y7j3T3/vn1ZzP7x2jHtKkY\nXVVdkOSKJDdOnYXxVdX3JHlzkj9Lku7+Wnc/M20qBrctySuqaluSs5L8x8R5GFB3fyrJ0y+4+cok\nN8+v35zkqk0NxbC+03jq7r3d/fX55gNJLtj0YAzrJZ6jkuTDSX4jiQV5F2AtJc6OJE8et30o/uFm\nQapqV5JLk3xm2iQsgT/M7MXh+amDsBRek+SpJH8+P0Xvxqo6e+pQjKm7Dyf5UGbvQB5J8l/dvXfa\nVCyRV3X3kfn1LyV51ZRhWCq/nORvpg7B2KrqyiSHu/vhqbMsCwsbM5mqOifJ7Umu7e6vTJ2HcVXV\nO5Mc7e59U2dhaWxL8oYkf9Ldlyb57zhFgZM0X6PkyszKwR9McnZV/eK0qVhGPfvYWe90c8qq6rcy\nWwLhlqmzMK6qOivJbyb57amzLJO1lDiHk+w8bvuC+W1w0qrqzMwKnFu6+46p8zC8y5Lsqap/y+yU\nz7dU1V9OG4nBHUpyqLu/OUvwtsxKHTgZb0vyr939VHc/l+SOJG+aOBPL4z+r6vwkmX89OnEeBldV\nv5TknUneNS8G4WS9NrM3MB6eH6dfkGR/Vb160lSDW0uJ82CS11XVa6rquzJbiO+ujY3FMquqymyd\nice7+w+mzsP4uvuD3X1Bd+/K7Dnq77rbu9yctO7+UpInq+rC+U1vTXJwwkiM7YtJdlfVWfPXwLfG\nQtkszl1J3j2//u4kn5gwC4OrqsszOz19T3d/deo8jK27H+nuH+juXfPj9ENJ3jA/zuIknbDEmS9s\n9WtJ7snsgOOvu/uxjQ7GUrssydWZzZZ4aH55x9ShAF7gvUluqaoDSS5J8rsT52FQ8xldtyXZn+SR\nzI6/PjppKIZUVbcm+XSSC6vqUFW9J8kNSX66qj6X2ayvG6bMyDheYjz9UZJXJrl3foz+p5OGZCgv\nMaZYsDJDDgAAAOD0Z2FjAAAAgAEocQAAAAAGoMQBAAAAGIASBwAAAGAAShwAAACAAWybOgAAwIlU\n1fcnuW+++eok30jy1Hz7q939pkmCAQBsIh8xDgAMpap+J8mx7v7Q1FkAADaT06kAgKFV1bH515+s\nqr+vqk9U1Req6oaqeldVfbaqHqmq1873O6+qbq+qB+eXy6Z9BAAAa6PEAQCWyeuT/EqSH01ydZIf\n7u43JrkxyXvn+3wkyYe7+8eT/Nz8ewAApz1r4gAAy+TB7j6SJFX1L0n2zm9/JMlPza+/LclFVfXN\n+3x3VZ3T3cc2NSkAwDopcQCAZfK/x11//rjt5/Pt456XJdnd3f+zmcEAAE6V06kAgK1mb759alWq\n6pIJswAArJkSBwDYat6XZKWqDlTVwczW0AEAOO35iHEAAACAAZiJAwAAADAAJQ4AAADAAJQ4AAAA\nAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAzg/wD7XYXWnq3bQAAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<pyannote.core.annotation.Annotation at 0x112c459e8>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reference = Annotation()\n",
    "reference[Segment( 0.0,  4.0)] = 1\n",
    "reference[Segment( 4.2,  6.0)] = 1\n",
    "reference[Segment( 9.0, 10.0)] = 2\n",
    "reference[Segment(11.0, 14.0)] = 1\n",
    "reference[Segment(12.0, 15.0)] = 3\n",
    "reference"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A segmentation algorithms processes the whole audio recording (even non-speech region) and outputs the following hypothesis segmentation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHEAAABnCAYAAACHKN7UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACPtJREFUeJzt3WuoZXUZBvDnzSnK7D5W5khHoptEN6awhLQrVpJBX4qK\nor4E3QmiC0SfQigyoSjESiExQo0iqBSL+tLFGSstze6XUUsjulh09e3D3tGkDbPPaZ1Z/vf5/eBw\n9tpnneGZzctZ6zx7rf+p7g4AAAAAd253mTsAAAAAAIenxAEAAAAYgBIHAAAAYABKHAAAAIABKHEA\nAAAABqDEAQAAABiAEgcAAABgAEocAAAAgAEocQAAAAAGsGszO+/evbs3Nja2KQoAAADAzrN///7f\ndPexh9tvUyXOxsZG9u3bt/VUAAAAAPyXqvr5Kvu5nQoAAABgAEocAAAAgAEocQAAAAAGoMQBAAAA\nGIASBwAAAGAAShwAAACAAShxAAAAAAagxAEAAAAYgBIHAAAAYABKHAAAAIABKHEAAAAABqDEAQAA\nABiAEgcAAABgAEocAAAAgAEocQAAAAAGoMQBAAAAGIASBwAAAGAAShwAAACAAShxAAAAAAagxAEA\nAAAYgBIHAAAAYABKHAAAAIABKHEAAAAABrCpEufGG2/crhzsUKeddtrcEVgj5ml1XisAdhLHPaZm\npphLdffqO1f1ZvaHw6mqmCmmYp5W57UCYCdx3GNqZoqpVdX+7t57uP3cTgUAAAAwgF2b/Yaq2o4c\n7GBmiimZp9V5rQDYSRz3mJqZYg6bLnFcMsaUXIbIlMzT6rxWAOwkjntMzUwxtVVLwU3dTnXcccdt\nKQwcyqmnnjp3BNaIeVqd1wqAncRxj6mZKeayqYWN9+7d2/v27dvGOAAAAAA7i4WNAQAAANaIEgcA\nAABgAEocAAAAgAEocQAAAAAGoMQBAAAAGIASBwAAAGAAShwAAACAAShxAAAAAAagxAEAAAAYgBIH\nAAAAYABKHAAAAIABKHEAAAAABqDEAQAAABiAEgcAAABgAEocAAAAgAEocQAAAAAGoMQBAAAAGIAS\nBwAAAGAAShwAAACAAShxAAAAAAagxAEAAAAYgBIHAAAAYABKHAAAAIABVHevvnPVH5Ncv31x2IF2\nJ/nN3CFYG+aJqZkppmammJqZYmpmiimZp9U9tLuPPdxOuzb5j17f3Xu3GAjuoKr2mSmmYp6Ympli\namaKqZkppmammJJ5mp7bqQAAAAAGoMQBAAAAGMBmS5xztyUFO5mZYkrmiamZKaZmppiamWJqZoop\nmaeJbWphYwAAAADm4XYqAAAAgAGsVOJU1elVdX1V/aiq3rbdoVhvVXVCVX25qq6tqu9V1RvnzsR6\nqKqjqupbVfW5ubMwvqq6b1VdXFXfr6rrquopc2diXFX15uUx77tVdVFV3X3uTIynqj5WVTdX1XcP\neu7+VXV5Vf1w+fl+c2ZkHIeYp/cuj3tXV9Wnq+q+c2ZkLP9rpg762luqqqtq9xzZ1slhS5yqOirJ\nh5I8N8lJSV5SVSdtdzDW2j+SvKW7T0pycpLXmikm8sYk180dgrVxTpIvdPejkjwuZostqqrjk7wh\nyd7ufkySo5K8eN5UDOr8JKff7rm3Jbmiux+e5IrlNqzi/Nxxni5P8pjufmySHyR5+5EOxdDOzx1n\nKlV1QpLnJPnFkQ60jla5EufJSX7U3T/p7r8l+WSSM7c3Fuusu2/q7quWj/+YxS9Gx8+bitFV1Z4k\nz09y3txZGF9V3SfJ05J8NEm6+2/d/bt5UzG4XUnuUVW7khyd5MaZ8zCg7v5qkt/e7ukzk1ywfHxB\nkhce0VAM63/NU3df1t3/WG5+PcmeIx6MYR3iZ1SSnJ3krUksyDuBVUqc45P88qDtA/ELNxOpqo0k\nT0jyjXmTsAY+kMXB4ba5g7AWTkxyS5KPL2/RO6+q7jl3KMbU3TckeV8W70DelOT33X3ZvKlYIw/q\n7puWj3+V5EFzhmGtvCrJ5+cOwdiq6swkN3T3d+bOsi4sbMxsquqYJJckeVN3/2HuPIyrqs5IcnN3\n7587C2tjV5InJvlwdz8hyZ/iFgW2aLlGyZlZlIMPSXLPqnrZvKlYR734s7Pe6eb/VlXvzGIJhAvn\nzsK4quroJO9I8q65s6yTVUqcG5KccND2nuVzsGVVddcsCpwLu/vSufMwvFOSvKCqfpbFLZ/PqKpP\nzBuJwR1IcqC7/32V4MVZlDqwFc9K8tPuvqW7/57k0iRPnTkT6+PXVXVckiw/3zxzHgZXVa9MckaS\nly6LQdiqh2XxBsZ3lufpe5JcVVUPnjXV4FYpca5M8vCqOrGq7pbFQnyf3d5YrLOqqizWmbiuu98/\ndx7G191v7+493b2Rxc+oL3W3d7nZsu7+VZJfVtUjl089M8m1M0ZibL9IcnJVHb08Bj4zFspmOp9N\n8orl41ck+cyMWRhcVZ2exe3pL+juP8+dh7F19zXd/cDu3liepx9I8sTleRZbdNgSZ7mw1euSfDGL\nE45Pdff3tjsYa+2UJC/P4mqJby8/njd3KIDbeX2SC6vq6iSPT/KemfMwqOUVXRcnuSrJNVmcf507\nayiGVFUXJflakkdW1YGqenWSs5I8u6p+mMVVX2fNmZFxHGKePpjkXkkuX56jf2TWkAzlEDPFxMoV\ncgAAAAB3fhY2BgAAABiAEgcAAABgAEocAAAAgAEocQAAAAAGoMQBAAAAGMCuuQMAABxOVT0gyRXL\nzQcn+WeSW5bbf+7up84SDADgCPInxgGAoVTVu5Pc2t3vmzsLAMCR5HYqAGBoVXXr8vNpVfWVqvpM\nVf2kqs6qqpdW1Ter6pqqethyv2Or6pKqunL5ccq8/wMAgNUocQCAdfK4JK9J8ugkL0/yiO5+cpLz\nkrx+uc85Sc7u7icledHyawAAd3rWxAEA1smV3X1TklTVj5Nctnz+miRPXz5+VpKTqurf33Pvqjqm\nu289okkBADZJiQMArJO/HvT4toO2b8t/znvukuTk7v7LkQwGAPD/cjsVALDTXJb/3FqVqnr8jFkA\nAFamxAEAdpo3JNlbVVdX1bVZrKEDAHCn50+MAwAAAAzAlTgAAAAAA1DiAAAAAAxAiQMAAAAwACUO\nAAAAwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADAAP4FVjj2uqp+IG0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Timeline(uri=None, segments=[<Segment(0, 2)>, <Segment(2, 4)>, <Segment(4, 5)>, <Segment(5, 11)>, <Segment(11, 13)>, <Segment(13, 15)>])>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hypothesis = Timeline()\n",
    "hypothesis.add(Segment( 0.0,  2.0))\n",
    "hypothesis.add(Segment( 2.0,  4.0))\n",
    "hypothesis.add(Segment( 4.0,  5.0))\n",
    "hypothesis.add(Segment( 5.0, 11.0))\n",
    "hypothesis.add(Segment(11.0, 13.0))\n",
    "hypothesis.add(Segment(13.0, 15.0))\n",
    "hypothesis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Two evaluation metrics can be computed that evaluate the purity and coverage of hypothesized segments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "purity = SegmentationPurity(tolerance=0.5)\n",
    "coverage = SegmentationCoverage(tolerance=0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Purity measures how pure (in terms of reference speaker) each hypothesized segment is."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8181818181818182"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "purity(reference, hypothesis)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Perfect purity (1) can be achieved easily by oversegmenting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHEAAABnCAYAAACHKN7UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAClFJREFUeJzt3W+sZHdZB/DvQ1ejBRV0C9Ru4xICaEPkT66k0sSugqYq\noSa+0SDBaNKYKKAhIaCJvX1jmkhEEo1mU7UkNhhTMBAStU0l+MY/vVulhdYKosK2iy0xKpUoYh9f\nzDSsLc2dO3funPub+XySzZ2ZPc+e7579zb2z3zvn3OruAAAAAHC8PWPqAAAAAADsT4kDAAAAMAAl\nDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIAAAAADODEQTY+efJk\nnz59+oiiAAAAAGyfc+fOfb67L9tvuwOVOKdPn87e3t7yqQAAAAD4f6rqnxfZzulUAAAAAANQ4gAA\nAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIA\nAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAM4EAlzsMPP7z0js6cObPWObObPTtV3t3d\n3a2aneJYTZV322anWFOO0+K26Vht23HaptnR8po93vs0e/xnR8tr9njvcxtnF1XdvfjGVX2Q7Z80\nm2Vml50zu9mzo+U1e7z3aXY9s6PlNbue2dHymj3e+zS7ntnR8ppdz+xoec0e731u6ey57t7Zbzun\nUwEAAAAM4MDvxDnCLDCEw7SyI+73MJZt+lmPEdfUFBynxXnOAwAsbaF34pw46J/qrVtmj8PslHkP\nY8T9TpF5pLU48uxhjPa8Hek4PbHfEWeXNeLf1ezx3KfZ9cyOltfsemZHy2v2eO9zW2cXcaDTqS6/\n/PKlwiTJtddeu9Y5s5s9O1XeG2+8catmpzhWU+Xdttkp1pTjtLhtOlbbdpy2aXa0vGaP9z7NHv/Z\n0fKaPd773MbZRR3odKqdnZ3e29s7wjgAAAAA28WFjQEAAAA2iBIHAAAAYABKHAAAAIABKHEAAAAA\nBqDEAQAAABiAEgcAAABgAEocAAAAgAEocQAAAAAGoMQBAAAAGIASBwAAAGAAShwAAACAAShx4AB2\nd3e3avYwlt3vVHm3zYhragqO0+I85wEAjl5198Ib7+zs9N7e3hHGgeOtqnKQ58zos4ex7H6nyrtt\nRlxTU3CcFuc5DwCwvKo61907+23nnTgAAAAAAzgxdQAYTVVt1X4PY8TM28S/z2Icp8U5VgAAR0uJ\nAwd0mFMrRtzvYSx7agXrMeKamoLjtDjPeQCAo3XJQS4oePbs2d0bbrjh6NLAAM6cObNVs4ex7H6n\nyrttRlxTU3CcFuc5DwCwnJtuuunC7u7u2f22c2FjAAAAgAm5sDEAAADABlHiAAAAAAxAiQMAAAAw\nACUOAAAAwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAA\nMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAygunvxjau+kOTBo4vDFjqZ5PNTh2BjWE+smjXFqllTrJo1\nxapZU6yS9bS4b+vuy/bb6MQB/9AHu3tnyUDwFFW1Z02xKtYTq2ZNsWrWFKtmTbFq1hSrZD2tntOp\nAAAAAAagxAEAAAAYwEFLnLNHkoJtZk2xStYTq2ZNsWrWFKtmTbFq1hSrZD2t2IEubAwAAADANJxO\nBQAAADCAhUqcqrquqh6sqk9V1TuOOhSbraqurKqPVNX9VfWJqnrr1JnYDFV1SVX9TVV9eOosjK+q\nnl1Vt1fV31XVA1X13VNnYlxV9Qvzr3kfr6r3VdXXTZ2J8VTV71bVI1X18Yse++aqurOqPjn/+Jwp\nMzKOp1lPvzr/undvVf1RVT17yoyM5autqYt+721V1VV1copsm2TfEqeqLknym0l+MMlVSX68qq46\n6mBstC8neVt3X5Xk6iQ/a02xIm9N8sDUIdgY70nyJ9397UleFmuLJVXVFUnekmSnu1+a5JIkPzZt\nKgZ1a5LrnvTYO5Lc1d0vSnLX/D4s4tY8dT3dmeSl3f2dSf4+yTvXHYqh3ZqnrqlU1ZVJfiDJZ9Yd\naBMt8k6cVyX5VHd/uru/lOQPklx/tLHYZN19obvvmd/+Qmb/Mbpi2lSMrqpOJfnhJLdMnYXxVdU3\nJfmeJL+TJN39pe7+t2lTMbgTSb6+qk4kuTTJwxPnYUDd/edJ/vVJD1+f5L3z2+9N8iNrDcWwvtp6\n6u47uvvL87t/meTU2oMxrKf5HJUk707y9iQuyLsCi5Q4VyT57EX3z8d/uFmRqjqd5BVJ/mraJGyA\nX8/si8PjUwdhI7wgyaNJfm9+it4tVfXMqUMxpu5+KMm7MvsO5IUk/97dd0ybig3yvO6+ML/9uSTP\nmzIMG+Wnkvzx1CEYW1Vdn+Sh7v7Y1Fk2hQsbM5mqelaS9yf5+e7+j6nzMK6qel2SR7r73NRZ2Bgn\nkrwyyW919yuS/GecosCS5tcouT6zcvBbkzyzqn5i2lRsop792Fnf6ebQquqXMrsEwm1TZ2FcVXVp\nkl9M8stTZ9kki5Q4DyW58qL7p+aPwdKq6msyK3Bu6+4PTJ2H4V2T5PVV9U+ZnfL5fVX1+9NGYnDn\nk5zv7ifeJXh7ZqUOLOO1Sf6xux/t7v9J8oEkr544E5vjX6rq8iSZf3xk4jwMrqp+MsnrkrxhXgzC\nsl6Y2TcwPjZ/nX4qyT1V9fxJUw1ukRLn7iQvqqoXVNXXZnYhvg8dbSw2WVVVZteZeKC7f23qPIyv\nu9/Z3ae6+3Rmn6P+rLt9l5uldffnkny2ql4yf+g1Se6fMBJj+0ySq6vq0vnXwNfEhbJZnQ8ledP8\n9puSfHDCLAyuqq7L7PT013f3F6fOw9i6+77ufm53n56/Tj+f5JXz11ksad8SZ35hq59L8qeZveD4\nw+7+xFEHY6Ndk+SNmb1b4m/nv35o6lAAT/LmJLdV1b1JXp7kVybOw6Dm7+i6Pck9Se7L7PXX2UlD\nMaSqel+Sv0jykqo6X1U/neTmJN9fVZ/M7F1fN0+ZkXE8zXr6jSTfkOTO+Wv03540JEN5mjXFipV3\nyAEAAAAcfy5sDAAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMIATUwcAANhPVX1Lkrvm\nd5+f5H+TPDq//8XufvUkwQAA1siPGAcAhlJVu0ke6+53TZ0FAGCdnE4FAAytqh6bfzxTVR+tqg9W\n1aer6uaqekNV/XVV3VdVL5xvd1lVvb+q7p7/umbavwEAwGKUOADAJnlZkp9J8h1J3pjkxd39qiS3\nJHnzfJv3JHl3d39Xkh+d/x4AwLHnmjgAwCa5u7svJElV/UOSO+aP35fke+e3X5vkqqp6YuYbq+pZ\n3f3YWpMCAByQEgcA2CT/fdHtxy+6/3i+8rrnGUmu7u7/WmcwAIDDcjoVALBt7shXTq1KVb18wiwA\nAAtT4gAA2+YtSXaq6t6quj+za+gAABx7fsQ4AAAAwAC8EwcAAABgAEocAAAAgAEocQAAAAAGoMQB\nAAAAGIASBwAAAGAAShwAAACAAShxAAAAAAagxAEAAAAYwP8B6X2GE2Ah9O0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Timeline(uri=None, segments=[<Segment(0, 0.2)>, <Segment(0.2, 0.4)>, <Segment(0.4, 0.6)>, <Segment(0.6, 0.8)>, <Segment(0.8, 1)>, <Segment(1, 1.2)>, <Segment(1.2, 1.4)>, <Segment(1.4, 1.6)>, <Segment(1.6, 1.8)>, <Segment(1.8, 2)>, <Segment(2, 2.2)>, <Segment(2.2, 2.4)>, <Segment(2.4, 2.6)>, <Segment(2.6, 2.8)>, <Segment(2.8, 3)>, <Segment(3, 3.2)>, <Segment(3.2, 3.4)>, <Segment(3.4, 3.6)>, <Segment(3.6, 3.8)>, <Segment(3.8, 4)>, <Segment(4, 4.2)>, <Segment(4.2, 4.4)>, <Segment(4.4, 4.6)>, <Segment(4.6, 4.8)>, <Segment(4.8, 5)>, <Segment(5, 5.2)>, <Segment(5.2, 5.4)>, <Segment(5.4, 5.6)>, <Segment(5.6, 5.8)>, <Segment(5.8, 6)>, <Segment(6, 6.2)>, <Segment(6.2, 6.4)>, <Segment(6.4, 6.6)>, <Segment(6.6, 6.8)>, <Segment(6.8, 7)>, <Segment(7, 7.2)>, <Segment(7.2, 7.4)>, <Segment(7.4, 7.6)>, <Segment(7.6, 7.8)>, <Segment(7.8, 8)>, <Segment(8, 8.2)>, <Segment(8.2, 8.4)>, <Segment(8.4, 8.6)>, <Segment(8.6, 8.8)>, <Segment(8.8, 9)>, <Segment(9, 9.2)>, <Segment(9.2, 9.4)>, <Segment(9.4, 9.6)>, <Segment(9.6, 9.8)>, <Segment(9.8, 10)>, <Segment(10, 10.2)>, <Segment(10.2, 10.4)>, <Segment(10.4, 10.6)>, <Segment(10.6, 10.8)>, <Segment(10.8, 11)>, <Segment(11, 11.2)>, <Segment(11.2, 11.4)>, <Segment(11.4, 11.6)>, <Segment(11.6, 11.8)>, <Segment(11.8, 12)>, <Segment(12, 12.2)>, <Segment(12.2, 12.4)>, <Segment(12.4, 12.6)>, <Segment(12.6, 12.8)>, <Segment(12.8, 13)>, <Segment(13, 13.2)>, <Segment(13.2, 13.4)>, <Segment(13.4, 13.6)>, <Segment(13.6, 13.8)>, <Segment(13.8, 14)>, <Segment(14, 14.2)>, <Segment(14.2, 14.4)>, <Segment(14.4, 14.6)>, <Segment(14.6, 14.8)>, <Segment(14.8, 15)>])>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oversegmentation = Timeline(SlidingWindow(start=0., end=15, duration=0.2, step=0.2))\n",
    "oversegmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "purity(reference, oversegmentation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Coverage is the dual metric where the role of reference and hypothesis are interchanged."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5454545454545454"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coverage(reference, hypothesis)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similarly, perfect coverage (1) can be achieved by undersegmenting:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHEAAABnCAYAAACHKN7UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACJ9JREFUeJzt3WuoZXUZBvDn1SlK7T5WM450JMoSqRQTS4guFnahCfpS\nVBj1JeiOEF0g+hRCUQlFEVYGiRFqFEE1YlFfungpL2mm2W2cMY3oYlFWvn3YW5y0cc457nPW/Pf8\nfnDYa+2zzvAMvJy9zrPX+u/q7gAAAABwcDts6gAAAAAAHJgSBwAAAGAAShwAAACAAShxAAAAAAag\nxAEAAAAYgBIHAAAAYABKHAAAAIABKHEAAAAABqDEAQAAABjAlrUcvHXr1l5ZWdmgKAAAAACHniuv\nvPIP3X30gY5bU4mzsrKSK664Yv2pAAAAAPgfVfWb1RzndioAAACAAShxAAAAAAagxAEAAAAYgBIH\nAAAAYABKHAAAAIABKHEAAAAABqDEAQAAABiAEgcAAABgAEocAAAAgAEocQAAAAAGoMQBAAAAGIAS\nBwAAAGAAShwAAACAAShxAAAAAAagxAEAAAAYgBIHAAAAYABKHAAAAIABKHEAAAAABqDEAQAAABiA\nEgcAAABgAEocAAAAgAEocQAAAAAGoMQBAAAAGMCaSpw9e/ZsVA4AAAAAHsCaSpy9e/duVA4AAAAA\nHoDbqQAAAAAGsGWtP1BVG5EDAAAAgAew5hKnuzciBwAAAMAhabUXzKzpdqpt27atKwwAAAAAD86a\nSpzt27dvVA4AAAAAHoCFjQEAAAAGoMQBAAAAGIASBwAAAGAAShwAAACAAShxAAAAAAagxAEAAAAY\ngBIHAAAAYABKHAAAAIABKHEAAAAABqDEAQAAABiAEgcAAABgAEocAAAAgAEocQAAAAAGoMQBAAAA\nGIASBwAAAGAAShwAAACAAShxAAAAAAagxAEAAAAYgBIHAAAAYABKHAAAAIABKHEAAAAABqDEAQAA\nABiAEgcAAABgANXdqz+46q9Jbty4OByCtib5w9QhWBrmiUUzUyyamWLRzBSLZqZYJPO0ek/q7qMP\ndNCWNf6jN3b3KesMBPdTVVeYKRbFPLFoZopFM1Msmpli0cwUi2SeFs/tVAAAAAADUOIAAAAADGCt\nJc5nNyQFhzIzxSKZJxbNTLFoZopFM1MsmplikczTgq1pYWMAAAAApuF2KgAAAIABrKrEqaozq+rG\nqrq5qt670aFYblV1bFV9t6qur6qfVdU7p87Ecqiqw6vqJ1X1jamzML6qenRVXVRVP6+qG6rqOVNn\nYlxV9e75a951VXVhVT1s6kyMp6o+X1W3V9V1+zz32Kq6tKpumj8+ZsqMjGM/8/SR+eveNVX11ap6\n9JQZGcv/m6l9vnd2VXVVbZ0i2zI5YIlTVYcn+VSSlyY5Iclrq+qEjQ7GUvt3krO7+4QkpyV5q5li\nQd6Z5IapQ7A0zk3yre5+WpJnxmyxTlV1TJJ3JDmlu09McniS10ybikGdn+TM+zz33iSXdfdTklw2\n34fVOD/3n6dLk5zY3c9I8osk79vsUAzt/Nx/plJVxyZ5SZLfbnagZbSaK3FOTXJzd9/S3Xcl+XKS\nnRsbi2XW3Xu7+6r59l8z+8PomGlTMbqq2pHk5UnOmzoL46uqRyV5XpLPJUl339Xdf5o2FYPbkuTh\nVbUlyRFJ9kychwF19/eT/PE+T+9M8sX59heTvGpTQzGs/zdP3b2ru/893/1hkh2bHoxh7ed3VJJ8\nPMl7kliQdwFWU+Ick+R3++zvjj+4WZCqWklyUpIfTZuEJfCJzF4c7p46CEvhuCR3JPnC/Ba986rq\nyKlDMabuvjXJRzN7B3Jvkj93965pU7FEntDde+fbtyV5wpRhWCpvSvLNqUMwtqrameTW7r566izL\nwsLGTKaqjkpycZJ3dfdfps7DuKrqFUlu7+4rp87C0tiS5OQkn+7uk5L8LW5RYJ3ma5TszKwc3J7k\nyKp6/bSpWEY9+9hZ73TzoFXVBzJbAuGCqbMwrqo6Isn7k3xw6izLZDUlzq1Jjt1nf8f8OVi3qnpI\nZgXOBd19ydR5GN7pSV5ZVb/O7JbPF1bVl6aNxOB2J9nd3fdcJXhRZqUOrMcZSX7V3Xd097+SXJLk\nuRNnYnn8vqq2Jcn88faJ8zC4qnpjklcked28GIT1enJmb2BcPT9P35Hkqqp64qSpBreaEufyJE+p\nquOq6qGZLcT39Y2NxTKrqspsnYkbuvtjU+dhfN39vu7e0d0rmf2O+k53e5ebdevu25L8rqqOnz/1\noiTXTxiJsf02yWlVdcT8NfBFsVA2i/P1JGfNt89K8rUJszC4qjozs9vTX9ndf586D2Pr7mu7+/Hd\nvTI/T9+d5OT5eRbrdMASZ76w1duSfDuzE46vdPfPNjoYS+30JG/I7GqJn86/XjZ1KID7eHuSC6rq\nmiTPSvLhifMwqPkVXRcluSrJtZmdf3120lAMqaouTPKDJMdX1e6qenOSc5K8uKpuyuyqr3OmzMg4\n9jNPn0zyiCSXzs/RPzNpSIayn5liwcoVcgAAAAAHPwsbAwAAAAxAiQMAAAAwACUOAAAAwACUOAAA\nAAADUOIAAAAADGDL1AEAAA6kqh6X5LL57hOT/CfJHfP9v3f3cycJBgCwiXzEOAAwlKr6UJI7u/uj\nU2cBANhMbqcCAIZWVXfOH59fVd+rqq9V1S1VdU5Vva6qflxV11bVk+fHHV1VF1fV5fOv06f9HwAA\nrI4SBwBYJs9M8pYkT0/yhiRP7e5Tk5yX5O3zY85N8vHufnaSV8+/BwBw0LMmDgCwTC7v7r1JUlW/\nTLJr/vy1SV4w3z4jyQlVdc/PPLKqjuruOzc1KQDAGilxAIBl8s99tu/eZ//u3Hvec1iS07r7H5sZ\nDADgwXI7FQBwqNmVe2+tSlU9a8IsAACrpsQBAA4170hySlVdU1XXZ7aGDgDAQc9HjAMAAAAMwJU4\nAAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAAwAD+\nC+P9vkUECKzyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Timeline(uri=None, segments=[<Segment(0, 15)>])>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "undersegmentation = Timeline([Segment(0, 15)])\n",
    "undersegmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coverage(reference, undersegmentation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's discuss the implementation details of these two metrics."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, reference is preprocessed to remove short gap between speech from the same speaker.\n",
    "The maximum duration of gaps is the parameter `tolerance` passed to `SegmentationPurity` and `SegmentationCoverage`.\n",
    "The defaults value is 0.5 (i.e. 500ms)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preprocessed_reference, preprocessed_hypothesis = coverage._preprocess(reference, hypothesis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHEAAACsCAYAAAD8BzCjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADVBJREFUeJzt3X+snXV9B/D3R8qiyDa3wdS1zWqMY2vIANOZTpLFqZto\nSWuyf8aEuMw/WOIPXFwMbsmEfxaWmanJzIZhCgkEY0SFaBwQR+Y/orQI5ddQxpy0w1VDnDIzEfns\nj3MMHch6W57bp9/T1ys5uc9z7nNv38/tN/ec+z7f53uquwMAAADAse05cwcAAAAA4NCUOAAAAAAD\nUOIAAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAA\nAxiuxKmqc6rq/qp6oKounjvPVKrqI1V1oKrunjvLlKpqc1XdUlX3VtU9VXXR3JmmUFXPraovV9Wd\ny/O6dO5MU6qqE6rqK1X1mbmzAAAAsDBUiVNVJyT5UJLXJ9ma5Lyq2jpvqslcmeScuUOsg8eTvKu7\ntybZnuStK/J/9oMkr+7uM5KcmeScqto+c6YpXZTkvrlDAAAA8KShSpwkr0jyQHc/2N2PJflYkl0z\nZ5pEd38hySNz55hadz/c3bcvt7+XRTGwcd5Uz14vPLrcPXF56xkjTaaqNiXZkeSKubMAAADwpNFK\nnI1JHjpof19WoBA4XlTVliRnJfnSvEmmsbzk6I4kB5Lc3N0rcV5JPpDk3UmemDsIAAAAT9rwbL54\n/8bNk8482Lj/oZry+x2p7e+9cdLzuvXS1x0T57Xz0zsmPa8b3vjZNZ9XVZ2c5Lok7+zu706ZI0ku\n33X1pOd24fXnH/LcuvtHSc6sqhck+VRVnd7d065pdElNO7vnkv5/z6uqzk1yoLv3VNWrJv23AQAA\neFZGm4mzP8nmg/Y3Le/jGFZVJ2ZR4FzT3Z+cO8/Uuvs7SW7JaqxpdHaSnVX19SwuV3x1VV09byQA\nAACSpLrHWcajqjYk+WqS12RR3tyW5A+6+55Zg01kebnRZ7r79JmjTKaqKslVSR7p7nfOnWcqVXVq\nkh9293eq6nlJbkryV929Mu/mtJyJ86fdfe7cWQAAABhsJk53P57kbUluzGKB3I+vUIFzbZIvJjmt\nqvZV1VvmzjSRs5NckMWMjjuWtzfMHWoCL05yS1XtzaJMvHmVChwAAACOPUPNxAEAAAA4Xg01EwcA\nAADgeKXEAQAAABiAEgcAAABgAEocAAAAgAEocQAAAAAGsOFwDj7llFN6y5Yt6xQFAAAA4PizZ8+e\nb3f3qYc67rBKnC1btmT37t1HngoAAACA/6Oq/n0tx7mcCgAAAGAAShwAAACAAShxAAAAAAagxAEA\nAAAYgBIHAAAAYABKHAAAAIABKHEAAAAABqDEAQAAABiAEgcAAABgAEocAAAAgAEocQAAAAAGoMQB\nAAAAGIASBwAAAGAAShwAAACAAShxAAAAAAagxAEAAAAYgBIHAAAAYABKHAAAAIABKHEAAAAABqDE\nAQAAABiAEgcAAABgAEocAAAAgAEocQAAAAAGoMQBAAAAGIASBwAAAGAAShwAAACAAShxAAAAAAag\nxAEAAAAYgBIHAAAAYABKHAAAAIABKHEAAAAABqDEAQAAABiAEgcAAABgAEocAAAAgAEocQAAAAAG\noMQBAAAAGIASBwAAAGAAShwAAACAAShxAAAAAAagxAEAAAAYgBIHAAAAYABKHAAAAIABKHEAAAAA\nBqDEAQAAABiAEgcAAABgAEocAAAAgAEocQAAAAAGcFglzg/37l2vHABwTNm/cXP2b9w8dww47mx/\n743Z/t4b544xBD+rtdn56R3Z+ekdc8cYwuW7rs7lu66eOwar4pJa3Di0w/g5mYkDAAAAMAAlDgAA\nAMAAlDgAAAAAA9hwuF9gfQAAANabtV6YmnVxDm1HzksS6+IwiQvPWm5YF2dSZuIAAAAADOCwZ+Js\n3P/QeuQAgGOKmacwr1svfd3cEY55Zisdnhve+Nm5IxzzLv/oYgbOhdefP3MSVsIlFyw/9rw5RuDd\nqQAAAABWixIHAAAAYABKHAAAAIABKHEAAAAABlDda19kaNu2bb179+51jAMAAABwfKmqPd297VDH\nmYkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIAAAAA\nDECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAA\nAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAA\nAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIA\nAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1Di\nAAAAAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ\n4gAAAAAMoLp77QdXfS/J/esXh+PQKUm+PXcIVobxxNSMKaZmTDE1Y4qpGVNMyXhau1/u7lMPddCG\nw/ym93f3tiMMBE9TVbuNKaZiPDE1Y4qpGVNMzZhiasYUUzKepudyKgAAAIABKHEAAAAABnC4Jc6H\n1yUFxzNjiikZT0zNmGJqxhRTM6aYmjHFlIyniR3WwsYAAAAAzMPlVAAAAAADWFOJU1XnVNX9VfVA\nVV283qFYbVW1uapuqap7q+qeqrpo7kyshqo6oaq+UlWfmTsL46uqF1TVJ6rqX6rqvqr6zbkzMa6q\n+pPlY97dVXVtVT137kyMp6o+UlUHqurug+77+aq6uaq+tvz4c3NmZBzPMJ7+evm4t7eqPlVVL5gz\nI2P5SWPqoM+9q6q6qk6ZI9sqOWSJU1UnJPlQktcn2ZrkvKraut7BWGmPJ3lXd29Nsj3JW40pJnJR\nkvvmDsHK+GCSf+zuX01yRowtjlBVbUzyjiTbuvv0JCck+f15UzGoK5Oc85T7Lk7y+e5+WZLPL/dh\nLa7M08fTzUlO7+5fT/LVJO852qEY2pV5+phKVW1O8rtJvnG0A62itczEeUWSB7r7we5+LMnHkuxa\n31issu5+uLtvX25/L4s/jDbOm4rRVdWmJDuSXDF3FsZXVT+b5LeS/EOSdPdj3f2deVMxuA1JnldV\nG5KclOQ/Zs7DgLr7C0keecrdu5Jctdy+Kskbj2oohvWTxlN339Tdjy93b02y6agHY1jP8DsqSd6f\n5N1JLMg7gbWUOBuTPHTQ/r74g5uJVNWWJGcl+dK8SVgBH8jiweGJuYOwEl6S5FtJPrq8RO+Kqnr+\n3KEYU3fvT/K+LF6BfDjJf3X3TfOmYoW8sLsfXm5/M8kL5wzDSvmjJJ+bOwRjq6pdSfZ3951zZ1kV\nFjZmNlV1cpLrkryzu787dx7GVVXnJjnQ3XvmzsLK2JDk5Un+rrvPSvLfcYkCR2i5RsmuLMrBX0ry\n/Ko6f95UrKJevO2sV7p51qrqz7NYAuGaubMwrqo6KcmfJfmLubOskrWUOPuTbD5of9PyPjhiVXVi\nFgXONd39ybnzMLyzk+ysqq9nccnnq6vq6nkjMbh9SfZ1949nCX4ii1IHjsRrk/xbd3+ru3+Y5JNJ\nXjlzJlbHf1bVi5Nk+fHAzHkYXFX9YZJzk7xpWQzCkXppFi9g3Ll8nr4pye1V9aJZUw1uLSXObUle\nVlUvqaqfymIhvhvWNxarrKoqi3Um7uvuv5k7D+Pr7vd096bu3pLF76h/6m6vcnPEuvubSR6qqtOW\nd70myb0zRmJs30iyvapOWj4GviYWymY6NyR583L7zUmunzELg6uqc7K4PH1nd39/7jyMrbvv6u5f\n7O4ty+fp+5K8fPk8iyN0yBJnubDV25LcmMUTjo939z3rHYyVdnaSC7KYLXHH8vaGuUMBPMXbk1xT\nVXuTnJnkL2fOw6CWM7o+keT2JHdl8fzrw7OGYkhVdW2SLyY5rar2VdVbklyW5Heq6mtZzPq6bM6M\njOMZxtPfJvnpJDcvn6P//awhGcozjCkmVmbIAQAAABz7LGwMAAAAMAAlDgAAAMAAlDgAAAAAA1Di\nAAAAAAxAiQMAAAAwgA1zBwAAOJSq+oUkn1/uvijJj5J8a7n//e5+5SzBAACOIm8xDgAMpaouSfJo\nd79v7iwAAEeTy6kAgKFV1aPLj6+qqn+uquur6sGquqyq3lRVX66qu6rqpcvjTq2q66rqtuXt7HnP\nAABgbZQ4AMAqOSPJHyf5tSQXJPmV7n5FkiuSvH15zAeTvL+7fyPJ7y0/BwBwzLMmDgCwSm7r7oeT\npKr+NclNy/vvSvLby+3XJtlaVT/+mp+pqpO7+9GjmhQA4DApcQCAVfKDg7afOGj/iTz5vOc5SbZ3\n9/8czWAAAM+Wy6kAgOPNTXny0qpU1ZkzZgEAWDMlDgBwvHlHkm1Vtbeq7s1iDR0AgGOetxgHAAAA\nGICZOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAA\nAMAA/hfBo6MiebyYpgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<pyannote.core.annotation.Annotation at 0x112d015c0>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notebook.reset()\n",
    "preprocessed_reference.rename_labels(generator='int')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note how the small gap between the first two speech turns have been removed.\n",
    "Note also how the overlapping segments at the end have been resegmented: the point is to not miss any boundaries."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then, hypothesis is also preprocessed to only consider speech regions, as this is speaker change detection that we are evaluating: speech to non-speech changes are not considered. It would probably be different if we were to evaluate the actual change detection with precision and recall metrics (see final note)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHEAAAC7CAYAAADi1AKAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADrFJREFUeJzt3XusZWdZBvDnpYPhUgGxF3BaGSRcWisUGEgBA1JQ0JKZ\nEkxKBYJRAybcNEQFjLT8o00kIEYCJUVabCkYLm0VpSVoQLQVZiq0lFouRWnHQsErFRRoX//Yu2Gc\n0sw503VmzbfP75fs7L3W7Dl51smXOXue833fqu4OAAAAAIe2u80dAAAAAID9U+IAAAAADECJAwAA\nADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAxgyBKn\nqk6tqq6qR8ydZUpV9YCqendVfbGqdlfVX1TVw+bONYWqurWqPrXX41VzZ5rK97m2bXNnmkJVHV1V\n76qq65fj8fKqevbcuQAAADarLXMHOECnJ/n48vmMmbNMoqoqyQeSnNfdz12ee1SSo5N8bs5sE/lW\nd584d4gNsnLXthyPF2UxHn9hee5BSXbMGgwAAGATq+6eO8O6VNXhSa5L8tQkf9bdD5850iSq6uQk\nZ3b3k+fOshGq6pbuPnzuHBthFa+tqp6W5LXd/ZS5swAAALAw4nKqnUk+1N2fS/KvVfXYuQNN5IQk\nu+cOsYHuuc+So9PmDjShva/tA3OHmciPJ7ly7hAAAAB8z11aTrVn67GTTuPZuueGWsPbTk/ypuXr\ndy+PJy8/dlx0yqTXdsmpH1zLtR0cZ9a006/O7LVc20FacrR94qllu2a/tgtPO27Sazr9PdeueyxW\n1ZuT/GSSb3f346bMAwAAwNoMtSdOVd0/yclJfqKqOslhSbqqfqNHWxd2R9ck+fm5Q8DSNUmec/tB\nd7+kqo5Ismu+SAAAAJvbUHviVNWLkjy2u1+817mPJvmd7v7YfMnuuuVGslckeXt3v2157pFJ7tvd\nfzNruAms4r4xt1vFa9trPJ7b3W9ZnvvRJB/r7m1zZgMAANisRtsT5/Qs7uC0t/ctzw9tOZPo2Ume\nvrzF+DVJfi/JV+ZNNpl998Q5a+5A3LnleDw1yVOq6ktV9Ykk5yX5rXmTAQAAbF5DzcQBAAAA2KxG\nm4kDAAAAsCkpcQAAAAAGoMQBAAAAGIASBwAAAGAAShwAAACAAShxAAAAAAawZe4A61VVtya5Okkl\nuTXJS7v77+ZNNY29ru3uSb6b5J1J3tjdt80aDAAAAJhddffcGdalqm7p7sOXr5+R5DXd/ZSZY01i\nn2s7Ksm7kvxtd58xbzIAAABgbqMvp7pPkn+fO8RG6O6bk7woyUurqubOAwAAAMxruOVUSe5ZVZ9K\nco8kD0xy8sx5Nkx3X19VhyU5KslX584DAAAAzOculTgnnXHppGuxrnjdM9Yy4+Rb3X1iklTVE5K8\ns6pO6InXhZ298/xJv96LL36+2TQAAADAARt6OVV3X57kiCRHzp1lI1TVj2WxefPNc2cBAAAA5jX6\nxsaPSPLxJEd3963zJrvr9rm2I5NckORyGxsDAAAAI++JkyxuM/7CVShwlm6/tttvMf4nSd4wbyQA\nAADgUDDcTBwAAACAzWjoPXEAAAAANgslDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwgHXdYvyI\nI47obdu2bVAUAAAAgM1n9+7dX+/uI/f3vnWVONu2bcuuXbsOPBUAAAAA/09V/fNa3mc5FQAAAMAA\nlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADA\nAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAA\nwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAA\nAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4A\nAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUO\nAAAAwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMYF0lzneuumqj\ncrAJnXTGpTnpjEvnjsGK2HHRKdlx0SlzxxjC2TvPz9k7z587xiHP9wlmcmYtHuyf79UabV8+YBoX\nnnZcLjztuLljsELWM57MxAEAAAAYgBIHAAAAYABKHAAAAIABbFnvX9iz9diNyMFm9CvnJIl9cZjE\nUY9ePNsXZ/9OyelJYr8X4NBmrxcmZ18cpmVfHOZgJg4AAADAANY9E2frnhs2Igeb0XIGzhWve8bM\nQVgFOy76wyTJJad+cOYkh76z37GYgfPii58/c5JDm5lKMLMze+4Ehz6zldZp19wBWBmLGTinv+fa\nmXOwKtydCgAAAGDFKHEAAAAABqDEAQAAABiAEgcAAABgANW99k3jtm/f3rt22RAMAAAAYCpVtbu7\nt+/vfWbiAAAAAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADAAJQ4\nAAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAAwACU\nOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAA\nlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAAwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADA\nAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAA\nwACUOAAAAAADUOIAAAAADECJAwAAADAAJQ4AAADAAJQ4AAAAAANQ4gAAAAAMQIkDAAAAMAAlDgAA\nAMAAlDgAAAAAA6juXvubq76R5LqNi8MmdESSr88dgpVhPDE1Y4qpGVNMzZhiasYUUzKe1u5B3X3k\n/t60ZZ1f9Lru3n6AgeAOqmqXMcVUjCemZkwxNWOKqRlTTM2YYkrG0/QspwIAAAAYgBIHAAAAYADr\nLXHetiEp2MyMKaZkPDE1Y4qpGVNMzZhiasYUUzKeJraujY0BAAAAmIflVAAAAAADWFOJU1XPrKrr\nquoLVfWqjQ7FaquqY6vqr6vqs1V1TVW9Yu5MrIaqOqyq/qGq/nzuLIyvqu5XVe+tqn+sqmur6glz\nZ2JcVfXry595n6mqC6vqHnNnYjxV9cdVdXNVfWavc/evqg9X1eeXzz80Z0bGcSfj6feXP/euqqoP\nVNX95szIWL7fmNrrz15ZVV1VR8yRbZXst8SpqsOSvDnJzyY5PsnpVXX8RgdjpX03ySu7+/gkJyV5\niTHFRF6R5Nq5Q7Ay3pTkQ939iCSPirHFAaqqrUlenmR7d5+Q5LAkz503FYM6N8kz9zn3qiQf6e6H\nJvnI8hjW4tzccTx9OMkJ3f3IJJ9L8uqDHYqhnZs7jqlU1bFJfibJlw92oFW0lpk4j0/yhe6+vru/\nneTdSXZubCxWWXff1N1XLl9/I4v/GG2dNxWjq6pjkpyS5Jy5szC+qrpvkicneXuSdPe3u/s/5k3F\n4LYkuWdVbUlyryT/MnMeBtTdH0vyb/uc3pnkvOXr85KcelBDMazvN566+7Lu/u7y8Iokxxz0YAzr\nTv6NSpI3JvnNJDbkncBaSpytSW7Y6/jG+A83E6mqbUkeneTv503CCviDLH443DZ3EFbCg5N8Lck7\nlkv0zqmqe88dijF1954kr8/iN5A3JfnP7r5s3lSskKO7+6bl668kOXrOMKyUX0ryl3OHYGxVtTPJ\nnu7+9NxZVoWNjZlNVR2e5H1Jfq27/2vuPIyrqp6V5Obu3j13FlbGliSPSfKW7n50kv+OJQocoOUe\nJTuzKAd/JMm9q+r586ZiFfXitrN+081dVlW/ncUWCBfMnYVxVdW9krwmyWvnzrJK1lLi7Ely7F7H\nxyzPwQGrqrtnUeBc0N3vnzsPw3tSkh1V9U9ZLPk8uarOnzcSg7sxyY3dffsswfdmUerAgXh6ki91\n99e6+ztJ3p/kiTNnYnV8taoemCTL55tnzsPgquoXkzwryfOWxSAcqIdk8QuMTy8/px+T5MqqesCs\nqQa3lhLnk0keWlUPrqofyGIjvks2NharrKoqi30mru3uN8ydh/F196u7+5ju3pbFv1F/1d1+y80B\n6+6vJLmhqh6+PPW0JJ+dMRJj+3KSk6rqXsufgU+LjbKZziVJXrh8/cIkF8+YhcFV1TOzWJ6+o7u/\nOXcextbdV3f3Ud29bfk5/cYkj1l+zuIA7bfEWW5s9dIkl2bxgeNPu/uajQ7GSntSkhdkMVviU8vH\nz80dCmAfL0tyQVVdleTEJL87cx4GtZzR9d4kVya5OovPX2+bNRRDqqoLk1ye5OFVdWNV/XKSs5L8\ndFV9PotZX2fNmZFx3Ml4+qMkP5jkw8vP6G+dNSRDuZMxxcTKDDkAAACAQ5+NjQEAAAAGoMQBAAAA\nGIASBwAAAGAAShwAAACAAShxAAAAAAawZe4AAAD7U1U/nOQjy8MHJLk1ydeWx9/s7ifOEgwA4CBy\ni3EAYChVdWaSW7r79XNnAQA4mCynAgCGVlW3LJ9/qqo+WlUXV9X1VXVWVT2vqj5RVVdX1UOW7zuy\nqt5XVZ9cPp407xUAAKyNEgcAWCWPSvKrSY5L8oIkD+vuxyc5J8nLlu95U5I3dvfjkjxn+WcAAIc8\ne+IAAKvkk919U5JU1ReTXLY8f3WSpy5fPz3J8VV1+9+5T1Ud3t23HNSkAADrpMQBAFbJ/+71+ra9\njm/L9z733C3JSd39PwczGADAXWU5FQCw2VyW7y2tSlWdOGMWAIA1U+IAAJvNy5Nsr6qrquqzWeyh\nAwBwyHOLcQAAAIABmIkDAAAAMAAlDgAAAMAAlDgAAAAAA1DiAAAAAAxAiQMAAAAwACUOAAAAwACU\nOAAAAAADUOIAAAAADOD/AOtaDVd394TJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<pyannote.core.annotation.Annotation at 0x11309f6d8>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notebook.reset()\n",
    "preprocessed_hypothesis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Purity and coverage are then computed based on their coocurrence duration matrix:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>j</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>G</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "j    0    1    2    3    4\n",
       "i                         \n",
       "A  2.0  0.0  0.0  0.0  0.0\n",
       "B  2.0  0.0  0.0  0.0  0.0\n",
       "C  1.0  0.0  0.0  0.0  0.0\n",
       "D  1.0  0.0  0.0  0.0  0.0\n",
       "E  0.0  1.0  0.0  0.0  0.0\n",
       "F  0.0  0.0  1.0  1.0  0.0\n",
       "G  0.0  0.0  0.0  1.0  1.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K = (preprocessed_hypothesis * preprocessed_reference.rename_labels(generator='int')).to_pandas()\n",
    "K"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For each segment in hypothesis, we get the duration of the most coocurring reference segment..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "i\n",
       "A    2.0\n",
       "B    2.0\n",
       "C    1.0\n",
       "D    1.0\n",
       "E    1.0\n",
       "F    1.0\n",
       "G    1.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.max(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.max(axis=1).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "... and divide by the overal hypothesis duration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'intersection duration': 9.0, 'total duration': 11.0}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "components = purity._process(preprocessed_hypothesis, preprocessed_reference)\n",
    "components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8181818181818182"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "components['intersection duration'] / components['total duration']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Coverage is the dual metric where the roles of reference and hypothesis are interchanged:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "j\n",
       "0    2.0\n",
       "1    1.0\n",
       "2    1.0\n",
       "3    1.0\n",
       "4    1.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.max(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.0"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.max(axis=0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'intersection duration': 6.0, 'total duration': 11.0}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "components = coverage._process(preprocessed_reference, preprocessed_hypothesis)\n",
    "components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5454545454545454"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "components['intersection duration'] / components['total duration']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final note: why not use precision & recall?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`pyannote.metrics` does implement `SegmentationPrecision` and `SegmentationRecall` metrics.\n",
    "Three reasons explains why we'd rather use purity and coverage:\n",
    " 1. precision and recall are very sensitive to the `tolerance` parameter.\n",
    " 2. purity and coverage can also be used for evaluating subsequent steps of the speaker diarization pipeline (which is very convenient to evaluate the impact of each step)\n",
    " 3. it is not clear what to do with speech/non-speech boundaries with precision and recall. purity and coverage are not impacted with this issue.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Impact of `tolerance` parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XuUXGWZ7/HvL50bISEhdAghF5r7TUAgRsBbVFBAIDji\nMTiCsNRwdBjUWTozOgx6XOMM6jjeZ5BBDgEV8ARlksgdBMUYSAdJyAUwYO63JoFcSEjS3c/5Y7/R\nSttJVzfVtXd1/z5r1UrV3m9VPbUrTz97v/utdysiMDMzK5o+eQdgZmbWHhcoMzMrJBcoMzMrJBco\nMzMrJBcoMzMrJBcoMzMrJBeoHkbSOElbJdXlHYuZ2evhAlUQkpZK2p6KyzpJt0ga3NnXiYjlETE4\nIlrS6z4q6eOVj9isuiR9WFJjypE1ku6V9Na847Lu4wJVLBdGxGDgNGA8cG1nniypb7dEZZYzSX8H\nfBv4V2AkMA74T2BSld5fkvz3ssq8wQsoIlYB9wJvSEdWZ+9eJ+nLkn6c7jdICkkfk7QceKRkWV9J\nXwXeBnw/7XV+X9IPJH2z9P0kTZf02Sp+RLOySRoKfAX4m4j4eUS8GhG7ImJGRHxe0gBJ35a0Ot2+\nLWlAeu5iSReUvFZfSU2STkuPz5A0S9IrkuZJmljS9lFJX5X0W2AbcISkK9NrbpH0oqSr2sT69+no\nbrWkj6dcPCqtGyDp3yUtT70kN0jar9s3YA1zgSogSWOB84Hfl/mUdwDHA+8tXRgR/wT8Brg6dftd\nDUwFLt29NyipHjgb+GmFwjertDOBgcAv9rL+n4AzgDcCpwAT+HPvw+3ApSVt3wu8FBFPSRoN/BL4\nF2A48DngLkkjStpfBkwBhgDLgPXABcABwJXAt0qK3bnA35Hl01HAxDZxXg8ck+I8ChgNXFfmNuiV\nXKCK5W5JrwCPA4+RdWeU48tpr3J7Rw0j4klgE/DutGgy8GhErOtKwGZVcBBZUWney/q/Br4SEesj\nogn4P2SFBbIdr4skDUqPP0xWtAA+AtwTEfdERGtEPAg0ku0c7nZLRCyMiOZ01PbLiHghMo8BD5D1\nUgD8L+D/pvbbgC/vfhFJIit0n42IjRGxhSy/J3d1o/QGPmdRLBdHxEOlC7L/1x1a0cn3mUqWnA+m\nf7/TyeebVdMGoF5S370UqUPJjm52W5aWERFLJC0GLpQ0A7gIODW1Owz4oKQLS57bD/hVyeM9ckvS\necCXyI6E+gCDgGdK4mjcy3NHpLZzS3JagEfb7oMLVPG9SvYfe7dD2mmzrynp21v3Y2CBpFPIugbv\n7np4Zt3ud8AO4GJgWjvrV5MVm4Xp8bi0bLfd3Xx9gEURsSQtXwHcFhGf2Md7/yl/0nmtu4DLgf+J\niF2S7iYrNABrgDElzx1bcv8lYDtwYjrHbGVwF1/xPQ1MltRP0njgkk4+fx1wROmCiFgJzAFuA+4q\np2vQLC8RsYnsXM0PJF0saVDKh/MkfZ2sAF0raUQ6p3od2U7YbncA7wE+yZ7nWn9MdmT1Xkl1kgZK\nmiiptMiU6g8MAJqA5nQ09Z6S9T8DrpR0fOpS/OeSz9AK/DfZOauDASSNlrTHeWPbkwtU8f0zcCTw\nMlnfemcHM3wHuETSy5K+W7J8KnASWZEyK7SI+CbZAIRryQrECuBqsqP/fyHrWptP1t32VFq2+7lr\nyI7CzgLuLFm+gmyY+hdLXvPz7OXvYjpvdA1ZIXqZ7HzW9JL19wLfJesiXALMTqt2pH//YfdySZuB\nh4BjO781eg/5goW9k6S3k+1BHhb+T2BWcZKOBxYAA/YxwMP2wUdQvZCkfsCngZtcnMwqR9L70++d\nDgS+Bsxwceo6F6heJu3VvQKMIvtlvplVzlVkv5V6AWghO+9lXeQuPjMzKyQfQZmZWSHl9juo+vr6\naGhoyOvtzTo0d+7clyJiRMct8+VcsqLrai7lVqAaGhpobGzsuKFZTiQt67hV/pxLVnRdzSV38ZmZ\nWSG5QJmZWSG5QJmZWSF1WKAk3SxpvaQFe1kvSd+VtETS/N3XRjGzPTmXzDqnnCOoW4Bz97H+PODo\ndJsC/NfrD8usR7oF55JZ2TosUBHxa2DjPppMAm5NF/CaDQyTNKpSAZr1FM4ls86pxDmo0ex5Ya6V\nadlfkDRFUqOkxqampgq8tVmP4lwyK1HVQRIRcWNEjI+I8SNGFP73j2aF5Vyy3qASBWoVe145ckxa\nZmad41wyK1GJAjUduDyNQDoD2JQuEGZmneNcMivR4VRHkm4HJgL1klYCXwL6AUTEDcA9wPlkV4rc\nBlzZXcGa1TLnklnndFigIuLSDtYH8DcVi8ish3IumXWOZ5IwM7NCcoEyM7NCcoEyM7NCcoEyM7NC\ncoEyM7NCcoEyM7NCyu2S7xte3cktv/0jLQEtra20tEJrBM0tQUsEra1Bc2vQGkFLa8ktgpY2bczM\nrOfJrUCtfmU7X56xaK/r6/qIOok+faBvnz70UVrWpw91faBOoq4uayOpipGbmVk15Fagjh91AI/8\n8zmp6GSFZvf9PsJFx3Knz+cdgVnvlluB6ttHDN+/f15vb2ZmBedBEmZmVkguUGZmVkguUGZmVkgu\nUGZmVkguUGZmVkguUGZmVkguUGZmVkguUGZmVkguUGZmVkguUGY17tm1W/jew39g3ebX8g7FrKJc\noMxq3IC+ffjmg89z1vWPMOXWRh59bj2tnuXfeoDc5uIzs8o4vH5/pn1uIrfPWc60xpU8sGgdYw7c\nj0snjOODp4/h4AMG5h2iWZf4CMqsB2io358vnHc8s77wLr536amMPXAQ37j/Oc66/hH+921z+fXz\nTT6qsprjIyizHmRA3zouPOVQLjzlUF5s2sodc1Ywbe5K7lu4lnHDBzF5wlg+ePpYRgwZkHeoZh3y\nEZRZD3XEiMF88fzj+d0X3sV3Jr+RUUMH8vX7nuPMf3uYT/1kLo//4SUfVVmh+QjKrIcb0LeOSW8c\nzaQ3juaFpq3c/sRypj21knueWcthBw3i0gnjuOT0MdQP9lGVFYsLlFkvcuSIwVx7wQl87r3Hct+C\ntfz0yeVcf++zfPOB53jHMQdz4KB+eYdo9idlFShJ5wLfAeqAmyLi+jbrrwC+AaxKi74fETdVME6z\nmlekPBrYr46LTx3NxaeOZsn6Lfz0iRU8tHgdzS2t3fF2Zl3SYYGSVAf8ADgHWAnMkTQ9Iha1aXpn\nRFzdDTGa1bwi59FRBw/hugtP4LoLT6jm21ovoi927XnlDJKYACyJiBcjYidwBzCpa29n1ms5j8w6\nqZwCNRpYUfJ4ZVrW1gckzZc0TdLY9l5I0hRJjZIam5qauhCuWc2qWB6Bc8l6h0oNM58BNETEycCD\nwNT2GkXEjRExPiLGjxgxokJvbdZjlJVH4Fyy3qGcArUKKN2TG8OfT+ICEBEbImJHengTcHplwjPr\nMZxHZp1UToGaAxwt6XBJ/YHJwPTSBpJGlTy8CFhcuRDNegTnkVkndTiKLyKaJV0N3E82PPbmiFgo\n6StAY0RMB66RdBHQDGwErujGmM1qjvPIrPMUkc9UJ+PHj4/GxsZc3tusHJLmRsT4vOPoiHPJiq6r\nueS5+MzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBc\noMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzM\nrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJBcoMzMrJDK\nKlCSzpX0nKQlkv6xnfUDJN2Z1j8hqaHSgZr1BM4ls/J1WKAk1QE/AM4DTgAulXRCm2YfA16OiKOA\nbwFfq3SgZrXOuWTWOeUcQU0AlkTEixGxE7gDmNSmzSRgaro/DXi3JFUuTLMewblk1gl9y2gzGlhR\n8ngl8Oa9tYmIZkmbgIOAl0obSZoCTEkPd0ha0JWgq6yeNp+jwGol1lqJ89gKv55zqTa+d8dZeV3K\npXIKVMVExI3AjQCSGiNifDXfvytqJU6onVhrKc68Y9gb51L3cZyV19VcKqeLbxUwtuTxmLSs3TaS\n+gJDgQ1dCcisB3MumXVCOQVqDnC0pMMl9QcmA9PbtJkOfDTdvwR4JCKicmGa9QjOJbNO6LCLL/WD\nXw3cD9QBN0fEQklfARojYjrwI+A2SUuAjWSJ15EbX0fc1VQrcULtxNor43QuOc4Kq5U4oYuxyjtn\nZmZWRJ5JwszMCskFyszMCqnbC1StTO1SRpxXSGqS9HS6fTynOG+WtH5vv3tR5rvpc8yXdFq1Y0xx\ndBTnREmbSrbnddWOMcUxVtKvJC2StFDSp9tpk/s2rZU8SrEUPpdqJY9SLIXPpW7Lo4jothvZieAX\ngCOA/sA84IQ2bT4F3JDuTwbu7M6YXkecVwDfr3Zs7cT6duA0YMFe1p8P3AsIOAN4oqBxTgRmFmB7\njgJOS/eHAM+3893nuk1rJY86EWvuuVQreVRmrLnnUnflUXcfQdXK1C7lxFkIEfFrstFdezMJuDUy\ns4FhkkZVJ7o/KyPOQoiINRHxVLq/BVhMNptDqby3aa3kEdRILtVKHkFt5FJ35VF3F6j2pnZpG/Qe\nU7sAu6d2qaZy4gT4QDo0nSZpbDvri6Dcz1IEZ0qaJ+leSSfmHUzqFjsVeKLNqry3aa3k0R5xJLWa\nS3l/551VmFyqZB55kET5ZgANEXEy8CB/3lu1rnkKOCwiTgG+B9ydZzCSBgN3AZ+JiM15xtILOJcq\nqzC5VOk86u4CVStTu3QYZ0RsiIgd6eFNwOlViq2zytnmuYuIzRGxNd2/B+gnqT6PWCT1I0uqn0TE\nz9tpkvc2rZU82iOOpFZzKe/vvGxFyaXuyKPuLlC1MrVLh3G26Su9iKyPtYimA5enETNnAJsiYk3e\nQbUl6ZDd50gkTSD7v1j1P6gphh8BiyPiP/bSLO9tWit5BD0nl/L+zstWhFzqrjzq1tnMo/umdskj\nzmskXQQ0pzivqHacAJJuJxu1Uy9pJfAloB9ARNwA3EM2WmYJsA24sqBxXgJ8UlIzsB2YnNMf1LcA\nlwHPSHo6LfsiMK4k1ly3aa3kUSdizT2XaiWPoGZyqVvyyFMdmZlZIXmQhJmZFZILlJmZFZILlJmZ\nFZILlJmZFZILlJmZFZILVBVJGibpUx20adjbrMVmlnEu9Q4uUNU1jGzW6YpJswaY9TbOpV7ABaq6\nrgeOTNds+Ua6LZD0jKQPtW0sqS61mZMm1rwqLZ8o6TeSpgOL0rK7Jc1Vdi2WKSWvsVXSV9NEkrMl\njUzLR0r6RVo+T9JZaflHJD2ZYvyhpLpqbBizTnIu9QbdeY0Q3/7imikNpGu6AB8gmyizDhgJLCe7\npkppmynAten+AKAROJzsV+WvAoeXvPbw9O9+wALgoPQ4gAvT/a+XvN6dZBM6kmIYChxPNpFnv7T8\nP4HL895uvvnW9uZc6h03H9Lm563A7RHRAqyT9BjwJmB+SZv3ACdLuiQ9HgocDewEnoyIP5a0vUbS\n+9P9sandhtR2Zlo+Fzgn3X8XcDlAimGTpMvIJu6ck6b22g9YX5mPa9ZtnEs9lAtUsQn424i4f4+F\n0kSyvb7Sx2cDZ0bENkmPAgPT6l2RduGAFvb9nQuYGhFfqEj0ZsXhXKpBPgdVXVvILocM8BvgQ6lv\nfATZZZ2fbNP+frJJIPsBSDpG0v7tvO5Q4OWUUMeRXU65Iw8Dn0yvWydpaFp2iaSD0/Lhkg7r3Ec0\nqwrnUi/gAlVFEbEB+G0a+nomWRfEPOAR4O8jYm2bp9xEduL2qfScH9L+Xtt9QF9Ji8lOHs8uI5xP\nA++U9AxZd8UJEbEIuBZ4QNJ8sn79XC5zbbYvzqXewbOZm5lZIfkIyszMCskFyszMCskFyszMCskF\nyszMCskFyszMCskFyszMCskFyszMCskFyszMCskFyszMCskFyszMCskFyszMCskFyszMCskFqodI\nl6ee2EGbcemy1b70tFkFpEvGryx5vFTS2XnG1JO4QFVB+k+7PRWHdZJukTS4ku8RESdGxKMdtFke\nEYPTVT/Nepw2uba2O3LNqscFqnoujIjBwGnAeLJrxfyJMv4+zF6/3bn2RuBUwFe1rVH+g1hlEbEK\nuBd4g6RHJX1V0m+BbcARkoZK+pGkNZJWSfqX0i45SZ+QtFjSFkmLJJ2Wlv+pa0HSBEmNkjanI7b/\nSMsbJIWkvunxoZKmS9ooaYmkT5S8z5cl/UzSrem9FkoaX70tZfb6pIsW3k9WqJA0QNK/S1qe8uIG\nSfvtbi9pkqSnU968IOnctPzKkpx7UdJV+Xyi3scFqsokjQXOB36fFl0GTCG7fPUy4BagGTiKbO/v\nPcDH03M/CHwZuBw4ALgI2NDO23wH+E5EHAAcCfxsL+HcAawEDgUuAf5V0rtK1l+U2gwDpgPf7+TH\nNcuNpDHAecCStOh64BiygnUUMBq4LrWdANwKfJ7s//vbgaXpeeuBC8hy7krgW7t3DK17uUBVz92S\nXgEeBx4D/jUtvyUiFkZEMzCcrHh9JiJejYj1wLeAyantx4GvR8ScyCyJiGXtvNcu4ChJ9RGxNSL+\n4rLVqVC+BfiHiHgtIp4muyz25SXNHo+Ie9I5q9uAU17vRjCrgrslbQFWkBWXL0kS2Y7gZyNiY0Rs\nIcvB3bn1MeDmiHgwIlojYlVEPAsQEb+MiBdSzj0GPAC8reqfqhdygaqeiyNiWEQcFhGfiojtafmK\nkjaHAf2ANZJeSQXth8DBaf1Y4IUy3utjZHuKz0qaI+mCdtocCuxO1N2Wke1V7ra25P42YODu7kGz\nArs4IoYAE4HjgHpgBDAImFuSW/el5bCP3JJ0nqTZqSv8FbKdyPpu/gwG+I9N/qLk/gpgB1Cfjqja\nWkHWZbfvF4z4A3BpGnTxV8A0SQe1abYaGC5pSEmRGges6uwHMCuiiHhM0i3Av5PlwXbgxHQeuK12\nc0vSAOAusp6F/4mIXZLuBtRtgduf+AiqQCJiDVn3wTclHSCpj6QjJb0jNbkJ+Jyk09Oov6MkHdb2\ndSR9RNKIiGgFXkmLW9u81wpgFvBvkgZKOpnsyOvH3fX5zHLwbeAc4CTgv8nOHx0MIGm0pPemdj8C\nrpT07pR3oyUdB/QHBgBNQLOk88jOC1sVuEAVz+VkSbEIeBmYBowCiIj/B3wV+CmwBbib7LxVW+cC\nCyVtJRswMbmkS7HUpUAD2dHUL4AvRcRDlfwwZnmKiCaywQ/XAf9ANmBitqTNwEPAsandk6QBEMAm\nsvPEh6XehWvIBhq9DHyYbMCQVYEiouNWZmZmVeYjKDMzKyQXKDMzKyQXKDMzKyQXKDMzK6TcfgdV\nX18fDQ0Neb29WYfmzp37UkSM6LhlvpxLVnRdzaXcClRDQwONjY15vb1ZhyS1N41U4TiXrOi6mkvu\n4jMzs0JygTIzs0LqsEBJulnSekkL9rJekr6bric039PQm7XPuWTWOeUcQd1CNnXO3pwHHJ1uU4D/\nev1hmfVIt+BcMitbh4MkIuLXkhr20WQScGtkcybNljRM0qg08amZJc6lnqG1NZgxfzVbXmvvggNW\nSZUYxTeaPa9ptDIt+4ukkjSFbM+QcePGVeCtzXoU51INeOz5Jj59x9N5h9ErVHWYeUTcCNwIMH78\neM9Sa9ZFzqX8zJi3mgMG9uX+z76duj6+LFQ5Rn6ta8+rRIFaRXY1yt3G4IvemXWFc6ngXtvVwgOL\n1nH+SYcwauh+eYfT41VimPl04PI0AukMYJP7zM26xLlUcI8938TWHc1ccPKheYfSK3R4BCXpdmAi\nUC9pJfAloB9ARNwA3AOcT3YhsG1kF/0yszacS7VvxrzVDN+/P2cdeVDeofQK5Yziu7SD9QH8TcUi\nMuuhnEu1bdvOZh5evJ6/Om00fes8x0E1eCubmZXhkWfXs31Xi7v3qsgFysysDDPnrWHEkAFMOHx4\n3qH0Gi5QZmYd2PLaLh55bj3vO2mUh5ZXkQuUmVkHHlq8jp3NrVxw8qi8Q+lVXKDMzDowc94aDh06\nkNPGHZh3KL2KC5SZ2T5s2raLX/+hifedPIo+7t6rKhcoM7N9uH/hWna1hEfv5cAFysxsH2bMX824\n4YM4eczQvEPpdao6Wazl6+bH/8hjzzflHYZZzdiwdQezXtjAVW8/Asnde9XmAtVLNG3Zwb/du5iD\nhwykfsiAvMMxqwn3LVxLS6u79/LiAtVL3P7kcna1BLd+bAJHjhicdzg1QVfnHYHlbca81RwxYn+O\nHzUk71B6JZ+D6gV2tbTykyeW8baj612czMq0fvNrPPHHjVxw8qHu3suJC1QvcN+CtazbvIMrzmrI\nOxSzmnHPM2uIgAv949zcuED1AlNnLWXc8EFMPPbgvEMxqxkz5q/huEOGcPRId+/lxQWqh1uwahON\ny17m8jMP8xxiZmVa/cp25i572VMb5cwFqoebOmsp+/Wr44Pjx3bc2MwA+OX87ELGHr2XLxeoHmzj\nqzv5n3mref9poxm6X7+8wzGrGTPnr+ak0UNpqN8/71B6NReoHuyOOcvZ2dzKR89syDsUs5qxbMOr\nzFu5yd17BeAC1UM1t7Ty498t48wjDuLYQ3yS16xcM1P33vtcoHLnAtVDPbR4Has3vcYVb2nIOxSz\nmjJz/hpOHTeMMQcOyjuUXs8Fqoe6ZdZSRg/bj7OPH5l3KGY1Y8n6rSxes5kLPTiiEFygeqBn125m\n9osbucxDy806Zeb81Uju3isKF6geaOqsZQzo24cPeWi5Wdkigpnz1/CmhuGMPGBg3uEYLlA9zqZt\nu/jF71dy8RtHc+D+/fMOx6xmPLduC0vWb/XURgXiAtXD3Nm4nNd2tfJRz7tn1ikz562hj+DcN7hA\nFYULVA/S0hrc+rtlTGgYzgmHHpB3OGY1IyKYMX81Zx1ZzwhfL60wXKB6kEeeXc/Kl7f76Mmskxas\n2syyDdv849yCcYHqQabOWsohBwzkPSd6aLlZZ8ycv5q+fcS5bzgk71CsRFkFStK5kp6TtETSP7az\n/gpJTZKeTrePVz5U25cl67fw+JKX+MgZ4+hX5/2OInIeFdPu0XtvPbqeYYM8sKhIOrzku6Q64AfA\nOcBKYI6k6RGxqE3TOyPCF8nOydRZy+hf14fJE8blHYq1w3lUXE8tf4VVr2zn7845Ju9QrI1ydrUn\nAEsi4sWI2AncAUzq3rCsMza/tou7nlrJBaeMon6wT/AWlPOooGbOX03/uj6c467xwimnQI0GVpQ8\nXpmWtfUBSfMlTZPU7i9EJU2R1CipsampqQvhWnumNa5k284WX9K92CqWR+BcqpTW1uCeZ9bwjmNH\ncMBAX5KmaCp1smIG0BARJwMPAlPbaxQRN0bE+IgYP2LEiAq9de/W2hrc+rulnDpuGCePGZZ3OPb6\nlJVH4FyqlDlLN7Ju8w6P3iuocgrUKqB0T25MWvYnEbEhInakhzcBp1cmPOvIY39oYumGbT56Kj7n\nUQHNmL+agf36eFLlgiqnQM0BjpZ0uKT+wGRgemkDSaW7HxcBiysXou3L1FlLGTFkAOf51+9F5zwq\nmOaWVu59Zi3vPm4k+w/ocLyY5aDDbyUimiVdDdwP1AE3R8RCSV8BGiNiOnCNpIuAZmAjcEU3xmzJ\nH196lUefa+IzZx9N/74eWl5kzqPimf3iRja8utPdewVW1m5DRNwD3NNm2XUl978AfKGyoVlHbv3d\nUvrViQ+/2UPLa4HzqFhmzl/N/v3reOdxB+cdiu2Fd7tr1Ks7mpnWuJLzTxrFwUN8aQCzztjZ3Mq9\nC9ZyzgkjGdivLu9wbC9coGrUz59ayZYdzZ53z6wLfrvkJTZt38UFvnJuoblA1aCI4JZZSzl5zFBO\nHeuh5WadNWP+aoYM7MvbjqnPOxTbBxeoGvT4kpd4oelVPnpmA5Iv6W7WGa/tauHBhet474mHMKCv\nu/eKzAWqBk2dtZSD9u/PBad49JFZZz32fBNbdjRz4Snu3is6F6gas3zDNh5+dj2XThjnvT+zLpg5\nfw0HDurHWUcelHco1gEXqBpz2+yl9JH46zM8tNyss7bvbOHhxes49w2jfFmaGuBvqIZs29nMnXNW\ncO6JhzBq6H55h2NWcx55dj3bdrZwoX+cWxNcoGrI3b9fzebXPLTcrKtmzFtN/eABvPkId+/VAheo\nGhERTJ21lONHHcCbGg7MOxyzmrN1RzO/em497zvpEOr6ePRrLXCBqhGzX9zIc+u2cMVZh3louVkX\nPLRoHTuaW7nAo/dqhqfwrRFTZy1l2KB+THpje9e4s95s8/Zd3L9wbd5hFN5Pn1jOIQcM5PRx7oGo\nFS5QNWDVK9t5YNFaPvH2IzxvmP2FZRu3cdVtc/MOoyZc9Y4j6OPuvZrhAlUDfjx7GQCXnXFYzpFY\nER118GB+ds1b8w6j8IQ4euTgvMOwTnCBKrjXdrVwx5PLOeeEkYw5cFDe4VgB7devjhMPHZp3GGYV\nl1uBWrFxG5+54/d5vX3NeGnrTl7etstDy82s18mtQG3b2cLvV7yS19vXlHNPPIQz/bsNM+tlcitQ\nxx4yhMc+/8683t7MzArOv4MyM7NCcoEyM7NCcoEyM7NCcoEyM7NCcoEyM7NCcoEyM7NCcoEyM7NC\ncoEyM7NCcoEyM7NCcoEyM7NCKqtASTpX0nOSlkj6x3bWD5B0Z1r/hKSGSgdq1hM4l8zK12GBklQH\n/AA4DzgBuFTSCW2afQx4OSKOAr4FfK3SgZrVOueSWeeUcwQ1AVgSES9GxE7gDmBSmzaTgKnp/jTg\n3ZJ82UqzPTmXzDqhnNnMRwMrSh6vBN68tzYR0SxpE3AQ8FJpI0lTgCnp4Q5JC7oSdJXV0+ZzFFit\nxForcR5b4ddzLtXG9+44K69LuVTVy21ExI3AjQCSGiNifDXfvytqJU6onVhrKc68Y9gb51L3cZyV\n19VcKqeLbxUwtuTxmLSs3TaS+gJDgQ1dCcisB3MumXVCOQVqDnC0pMMl9QcmA9PbtJkOfDTdvwR4\nJCKicmEJKOlvAAAFSUlEQVSa9QjOJbNO6LCLL/WDXw3cD9QBN0fEQklfARojYjrwI+A2SUuAjWSJ\n15EbX0fc1VQrcULtxNor43QuOc4Kq5U4oYuxyjtnZmZWRJ5JwszMCskFyszMCqnbC1StTO1SRpxX\nSGqS9HS6fTynOG+WtH5vv3tR5rvpc8yXdFq1Y0xxdBTnREmbSrbnddWOMcUxVtKvJC2StFDSp9tp\nk/s2rZU8SrEUPpdqJY9SLIXPpW7Lo4jothvZieAXgCOA/sA84IQ2bT4F3JDuTwbu7M6YXkecVwDf\nr3Zs7cT6duA0YMFe1p8P3AsIOAN4oqBxTgRmFmB7jgJOS/eHAM+3893nuk1rJY86EWvuuVQreVRm\nrLnnUnflUXcfQdXK1C7lxFkIEfFrstFdezMJuDUys4FhkkZVJ7o/KyPOQoiINRHxVLq/BVhMNptD\nqby3aa3kEdRILtVKHkFt5FJ35VF3F6j2pnZpG/QeU7sAu6d2qaZy4gT4QDo0nSZpbDvri6Dcz1IE\nZ0qaJ+leSSfmHUzqFjsVeKLNqry3aa3k0R5xJLWaS3l/551VmFyqZB55kET5ZgANEXEy8CB/3lu1\nrnkKOCwiTgG+B9ydZzCSBgN3AZ+JiM15xtILOJcqqzC5VOk86u4CVStTu3QYZ0RsiIgd6eFNwOlV\niq2zytnmuYuIzRGxNd2/B+gnqT6PWCT1I0uqn0TEz9tpkvc2rZU82iOOpFZzKe/vvGxFyaXuyKPu\nLlC1MrVLh3G26Su9iKyPtYimA5enETNnAJsiYk3eQbUl6ZDd50gkTSD7v1j1P6gphh8BiyPiP/bS\nLO9tWit5BD0nl/L+zstWhFzqrjzq1tnMo/umdskjzmskXQQ0pzivqHacAJJuJxu1Uy9pJfAloB9A\nRNwA3EM2WmYJsA24sqBxXgJ8UlIzsB2YnNMf1LcAlwHPSHo6LfsiMK4k1ly3aa3kUSdizT2XaiWP\noGZyqVvyyFMdmZlZIXmQhJmZFZILlJmZFZILlJmZFZILlJmZFZILlJmZFZILVBVJGibpUx20adjb\nrMVmlnEu9Q4uUNU1jGzW6YpJswaY9TbOpV7ABaq6rgeOTNds+Ua6LZD0jKQPtW0sqS61mZMm1rwq\nLZ8o6TeSpgOL0rK7Jc1Vdi2WKSWvsVXSV9NEkrMljUzLR0r6RVo+T9JZaflHJD2ZYvyhpLpqbBiz\nTnIu9QbdeY0Q3/7imikNpGu6AB8gmyizDhgJLCe7pkppmynAten+AKAROJzsV+WvAoeXvPbw9O9+\nwALgoPQ4gAvT/a+XvN6dZBM6kmIYChxPNpFnv7T8P4HL895uvvnW9uZc6h03H9Lm563A7RHRAqyT\n9BjwJmB+SZv3ACdLuiQ9HgocDewEnoyIP5a0vUbS+9P9sandhtR2Zlo+Fzgn3X8XcDlAimGTpMvI\nJu6ck6b22g9YX5mPa9ZtnEs9lAtUsQn424i4f4+F0kSyvb7Sx2cDZ0bENkmPAgPT6l2RduGAFvb9\nnQuYGhFfqEj0ZsXhXKpBPgdVXVvILocM8BvgQ6lvfATZZZ2fbNP+frJJIPsBSDpG0v7tvO5Q4OWU\nUMeRXU65Iw8Dn0yvWydpaFp2iaSD0/Lhkg7r3Ec0qwrnUi/gAlVFEbEB+G0a+nomWRfEPOAR4O8j\nYm2bp9xEduL2qfScH9L+Xtt9QF9Ji8lOHs8uI5xPA++U9AxZd8UJEbEIuBZ4QNJ8sn79XC5zbbYv\nzqXewbOZm5lZIfkIyszMCskFyszMCskFyszMCskFyszMCskFyszMCskFyszMCskFyszMCun/A6sr\nZtqvAkKOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112c6a1d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "subplot(221)\n",
    "plot(np.linspace(0, 2, 10), [SegmentationPurity(tolerance=tolerance)(reference, hypothesis) for tolerance in np.linspace(0, 2, 10)])\n",
    "xlabel('tolerance'), xlim(0, 2), ylim(0, 1.1), title('Purity');\n",
    "subplot(222)\n",
    "plot(np.linspace(0, 2, 10), [SegmentationCoverage(tolerance=tolerance)(reference, hypothesis) for tolerance in np.linspace(0, 2, 10)])\n",
    "xlabel('tolerance'), xlim(0, 2), ylim(0, 1.1), title('Coverage');\n",
    "subplot(223)\n",
    "plot(np.linspace(0, 2, 10), [SegmentationPrecision(tolerance=tolerance)(reference, hypothesis) for tolerance in np.linspace(0, 2, 10)])\n",
    "xlabel('tolerance'), xlim(0, 2), ylim(0, 1.1), title('Precision');\n",
    "subplot(224)\n",
    "plot(np.linspace(0, 2, 10), [SegmentationRecall(tolerance=tolerance)(reference, hypothesis) for tolerance in np.linspace(0, 2, 10)])\n",
    "xlabel('tolerance'), xlim(0, 2), ylim(0, 1.1), title('Recall');\n",
    "tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Purity and coverage are much more stable..."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}