Tensorflow / Keras AUC calculations

tensorflow-auc-calculations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from plotnine import *\n",
    "\n",
    "from sklearn.metrics import roc_curve, roc_auc_score, average_precision_score\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "scores = np.array([random.uniform(0, 1) for _ in range(1000)])\n",
    "y_true = np.array([np.random.binomial(1, x) for x in scores])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr, tpr, thresholds = roc_curve(y_true, scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAq8AAAIzCAYAAAAuxfClAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAABy+ElEQVR4nO3deXwU9f3H8ffs5r4DJCQQ7vvwAFRARcEDxRME0SIgtgpab60Vf4pXqbVq64mKtVLFG+t9VLCKB7VoOeQQBAEJhISEQC5y7e7M748km2yyOUl2s9nX8+FKdmZ25jOZLLzz3e98v4ZlWZYAAACAAGDzdwEAAABAUxFeAQAAEDAIrwAAAAgYhFcAAAAEDMIrAAAAAgbhFQAAAAGD8AoAAICAQXgFAABAwCC8AgAAIGAQXgEAABAwQvxdQGv78MMP9fnnn+uXX37R2LFjddttt/m7JAAAALSSDhdeO3XqpOnTp2v9+vUqLCz0dzkAAABoRR0uvJ544omSpJ07dxJeAQAAOhj6vAIAACBgdLiW16bIzMxUZmam+3lUVJTi4uL8WBEAAGiuqKgoJSQk+LsM+FhQhtfFixfrvvvucz+/6qqrdO+99/qvIABAu2dZlr54f5P+9eZalZc5NWxkD112w6nas/OAHrrlbcny3D61V6KK80uVn3vYPwV3UM70PVK5Q5I0/g/H6tprryXABpmgDK/z5s3TBRdcIEnKyclRcnKyDh48qE6dOvm5Mt+x2+1KSEhQXl6eXC6Xv8tBG+N6Bxeud+tI35GjV5/+Wgf2FyqlR6IGDEnRB0u/k1UZUld/sU05WQU677Lj6gRXScrcfUhymV73PWnGcZpw4VGtUqfdbldcXJwKCgqC4np//tLX+vipz2S6TDkcDhUXFxNeg0yHC68ul0sul0umaco0TZWXl8tmsykkpPpUU1NTlZqaKknat2+fe3m3bt18Xq+/WJYlp9Op5ORkGYbh73J8xjRNZWVlKSUlRTZb8HT55npzvYNBU6+3ZVn64I3vtPzdtbJMS6ede4ymzBrr8Zq9vxzQH296W45ypyTp4IEibflfukdItSxpx49ZKjhQ7vU4CZ1jdOm8CXp24ftyJ15J9lC7Lv7NaUrt0fkIz7j6fJxOp7r3Co7rPeyYgYoKi9Bbf/nA36XATzpceH3jjTf0+uuvu5+vWrVKp512mm666Sb/FQUAaBf+++VWPfvnj5SVkedetmt7lg7lFurKW852L3v7xW/cwVWSZFXkT2/R8IVHl3tdHhMfqQtnnajS4jK9+OinskxLUbERuvOJma0WXIORzWbTVX+epbN+O14vLPm7v8uBH3S48DpjxgzNmDHD32UAAPwsOzNPrzy3Uvsz8tR3YFf16pesR+97t85H/KZp6a2XVumci49XSIhdkrR53W6v+7RUHWCrdnPzwql69o/vq7S4ugU2JCxEo08dLEm6ZN4ETb78ZBXkFatTlxjZK4+BIxMSyvcxWHW48AoA6Hh2bMvS+29+r8OFJTpqVG+dP+142Wz1f0R+YH+Brpn+tIqLSmVZ0ob/7apY4aVvatXyX5/3WPVz03tf1dpHjIgO1ylnDVe3np1037VLdbiwVJJ03LiBmn3TRPd24RGhSkqJb+QsATQF4RUA0G6UlTr07pvfKX33AXVNideUS0YrY89B3fKbF2SapixLWrVyq7b9uE+/u+fCevt4vv73r9xBUqpoXW3M02/+VjFxkZKkb1Zs0nMPf+K5gc1Q915dtHdXjiQpKiZC9z8zW5HR4TrquD56+Yv52vvLAUXHRiglLTEo+p8C/kB4BQC0uX17D2r5xxtUXFyuY0b00kmnDqqzjcPh0i1X/0M7tmXJNC3ZbIY+/XC9YqMj5Kpx175pWvrsox902qSj1KNXlxrLTeXmFMmmfK1bvaPO/i2pIlBadYNsYudo9RmY4g6cU2adpL27c/Xxm9/JsBmyTEuX/fY0zZg3Qbu2Zam0uFx9BqYoOjbCvY+IqDD1Hxo8N/4C/kJ4BQC0CtO09O/lG5X+ywF1SYrTWeceo4iIUO3Yvl83zl0ih8Mly7L07rLvdNmccbr8qlM9Xv/ZJz/o558y3dnSNC3l7C/QQaugzrEsSXdc97LXG6UkSc6KIaNqrq9v26iYcN3/1CyPllLDMHTDggt1/iWjlZ2Vp249OqtHnyRJUv8hBFTAnwivAIAjZpqW7r3jTX337Q7ZbIYsSe+//T89+dwV+ssDH6iszOmx/ctLvtZxo/sqKbl6dsNvPt/irVFUrsqP/GsH0atvOUsnTRhSowZTOTk5SkpK0kdv/U/LlnztsT9DUkxchIqLyiqemJYuvmKcLpp1ouITo72eV5+BKeozMKV53wwAbYrwCgBolqzMPP1v9U4ZhnT86H5KTonXqq9+0nff7pDLZapqnPx9GYf0zzdWa9fOHK/7uenqFz0XVN4kVbuFNDIqTKXF5R73WhmGNPbUwUqucROUaZoyVaLklHjNunqCNq9P15b1e2SzGXKZpiZeMEJzbzlLa7/dobIyh4aN6KVuPYJnchqgoyC8AgCabMP6dN1x62uyKltD7XabHnx0hvbtPSirVrOp0+HStq2ZXvuYStKNt52tE8YOcD9f9eVWPfPop3W2O3vKSH3+8UYVFZRIhmS6TF15w5lK6ZZQb51hYSF6+Lk5+s/Kn3QgO189eidp1Nh+MgxDp5w1vAVnDqC9ILwCAJqk+HCZ7p7/psprdAFwOl1aePfbOmPicK939P931fZ6h6c6fkx/j5bTC6Ydr9Wrtmt91bBWknr26aIrrj5NM+aM05efbdbhojINO7qHjh7Zq9F67SF2jTtjaDPOEEAgILwCALQ/K1+7duUoITFKgwal1hnmKSszTzde+5IOF5V5LLcs6UBOoZwOl7zdyH/C2H6aPPV43XPHMrmcpizLkmVJl11xsrqmJnhsa7fb9MdHZ+jfn2zQnvRcJSXH6azzjlV4RKgiIkJ1wbTj2+LUAQQYwisAdEAZ+w4pL69E3bt3UlKX2Aa3/eTjH/TXv1SMaWqalkaP6af77p+q0BozGP3h3reVm1MoqW6fVEvSW2+srrNfu91Q7z7JOn5sfz3/8rxGh8qqeI1NE887tsnnCSD4EF4BoIMoKCjRvqw8vff+Wv1r+SZJlUM+XXuGJl8w0utrdu8+oL888rFHi+l3q3fo1Vf/o8svH+detm1rlju11unCakjPvThXf7znHWXsyZXTacputyk2LkJTLx0tSeqW1klz5o5vrVMFEMQIrwDQAXzw8Xo99uRymS7PZGlZlh5/aoUGDkjRUC/jk377n+11wqhlSf/6ZIPOPvtoj/1Iqm52rXqNIYWF29W7b5Ke/NsVevOVb7VrZ7a6do3X9MvGqlPnmFY6QwCoQHgFgAC37J3v9fSznze4zYcfr1dnL0Eyfc9Br9tn7y/QjEuf9lhm1Pyi8onNZmj2FafIMAxFRobp8is9Jx4AgNZGeAWAAFVSWq79+wv0zHOfV0x92sC2n3y6UZ98urHuCpdZM4u6DR7aTXffPdn9/LNPN2rJ37/0GDkgISFKv5k3QWefc0zLTwIAmonwCgDtlGVZcpmWQuw2j+XZOYVa8Id39NP2LEmWDLPxfc24dIwuOG9EneWlpQ7dfttryskpdAdTwybNm3eaunatHsbqstknKykpTu+98z+Vl7t00riBmnX5ONlDbHX2CQBtifAKAO3Qux+v0+J/fKXiknL17N5J995+vvr1SZbLZerW/3tDe/ZWftxv1TuMqlt0dJimTjlOneqZAvXpZ67QE49/qp9+ylRCQpSuvGq8jj66R53tJk46WhMnHe1lDwDgO4RXAPCDnNxCLXz0I23ckqHIiDDNuniMLrnwOBmGoS+++UmPPfOZO5SmZxzU9fNf16vPXan8gpLq4FqLJbnHWjUkRUWFqW+fJN12y6R6g6skdeoUrXvvu6jVzxEA2gLhFQB8rNzh1E0L3lBmVr6cLlOFRaV69sUvFREeosmTRujVf66u05p6uLhMn3y2SV0Svdy9b8jd/GpVPh86JFVPPTqrzmQDABDoCK8A4ENlZQ794dGPtCfjkMdy07T01gdrNfa4fsrcn1/ndZakZ/7xpYwaU7DWHLWqZkY1DEPTLjqu9YsHgHaAnvYA4EP3/fUjff3ddq/9VNMzDuriKxeroLDUY33VSAK/vuwkvfb8XHXtGudubLUMQ4YhRUaGSpLCw0N007Vn6OSxA9r8XADAH2h5BYA2smZjul5861sVFpVpxPAemnbOCH29ertkeR/WKj4uQs//9XIt/2Kznn/5G3ffVUkKCbVr0unD1TUpTk/8+Vf6v/vf1o6dOZKk0ycM1e9vmiRZlsLCKv5adzqdvjlJAPAxwisAtKK9WXn6+xurtH1Xtn7Zk+teviM9Rxu27HU/9zYua3RUuLomxWnmxWN0uKRcr771nSxVjBaw8I7J6poUJ0lK6RqvFxZdocLCUoWG2hUREeqxH6vO/K0A0HEQXgGglWRm5+uK215SWblTLqfpEU5N09LW7Vkey2oG2JAQm8aM6iupos/q1ZefqlkXj1FBYamSOscoJMRe53ixsRFtdSoA0G4RXgGgFRwuKdc9j36oktJy1dvwWdVRtcboAFVGDO+pay73nFo1Oipc0VHhbVAtAAQuwisAHAGny9Rbn6zTkmXfquhwaaPbh4eHqKzMWR1gbdKgfil65N5pDGsFAE3AaAMA0EKmaen2h97VUy+tVFFxWZ31tRtgbTZDd910jqIiw2S3GbLZDKUmx2vh7RcSXAGgiWh5BYAW+nb9Lq3+4RdVDb1aX/y0GRVB9o7rJmn82EE6ZkiaftyepbBQu44a0l0R4aH1vBIAUBvhFQBa4MefM/Xh5xtlMwyZ3kZtrdEtYPp5x+mCM49Wz26dJEmJCdE66fh+Pq0XADoKwisANNO7K37Qw89/JrvNJqfLrHc7wyadfeowXTv7VLoFAEArIbwCQDP8tGu/Hn7+M1mW6gRXyzAky5IhKSUpTjfOmaBxJ/QnuAJAKyK8AkAjSkod2rAtQ1t37tff3/xP3aGwag59ZRjq26uLljw40+vYrACAI0N4BQAvTNPStl+y9cu+g3rmta+Uc7BIMlV3CIEqlY2rdrtNRw/uTnAFgDZCeAWAWkrLynXDA29p4/bM6rBq1j+aQJUQu01RkWG6fMroti4RAIIW4RUAanCZpmbNX6q9+/Oa9brY6HCdN2G4Lj1vlJI6xbZNcQAAwisAVLEsS9f/cZn34OplSlf3KsPQfTeepzHH9m7D6gAAEuEVANw+/HKT1m3Z631lxUACkiEZNUJsv55ddPMVp2nksB4+qREAgh3hFQAkffTlJv3p+RV1V3iMJFDxNC4mQr26d9KUM4/R2eOG+rBKAADhFUBQMk1LhiH9sG2fbn/0PeUXlrhDap0bswzJbjPUvWuCJpwwUFdOHctoAgDgJ4RXAEElN++w7nr6I/2wLUMhNpucTlNm7ckG5BlgbTZDb/7l1+rWNcGXpQIAvCC8Agga+YUlmvb7JSouLZckuZwuSTWCao0uAjV6CmjuxScRXAGgnSC8Auhw/v39Nj391jcqKCpVv+4Jum/eeUruFKvrH/6nO7jWq1afgciIUE06mX6tANBeEF4BBLS92Xn688v/1s6MXHXtFKszRg3QY69/5V6/bluW5j7wpp743UX66Zdsr/uwDENGnTlfpaTEaP355slK7sy4rQDQXhBeAQSsQwXF+vUfX1PB4VK5TEsH8g5r0/ZMj20sSZm5Bfrgq011d+AxkoBR0dfVshQeHqq7552l00cPauMzAAA0F+EVQMD66D8/Kr+oVGZlq6lpWfVO4frSR//zOoqAVDGSgMtVMfrAvItP1pwLTpBhNDYZLADAHwivAAJSablDL378vTu4Vqk9UkCVeVNP1LLl63SosMRjeaf4KC17+AoVl5QrPiZS4WH8tQgA7Rl/SwNod3buy9Wrn61VXlGpju3fTZeePkIhdpvHNl+v36n8opJ69uDJZjN0zklDNWFUf13/0D91IO+wJGlgryQ9dfs0xUSGKyYyvNXPAwDQ+givAPzC6TK1bOUP2pqerc5xUbr0tBFKTozRtj05mv3Aa3K6XLIs6cv1O7T+5ww9fM35Hh/lHyooVt1brCQZUnJijLIPFkmSQmyGFl5zrlK7xEmS3vnLb5SedUhhISFK65ogm43uAQAQSAivAHzKsiwt/982PfHPr7X/YKEsSTbD0Ntfb9Trd8/SQ699IUfl+KtSRT/WL9bt0Jfrd2hQz2T38uy8Iu8HMKTZ5x6v4wb3VF5hsaLsTg3s18u9Oiw0RP17JLXV6QEA2hjhFYBPzX/uI322ZrvHMtOyVFhcpode+1xb070PZ3Xr0x+o1otkqG7/1s5xUZo64RjZbTaZZqKysrJar3gAgN8RXgH4xPaMA3r4tS+0Ztveerf58oed8t4XQLr6wrE6b2z1ZAGbdmbqzmc/Us37teyGdMG44bLbbF72AADoCAivANrMwYJifbVxpzbtytQ7X2+qN5hW6RIfpf7dk/TfzbvrrDtj1ACldo5zP0/tHKfNO7L06oo1CgsJkcs0NaBHkn593ujWPg0AQDtCeAXQqsodTq368RdtzzigpcvXqKSsXKYpyfI+hFVN447qq3kXjNXMP76qQwXFMiQ5TUtXXzBWfVI719n+pktP1bhj+2r7nhx1jo/W+JH9FRpib4vTAgC0E4RXAK2msKRMVz76pnZkHpTpNCtaWhtpba3SNTFG1045WYmxkXrrvtla/v02FRSXanifVB0/uEe9rxs1uIdGNbAeANCxEF4BtIptGTl68r1vtDProFym2aSW1ipXnH28Lj/7eMVGVYy1GhsVoamnHt12xQIAAhbhFUCzHSoq0Y7MXMVFhatfSmc9uOwLLftmg4yGWlkNedxcVTVSwENXn6fTRg5o44oBAB0F4RVAk5WWO/X2txv16Ltfu8di7Z/SWTsycyVV9xDwaHE15LXrwInDe+vGqePUv3uXtiwZANDBEF4l2Ww22Ww2WVYTO+d1AFXnGkznLFWcb0hIiCzLCqpzb43rvXVvtq5e9LbyDpd6hNOfK4NrFfc6m2SZlcsqF/bqmqDhfVJ17ughGj2k5xHX1Biud/Ccs8T1DqZzlqSQECJMsOLKS4qIiFB0dLScTqe/S/E5l8vV+EYdTGJiokzTlGma/i7F55pzvYtKy1VSVq7OsdFyuFz67bPvKP9waaOvs1QZYA1JNskwpaG9knXe6CG66OTh7ileffV+43oHF6538EhMTPR3CfATwquk0tJSlZeXKykpeKaMtCxLLpdLdrvdY774js40TeXm5qpz586yBdFA9s253g6XS/e/8Zk++H6LJCk1MVa/u/AUHSwsafrxVJFdQ0Ns6teti1743SUKtft+CCuuN+/vYBCs1zsnJ8ffJcBPCK+S+7f0YHrTVzEMI6jO2zAMOZ3OoDvvKo2dt2VZ+s2Tb2nD7kz3ssxDhbr39RVNPIAkS7LbDKV1iddx/dN080WnKMxPH+9xvYPrvLnewXXewfhpKSoQXgG4vfrVOo/gWqWwuNzjubtrgCq+uGXyKVq5cYf25Raod9dE3TH9NPVK5iM9AEDrI7wCQe6/29J17xsrtD+vUPb6Wm2qRgyoMXKApYoW1oWzztY5xw3W7NNH+aZgAEBQI7wCQSK/uFQL3/5C323fo4jQUF1x2nE6oX+arln8tlxmRSJ1WVa9EwukdYlX1qFCOStvhAmx2XTlmSfonOMG++gMAAAgvAJBwWWauv6FD7RtX44cLlNFpeX66wdfaXiPFHdwreLRJaBSQnSE/n79xbr39RX6ftsehYbY9atTjtXcs0f77BwAAJAIr0BQ2Ln/oDbv2e+xzGVa+rHWsiq1A+xNF45TSmKsnr3mIlmWFVQ3hQAA2pfgGUsECGIlZQ6vyx2uWmNhVs3ZqsqurYYUExWuCcP7VW9CcAUA+BHhFejgLMvSn97+QpY8Z2m1JPVL6eT9RZUhNjEmSs/Ou0gJ0ZFtXygAAE1AtwGgg/vzu19qc0a213VOy9LtU8broXdWypJkM6Q7pp6mc0YOVlFpmZLiYhRi53dcAED7QXgFOrCScode+Xqdx7Kara9FJWW67JQROv+4IdqfX6SuCTGKi4yQJMVGhvuwUgAAmobwCnRgBSVlDa4f0aebJCkuKkJxURG+KAkAgCNCeAU6oD25efpwzRZtTM/ymFigpvBQuxZMO8PntQEAcCQIr0AHUFLu0PtrflR2/mFFhoVq8Yr/qtzhlHsIVy8BdtmtM5UYw41YAIDAQngFAsiG9Eyt2ZWh6PAwTTxqgBKiI3W4tFwznnxNvxw4JEkyXV6aWaU6Mw/06pLYxtUCAND6CK9AAEg/kKenlv9HH6//SXabTZKlp5b/R69fP0NvfrtBO7IPemzf2EisqYmxstkYrxUAEHgIr0A799WWXbrhxffkrOwD4DIrJhY4WFSiu978VAcLS+p/cT39Xf804+w2qBQAgLbHAI5AO1budOrWVz5yB9fa1u/OVJnTWWe5x9aVDaxVE2PdNfU0jeqb1rqFAgDgI7S8Au3Yqp92q6Tc+9SuklTudGlPbr7XdTUbXUf27aaRvbvpjKMHaFiPlNYvFAAAHyG8Au2UZVm6953PZKn+Pqy9uiTo4V+do6uef1uFJWUyLUs2m6H+KZ110sBeKi13avSAHjp9eH85nU6FhPCWBwAENv4lA9qpnzIP6EBhsSS5A2zNIGszpPsvPlNDe3TVe7+brZe+Wqv9+UUakNJZs8aNVHho9dvbsuoZgQAAgABDeAXaqQX/XO7x3KrxZ3iIXS9dM13DK7sAdImN1i3njvNtgQAA+AHhFWiHDh0u1o8Z2RVPvPQZuO28U9zBFQCAYMJoA0A7U+ZwasqTL3sb4UqSZDMMXTr2GJ/WBABAe0HLK9COuExTlz77mrILDkuG6tysZUm67KRjZRhMMAAACE6EV6CdsCxLV7/0jn7KOuC5vNZ210880XdFAQDQzhBeAT/7atsu/eHDz5VZUCiX06poaa1qWK3VwDp+UB9Fh4f5uEIAANoPwivgJ1uzcnTrso+1M+dgxYJGRrOKiwzXI5ee0/aFAQDQjhFeAT/Yl1egGX9/QyVlDq/9Wmu2vlYNjfXRTXMURasrACDIMdoA4AefbNomp8ust7G15nKbpHeum6nOsVE+qAwAgPaNllegjViWpTfXbtR7P2yRJE0+ZqguHjlchmGo1OmU5S26GnJPpWVZkmFIS+dOV5+kTj6tHQCA9orwCrSR3739iT7a9JP7+fq9mTp4uFhXnzJaw7p1lcNl1v9iQ4qNDNOzMydrZM/uPqgWAIDAQHgFWtGWrGwt/W69Nmfu19asAx59WU3L0hMrv9V5Rw/WxoxM2QxDplWj9bVy4zvPGa9xA/ooNT5WYSF2n9YPAEB7R3gFWskPGZm67B9vymVZMl2Wt1ldZVqWTn/8BRlVja5VN2VVdhGYOmKYZo4Z4auSAQAIOIRXoAXKXS69vnaDdh44pNS4GF123DF6+LOv5bQsWaZVMTtWZaNqVYitamP95NrLtW7PPi1477PqlldDsttsGtmLLgIAADSE8Ao0U5nTqfOfW6rdB/NkSbIbht5ct0kOp0uWVfcmrJpLOkdHqm+XTurbpZN+3p+rJd+uVajdJqfL1HlHDdaUY4f67DwAAAhEhFegib5P36vv0zP0xtoNyiwoci93WZb25OUrwl7ZP7VytAD3nzU8+6sL3V/ffvapOv+YIUo/mKfU+Fgd3T1FhuGtswEAAKhCeAWaYMnqNXrws69kNww5Te+js5Y6XXX7uRoVGdZmGHpkyiQd3T3VY/XQ1GQNTU1uk5oBAOiICK9AI/blF+jBz76SJdUbXCUpOixUpeUOmZJHq+sN48fqvOGD1bNTQtsXCwBAB8cMW0Aj3vphc70zYdU0+eghiouMUJjdrjC7XYatIrj+9pQxBFcAAFoJLa9APYrKyvXtL+l65pv/NrptTFiobj1tnH47bow+2LhVhWVlGtWju07u18sHlQIAEDwIr4AXu3IPaeary5RTdLjRbSNDQ/TulTMVEx6mmPAw/XrsKB9UCABAcCK8Al5c+/YHOnC4uGLygJoravRlTYyK0K9GHqNZxx+rztFRfqgSAIDgQ3gFatmYmaXtB3I9Fxp1v35y6vk6oVeaz+oCAACEV8CtzOnUDe9+pH9v3ylJXqd3rdI1JprgCgCAHxBeAUku09SUF1/VtgO5FWOz1hheoOpLo8afvzttnI8rBAAAEuEVkCQ9t/r7iuBaW40+rpakEd1SdMXoUZo0ZKAvywMAAJUIr4CkT7Zu91zgpY/rBUMH6y8XTPJZTQAAoC7CK4LaZz/v0D0rPtP+gmKP5ZY882taXKwePHeiT2sDAAB1EV4RtNbszdA1774vy6rdq7WCpYop6C4YNlj3nXW6Qu12X5cIAABqIbwiaL23ZWvNLq3VamTYxy48R+cMHuTDqgAAQEMIrwg6+w8f1r3vva//7t4js6rVtWaja+Uim81QTHi4z+sDAAD1I7wiqOSVlmre8hXKLy+X02l66ShgVPxnGIoJC9PRKSn+KRQAAHhFeEVQeX/LFh0sLXV3Fah9Y1aVzpGRem7qhUqIjPBhdQAAoDGEVwQFp2lq8fff64n/fFu3j6vkTrBhdkO/Oe443XTyibLbbL4sEQAANAHhFR2eaZqa8uor2pydI1k1Wlq93K3lcJma0K8vwRUAgHaqXYTXoqIiLVq0SGvXrlVkZKSmTJmiCy+8sM52K1eu1NNPP+1+blmWysrKNH/+fJ144onauHGj7rrrLoXXuMlm2rRpmj59uk/OA+3TjR9/pM05OY1uZ0i687TxGtm9W9sXBQAAWqRdhNfFixfL4XBoyZIlys7O1oIFC5SWlqZRo0Z5bDd+/HiNHz/e/XzNmjV6+OGHPbaLj4/XSy+95KvS0c7tyc/XR9u3e19pVDwsS7rsmGN09ejj1S0uzqf1AQCA5vF7eC0tLdWqVav06KOPKioqSr1799bEiRO1YsWKOuG1thUrVujkk0/2aGkFJGnHwYP618/b9fnOndULK/sLWJbnol6J8br79AkKoasAAADtnt/Da0ZGhizLUq9evdzL+vTpo2+//bbB1xUUFOi7777TAw884LG8sLBQs2fPVmhoqEaOHKnZs2crNja2TWpH+3OguFjPfP+dXly/TpZleQRVSdUdXiv7vvbtlKj3LruM4AoAQIDwe3gtLS1VVFSUx7Lo6GiVlJQ0+Lovv/xSqampGjx4sHtZWlqaHn/8caWlpSk3N1fPPPOMHnvsMS1YsMDjtZmZmcrMzJQk5eTkKDk5WV27dpVpmq10Vu2fZVkyTVOmacowvA0WFXh2HTqki996Q3klpdULvU2hVdld4E9nnqkLBw1WqN3e4a99R7zeTVF1XTv69a2N6831Bjoyv4fXiIiIOkG1uLhYkZGRDb7us88+0+mnn+6xLDExUYmJiZKkpKQkzZ07V1dffbXKyso8uhYsXrxY9913n/v5LbfcoltvvVVZWVlHejrwk3KXS/M+W678msG1ipcAO7RzJ53cqbNym3AjFwJfdna2v0uAD3G9gY7N7+G1e/fukqT09HT17NlTkrRr1y73197s2LFD6enpmjBhQoP7ttlslR8deyaXefPm6YILLpBU3fIqSSlBNJuSZVlyOp0KCQkJ+N/U80tLddmbr2l3QX79GxmeXz501iSlJCW1eW3tRUe63s1hmqays7OVnJwsWxB1DeF6c72DAQ1Owcvv4TUiIkInnXSSli5dqptvvlk5OTlavny5brzxxnpf8+9//1ujRo1yt7JW2bBhg7p27ark5GTl5eXpueee07HHHquICM9ZklJTU5WamipJ2rdvn3t5sP1lZ7PZZLPZAvovu1KnQ2e+/A8dLClR1dSu3mchqGCTdO/YsRqclMT1DiJV5x4suN5cb6Aj83t4lSpaQp966inNmTNHkZGRmjp1qnukgenTp+uee+7RsGHDJEkOh0Nffvmlrr/++jr72blzpx577DEVFBQoJiZGI0eO1OWXX+7Tc4HvmJalKW+8Wh1cGzA6LU3HdE3RhYMGK97p9E2BAACg1bWL8BoTE6P58+d7Xffmm296PA8NDdUrr7ziddvJkydr8uTJrV0e2qmXN6zXT7kHKp9ZcgfYWq2vD55xpqYPP0pSxceKfNQEAEDgahfhFWiJVzf+UGtJVYC1JMOQ3TA0efAQd3AFAACBj/CKgLR8x3ZtO5hbEVc9WlorvogIsevG0SfqypENT3QBAAACC+EVAenrPbvdedUdYCuF2ez6+oqr1LnW+MEAACDwBc/tmOhQwu0hHoHVUnXj620nnUxwBQCggyK8IiBNGzJMNsNwz5ZV9YiLCNfkwUP9XB0AAGgrhFcEpMLyMsWHe47f2yc+Qf+8eIY6R9LqCgBAR0WfVwScFTt/1tyP3quzfF9RocJsdj9UBAAAfIXwinavoKxM9339uVbtTVdkSIjS8/K8bucwTX2yY5vmjjzetwUCAACfIbyiXSt2lOv0V19QTnFxxQJLMuqZ/tWyLDlcpu+KAwAAPkefV7RbTpdLE19bUh1cK9WTXWVJGtezV5vXBQAA/Ifwinbr1s8/0d7CQtUfVz1dPep4Hd01pW2LAgAAfkV4Rbt0qLRE7/28tcaSGgHWS7+BCwYM0u0nntL2hQEAAL8ivKJduufrf1d8YdRaYajO8tlHH6vHzjrXF2UBAAA/44YttDtv/bRJ71e1ulaFVMv9P8WEh+r9i2cpt7hYqTGxSouL90eZAADADwivaFd+yT+o+V9+WhFaq3oH1Gp9Xf6rX6tbTKz6JnTydXkAAMDP6DaAdmNDTpbOWvYPOc3K1FoztFZO/zq4Sxd1i4n1R3kAAKAdoOUV7cZtKz9RuemqscTwCLCGpCfOOM/XZQEAgHaE8Aq/cpgurdyzS9sOHdBPhw5ULHR3GfAcVeDx08/TwE5dfF0iAABoRwiv8JtSp0O/+vhN/ZCTKdOyqru4etykVSEiNETn9R/s6xIBAEA7Q3iF3yz6YbU25GTJZVWnVKPmF0b1l4+MnySbUXvcLAAAEGwIr/AL07L0xk8b5LTM6oWGZFl1h3Z94eyLNKFnP5/WBwAA2ifCK/zitZ9+0P7iw6roG+B5V1bNnq4zBh9NcAUAAG4MlQW/eHnrelXH1KqhsTy7D4xK7qY/nnSmjysDAADtGS2v8LnP0n/WjwezPSciUPXYrsM6J+mi/sN1xbCRstn4/QoAAFQjvMLnVmXurs6ttTq4JoaH65/nX6bIkFA/VAYAANo7mrXgc1nFhR6jCdR0zTGjCa4AAKBetLzCp97cvkGf/PKTpMrxXGsMfxUdGqap/Yf7qTIAABAIaHmFz+wpzNMd//lXrRZXS1X9Xaf1H6YukdH+KA0AAAQIWl7hE3uL8jVzxRvVExLU6jJgyFLnyCjfFwYAAAIK4RVt7rCjXNM/fVWZRQUNbjex50AfVQQAAAIV3QbQ5j7f+7MyDxd4HV2gyh3HjdfgxCRflgUAQLO9++67evrppz2WzZkzR8OH+/eejXvvvVcxMTE+3dfkyZM1fvz4VjlmcxBe0aacpqmH130lq+a8WbVGGjiha5rmDh/t89oAAGgub+EVvkV4RZtxmabu+36F0ovyKpdYnhsYFYMNzB4y0telAQDgdyUlJf4uISARXtFm7lz9qV75aV31Ai9dBm469mSd13uI74oCAKCF5syZoxdffFGbN2+WYRgyDENz5sxxr1+5cqVGjBih6OhonXDCCVqzZo3H6w3D0IMPPqjbb79dKSkpSk5OliRZlqVHHnlEAwcOVHh4uPr27atHH33U47V79+7V9OnT1bVrV0VERKhPnz66+eab69S4ceNGnXzyyYqKitLw4cP16aefeqw3TVMLFy5U7969FR4ersGDB2vx4sWNnvuWLVt06qmnKiIiQv369dOLL77Y1G9bq+OGLbSJ7JIivf7zD3VXGJa7AXbusBN04zEn+bYwAABaaMGCBcrJydHWrVv1yiuvSJKSkpL0hz/8QVlZWbrhhhs0f/58xcfH64477tCUKVO0Y8cOhYZWT77z+OOPa8yYMfr73/8up9MpSbrxxhv1/PPP684779To0aP1n//8R7fffrsiIyN19dVXS5Jmz56tffv26YknnlDXrl2Vnp6u//3vfx71ORwOXXbZZbrhhhu0YMEC/fnPf9bUqVO1e/dude7cWZJ022236fHHH9ddd92lE088UR9++KGuvvpqORwOXXfddV7Pu7S0VBMnTlR0dLSWLl0qSbr77rtVUFCgAQMGtO43uQkIr2gT9363ov6VhhRlD9XvRpzqu4IAADhC/fr1U1JSknbv3q0xY8Z4rDt48KC+/PJLDRs2TJIUHR2tCRMmaPXq1Tr55JPd23Xq1Elvv/22jMpJenbs2KGnnnpKzz77rObOnStJOuOMM1RcXKz77rtPc+fOlc1m03fffac//elPuuSSS9z7mj17tkcN5eXlevDBB3XOOedIkgYNGqQ+ffrok08+0cyZM3XgwAE9+eSTuu2223TvvfdKkiZOnKgDBw7o/vvv1zXXXCO73V7nvP/xj39o37592rp1qzusjhgxQoMGDfJLeKXbAFrdX9Z/qY/Tt9a73i7pnF6DFOblDQIAQCDq1q2bO7hK0tChQyVVfNxf06RJk9zBVZI+++wzSdLUqVPldDrdjzPOOENZWVnas2ePJGnkyJF65JFH9Mwzz+jnn3/2WoPNZtMZZ5zhft67d29FRka6a1i9erUcDocuvvhij9ddcsklysnJ0bZt27zud/Xq1Ro+fLhHUO3fv7+OOeaYhr8pbYTwilb1+d6f9eTG/8jdwdVLP9eBiclacMIZdVcAABCgEhISPJ6HhYVJqvjIvaauXbt6PD9w4IAsy1KXLl0UGhrqfpx55pmS5A6vb7zxhk4//XTdeeedGjBggAYPHqy3337bY1+RkZHu49aso6qGQ4cOea2h6vnBgwe9nltmZqa7f25D5+IrdBtAq/r9tx9WflU1soBREWArn/aN66R3z5mtcDs/egCA4FOz1VWq6EZgGIa++eabOsFTqvjoX5JSU1P1wgsv6Pnnn9eaNWu0cOFCXXLJJfrpp5/Ut2/fJh27U6dOkqTs7Gx1797dvXz//v0e62tLTU3V2rVr6yzfv3+/4uLimnTs1kTLK1qNyzSVW1Yiz/elVfEwLBmG9NyEiwiuAICAVbMlszWcfvrpkqTc3Fwdd9xxdR6xsbEe29tsNh1//PFauHChnE5nvV0IvDnhhBMUGhqqZcuWeSx/8803lZycrIEDvc90ecIJJ2jTpk0ex/r555/1ww9ebsz2AVIEWo3dZlNSZJRySoplGJJVa1jXv0+Ypv7xXfxTHAAArWDIkCF64YUX9Nprr2nAgAHq0uXI/l0bOHCgrr32Ws2aNUu33XabRo8eLYfDoW3btumLL77Qu+++q/z8fJ111lmaNWuWBg0apPLycj355JNKSEjQyJFNHyu9S5cuuv766/Xwww8rIiJCY8aM0ccff6xXX31VTz75pNebtaSKIcIWLlyo8847T3/4wx8kVYw2kJKSckTn3lKEV7SaXwoPqtTlcA+HVdECW9EM++CYSTotrb9f6wMA4Ej95je/0Xfffafrr79eubm5uvzyy494n0888YQGDRqkxYsX6/7771dMTIwGDRrkvrEqIiJCRx11lJ588kmlp6crMjJSxx13nJYvX97s8Pzwww8rISFBzz//vHu812effVbz5s2r9zWRkZFavny5rrnmGs2cOVPdu3fXggUL9N577ykvL+9ITr1FDMuq3T4WXPbt2+f+ulu3bn6sxLcsy5LT6VRISEid/jctNfXTF7UuN6NOi2tcWLjWTbtFtlY6zpEwTVNZWVlKSUmRzRY8vWba4noHAq431zsYBOv13rdvn5577jnNnTs3qP79Bn1e0Yp+PJQlqaLFteaj0FGm3NLDfq4OAAB0BIRXtJrE8Civy22GobiwCB9XAwAAOiLCK1rNHSNOq7PMkHTD8JMZYQAAALQKEgVazfm9hykmNFwP/7BSW/P2q3dMJ10z7ERN63u0v0sDAKDFLMd2uXKnVjwxwiSFyjDauv3PkiyHJIcsq1xG7O9kj/51Gx8zMBBe0aomdO+vQQlJGvfhU3r59BnqFh3v75IAADgipnOnnFZxxZOqPysZtf5sLTXvfbYsSzbHLjGpegXCKwAAQAMsSS41PjiTUc/XzTlOzT9r7qztW3oDB+EVAACgAZYsuSyz2a9zt8rWM4RZ1WilTRmz1NaC43dUhFcckfd2b9SDP/xb+eUlGhiXpKGJqfow/UdJ0l3/+0SPjZ3MSAMAgIBmSXI2KWLWtwOrMsgaqojCzUeXgWqEV7TYF/u263er33O/CTcd2q+Nh/a716/av0vXrHpLL4+/LKgGzgYAdCyWLJktiJwVDavVYbWxltjGakAFwita7PWdaz07lMtSzV4+TsvUf7N3K6ukUKlRcT6vDwCA1mDKkrOJH9s3KWLWmoqyKVG2JeG5oyK8osUOlhU3vpEkp0k/HQBAYKvvhq3WbhE16omy/EtajfCKFvkpP1s/HNznZU1166vdMNQjOkHdaHUFAAQw0yyXy31zVdNvsmqZij1XD8FV8VW5M7PNjhhoCK9okaU/fy/Tavit2yM6US+ceqnsNob3AAAELpekcsuUVXnDVU1tfUuHVXmzl72Rf3ODCeFVks1mk81mcw9ZEQzcw3O08JwLykvdb9iqXRhGRWvrHcecqTO7DVRKVJxshtGuvq+WZSkkJESWZbWrutrakV7vQMX1Dp5zlrjewXTOkhQS4sMIY4SrrL7IZNVuKT0y3sd6NRQV0u0I99xxEF4lRUREKDo6Wk6n09+l+JzL5WrR60Z26q5P926VKcvjt05L0pguPZQcHi3T5WqXfXQSExNlmqbMIOyL29LrHci43sGF6x08EhMTfXYsS5LLqi+W1l5eGWYN72u97bsqqVoNbG3Ve/zgQ3iVVFpaqvLyciUlJfm7FJ+xLEsul0t2u71FQ3bMGnCCNhzK1Ad7NruXGTL0p+PO1aDElNYstVWZpqnc3Fx17txZtiDqznCk1ztQcb253sEgWK93Tk6Oz45lyZBTzfyZsqq/qN0q625ddY+h1fh1c7X6BLSBi/AquX9LD6Y3fRXDMFp03nbD0F9GT9bkXkfpN9+8rv875gydnTa03Q+JZRiGnE5ni8870AXbeXO9g+u8ud7Bdd6+/LTUtBpqeW1MzX6y9eyjCT0+gqxXSIMIr2gyy7K0fN9WbcvPVnJkrCZ2G6wt+RWTEkTYQ5USGevnCgEAaAOGIVczWl4t1e67atR5Zqh5/WPN5rb8dmCEVzSJZVm67ft39dHezbIbNpmWpfvXfuoeceC+df/S/w7s0SMnXBhUv/kDADo+05KcDbS81neTVV319I9t4BXuGhoqMMgQXtEkX+/foQ/3bpZpWXJZLlVMNFL9NnVZlj7as1nn9xyu8an9/VYnAACtzZI8Wl6tWusqtEbDTf1h1tXEGb6CAeEVTbKzMFehhl1lVv19jEIMu3YV5hJeAQAdisMsk8OsCK8eraut/kGj9x0akvLL01v7YAGL8IomibCHyGHVHoalejYtSXJaZru/YQsAgOYybJEqVWjdFVbjHQVaqnaLbtewAa2498BGeEWjPtizUfet/1imezA6QzIsybIpxLDJZZkKMWwa1aWHzuw+yM/VAgDQuizLJqdlb2wrSS2frKCx4bNMixu2qhBeUa9Sl0Pf7t+l2//3rntO56q3o2EYskkak9xL3aMSNCA+STP6jZLd4M0FAOhYLKn547zWGt/V+61aNUcmaHhWA5NxXt0Ir6jDtCx9nfWz5q95TwfLi+vfzrAUGxauhced68PqAADwLUuGXI22vDbGrBzxtWYIbXogZaisaoRXeMgpLdKV37ysrQXZjW5rMwwlhEX6oCoAAPzHtAw5W/Cxfd2xXo0Wdyto+SQJHQ/hFR5uWv2WtjcYXKtv0go17Jrd/wSf1AUAgL9YRtOmh63qYVd3Qtg6W1aurfyzCbmUbgPVCK9wKzddWpub7nUgZMOoeVOlpfjQSL10yiz1j0vyYYUAAPheRctr3W4DdWdsbWHAbMKoBdywVY3wCje7YchmGO5Zs2oz3H3JDf35+As0JCHFh9UBAOAf5WapXLLVaFmt0latobVGLjCk3PK9bXSswEN4hZvdsGlq7xH65651cnr5fTLKHqrUqHjdMHS8TktlSCwAQHCw26JUalZFporbrmpqyw/0K0aptBRqS27DowQWwivcnKapSHuIrBpvSpsMxYdFama/4/XbwafI1pSOOQAAdCCWZchpNRSZWi/M1tcVIdQecwR77VgIr9AHezboqS1fKqu4QGVmxSxa1W88S6d07afrhpzqr/IAAPArS2rmaAO1BsRqIM02tSuCyWgDboTXILbmYLru3fCRdhcdklT1BvJ8c1iS3t+zUQtHnq8wOz8uAIDgUzHO6xHcMGVVhNmaNz83t98s4bUaaSRIbTyUobmrX5bDVT30VUMclqmwti8LAIB2x5QhVwsnCag5Uo+sihkqW1YDow1UIbwGqVd2ra5nVIHaYdbS4PgURYcQXQEAwcmy1OSW17otq5LHv6tW3ZEEmoJxXqsRXoNUgaO0xhur/tbXEMOuZ8Ze6qOqAABojxruNmDV+rNZt2zVDrP1bkZ4rUJ4DVJjuvTRf7J3qLzGlAQVfXGq33pJ4TH6x7jZSo2K90eJAAC0C6Y8p4dttckJ6qh/1ILDzpJWOkbgI7wGqcv6jtbmQ/v0YcYmj8AaYbNp/jHn6NhOaeob04WbtAAAQa/UWSaHWRFeKz6r9FUraHWrbJ6j0EfHbP9IJkHKbti08JgLNKXXCF317Yt6cORUdY2M04C4ZCWERfm7PAAA2o1Qe6TKrVAva5r2kX9zeeuG0Cm8eyvtPfARXoNUiatcr/+yWl/lbJPNJhU6inVu2lEtvgsSAICOquk3bLUszDalzyxDZVVj3IUgVOws06VfPavHfvpMaw+mS5L+tPlj/d+6f3p0IQAAANXjvDb+sMtl2eWsfDgsmxyWTc7KR81tnZXrHKZNTtMmZ+VrKx51913faAN5eXmaPn26YmNj1a1bNz322GNet3vllVcUExPjfkRHR8swDL399tuSpJUrV8pms3ls88ADD7TVt/SI0PIaZCzL0sINH+iXw7l11n2csVEX9hihMUn9/FAZAADt0xFPUiBVTFRQY5KCOmNkNdJ2ZNVz/Ouuu05lZWXKyMjQ7t27dfrpp2vQoEGaNGmSx3aXXXaZLrvsMvfzTz75RJdeeqnOPvts97Lk5GRlZWU1+ZT8hfAaREzL1P+tf1OfZGyWt48kbIah9MMHCa8AANTQnHFea75GqpVJrZb3j/XW8nr48GEtW7ZMa9asUVxcnI466ihdddVVeuGFF+qE19peeOEFXXLJJYqKCrz7XOg2EESWZ27Uv7N+rHwj1f0Vz7RMpUYyLBYAADU1pduAs/Ljf0flo6rrgKvWw92lwKzRZcBsvEuCtz6v27Ztk2maGj58uHvZscceq02bNjV4Prm5uXr//fd1xRVX1FmekpKiXr16ad68eTp48GDrfANbGS2vQeTnwmzZZEgyVfE7X80Aa6hPTBedlNzfP8UBANBOWTJ0vf23DW7zV8ezzdxr7W4DFf8m3xJ6jffNcyX19lxUVFSk+HjPRqeEhAQVFjY8rNYrr7yifv36aezYse5lgwcP1vr16zVkyBBlZGTo6quv1uWXX64PPvigKSfjU4TXINIlPFaWJJuhyqlhq984NklX9B8nm0FjPAAANZU6yxUlb0NlVTviPrEtEBMTo4KCAo9l+fn5io2NbfB1S5YsqdPqmpKSopSUFElSjx499NRTT6l///4qLi5ud10LSCpB4rCzTJklh+S0XDKMik7jVX+G2mwaGN9Vk7od5e8yAQBod0Lt4Y1uUzWiwJE+mmPgwIEyDEObN292L1u/fr1HN4La1q1bp02bNmnWrFkN7ttms8myrHY5ChEtr0FgVc5P+t3aV1TmckmSDMOQTRWfUFiyNCFliO4/5iJm0wIAwAvLkv5Q8kIjW3kGz5YMm97QcSalnqJRGuWxLDo6WtOmTdOdd96ppUuXavfu3Xr++ee1ZMmSeo+xZMkSTZo0yd3KWuWLL75Qnz591KtXL+3fv1833HCDJk6cqOjo6OafSBuj5bWDyyw+pFvXvqJy0+Wx3DAkm02y26SukbGKCgnzU4UAALRvlgyZlq1ZD5dpeDxMq+6jer1NLrPh/Vn1TFKwaNEihYaGKjU1VWeeeabmz5/vHmkgJiZGX3/9tXvb8vJyvfrqq/r1r39dZz/r1q3TuHHjFB0drVGjRqlLly5aunRp23xDjxBNbR3YrqJsXb36b3KYTjU0KEekneAKAEB9LEmuZs9w5bm9UePT95Z8EG/W86KEhAQtW7bM67qioiKP52FhYTpw4IDXbW+55RbdcsstLajM9wivHVSpy6Hrvn9BB8sPe1lbdbOWJZtsuiBtpI+rAwAgcFiVLaUteq2qw2pLx3it2A/Tw1YhvHZQOwqzlFNWUOO3O0uGYVTP7FEZYBceM009ozv7o0QAAAJCc2bYqttA6i10Wg2u9aal4bkjIrx2UPbKIa+qftSrJiao6kAeaQ/TI8fM0OjkAX6oDgCAwNFQt4GmhdWGtrHqXVOTtxm2ghXhtYNKjUxQlD1Mh53lMuT5ZrDJ0ILhUzSqUx9/lQcAQMAodzndLZ8tC6sNqf1672F2f0neER6n4yC8dkCWZemO9a+qxFUmw5BqDtEWbgvVjYPP0cTUo+V0Ov1XJAAAAcKwbHK46uk20OoNorVn3qr80ySyVeE70cGYlqmnt/1Law/trJiIQJ6/w3WLitfFvca0y0GHAQBoj+w2e/19Xmv9c9qS8V1r78/bv9CJEfFelgYnwmsHs2jbJ3p99zcey2q+kcpMh48rAgAgsFWN89q0jWt97N+EMFu3Panui+ob5zUYEV47EKfp0uu7v5HL413g+cN+UtJg3xYFAECAsyzJNJsaHuvpw2rUuInaqm/bhmtABcJrB1JmOmRWjijg7YfcLkM3DDrX94UBABDAjmScV3dAPcLwSctrNcJrBxIdEqHk8Hhll+XXCLAV7xa7DI1JGqAIe6hfawQAIBA1J7zW3w2g+V0K3PtkqCw3wmsH8lnWD8opy3M/r35TGOoT01V3DrvYH2UBABDQmtJtoGmjtdYeSaBpr/KyaVAjvHYgD/34z4rAWvkDXvVzHmkP0Qujr1UYra4AADRbxQ1bRq1ltbWkZbRpY7xKUrnLbMH+OybCawfxxf4fdNhVJqm6xbXqh77MdBxpVxsAAIJWidMpV2XLa82brVr/g/x6Zt4ypF2FB1v9aIGqWeHVsiwdOnRI0dHRCg8Pb6ua0Ey7irJ076aXVf1D7/l26hwWq3BaXQEAaJEIe4hMs+5QWbUbhlozzHr0cbWkfrHJrbj3wNbEQcsqOBwOJScn67PPPmuretACD215S6bHO8jyePxu8GR/lAUAQIdgyZBlNv4wKx/uZVbTH6bp+aizf0YbcGtWy2tYWJjS0tLkcrnaqh60wJ7inMrZtKzKjzOqf8BDDZtOSh7qt9oAAAh4VvOGqrJqfWG4/1djm2b25yO8VmtWy6skXXvttfrrX/+q0tLStqgHzZRXflglVX1dVdHf1TAs9+PWIRfJbjT7MgMAgJqslj8sS7LMiodZ+bBasi9IasENW+np6dq2bZt69uyp8ePHq2vXrjJqDFRmGIYef/zxVi0S9Xvsp3fksCpawqtGGqj6+Z7cfYzO736C32oDAKBjaNnH9vW1rla1xJJHW6bZ4fXDDz9UeHi4wsPD9f3339dZT3j1rR8L0j2eGzWmnzu/+2jfFwQAQEfTxG4DTe0KUGMOIUlNm6yAbgPVmh1ed+3a1RZ1oAUcplOHnSWqeAfU/aGOD432eU0AAHQ4lmQ1MknBEe2+KaGXZlo3xnkNYC/uWl4ZXqXaAfb05GOUEpnol7oAAOhIDjvL6wmYbRVo6x7s57zcNjpW4GlReD1w4IAeffRRrV69WpmZmUpNTdWYMWN04403Kikpqdn7Kyoq0qJFi7R27VpFRkZqypQpuvDCC71ue8EFFyg8PNzdz3bo0KG699573etXrVqlF198UQcPHtTgwYN1ww03KDm5442NZlmWPt63WhUDeBiVP+bVP+zn0dcVAIBWEWUPl+VqIKi2eoat3GGNDJsSFdfaBwlYzQ6vq1ev1tlnny3TNHXGGWdo4MCB2r9/v5588kk9+eSTWr58uUaPbl5fy8WLF8vhcGjJkiXKzs7WggULlJaWplGjRnnd/tFHH1VaWlqd5Xv27NHjjz+uO+64Q0OHDtXSpUv10EMP6ZFHHmnuabZ7929+SYccRZIqRxmotT46JNLnNQEA0GE11Oe0LT7Sr9pn5WGj7EwOVaXZ4fXaa6/VsGHD9PHHHysurvq3gPz8fE2aNEnXXXed1xu56lNaWqpVq1bp0UcfVVRUlHr37q2JEydqxYoV9YbX+qxcuVIjR47UiBEjJEkzZszQrFmzlJ6erp49ezZrX+3Z1zkbtDJ7Q40lNbsMWEqLTFL/mG5+qAwAgA7IktTSPq+Nvayx4GvV+hPND6+bN2/WsmXLPIKrJMXHx2v+/Pm65JJLmrW/jIwMWZalXr16uZf16dNH3377bb2vueuuu+RyuTRgwADNmTPHHUx3796tAQMGuLeLiopSSkqKdu/e3aHC69t7vlJVYDWMqo7e1T/VV/WbpBCb3U/VAQDQAbU0PNZ8ndFK+wlyzQ6v/fv3V15entd1+fn56tu3b7P2V1paqqioKI9l0dHRKikp8br9Aw88oEGDBsnhcOjtt9/W3XffraefflpRUVEqLS1VdHR0o/vKzMxUZmamJCknJ0fJycnq2rWrTNNsVu3+sKVgtzbk7/RYVnOIjbiQKB3XaVCj52JZlkzTlGmaHuP0dnRV35dAuNatievN9Q4GXO/gut6+VDWFa8teXOt5iy8R17ZKs8Prww8/rGuvvVY9evTQqaee6l6+cuVK3XvvvXrqqaeatb+IiIg64bK4uFiRkd77bA4fPlySFBoaqpkzZ+qLL77Qli1bNGrUKEVERKi4uLjRfS1evFj33Xef+/ktt9yiW2+9VVlZWc2q3R8+yv5GUuVUsJVfVai4ceu27lOUn3NQ+X6qL1BkZ2f7uwT4ENc7uHC90fosqQW/ExneAqcl97/gzS0BFZodXm+77Tbl5+frtNNOU3x8vJKSkpSTk6P8/HwlJibq9ttv1+233y6pYsKCH374ocH9de/eXZI8+qXu2rWryR/z1/wts1evXtq5s7pVsqSkRFlZWR5dEiRp3rx5uuCCCyRVt7xKUkpKSpOO6U8RhREyDsn90YPl7u9qaEav03Rin2ObtB/LsuR0OhUSEhJUv6mbpqns7GwlJyfLZgueaXO53lzvYMD1Dq7r7dMGJ8to+Iat+l5W75oWXCfCq1uzw+txxx3XqgVERETopJNO0tKlS3XzzTcrJydHy5cv14033lhn2/T0dDkcDvXu3VtOp1P//Oc/VV5erkGDBkmSxo8fr1tvvVXr16/X0KFD9eqrr6p37951gnBqaqpSU1MlSfv27XMvb+9/2WWXHtKu4kxV/QQb7r4zlkJsNp2dekKTz8GyLNlsNtlstqD6y65K1bkHC6431zuYcL3R+izP8FgnSLbW973WjmvsNr+0zOsr8vLyNHfuXH3yySeKjY3V73//e910001etzUMQ1FRUe6fk3HjxumTTz5xr3/rrbd0++23KzMzU2PHjtULL7xQpwGwPWh2eDUMQwsWLFCfPn3qrNu9e7fuu+8+vfDCC83a57x58/TUU09pzpw5ioyM1NSpU90jDUyfPl333HOPhg0bpry8PD3zzDM6cOCAwsLC1L9/f913332KiYmRJPXo0UM33HCDFi1apEOHDmnQoEH6/e9/39xTbJcKHId1w9q/Kt9RJMOwJKui40DVD+CdQ2cqLar5Y+wCAICGFZc7pYbGeW019R/jULH3e4Guu+46lZWVKSMjQ7t379bpp5+uQYMGadKkSV63X7NmjQYPHlxn+ZYtWzRnzhy9/fbbGjdunP7v//5P06dP1+rVq1t2Km2o2eH1xRdf1NVXX+01vB44cEAvvvhis8NrTEyM5s+f73Xdm2++6f766KOP1jPPPNPgvk4++WSdfPLJzTp+IPgie40KncVyWnVbXUMNm/Iqx3wFAACtKzo0VEZT+7y2Rsb10kWgd3zdWTMPHz6sZcuWac2aNYqLi9NRRx2lq666Si+88EK94bU+L7/8ss4++2xNnDhRknT//fcrKSlJmzdv1rBhw1p0Gm2l2Z+rWJZV78cS27dvV+fOnY+4KNSVV14ks9bcdIZR8bAZNhU5vf9GBgAAjlBVn9emPMwaj5a8pt7X1S1r27ZtMk3TfTO7JB177LHatGlTvady2mmnqWvXrjr33HO1efNm9/JNmzbp2GOPdT+PjY1Vv379GtyXvzSp5fWZZ55xt3gahqEZM2bUuYO/tLRUv/zyiy6++OLWrzLIOU2XvsxeK1Om1yHiHJZLQ+N6+6EyAACCw1snjm9w/cWrVtZZZjU0TFY93VuXndTwcWoqKipSfHy8x7KEhAQVFhZ63X7lypUaO3asysrK9Oc//1kTJ07Uli1bFBcXp6KiIiUkJDR5X/7UpPDarVs3dx/UTZs2adCgQUpK8uxfGRYWpiFDhug3v/lN61cZ5L458IP2lR7wsqZipIHZvc/WiMQBXtYDAACf8NIyatReX7MFqhW6F8TExKigoMBjWX5+vmJjY71uXzXEaVhYmBYuXKilS5fqP//5j84++2zFxMQoP99zoM2G9uVPTQqvF154oS688EL387vvvttrn1e0Psuy9Gb6CveQWIZR+bNf+evcuKSjNbP3RL/WCABA0KsdXr0NbdXKw10NHDhQhmF49Etdv369RzeChthsNneeGD58uNavX+9eV1RUpB07djR5X77U7Bu2lixZ0hZ1oB4b8rZr1+EM1f4Vrarb8VHxzZvRDAAANI9hSdO/XNnwNk1Y4qFOn4IK9R1n5jHHaFStZdHR0Zo2bZruvPNOLV26VLt379bzzz/vNatt3rxZZWVlOvroo1VeXq6HHnpIJSUlGjt2bMX+Z87U8ccfr88++0wnn3yy7rnnHh199NHt7mYtqQU3bMG39pflKsR9g5zl8YgPidZ53TreyAoAALQ7zblpq+oGqwYfzd2f9zC8aNEihYaGKjU1VWeeeabmz5/vHmkgJiZGX3/9taSKmedmzJih+Ph49ezZU//973/16aefuvu5DhkyREuWLNHcuXPVqVMnrVu3zmPEp/ak2S2v8K1tBbtlylSIYchV45c0Q4YmpY5WiM3uv+IAAAgCpQ5n04fKaiPbc7zd+1JxU9WyZcu8risqqh5Gc8KECdq6dWuDx7j44osD4sZ7wms7tqc4S59krXI/t9f4pSvKHq7pPc/0Q1UAAASXCHuI5M/wakn9Ezv5sYD2hfDaju0pzpLdsMll1X3H9I7uruiQSC+vAgAArc1o5ZutmiuYpjxuDOG1HesUFi/TS3CVpJQIJoMAAMAnLB1BtwHD4w/P/TYjEfu520J7Qnhtx0pcpTWGhKsaIE4Kt4VqRq+z/FcYAADBpsUtr9YRvh61EV7bqQJHkR748TkZhiWbVX1zoiHpzJTjlRqZ1MgeAABAazAsyTBbYVaBI6rBv8dvTwiv7dQvhzNUbjkqJiVw/7xW/Nr2c1G63+oCACDYWO7/NV1jUbPZDbG03LoRXtupSHtEi9YBAIDWZaiBG7ZaGCrrDbf1rPD3DWPtCbeutVN9Y3poUExvr+suSjvDt8UAABDESsudFTdMeXkYVus+6jtOxqECP519+0N4bafshk33Dr9W47qMUpgRKslS57AE3TV0nkYmDvV3eQAABI0wm02GS14fcqnewNmSh1H7UXmcCCYlcqPbQDsWFRKp3w2+QjllB3X1mrv052NuUVI4gxQDAOBLdput4Y/ta6470vuq6jlOl5joI9xxx0F4BQAAaEjVR/pN3PZIjiOpnjFhj2C/HQzhtZ0qcZXquZ9f1X8PrpfTckmy9N8D63R+99P9XRoAAMGnLcJjfWGVoNogwms7ZFmW/vzjM9pUuF2We3RXQ//Y/U8lhMVpXNLx/i4RAICgckR3+zfUolpzfVsdv4Phhq12aF/pfm0q3Fb5s+z5k/7evhX+KAkAgODVwCgATboJq5GRBJr0ILy60fLaDpW5yiu/qp4StkqJq8zn9QAAEMwaHOdVanKwbPReroY2ILy6EV7biR/zf9K/s79SuVmuYXFDFWOPUqGrWDUDrCFDIxIYJgsAAJ+rHR7bsg9slRph1uF0tcEBAxPhtR3438H1+uu2Z6TKHq5rD21U57AkFbmKPX6Oe0V116xeU/xUJQAAwam0zCHDrHzijxZQQ9qbc8gPB26fCK/twNLdyypjawWXZWp/WY5sqnqPVK6zHAq3h/mhQgAAgldEaGjFhASNOdIxXmuqFZL7JXdpxZ0HNsJrO5Dv8D7lm2F4vg/ynYW+KQgAALg12ue1ktUKrbIGw2c1ivDqZ+nFe1Vulje6nd2wqXd0mg8qAgAAHqpGCmhEgw2vzQ2jvuhjG6AIr372wq5XPboMSBUtrrJMGbLLbthlyVJsSIzm9ZvlnyIBAAhyRzzO6hG+vjV7JAQ6wquf7S/N9rrcMKRbBs5TgaNQ4bZwjUgcruiQKB9XBwAAZKllfQLcLzEqxw6qsY/mptHW6JPQQRBe/axrRLLyvPR5NWRoaNxAAisAAO2A0YRuA/W3rlrN61LQrH0HH2bY8rNf95mhMCO08ln1T+ac3pcSXAEAaAeqZsiq8zBrPerbrrFHE/eDCrS8+lnPqDQ9csx9+mfGB/oy5z8akXCUzkk9U8PjB/u7NAAAIKmkzOH1hq226ofqrYfAL5kH2+hogYfw6meWZanMLFPfqJ5aZazSjJ5TlRbVzd9lAQCASpFhTRzntZXUCcWWlBof67sC2jnCqx+Vmw49vPVRbS3c5l52z+aFun3wzRoYO8CPlQEAADeriX1eqxxJk2w93QMiwkK9rwhC9Hn1k2Jnse7YcI9HcJWkcrNcj/z0hEpcpX6qDAAA1FQ1SUFr92Ft1rb+/ia0I4RXP/ho3780b80Nyi7zPkxWiatEGcUZPq4KAAB45S1kNuMhV+Wj5teVjybvo56W37y8PE2fPl2xsbHq1q2bHnvsMa/b/fe//9VZZ52lzp07q3Pnzjr33HO1fft29/qVK1fKZrMpJibG/XjggQda8ZvYeug24GPfH1yjN/b8s9Htwu0RPqgGAAA0zmr5OKu1W01b3ITq/fjXXXedysrKlJGRod27d+v000/XoEGDNGnSJI/tDh06pF//+td68803FRkZqQULFuiCCy7Qli1b3NskJycrKyurpQX6DOHVx9Yd+qHRbdIiu6t7ZKoPqgEAAE3R5KGqPLbzklStWhs1Mcx6O/7hw4e1bNkyrVmzRnFxcTrqqKN01VVX6YUXXqgTXms/v/XWW/XQQw8pNzdXnTt3bloR7QTdBnwsxGj494VQI1R3DPmdbAaXBgCA9sCwVPGxfVMeHn1VrXoeNcZubcZ+a9u2bZtM09Tw4cPdy4499lht2rSp0XP68ssvlZKS4hFcc3NzlZKSol69emnevHk6eLB9Ds9Fy2sbyy7N1t92/U27D+9WdEi0Tu58iupr+g9RiP501L2KC2U4DAAA2gvTlBb9bnyD21z30MoW7bt2w+tTv2/4ODUVFRUpPj7eY1lCQoIKCwsbfN3OnTt13XXX6YknnnAvGzx4sNavX68hQ4YoIyNDV199tS6//HJ98MEHTa7HVwivbajEVaIHtz6ofEe+TJnKc+Tpw8wPZMgmyx1gK35se0X10PUDrlHXiGT/FQwAAOooL3VIjYxU1ayhtFpJTEyMCgo8p5jPz89XbGz9jWB79uzRGWecodtvv12XXHKJe3lKSopSUlIkST169NBTTz2l/v37q7i4WFFR7WvGTz6bbkPrD63XIcchmTVuETQlybBkM1T5sBRms2tC8jiCKwAA7VBERONjrBqm1SqP5hg4cKAMw9DmzZvdy9avX+/RjaCmvXv36rTTTtPcuXN1yy23NLhvm80my7JktfRGtTZEy2sbcZpOvZvxliq6CNT+UMBzmSFDDsvpw+oAAECTWdKNC79ocBOv9101dWSBGvmwvuNMOftYjRrluSw6OlrTpk3TnXfeqaVLl2r37t16/vnntWTJkjqv37dvnyZMmKCZM2dq/vz5ddZ/8cUX6tOnj3r16qX9+/frhhtu0MSJExUdHd3Ek/AdWl7byK7DO5RbfqDyWfVPpSFTtX+aXZZLw+KG+q44AADQZBWTFNR381UDj9otqzVv5PJY3oRHPffLLFq0SKGhoUpNTdWZZ56p+fPnu0cWiImJ0ddffy1J+tvf/qaff/5ZDz/8sMdYrunp6ZKkdevWady4cYqOjtaoUaPUpUsXLV261Cff3+ai5bWNlJqlshmGLMuSWbOV1ZA6hyYotzxfliyFGqGa1+9K9YhK82O1AACgXs2dHrahHbXySxMSErRs2TKv64qKitxf33PPPbrnnnvq3f0tt9zSaFeC9oLw2kZ6RfVWiOwyDadsNcZ0sxmGxiefotOTz1S+o0CdwhIVbg/3a60AAKARzc2dVs1PXb3sxmjmbAXtr+up39BtoI3EhcbrmgE3KNQIkWFYlQ/p2IQRmpRyrqJCopQamUJwBQCgnTOs5t+QZbPkftQc19W9rLk3cxFe3Wh5bUNHxR+jR455XOvz1uilX/6mGwfM15C4YTKa+9sWAADwqzrh8QjDZKNJoNYGhNdqhNc2Fh0SoyFxw2S3SSmRKQRXAAACTFmpQ6o1jFVb/2tee4SqXbuy2/iIgYPwCgAA0IDwsBAZzkbu2GrlxilD8kiwfXp0rnfbYEN4BQAAaESjH9vXbiptSZZt4BhGm7f1Bg7CKwAAQAOqxmVtjjpbe8uezcm77XCmK38hvAIAADSmmeGxThAle7YawmsbynfkaUvBRmWVZkiSTKtVRjgGAAC+ZElq0T/hlsdXNSeGbzYihBvhtY1szv9Bz+54VC453cue2/mYbhl4tyLsEX6sDAAANJfR1JbXejYzGtqoCVm2vulhgxGTFLSBw84iPbfzMY/gKkkZJXv0bsbrfqoKAAC0iNWcyQRa8GjCfsmu1Wh5lWSz2WSz2WS1UmfofSV75bAcdZZbsrS9aEurHedIVNXQHmrxJcuyFBISIsuygurcud5c72DA9Q6ec5akkBDfRZiyUkd1ePT197lyCK6sjDzfHrcdI7xKioiIUHR0tJxOZ+MbN0FhaX696yJtrXec1uByufxdgs8lJibKNE2ZZvB1IOJ6Bxeud3AJtuudmJjos2OFhdgbH+e1zVSE5cgwu5+O3/4QXiWVlpaqvLxcSUlJR7yvnNL9WrLnSVX/iubZkeWc1Mk+/W2xPpZlyeVyyW63B9WsX6ZpKjc3V507d5bNFjy9ZrjeXO9gwPUOruudk5Pjs2PZbIYMl/fwWndIrCO8BrVadqv2lpgQdWT77UD8n6Lagarf0o/0TV/kKNSftsyXZMmmqhsDq38I40PiNDT+6CM6RmszDCOo/rIzDENOpzPozrtKsJ031zu4zpvrHVzn7dNPMS3V212g7pBYtW/GasI1aUpXhODqFdIgwmsr+tuOv8ipir6uhqTaDfxR9mif1wQAAI6UJTVzkgKP11apCrIt6TcbZH2aG0J4bSWHHUXaWby9wW1OTZ7oo2oAAECrsXQE4VU1Bnm1ag/42rx9QBLhtdW8u+/lBtcPihmucV3O9FE1AACg1VjNbHmtZxjXJk0Z21ANkER4bRUu06XvDn5VY4nnT2NqRJp+O+D2oOqLBABAR9LgJAVNDJaNThnbUE4gvLoRXlvBv/e/L8mSUeP/Nd0y6B7ZjOC58xUAgI7EqNNtoHrSgJb2AvDGqkwR1QeufuYorTt+fLAivB6hLQXr9HHWW5IqfsYMWbJqdGkZGDtMEXaGtwAAIFCVlpRXDJXlpfWzNT9TrXdfhqGs3bmteKTARng9Apvz1+jvux6RVev3rooQW2F6j1/7pTYAANA6IiJCpSZNAtGaUdby+KNnvyMfi76jILy2UHbpPr2W/kxlcPWuc2iyksJTfVgVAABodU2+YaueO7WadIzaX9R68ZGMdtDBEF5bYEfRj1q84w9yWKYa+sm8tNdV3KQFAECgsyS1xpTDNTNBozdg1VrPDVtuhNdmynfk6rkdf5Cpqh/iqi4Dnl0Homwx6hs92A8VAgCA1mW1TngkgLYKwmszrcx+T67K4Fo1Bazhvj+w4ocyzBahuf1uU4iNby8AAAHPslrW8tpQWG3uJ7NWK7T8dhCkq2Y6WJatqlZWw5BslQ2ullXR+9Vm2HTzwPvVLbKnfwsFAACto6ndBprVsFpz2tgmbE6fVzfCazM4zDJlFG9Xzc7UVb84VYwwYGh6j3kEVwAAOhqv4bGVAmVTpt4iu7oRXpvh2wMf6bAr3+tkBN0j+urSnr9VKsEVAIAOpfRwaevcsNVkdZPqnp/2+fD47RvhtRlyyvbJMizZLEOWTPc8GDZJJ3QeR3AFAKADCo8Mk+V01ru+rUYWsmr0mU1OS2yTYwQiwmszJIYlyyZDpmHJqNHyashQQiiDBwMA0CE1csNW3U/9jyDM1nOTV1g4ka0K34lmOLHLufoud7nynHmqOTRW/5ijNCTuOH+WBgAA2owl6wi6DTTUMms1dfgsbthyI7w2w7+zXlKZK1uhMuSSTZI0JO54zeh1u2yGzc/VAQCANnGEkxRYUuWd3UZ1y2pzx3xljFg3wmsTfZDxhNYdWiHDMBRqWApVxRzHu4r+qz3FW9Q7erifKwQAAG2ipeO8qm7Laov7xzLOqxvhtQnSD2/SukMrvK4zZGjP4R8JrwAAdFhH1m3AY08tbEFt6es6IsJrE6QXb5JROY9WbZakCHu074sCAAC+cYTdBlqnBsJrFcJrE4QaEQ2sC9Ow+HE+rAYAAPhUrW4DvoyRVZ0MykvKfXjU9o3w2gRD40/RV9mvqsQsVc1RBiRDl/W+X1EhcX6sDgAAtKWjxw/V0eOGKCI6XIld42UPsfvkuJZlqSC3UAcyDurMWTSUVSG8NkFsaCf9ut9f9e7evyizZJdMudQlvKem9rhNXSN6+7s8AADQhuI6xeqRL+71dxmoxPhOTdQ5PE2/6feorh/0nEJtLl3W+x6CKwAAgI8RXgEAABAw6DbQBPnl+/XBvr9oX/FPslTRYTun9BfFhyb7uTIAAIDgQstrIw6V7dPin6/SnuJNcskhUy5Jlv65d6Fyy/b6uzwAAICgQnhtgMMs08u/3CqXHLXWGDItl37I+9QvdQEAAAQrwmsDdhX9T8Wu/HrWWio3S3xaDwAAQLAjvDagzCxWQzMQ94w6yme1AAAAgPDaoNTIgZKqpiTwnE+jd9QxGhJ3is9rAgAACGaE1wZ0Ce+ls1NvlE0V3yhDlgxZGhgzWpf2+qMMo6F2WQAAALQ2hspqxNGJZ6l3zAhtzf9KX2U/p4t6/FF9Yo4juAIAAPgBLa9NEBearMHxp8huSF0iehFcAQAA/ISW10bsK96snwq+UKEjR5Ilp1nu75IAAACCFuG1Ad8feE3f5Lwg1Rhz4N09d+myPosUbo/2X2EAAABBim4D9cgt261VOX+vtdRQviNT/z3wsl9qAgAACHaE13ocLNstQ3YvayzllO7weT0AAAAgvNYrOqSzLLm8rosNTfZxNQAAAJAIr/VKjRyqfjEnqqK/a/UEBaFGhEZ3ucxvdQEAAAQzwms9DMPQuWn3aFzyb9QpLE02udQ/5kTN6vucEsJS/V0eAABAUCK8NsBm2HVc50s0teefFGsv04SUqxUfluLvsgAAAIIW4RUAAAABg/AKAACAgEF4BQAAQMAgvAIAACBgEF4BAAAQMEL8XUB75DCL9W32o9pb9K0cVomMyoy/9/B/NThhsn+LAwAACGKE11osy9SKjNu1v2RDnRm2/pP9F0WFJKlnzEl+qg4AACC40W2glnzHHmWVrKtnalhLmw+94fOaAAAAUIHwWovTLGlwvcMs9lElAAAAqI3wWkt8WC+F2+K8rrMpRGnRY31cEQAAAKoQXmsJtUVqWMLF7pu0auoZM07Hdr7cD1UBAABAaic3bBUVFWnRokVau3atIiMjNWXKFF144YV1ttu6datee+01/fzzz5KkQYMG6corr1S3bt0kSRs3btRdd92l8PBw92umTZum6dOnN7mWH3L/ro2H/ia7JMu91KZxKfeoT+xEGYbRwrMEAADAkWoX4XXx4sVyOBxasmSJsrOztWDBAqWlpWnUqFEe2x0+fFhnnHGGfv/73yssLEyvvPKKFi5cqKefftq9TXx8vF566aUW1VHuKtSGg3+TJBmGVB1TTf146GX1jTurRfsFAABA6/B7t4HS0lKtWrVKs2bNUlRUlHr37q2JEydqxYoVdbYdNWqUxo0bp+joaIWGhmry5Mnau3evCgoKWqWWElduveuKXTmtcgwAAAC0nN9bXjMyMmRZlnr16uVe1qdPH3377beNvnbTpk1KTExUXFz1DVaFhYWaPXu2QkNDNXLkSM2ePVuxsbFNqiU6JEU2hclUeZ11CWH9mrQPAAAAtB2/h9fS0lJFRUV5LIuOjlZJScNDVmVlZWnx4sWaO3eue1laWpoef/xxpaWlKTc3V88884wee+wxLViwwOO1mZmZyszMlCTl5OQoOTlZXbt2lU1hOqnrPfp6/12q6PFqSTIUZovV6KTbZZpma5xyu2BZlkzTlGmaQdWPt+oadqRr2RRcb653MOB6B9f1RvDye3iNiIioE1SLi4sVGRlZ72tycnK0YMECTZ06VePGjXMvT0xMVGJioiQpKSlJc+fO1dVXX62ysjKPm7gWL16s++67z/38lltu0a233qqsrCyFaYhGRz6qvY6Plen8RD1DpqtH2Pk6nBuiw8pqrdOGn2VnZ/u7BPgQ1zu4cL2Bjs3v4bV79+6SpPT0dPXs2VOStGvXLvfXtR04cEB33XWXzjrrLE2ePLnBfdtsNlmWJcuyPJbPmzdPF1xwgaTqlldJSklJqdwiRd2cPfXJno80ovtligpJUUdjWZacTqdCQkKC6jd10zSVnZ2t5ORk2Wx+7/LtM1xvrncw4HoH1/XOyqJBKVj5PbxGRETopJNO0tKlS3XzzTcrJydHy5cv14033lhn29zcXN15550aP368pk2bVmf9hg0b1LVrVyUnJysvL0/PPfecjj32WEVERHhsl5qaqtTUVEnSvn373Mtr/mVnq/wLwGYYHfIvQcuyZLPZZLPZguovuypV5x4suN5c72DC9QY6Nr+HV6miJfSpp57SnDlzFBkZqalTp7qHyZo+fbruueceDRs2TMuXL1dmZqbeeecdvfPOO+7XL1q0SElJSdq5c6cee+wxFRQUKCYmRiNHjtTllzOpAAAAQEfRLsJrTEyM5s+f73Xdm2++6f76V7/6lX71q1/Vu5/Jkyc32pUAAAAAgSt4PlcBAABAwCO8AgAAIGAQXgEAABAwCK8AAAAIGIRXAAAABAzCKwAAAAIG4RUAAAABg/AKAACAgNEuJiloLyzLpa0Hn9f2gjflMA9LspReuEKDO832d2kAAAAQ4dXDDwce1s8Fy2TJkmRIMrTx0BMKsUeof/x0f5cHAAAQ9Og2UMlllWpHwRuyJFUE1yqWthx63j9FAQAAwAPhtZJLJfWuc5jFPqwEAAAA9SG8VgpVgiJtyZXPLI91XSKO9n1BAAAAqIPwWskwDJ3U7UmFG9GVSyoCbHRImo5Pvs9/hQEAAMCN8FqDJVPdo8cpPiRNdpk6qtN1OqvnMkWGJPm7NAAAAIjRBtzyXKv1fcb9qmpxtRvST4eeVKfwIUqOGuPf4gAAACCJlldJkmVZ2uF4RLX7ukqm1uXc7Y+SAAAA4AXhVZKpMpk67HVdqStbluXycUUAAADwhvAqyaZw2RTldV24PUmGYfdxRQAAAPCG8CqpzNoru5yq6DZQ9ZAkQyOTGGkAAACgvSC8Strt/JNkFMsulwz3HFumou0JSo460c/VAQAAoAqjDUgqsXZIkmyGZJPpXl5u5shpHlaILbq+lwIAAMCHaHmV6u3vaihEdiPCx9UAAACgPoRXSd1CrvSy1NCAhGu5WQsAAKAdoduApC72c2WzonXAeFnFzt2KtKepX8I89Yid4u/SAAAAUAPhtVKi/RT1ThqjbzPO1ZhuSxQZ0s3fJQEAAKAWug1Iyij/i34sP1PfZpwrSdqQfYscrjz/FgUAAIA6CK+S8qxPPJ4XlP2g9fuvl2XVni4WAAAA/kR4lVQ9KUG1vLL/yWHm+b4UAAAA1Ivw2iCz8U0AAADgM4RXSZJRZ0lc2NEKtXXyQy0AAACoD+FVUpxxqufzsOE6NmWRDKNuqAUAAID/MFSWpB5hC1RmztEOx2yN7f6xosP6+bskAAAAeEHLayWbESabIdltkf4uBQAAAPUgvEra47hLux03SJKyD69giCwAAIB2ivAq6bC1Sk5lS5J+PvSAduY97ueKAAAA4A3h1Yvd+c/I4Trk7zIAAABQC+G1HmWuHH+XAAAAgFoIr14YClVESHd/lwEAAIBaCK9uVTdpGRqW9LBCbNF+rQYAAAB1Mc6rpCTbTJVbSxQbPUvd4i5TTNhAf5cEAAAAL2h5ldQ55Fz1Di1Xv4S5BFcAAIB2jPAKAACAgEF4BQAAQMAgvAIAACBgEF4BAAAQMAivAAAACBiEVwAAAAQMwisAAAACBuEVAAAAAYPwCgAAgIBBeAUAAEDAILwCAAAgYBBeAQAAEDAIrwAAAAgYhFcAAAAEjBB/F9Ae2AybZEmWLFmW5e9yfKLqPIPlfKtYlqWQkBBZVvBca4nrzfUODlzv4DlnSQoJIcIEK668pPDwcJmlksvplGU5/V2OT7lcLn+X4HOJiYkyTVOmafq7FJ/jegcXrndwCbbrnZiY6O8S4CeEV0llZWUKlWQPCZHdHhzfEsuy5HK5ZLfbZRiGv8vxGdM0lZubq86dO8tmC55eM1xvrncw4HoH1/XOycnxdwnwk+BIao0wrYrf0A0ZQfXGlyTDCK5zNgxDTqcz6M67SrCdN9c7uM6b6x1c5+10BtcnpagWPL+aAgAAIOARXgEAABAwCK8AAAAIGIRXAAAABAzCKwAAAAIG4RUAAAABg/AKAACAgEF4BQAAQMAgvAIAACBgEF4BAAAQMAivAAAACBiEVwAAAAQMwisAAAACBuEVAAAAAYPwCgAAgIBBeAUAAEDAILwCAAAgYBBeAQAAEDAIrwAAAAgYhFcAAAAEDMIrAAAAAgbhFQAAAAGD8CopRHdKkhzl//NzJQAAAGgI4VWSTRslSYfzrlVx4WI/VwMAAID6EF4lGTW+LilcKNOV5bdaAAAAUD/Cax2WXK69/i4CAAAAXhBevbDZu/m7BAAAAHhBeK1kVf4ZGXOz7IRXAACAdinE3wW0B3YZcmikYhOvVnjkJH+XAwAAgHoQXiXF2SKVbd6r8MgR/i4FAAAADaDbAAAAAAIG4RUAAAABg/AKAACAgEF4BQAAQMAgvAIAACBgEF4BAAAQMAivAAAACBiEVwAAAAQMwisAAAACBuEVAAAAAYPwCgAAgIBBeAUAAEDAILwCAAAgYBBeAQAAEDAIrwAAAAgYhFcAAAAEDMIrAAAAAgbhFQAAAAGD8AoAAICAQXgFAABAwCC8AgAAIGAQXgEAABAwCK8AAAAIGIRXAAAABIwQfxfQ2oqKirRo0SKtXbtWkZGRmjJlii688MImvNJq89oAAABwZDpceF28eLEcDoeWLFmi7OxsLViwQGlpaRo1alSDr+tsXC/LsURG6EAfVQoAAIDm6lDdBkpLS7Vq1SrNmjVLUVFR6t27tyZOnKgVK1Y0+lqbcmUdulyWWeSDSgEAANASHSq8ZmRkyLIs9erVy72sT58+Sk9Pb/S1hmFK5kHJsbEtSwQAAMAR6FDdBkpLSxUVFeWxLDo6WiUlJR7LMjMzlZmZKUnKyclRyjEVyy1LOph7UOXa55N6/clutyshIUHZ2dlyuVz+LsfnsrKy/F2CT3G9ud7BhOsdHA4ePOjvEuAnHSq8RkRE1AmqxcXFioyM9Fi2ePFi3Xfffe7nvXv31kUXXaTY2FhJ31c+0BEVFhZqzZo1GjVqVOX1RkfG9Q4uXO/gUlhYqHXr1unSSy9Vt27d/F0OfMiwLKvD3GZfWlqqGTNm6LHHHlPPnj0lSS+//LL27t2r+fPnu7er2fK6ZcsWzZw5U//617901FFH+aVu+M7GjRt19tlnc72DBNc7uHC9g0vV9V6zZo1Gjhzp73LgQx2u5fWkk07S0qVLdfPNNysnJ0fLly/XjTfe6LFdamqqUlNTPZYlJSXxm1sQqPo4kesdHLjewYXrHVyCrXsIqnWo8CpJ8+bN01NPPaU5c+YoMjJSU6dObXSYLAAAAASGDhdeY2JiPLoINCY1NVX33HNPnZZYdExc7+DC9Q4uXO/gwvUOXh2qzysAAAA6tg41zisAAAA6NsIrAAAAAkaH6/PaHEVFRVq0aJHWrl2ryMhITZkyRRdeeKG/y0IzNfU6bt26Va+99pp+/vlnSdKgQYN05ZVXuu9K3rhxo+666y6Fh4e7XzNt2jRNnz7dNyeCJmnO+/aCCy5QeHi4DMOQJA0dOlT33nuve/2qVav04osv6uDBgxo8eLBuuOEGJScn++I00ERNvd4rV67U008/7X5uWZbKyso0f/58nXjiiby/O5gPP/xQn3/+uX755ReNHTtWt912m79Lgg8FdXhdvHixHA6HlixZouzsbC1YsEBpaWmMThBgmnodDx8+rDPOOEO///3vFRYWpldeeUULFy70+AcvPj5eL730kq9PAc3Q3Pfto48+qrS0tDrL9+zZo8cff1x33HGHhg4dqqVLl+qhhx7SI4880tangGZo6vUeP368xo8f736+Zs0aPfzwwx7b8f7uODp16qTp06dr/fr1Kiws9Hc58LGg7TZQWlqqVatWadasWYqKilLv3r01ceJErVixwt+loRmacx1HjRqlcePGKTo6WqGhoZo8ebL27t2rgoICP1SOlmjN9+3KlSs1cuRIjRgxQuHh4ZoxY4Z27dql9PT0NqgcLXEk13vFihU6+eSTPVpa0XGceOKJGjNmjOLi4vxdCvwgaFteMzIyZFmWevXq5V7Wp08fffvtt36sCs11JNdx06ZNSkxM9PjLr7CwULNnz1ZoaKhGjhyp2bNnM81kO9KS633XXXfJ5XJpwIABmjNnjnv2vd27d2vAgAHu7aKiopSSkqLdu3e7t4F/tfT9XVBQoO+++04PPPCAx3Le30DHENQtr1FRUR7LoqOjVVJS4qeK0BItvY5ZWVlavHixrrzySveytLQ0Pf744/rHP/6hBx98ULm5uXrsscfaomy0UHOv9wMPPKC//e1vWrx4sfr27au7775bxcXF7n1FR0c3eV/wvZa+v7/88kulpqZq8ODB7mW8v4GOI2jDa0RERJ2/AIuLixUZGemnitASLbmOOTk5WrBggaZOnapx48a5lycmJqpnz56y2WxKSkrS3LlztWbNGpWVlbVZ/Wie5l7v4cOHKzQ0VFFRUZo5c6bsdru2bNni3ldVkG3KvuB7Lf17+rPPPtPpp5/usYz3N9BxBG147d69uyR59G/btWsXHxcGmOZexwMHDuiuu+7SWWedpcmTJze4b5vNJsuyxDwe7ceRvm+rRh2QpF69emnnzp3u5yUlJcrKyvL4iBr+1ZLrvWPHDqWnp2vChAkN7pv3NxC4gja8RkRE6KSTTtLSpUtVXFys3bt3a/ny5TrzzDP9XRqaoTnXMTc3V3feeafGjx+vadOm1Vm/YcMG7d+/X5Zl6dChQ3ruued07LHHKiIiwhengiZozvVOT0/Xjh075HK5VFZWpldffVXl5eUaNGiQpIq709euXav169ervLxcr776qnr37s0vsO1IS/6e/ve//61Ro0YpMTHRYznv747F5XKpvLxcpmnKNE2Vl5fL6XT6uyz4SFBPD1tUVKSnnnrKPX7gRRddxDivAaih6zh9+nTdc889GjZsmF577TW99tprdf6xWrRokZKSkvTuu+/q/fffV0FBgWJiYjRy5Ehdfvnlio+P98dpoR5Nvd4bNmzQM888owMHDigsLEz9+/fXnDlz1KdPH/e+vvnmG7344os6dOiQBg0apBtvvJFxXtuZpl5vSXI4HJozZ46uv/56jRkzxmM/vL87lldffVWvv/66x7LTTjtNN910k38Kgk8FdXgFAABAYAnabgMAAAAIPIRXAAAABAzCKwAAAAIG4RUAAAABg/AKAACAgEF4BQAAQMAgvAIAACBgEF4BAAAQMAivANq1Rx99VD179pTdbtfkyZP9XQ4AwM+YYQtAu7V9+3YNGjRIt99+u84//3x16dJFAwcO9HdZAAA/CvF3AQBQn59++kmWZemqq65S3759j2hfJSUlioyMbKXKAAD+QrcBAO3SnDlzdP7550uS+vXrJ8Mw9I9//EOGYejjjz/WRRddpOjoaKWmpuqBBx7weO29996rmJgYfffddxo7dqwiIiK0aNEif5wGAKCVEV4BtEsLFizQn//8Z0nS22+/rW+//VbR0dGSpLlz56pfv356++23NXPmTN1555169tlnPV5fXl6uGTNmaObMmfrkk080ceJEn58DAKD10W0AQLvUr18/d//WESNGqHfv3iotLZUknXbaaXr44YclSWeddZb279+vhQsXau7cubLZKn4ndzgc+uMf/6hLLrnEPycAAGgTtLwCCDhTpkzxeD5t2jRlZGRo7969HsvPPfdcX5YFAPABwiuAgJOcnOzxvGvXrpKkzMxM97KoqCjFxMT4tC4AQNsjvAIIONnZ2R7P9+/fL0lKTU11LzMMw6c1AQB8g/AKIOC88847Hs/feustdevWTWlpaX6qCADgK9ywBSDgfP7557rtttt05plnasWKFVq6dKkWLVrkvlkLANBx8Tc9gICzePFibdu2TVOmTNHSpUv1hz/8Qb/97W/9XRYAwAeYHhZAwFi5cqUmTJig77//Xscdd5y/ywEA+AEtrwAAAAgYhFcAAAAEDLoNAAAAIGDQ8goAAICAQXgFAABAwCC8AgAAIGAQXgEAABAwCK8AAAAIGIRXAAAABAzCKwAAAAIG4RUAAAABg/AKAACAgPH/Z8tZoidXlp8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x644.156 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<ggplot: (-9223372036464088060)>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'fpr': fpr, 'tpr': tpr, 'threshold': thresholds}).loc[lambda x: x.threshold <= 1]\n",
    "(\n",
    "    ggplot(df) +\n",
    "    geom_line(aes('fpr', 'tpr', color='threshold')) +\n",
    "    geom_point(aes('fpr', 'tpr', color='threshold')) +\n",
    "    theme_bw() +\n",
    "    coord_equal(xlim=(0, 1), ylim=(0, 1), expand=False)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ROC-AUC scores computed via SKLearn when the scores have different magnitudes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8325792944120902"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8325792944120902"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8325792944120902"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8325792944120902"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_true, scores)\n",
    "roc_auc_score(y_true, scores / 10)\n",
    "roc_auc_score(y_true, scores / 100)\n",
    "roc_auc_score(y_true, scores / 1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ROC-AUC scores computed via TF\n",
    "\n",
    "Note how the calculated AUC value differs from the actual when scores have smaller magnitude (as they might be, in case of a high imbalanced problem with calibrated scores). Also note the effect that various parameters to the method have."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_roc_auc(y_true, scores, summation_method='interpolation', num_thresholds=200, thresholds=None):\n",
    "    m = tf.keras.metrics.AUC(curve='ROC', summation_method=summation_method, num_thresholds=num_thresholds, thresholds=thresholds)\n",
    "    m.update_state(y_true, scores)\n",
    "    return m.result().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.83256525"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.83324707"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.7551299"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# with default parameters\n",
    "calculate_roc_auc(y_true, scores)\n",
    "calculate_roc_auc(y_true, scores / 10)\n",
    "calculate_roc_auc(y_true, scores / 100)\n",
    "calculate_roc_auc(y_true, scores / 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.83260936"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8328159"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8295231"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# with number of thresholds increased from 200 to 1000\n",
    "calculate_roc_auc(y_true, scores, num_thresholds=1000)\n",
    "calculate_roc_auc(y_true, scores / 10, num_thresholds=1000)\n",
    "calculate_roc_auc(y_true, scores / 100, num_thresholds=1000)\n",
    "calculate_roc_auc(y_true, scores / 1000, num_thresholds=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8325773"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8325733"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8327377"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.82940674"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# with number of thresholds increased from 200 to 10,000\n",
    "calculate_roc_auc(y_true, scores, num_thresholds=10000)\n",
    "calculate_roc_auc(y_true, scores / 10, num_thresholds=10000)\n",
    "calculate_roc_auc(y_true, scores / 100, num_thresholds=10000)\n",
    "calculate_roc_auc(y_true, scores / 1000, num_thresholds=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.83256525"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.83256525"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.83256525"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.83256525"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# if we know the general extent of score values, we can manually specify the thresholds\n",
    "calculate_roc_auc(y_true, scores, thresholds=np.linspace(0, 1, 200))\n",
    "calculate_roc_auc(y_true, scores / 10, thresholds=np.linspace(0, 1/10, 200))\n",
    "calculate_roc_auc(y_true, scores / 100, thresholds=np.linspace(0, 1/100, 200))\n",
    "calculate_roc_auc(y_true, scores / 1000, thresholds=np.linspace(0, 1/1000, 200))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8307423"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.81589824"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.5699411"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8343882"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.850596"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.9403186"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# effect of minoring and majoring summation methods\n",
    "calculate_roc_auc(y_true, scores, summation_method='minoring')\n",
    "calculate_roc_auc(y_true, scores / 10, summation_method='minoring')\n",
    "calculate_roc_auc(y_true, scores / 100, summation_method='minoring')\n",
    "calculate_roc_auc(y_true, scores / 1000, summation_method='minoring')\n",
    "\n",
    "calculate_roc_auc(y_true, scores, summation_method='majoring')\n",
    "calculate_roc_auc(y_true, scores / 10, summation_method='majoring')\n",
    "calculate_roc_auc(y_true, scores / 100, summation_method='majoring')\n",
    "calculate_roc_auc(y_true, scores / 1000, summation_method='majoring')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PR-AUC scores computed via SKLearn when the scores have different magnitudes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8224200411143863"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8224200411143863"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8224200411143863"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8224200411143863"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average_precision_score(y_true, scores)\n",
    "average_precision_score(y_true, scores / 10)\n",
    "average_precision_score(y_true, scores / 100)\n",
    "average_precision_score(y_true, scores / 1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PR-AUC scores computed via TF\n",
    "\n",
    "Note how the calculated AUC value differs from the actual when scores have smaller magnitude (as they might be, in case of a high imbalanced problem with calibrated scores). Also note the effect that various parameters to the method have."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_pr_auc(y_true, scores, summation_method='interpolation', num_thresholds=200, thresholds=None):\n",
    "    m = tf.keras.metrics.AUC(curve='PR', summation_method=summation_method, num_thresholds=num_thresholds, thresholds=thresholds)\n",
    "    m.update_state(y_true, scores)\n",
    "    return m.result().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8223197"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8215739"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.7005306"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.474"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# with default parameters\n",
    "calculate_pr_auc(y_true, scores)\n",
    "calculate_pr_auc(y_true, scores / 10)\n",
    "calculate_pr_auc(y_true, scores / 100)\n",
    "calculate_pr_auc(y_true, scores / 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8222167"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8227184"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.81962895"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.474"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# with number of thresholds increased from 200 to 1000\n",
    "calculate_pr_auc(y_true, scores, num_thresholds=1000)\n",
    "calculate_pr_auc(y_true, scores / 10, num_thresholds=1000)\n",
    "calculate_pr_auc(y_true, scores / 100, num_thresholds=1000)\n",
    "calculate_pr_auc(y_true, scores / 1000, num_thresholds=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8221786"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.82214785"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.82241213"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.81737304"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# with number of thresholds increased from 200 to 10,000\n",
    "calculate_pr_auc(y_true, scores, num_thresholds=10000)\n",
    "calculate_pr_auc(y_true, scores / 10, num_thresholds=10000)\n",
    "calculate_pr_auc(y_true, scores / 100, num_thresholds=10000)\n",
    "calculate_pr_auc(y_true, scores / 1000, num_thresholds=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.82231975"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.82231975"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.82231975"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.82231975"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# if we know the general extent of score values, we can manually specify the thresholds\n",
    "calculate_pr_auc(y_true, scores, thresholds=np.linspace(0, 1, 200))\n",
    "calculate_pr_auc(y_true, scores / 10, thresholds=np.linspace(0, 1/10, 200))\n",
    "calculate_pr_auc(y_true, scores / 100, thresholds=np.linspace(0, 1/100, 200))\n",
    "calculate_pr_auc(y_true, scores / 1000, thresholds=np.linspace(0, 1/1000, 200))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.814259"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.7288617"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.12399999"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.8240797"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.83588326"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.7446808"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.474"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# effect of minoring and majoring summation methods\n",
    "calculate_pr_auc(y_true, scores, summation_method='minoring')\n",
    "calculate_pr_auc(y_true, scores / 10, summation_method='minoring')\n",
    "calculate_pr_auc(y_true, scores / 100, summation_method='minoring')\n",
    "calculate_pr_auc(y_true, scores / 1000, summation_method='minoring')\n",
    "\n",
    "calculate_pr_auc(y_true, scores, summation_method='majoring')\n",
    "calculate_pr_auc(y_true, scores / 10, summation_method='majoring')\n",
    "calculate_pr_auc(y_true, scores / 100, summation_method='majoring')\n",
    "calculate_pr_auc(y_true, scores / 1000, summation_method='majoring')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}