IMDB Data Example SpaCy and SentencePiece

IMDB_data tokenize LM build SpaCy (default).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.text import *\n",
    "import re\n",
    "import sentencepiece as spm #https://github.com/google/sentencepiece\n",
    "\n",
    "np.random.seed(20180311)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentencepiece tokenizer & LM Build"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data processing to DataBunch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get IMDB Data and build DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/home/farzin/.fastai/data/imdb')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "IMDB_PATH = untar_data(URLs.IMDB)\n",
    "IMDB_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75000, 25000)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## raw data does not have the unsup in the train directory.  Just move it there\n",
    "#  ~/.fastai/data/imdb$ mv unsup/ ./train/\n",
    "CLASSES = ['neg', 'pos', 'unsup']\n",
    "\n",
    "def get_texts(path):\n",
    "    texts,labels = [],[]\n",
    "    for idx,label in enumerate(CLASSES):\n",
    "        for fname in (path/label).glob('*.*'):\n",
    "            texts.append(fname.open('r', encoding='utf8').read())\n",
    "            labels.append([0]) ## all unsupervised case\n",
    "\n",
    "    return np.array(texts),np.array(labels)\n",
    "\n",
    "train_texts,train_labels = get_texts(IMDB_PATH/'train')\n",
    "valid_texts,valid_labels = get_texts(IMDB_PATH/'test')\n",
    "\n",
    "len(train_texts),len(valid_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## we can combine the \"train\" and \"valid\" for the LM step\n",
    "all_texts = np.concatenate([train_texts,valid_texts])\n",
    "all_texts_df = pd.DataFrame({'text':all_texts, 'labels':[0]*len(all_texts)}, columns=['labels','text'])"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "raw_text = all_texts_df.loc[:,'text']\n",
    "\n",
    "print(\"Default Rules:\\n\",[x.__name__ for x in defaults.text_pre_rules],\"\\n\\n\")\n",
    "\n",
    "for rule in defaults.text_pre_rules:\n",
    "    print(rule)\n",
    "    raw_text = raw_text.apply(lambda x: rule(str(x)))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "## SentencePiece expects <s> ... </s> so we add that here:\n",
    "all_texts_df['new_text'] = '<s>' + raw_text + '</s>'\n",
    "all_texts_df['new_text'].head()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "## save as a file for SP\n",
    "formatted_text_file = 'tmp_IMDB_SP_example'\n",
    "all_texts_df['new_text'].to_frame().to_csv(formatted_text_file, header=False,index=False,quotechar=' ')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SP Tokenizer SWIG wrapper"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# m.model and m.vocab created in the pwd\n",
    "vocab_size = 500\n",
    "model_prefix = 'wk103m_example'\n",
    "\n",
    "spm.SentencePieceTrainer.Train(f'--input={formatted_text_file}'\\\n",
    "                               f' --model_prefix={model_prefix}'\\\n",
    "                               f' --vocab_size={vocab_size}')\n",
    "#                               f'--unk_piece={UNK} --bos_piece={BOS} --eos_id=-1 --pad_piece={PAD}')"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "## Vocab in order of frequency\n",
    "!head -n50 {model_prefix}.vocab | nl"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "## load up the Processor\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.load(f'{model_prefix}.model')\n",
    "\n",
    "## itos from m.vocab file: just read directly and populate the dictionary\n",
    "itos = {}\n",
    "with open(f'{model_prefix}.vocab','r') as f:\n",
    "    for line_num,line in enumerate(f):\n",
    "        itos[line_num] = line.split(\"\\t\")[0]\n",
    "        \n",
    "stoi = {v:k for k,v in itos.items()}"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "#Based on Tokenizer() class: https://github.com/fastai/fastai/blob/master/fastai/text/transform.py#L87\n",
    "class CustomTokenizer():\n",
    "    '''Wrapper for SentencePiece toeknizer to fit into Fast.ai V1'''\n",
    "    def __init__(self,sp_processor,pre_rules:ListRules=None,post_rules:ListRules=None):\n",
    "        self.sp = sp_processor\n",
    "        self.pre_rules  = ifnone(pre_rules,  defaults.text_pre_rules )\n",
    "        \n",
    "    def __repr__(self) -> str:\n",
    "        return \"Custom Tokenizer\"\n",
    "\n",
    "    def process_text(self, t:str) -> List[str]:\n",
    "        \"Processe one text `t` with tokenizer `tok`.\"\n",
    "        for rule in self.pre_rules: t = rule(t)                                                                                 \n",
    "        toks = sp.EncodeAsIds(t)\n",
    "        \n",
    "        return toks \n",
    "    \n",
    "    def _process_all_1(self,texts:Collection[str]) -> List[List[str]]:\n",
    "        'Process a list of `texts` in one process'\n",
    "        return [self.process_text(t) for t in texts]\n",
    "                                                                     \n",
    "    def process_all(self, texts:Collection[str]) -> List[List[str]]: \n",
    "        \"Process a list of `texts`.\"                                 \n",
    "        return self._process_all_1(texts)\n",
    "    \n",
    "mycust_tok = CustomTokenizer(sp)\n",
    "#setup Vocab object for use in LM  \n",
    "# Vocab source: https://github.com/fastai/fastai/blob/master/fastai/text/transform.py#L122\n",
    "sp_vocab = Vocab(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labels</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Detective Frank Keller(Al Pacino, in a perfect...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>This is a VERY good movie. I give it a 10.&lt;br ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>I can't say I enjoyed this as much as \"The Big...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>I sat last night to see this film being played...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>First of all, I agree that the plot left somet...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   labels                                               text\n",
       "0       0  Detective Frank Keller(Al Pacino, in a perfect...\n",
       "1       0  This is a VERY good movie. I give it a 10.<br ...\n",
       "2       0  I can't say I enjoyed this as much as \"The Big...\n",
       "3       0  I sat last night to see this film being played...\n",
       "4       0  First of all, I agree that the plot left somet..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = np.random.permutation(len(all_texts))\n",
    "cut = int(0.1 * len(idx))\n",
    "LM_train_df = pd.DataFrame({'text':all_texts[idx[cut:]], 'labels':[0] * (len(all_texts)-cut)}, columns=['labels','text'])\n",
    "LM_valid_df = pd.DataFrame({'text':all_texts[idx[:cut]], 'labels':[0] * cut}, columns=['labels','text'])\n",
    "LM_train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path(\"./tmp_IMDB_example\")\n",
    "\n",
    "data = TextLMDataBunch.from_df(PATH, LM_train_df, LM_valid_df, \n",
    "#                                tokenizer=mycust_tok, vocab=sp_vocab, \n",
    "                               text_cols='text', label_cols='labels')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60003"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data.vocab.itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['xxunk',\n",
       " 'xxpad',\n",
       " 'xxbos',\n",
       " 'xxfld',\n",
       " 'xxmaj',\n",
       " 'xxup',\n",
       " 'xxrep',\n",
       " 'xxwrep',\n",
       " 'the',\n",
       " '.',\n",
       " ',',\n",
       " 'and',\n",
       " 'a',\n",
       " 'of',\n",
       " 'to',\n",
       " 'is',\n",
       " 'it',\n",
       " 'in',\n",
       " 'i',\n",
       " 'this',\n",
       " 'that',\n",
       " '\"',\n",
       " \"'s\",\n",
       " '-',\n",
       " '\\n \\n ',\n",
       " 'was',\n",
       " 'as',\n",
       " 'with',\n",
       " 'for',\n",
       " 'movie',\n",
       " 'but',\n",
       " 'film',\n",
       " 'you',\n",
       " ')',\n",
       " 'on',\n",
       " \"n't\",\n",
       " '(',\n",
       " 'not',\n",
       " 'are',\n",
       " 'he',\n",
       " 'his',\n",
       " 'have',\n",
       " 'one',\n",
       " 'be',\n",
       " 'all',\n",
       " 'at',\n",
       " 'they',\n",
       " 'by',\n",
       " 'an',\n",
       " 'who']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.vocab.itos[:50]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  LM Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(data, AWD_LSTM, drop_mult=0.5)\n",
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  LM Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xuc3HV97/HXZ2b2fsnuJpt7QrhEMFzCZQlgKYiXFCjKsaUWj20R9aR4hNa2eoqn59hW7cV6etFSiSmK1QK2IigqhqAVUQEhIQnkJrmASTa3zd6vszszn/PH/HYZlt3NJJnfXDbv5+Mxj535/r6/+X1mMpnPfC+/78/cHRERkWOJFDoAEREpDUoYIiKSFSUMERHJihKGiIhkRQlDRESyooQhIiJZUcIQEZGsKGGIiEhWlDBERCQrsUIHkEuzZs3yJUuWFDoMEZGSsWHDhqPu3pxN3WmVMJYsWcL69esLHYaISMkws19mW1ddUiIikhUlDBERyYoShoiIZEUJQ0REsqKEISIiWVHCEBGRrChhiIhIVkJNGGbWYGYPmtkOM9tuZleM2/4xM9sU3LaYWdLMmoJtr5jZi8E2nVwhIjKBH2w7zOof787LscJuYXwOWOvu5wDLge2ZG939s+5+obtfCHwc+LG7d2RUuSbY3hJynCIiJemxrYf4ys9eycuxQjvT28xmAFcB7wNw92FgeIpd3gM8EFY8IiLTUdfgCA3VZXk5VpgtjNOBNuBeM9toZveYWc1EFc2sGrgW+GZGsQPrzGyDma0KMU4RkZLVNTA8LRJGDLgYuNvdLwL6gTsnqfsO4GfjuqOudPeLgeuAD5vZVRPtaGarzGy9ma1va2vLYfgiIsWvc2CExuryvBwrzISxH9jv7j8PHj9IOoFM5GbGdUe5e2vw9wjwMLBioh3dfY27t7h7S3NzVgsuiohMG+kWRoknDHc/BOwzs7ODorcC28bXC8Y6rga+nVFWY2Z1o/eBlcCWsGIVESlF7k7XwAiNeeqSCnt58zuA+8ysHNgD3GpmtwG4++qgzruAde7en7HfHOBhMxuN8X53XxtyrCIiJaUvniCR8ryNYYSaMNx9EzB+SuzqcXW+AnxlXNke0tNwRURkEl0DIwCl3yUlIiLh6hxIn6kwHQa9RUQkRJ1BCyNfYxhKGCIiJaoraGFMh/MwREQkRBrDEBGRrIyOYTRUqYUhIiJT6BoYoa4yRiyan69yJQwRkRKVz3WkQAlDRKRk5XMdKVDCEBEpWflcRwqUMERESlZnHteRAiUMEZGS1TUwnLcZUqCEISJSkhLJFD1DCXVJiYjI1LoH87ssCChhiIiUpLF1pGrUwhARkSl0D6bP8p6hMQwREZlKZ/9ol5RaGCIiMoV8XwsDlDBERErS2Eq1NdOkS8rMGszsQTPbYWbbzeyKcdvfbGbdZrYpuH0iY9u1ZvYLM9tlZneGGaeISKnpGhwmGjHqKkK90vZrhH2kzwFr3f0mMysHqieo8xN3vyGzwMyiwL8Abwf2A8+Z2SPuvi3keEVESkLnwAgNVWWYWd6OGVoLw8xmAFcBXwJw92F378py9xXALnff4+7DwNeBG8OJVESk9OR7pVoIt0vqdKANuNfMNprZPWZWM0G9K8xss5l938zODcoWAPsy6uwPykREhPQsqXwOeEO4CSMGXAzc7e4XAf3A+LGI54HT3H058M/At473IGa2yszWm9n6tra2k41ZRKQkdA2OTKsWxn5gv7v/PHj8IOkEMsbde9y9L7j/KFBmZrOAVmBRRtWFQdnruPsad29x95bm5uZcvwYRkaKU76XNIcSE4e6HgH1mdnZQ9FbgNYPWZjbXghEbM1sRxNMOPAcsNbPTg8Hym4FHwopVRKTUdA4M53UdKQh/ltQdwH3Bl/4e4FYzuw3A3VcDNwEfMrMEMAjc7O4OJMzsduAxIAp82d23hhyriEhJGBpJMjSSynsLI9SE4e6bgJZxxasztt8F3DXJvo8Cj4YXnYhIaRo7aW8ajWGIiEgICrEsCChhiIiUnNGEoRaGiIhMabRLSi0MERGZksYwREQkKxrDEBGRrHQNDFNZFqGyLJrX4yphiIiUmM6B/K8jBUoYIiIlp2tgOK/X8h6lhCEiUmK61MIQEZFsdA4M05jHS7OOUsIQESkxXQMjeV9HCpQwRERKirunr4WhMQwREZlKbzxBMuUawxARkal19RfmLG9QwhARKSmFOssblDBEREpKe38cgJm1ShgiIjKFtt50wmiuq8j7sZUwRERKyGjCmFU7zRKGmTWY2YNmtsPMtpvZFeO2v9fMXjCzF83sKTNbnrHtlaB8k5mtDzNOEZFS0dYbp74ylveFByHka3oDnwPWuvtNZlYOVI/b/jJwtbt3mtl1wBrgsozt17j70ZBjFBEpGW19cWYVoDsKQkwYZjYDuAp4H4C7DwPDmXXc/amMh88AC8OKR0RkOjjaO0xzAbqjINwuqdOBNuBeM9toZveYWc0U9T8AfD/jsQPrzGyDma2abCczW2Vm681sfVtbW24iFxEpUm198YIMeEO4CSMGXAzc7e4XAf3AnRNVNLNrSCeMP80ovtLdLwauAz5sZldNtK+7r3H3FndvaW5uzukLEBEpNm290zNh7Af2u/vPg8cPkk4gr2FmFwD3ADe6e/toubu3Bn+PAA8DK0KMVUSk6A0OJ+mLJwoyQwpCTBjufgjYZ2ZnB0VvBbZl1jGzxcBDwO+6+0sZ5TVmVjd6H1gJbAkrVhGRUnC0r3DnYED4s6TuAO4LZkjtAW41s9sA3H018AlgJvAFMwNIuHsLMAd4OCiLAfe7+9qQYxURKWpHCnjSHoScMNx9E9Ayrnh1xvYPAh+cYL89wPLx5SIip7Kxs7ynW5eUiIjkVluBu6SUMERESsTR3jhm0FST/4UHQQlDRKRktPXFaaoupyxamK9uJQwRkRJRyHMwQAlDRKRktPXGC3YOBihhiIiUjKMFXBYElDBEREqCu6tLSkREjq03niCeSBXsHAxQwhARKQljV9qrK8yUWlDCEBEpCUfHzvKuLFgMShgiIiWg0Gd5gxKGiEhJaCvwwoOghCEiUhLaeuNEI0ZDVVnBYlDCEBEpAUf74syqLScSsYLFoIQhIlICCn0OBihhiIiUhLa+eEHPwQAlDBGRklDodaRACUNEpOilUk573/D07pIyswYze9DMdpjZdjO7Ytx2M7PPm9kuM3vBzC7O2HaLme0MbreEGaeISDHrGhwhkfKCJ4xQr+kNfA5Y6+43mVk5UD1u+3XA0uB2GXA3cJmZNQF/Tvp64A5sMLNH3L0z5HhFRIpOMZyDASG2MMxsBnAV8CUAdx92965x1W4EvuppzwANZjYP+DXgcXfvCJLE48C1YcUqIlLMxtaRmsZjGKcDbcC9ZrbRzO4xs5pxdRYA+zIe7w/KJit/HTNbZWbrzWx9W1tb7qIXESkSR4tgWRAIN2HEgIuBu939IqAfuDPXB3H3Ne7e4u4tzc3NuX56EZGCm/ZdUqRbBfvd/efB4wdJJ5BMrcCijMcLg7LJykVETjltfXEqYhHqKsIedp5aaAnD3Q8B+8zs7KDorcC2cdUeAX4vmC11OdDt7geBx4CVZtZoZo3AyqBMROSUM3qWt1nhlgWB8GdJ3QHcF8yQ2gPcama3Abj7auBR4HpgFzAA3Bps6zCzTwHPBc/zSXfvCDlWEZGiVAwn7UHICcPdN5GeGptpdcZ2Bz48yb5fBr4cXnQiIqWhrTfOoqbxZyXkn870FhEpYu5Oa9cgCxurCh1KdgnDzM40s4rg/pvN7A/MrCHc0EREpHNghL54gsUl1ML4JpA0s7OANaRnMN0fWlQiIgLA3o4BgJLqkkq5ewJ4F/DP7v4xYF54YYmICLyaMEqphTFiZu8BbgG+G5QV7jqBIiKniH1jLYwSGcMgPd31CuCv3P1lMzsd+Fp4YYmICMDe9gFm1VZQXV7Yk/Ygy2m17r4N+AOA4ES6Onf/TJiBiYhIuktqcRG0LiD7WVJPmFl9sOz488C/mtk/hBuaiIikE0bhxy8g+y6pGe7eA/wG6eXILwPeFl5YIiIynEhxsHuw5BJGLLhOxbt5ddBbRERCdKBrkJQXx5RayD5hfJL04n+73f05MzsD2BleWCIiUkxTaiH7Qe9vAN/IeLwH+M2wghIRkYyEMbM4Eka2g94LzexhMzsS3L5pZgvDDk5E5FS2r2OA8miEOXWVhQ4FyL5L6l7S166YH9y+E5SJiEhI9nYMsLCpikiksNfBGJVtwmh293vdPRHcvgLoeqgiIiEqpim1kH3CaDez3zGzaHD7HaA9zMBERE5l7s7e9tJMGO8nPaX2EHAQuAl4X0gxiYic8roHR+gtkmXNR2U7S+qXwDszy8zsI8A/TbWfmb0C9AJJIOHuLeO2fwx4b0YsbyTd/dVxrH1FRKazYlrWfNTJrGb1xxwjYQSucfejE21w988CnwUws3cAfzTu2t2T7isiMp0V2zkYcHKXaM31sP17gAdy/JwiIiWpGFsYJ5MwPMs668xsg5mtmqySmVUD15K+st9x7SsiMh3t6xhkZk05tRWFX9Z81JSRmFkvEycGA7JZb/dKd281s9nA42a2w92fnKDeO4CfjeuOymrfIJmsAli8eHEWIYmIFL99HQNF1bqAY7Qw3L3O3esnuNW5+zHTnru3Bn+PAA8DKyapejPjuqOy3dfd17h7i7u3NDfr1BARmR6K7RwMOLkuqSmZWY2Z1Y3eB1YCWyaoNwO4Gvj28e4rIjIdJZIpWruKZ1nzUWF2js0BHjaz0ePc7+5rzew2AHdfHdR7F7DO3fuPtW+IsYqIFI2D3UMkU37qJIxgRdvlE5SvHvf4K8BXstlXRORUUIwzpCDELikRETkxxbas+SglDBGRIvPS4V6qyqLMrS+OZc1HKWGIiBSZrQd6OGdeHdEiWdZ8lBKGiEgRcXe2H+jh3Pn1hQ7ldZQwRESKyL6OQXrjCc6dP6PQobyOEoaISBHZeqAbgGXz1MIQEZEpbD3QQzRinD23rtChvI4ShohIEdl2sIezmmupLIsWOpTXUcIQESkiWw90F+WANyhhiIgUjaN9cQ73xFmmhCEiIlPZeqAHQAlDRESmNjpD6tx5xTelFpQwRESKxrYDPSxsrGJGdVmhQ5mQEoaISJHYVqRneI9SwhARKQL98QQvt/ezrEi7o0AJQ0SkKGw/2IM7amGIiMjURmdInbtACUNERKaw7UAPTTXlRXcNjEyhJgwze8XMXjSzTWa2foLtbzaz7mD7JjP7RMa2a83sF2a2y8zuDDNOEZFC23owfYa3WXFdAyNTaNf0znCNux+dYvtP3P2GzAIziwL/Arwd2A88Z2aPuPu2EOMUESmIkWSKlw71ceuvLCl0KFMq1i6pFcAud9/j7sPA14EbCxyTiEgoXmztZjiZ4oKFDYUOZUphJwwH1pnZBjNbNUmdK8xss5l938zODcoWAPsy6uwPykREpp2nd7cDcPkZTQWOZGphd0ld6e6tZjYbeNzMdrj7kxnbnwdOc/c+M7se+Baw9HgOECSiVQCLFy/OVdwiInnz9O52zplbx8zaikKHMqVQWxju3hr8PQI8TLqrKXN7j7v3BfcfBcrMbBbQCizKqLowKJvoGGvcvcXdW5qbm0N4FSIi4Yknkjz3SgdXnDmz0KEcU2gJw8xqzKxu9D6wEtgyrs5cC6YEmNmKIJ524DlgqZmdbmblwM3AI2HFKiJSKBv3dhFPpHjTmbMKHcoxhdklNQd4OMgHMeB+d19rZrcBuPtq4CbgQ2aWAAaBm93dgYSZ3Q48BkSBL7v71hBjFREpiKd3txMxWHF6cY9fQIgJw933AMsnKF+dcf8u4K5J9n8UeDSs+EREisHTu9s5b8EMZlQV5wq1mYp1Wq2IyLQ3OJxk475Orjij+McvQAlDRKRg1v+yg5Gkl8SANyhhiIgUzFO724lFjEuXFP/4BShhiIgUzNO721m+qIGainys0nTylDBERAqgd2iEF1u7eVOJdEeBEoaISEE890oHyZSXzIA3KGGIiBTEU7vaKY9FuPi0xkKHkjUlDBGRAvjxS220nNZIZVm00KFkTQlDRCTP9rT1sfNIHyuXzSl0KMdFCUNEJM/WbTsMwNvPnVvgSI6PEoaISJ6t23qI8xbUs6ChqtChHBclDBGRPDrSM8TGfV2sXFZarQtQwhARyasfbD+CO6w8t7TGL0AJQ0Qkr9ZtO8TipmrOnlNX6FCOmxKGiEie9A6N8NSudlYum0NwraCSooQhIpInP36pjeFkipUlNjtqlBKGiEierNt6mJk15VxSQmd3Z1LCEBHJg+FEih/tOMLb3jiHaKT0uqMg3Gt6Y2avAL1AEki4e8u47e8F/hSwoN6H3H1zNvuKiJSSn+0+Sm88UZKzo0blYxH2a9z96CTbXgaudvdOM7sOWANcluW+IiIl44Gf72VmTTlXLp1V6FBOWEG7pNz9KXfvDB4+AywsZDwiImE40DXID7Yf5t2XLqIiVjqLDY4XdsJwYJ2ZbTCzVceo+wHg+ye4r4hI0Xrg2b048N9XLC50KCcl7C6pK9291cxmA4+b2Q53f3J8JTO7hnTCuPIE9l0FrAJYvLi0/zFEZPoZTqR44Nl9vOXs2Sxqqi50OCcl1BaGu7cGf48ADwMrxtcxswuAe4Ab3b39ePYNtq9x9xZ3b2lubs79ixAROQmPbT3E0b44v3PFaYUO5aSFljDMrMbM6kbvAyuBLePqLAYeAn7X3V86nn1FRErB1575JYuaqrh6aen/oA2zS2oO8HBw+nsMuN/d15rZbQDuvhr4BDAT+EJQb3T67IT7hhiriEjOvXS4l2df7uDj151DpETPvcgUWsJw9z3A8gnKV2fc/yDwwWz3FREpJf/+zC8pj0X4rZZFhQ4lJ3Smt4hICI72xfnmhv3ccME8mmrKCx1OTihhiIiE4B8ff4l4IsXt15xV6FByRglDRCTHdh7u5YFn9/LeyxZzRnNtocPJGSUMEZEc++tHt1NTEeMP3/aGQoeSU0oYIiI59NOdR/nRL9q4/Zqzps3YxSglDBGRHEmmnE9/bxsLG6u45U1LCh1OzilhiIjkyDfW72PHoV7+9NpzqCwr3UUGJ6OEISKSA3vbB/j097azYkkTN1wwr9DhhEIJQ0TkJCWSKf7wPzZiBv/w28sJVqmYdvJxASURkWnt8z/cyca9XXz+PRexsLG0V6SdiloYIiIn4dmXO7jrR7v4jYsX8M7l8wsdTqiUMERETlD3wAh/9B+bWNhYzSdvPK/Q4YROXVIiIicgkUxx+wPPc6R3iP/8/SuorZj+X6fT/xWKiITgM2t38JOdR/nb3zifixY3FjqcvFCXlIjIcXro+f38609e5pYrTuPmEr9O9/FQwhAROQ6b93Vx50MvcvkZTfyfG5YVOpy8UsIQEcnS83s7ufUrzzG7roIvvPcSyqKn1lfoqfVqRURO0NotB3nPmmeoq4zxtQ9cNu0WFsxGqAnDzF4xsxfNbJOZrZ9gu5nZ581sl5m9YGYXZ2y7xcx2BrdbwoxTRGQqX/rpy3zovudZNr+ehz70Jk6fVVPokAoiH7OkrnH3o5Nsuw5YGtwuA+4GLjOzJuDPgRbAgQ1m9oi7d4YR4Hc2HyAWMarKo1SXx2isLuOs2bXT9vT+TO19cXYd6WN3Wz9DI0lm1pbTVFPOzJoKzpxdQ0Xs+BZQc3eAon3vkimne3CE/niC3qEEA8MJHIiYETGIRoxYJEJZ1IhFI0TNGH0pZpBKQdKdlDt9QwkOdg9xqHuQQz1xhkaSJFNOIuXEIsa8hkoWNVazsLGKBY1VzKqpIBIpzvdFJtbZP8xffmcr39p0gGvPncs/3XzhtFxUMFuFnlZ7I/BVT3/LPGNmDWY2D3gz8Li7dwCY2ePAtcADYQTx0W9sJp5IvaZsYWMVv37BPG44fz7nLagv2i/A49XeF+eJX7TxXzuO8PSedjr6hyetWxY1ls2r58JFDZw1u5aRpDOUSDI0kqKzf5jDPUMc7o1ztDfO4EiSoeAWi0aYN6MyuFVRFjUSSWck5SSSqaBeiqFEkspYlDn1Fcypr2ROfSXzG6pY0JD+gq2rjNHWG08fpyfOcDJFxNJf7u7QH0/QMzRC71CCvniCgeEkA8PpvwaUxSKURyOk3DnYNURr1yCHeoZIpjzn72tZ1KgqixKLRoiYkUil6BoYeV2dOfWVzJ9RxcKmKk5rquG0mdUsaKyisbqchuoyGqrKiGXRL55KOcPJFBWxyLT5bBabtVsO8n++tZWugWE+8ral3PGWpURP8YQfdsJwYJ2ZOfBFd18zbvsCYF/G4/1B2WTlr2Nmq4BVAIsXn9j0tsc+chUDw0kGR9JfNge6Bvn+lkN86Scv88Uf72FRUxXXnzeP68+fxwULZ+T8P2gy5fQMjlBVHs36C8Dd6RlK0NY7RPdggqGRJIPDSQZGkvTHE/QFX6I9QyN09A/T0T9MW2+cXxzuxR1m11XwlnNm88Z59ZzZXMNZs2upKY/RHtQ90jvEi63dbNrbxTc27GdgOPma4zdUlzGnrpLZ9RWcOauG6ooolbEolWVRhpMpDnYPcbBrkGdf7iDlTjRilEUjxCJGZVmUyrIItRUxBoeTbNjbmU4I45L28agpj1JdEaOmPDr2C3AkmWIk6TjOvPoqVpzexPyGSmbVVlBbEaOuMkZ1eSzdcnBIuZNMOolUer9EKkUylX6vHcAhEjGikXTSqimPMTdIjE015a/7d+uLJ9jfOcC+jkEOdg9yoCvdGjnQNcRTu9p5qKd1wtdSEYtQHotQEYtQFiS8ZCod30gyRTyRGnuvqsujzG+oGkvQjdXl1FeV0VBdRlN1Oc11FcyqraC5roLq8qiSSxYOdg/y6e9u53svHuTc+fV89f0rWDa/vtBhFYWwE8aV7t5qZrOBx81sh7s/mcsDBEloDUBLS8sJ/XRcMkF/5G9fupiugWHWbT3Mo1sO8qWfvswXn9zD/BmVNNWWk0g6yVT6i7C5roJ5MyqZW19JLBqhfzjBQDzJSDLFufPrufyMmWNdXK1dg/xs51Ge2n2UV9oHONwzxJHe+Niv3mjEqCmPUh6LjH2JpVJOWTT95ZEud9p6469rFU2ktiJGU026m2l+QxXXnTePt75xNsvm1U/YPdKYMZB3wwXpdXESyRQd/cNUxKJUlEVC+VXr7nQOjHCga5D9nYO0dg3SN5Rgdn0Fc+ormF1XSWVZFHdntIFQUxGlrrKM2opYUf7yq62Icc7ces6ZO/GXzdBIkv2dA7R2DdE1MEzXwAidA8MMDifTSSGZYiSRSneXBYkqFolQWRYNkonR0Z9+zw50D/KLQ710DY5MmngrYpGxz0JzXcVYa25hYxXzZqSTzuz6iuPuhpwuhkaSrHlyD3c/sZukOx9d+QZ+/+ozT7mZUFMJNWG4e2vw94iZPQysADITRiuwKOPxwqCslXS3VGb5E2HGOpGG6nLefeki3n3pIroHRnh8+2H+a8dhhkZSwS9mYyTpHOkZ4qXDvbT1xkk5lMciY8sEfP25dENpZk36l9/LR/sBmFVbwTlz6zjrrFljvwyHEunWQX88yXAyRTToVzez4Ndy+pelWTpJNddWMLu+gvqqMqrLolSVR6kqi1JbGaOmIkZNeW6+SGPRCLPrK0/6eaZiZmNfZuctmBHqsYpFZVmUs2bXcdbsupw+79BIku7BEdr7hmnrS3cZtvXF6egfpr1vmI7+OEd642ze10XnuG4zYOzfIT2WVU5FLBK0wNNdjouaqrlgwQzOX9jAufPrS75Pf2A4wXc3H+RzP9xJa9cg158/l49f90YWNU3fVWdPlI0OUub8ic1qgIi79wb3Hwc+6e5rM+r8OnA7cD3pQe/Pu/uKYNB7AzA6a+p54JLRMY3JtLS0+Pr1r5uMlTeJZAqHsV8k7s6+jkGe2dPO03va6R0a4fIzZvKrS5t5w5xTY1Bdilt/PMGBYGznYPcQh7uHONQzNNaN2dE/TDyRoro8/YOkLBphT1s/R/viAEQMFjVVs3R2LWfOrmXJzJqxbrp59VXUV8WK9nO+pbWbrz+3l29vPEBvPMGyefX83xuWccWZMwsdWl6Z2QZ3b8mmbpgtjDnAw8GHJQbc7+5rzew2AHdfDTxKOlnsAgaAW4NtHWb2KeC54Lk+eaxkUQzGD1aaGYtnVrN4ZjXvvnTRJHuJFE5NRYylc+pYOif7Vo67c7gnzub9XWw90MPuI33sPNLLj19qYyT52h+gZVFjZk0FM2vLmVtfydlz6zh7bh3nzK1nyazqvHd/7W0f4JHNrTyy+QAvHe6jIhbh18+fx80rFnPpksaiTW7FIrQWRiEUuoUhcipLJFMc7o1zqHswmG48RHv/MO19cdr7htnfOcjutj4SqdGp1zCnrpKFjVUsnlnN8oUNXLS4gXPm1lMey924QVtvnO+9cIBvbz7Axr1dAFy6pJF3Lp/PO5cvYEZ1Wc6OVYqKpYUhIqeQWDSSnhLdUDVpneFEij1H+9hxsJdftg+wr3OAfR0D/GTnUR56Pj1rrCIW4Q1z6lg8s5rTmqo5bWY1c2dUjU1+aKwum7Il0BdPsLW1mxdbu3ly51F+tusoyZTzxnn13HndObxj+fwpY5TJKWGISN6UxyITzhxzdw52D7Fxbxcb93by0pE+trZ289iWQ2MtklERI5iaHaUyFqEslj73xSw9lXp/5yCjHSeLm6q57eozuPHCBbzhOLrdZGJKGCJScGbG/IYq5jekT5gdlQjO6Rmdfn64Z4j2vuH0yZ/BSaSJZGpsCrqZcdPFizh/YT3nLZjB7LpwZ/edapQwRKRoxaIRFjVVa4prkdAZKSIikhUlDBERyYoShoiIZEUJQ0REsqKEISIiWVHCEBGRrChhiIhIVpQwREQkK9Nq8UEz6wZ2TrBpBtCd5ePR+xOVzQImuz75ZMYfK9vtE5VPFNNk908m5qniyja+Uol5ovJS/HxkE3PmfX0+st8+3T8fS909u4vQuPu0uQFrsimf6vHo/UnK1ucqpuONebKYjhX/icR8onGXYszT5fORTcyFfq/1+Sj+z8exbtOtS+o7WZZP9fg7U5TiKMGOAAAHWUlEQVTlMqZjbZ+ofLKYjhX/iTiRuEsx5onKS/HzkU3Mmff1+ch++6n0+ZjStOqSCpuZrfcs140vFoo5f0oxbsWcP6Uad6bp1sII25pCB3ACFHP+lGLcijl/SjXuMWphiIhIVtTCEBGRrJyyCcPMvmxmR8xsywnse4mZvWhmu8zs85ZxvUgzu8PMdpjZVjP7u2KP2cz+wsxazWxTcLu+2GPO2P4nZuZmNit3EY89dxjv9afM7IXgfV5nZvNLIObPBp/nF8zsYTNrKIGYfyv4/5cys5yNGZxMrJM83y1mtjO43ZJRPuXnvqBOZHradLgBVwEXA1tOYN9ngcsBA74PXBeUXwP8AKgIHs8ugZj/AvhoKb3PwbZFwGPAL4FZpRA3UJ9R5w+A1SUQ80ogFtz/DPCZEoj5jcDZwBNAS6FjDeJYMq6sCdgT/G0M7jdO9bqK4XbKtjDc/UmgI7PMzM40s7VmtsHMfmJm54zfz8zmkf6P/4yn/3W/Cvy3YPOHgL9193hwjCMlEHOoQoz5H4H/BYQyCBdG3O7ek1G1JtexhxTzOndPBFWfARaWQMzb3f0XuYzzZGKdxK8Bj7t7h7t3Ao8D1xby/2o2TtmEMYk1wB3ufgnwUeALE9RZAOzPeLw/KAN4A/CrZvZzM/uxmV0aarRpJxszwO1Bl8OXzawxvFDHnFTMZnYj0Orum8MOdJyTfq/N7K/MbB/wXuATIcY6Khefj1HvJ/2LN2y5jDls2cQ6kQXAvozHo/EXy+uakK7pHTCzWuBNwDcyugwrjvNpYqSbmJcDlwL/aWZnBL8Uci5HMd8NfIr0r91PAX9P+oshFCcbs5lVA/+bdFdJ3uTovcbd/wz4MzP7OHA78Oc5C3KcXMUcPNefAQngvtxEN+lxchZz2KaK1cxuBf4wKDsLeNTMhoGX3f1d+Y41V5QwXhUButz9wsxCM4sCG4KHj5D+gs1sli8EWoP7+4GHggTxrJmlSK8f01asMbv74Yz9/hX4bkixjjrZmM8ETgc2B/9JFwLPm9kKdz9UxHGPdx/wKCEmDHIUs5m9D7gBeGtYP34y5Pp9DtOEsQK4+73AvQBm9gTwPnd/JaNKK/DmjMcLSY91tFL41zW5Qg+iFPIGLCFjAAt4Cvit4L4ByyfZb/yg1PVB+W3AJ4P7byDd5LQij3leRp0/Ar5e7O/zuDqvEMKgd0jv9dKMOncAD5ZAzNcC24DmMN7jMD8f5HjQ+0RjZfJB75dJD3g3Bvebsv3cF+pW8AAK9sLhAeAgMEK6ZfAB0r9c1wKbg/8kn5hk3xZgC7AbuItXT4AsB/492PY88JYSiPlrwIvAC6R/uc0r9pjH1XmFcGZJhfFefzMof4H0+j0LSiDmXaR/+GwKbrme2RVGzO8KnisOHAYeK2SsTJAwgvL3B+/vLuDW4/ncF+qmM71FRCQrmiUlIiJZUcIQEZGsKGGIiEhWlDBERCQrShgiIpIVJQyZ1sysL8/Hu8fMluXouZKWXtl2i5l951grxZpZg5n9z1wcW2QimlYr05qZ9bl7bQ6fL+avLsYXqszYzezfgJfc/a+mqL8E+K67n5eP+OTUoxaGnHLMrNnMvmlmzwW3XwnKV5jZ02a20cyeMrOzg/L3mdkjZvZfwA/N7M1m9oSZPWjpa0XcN3rNgqC8JbjfFyw2uNnMnjGzOUH5mcHjF83s01m2gp7m1cUXa83sh2b2fPAcNwZ1/hY4M2iVfDao+7HgNb5gZn+Zw7dRTkFKGHIq+hzwj+5+KfCbwD1B+Q7gV939ItIryf51xj4XAze5+9XB44uAjwDLgDOAX5ngODXAM+6+HHgS+B8Zx/+cu5/Pa1cmnVCwjtJbSZ+JDzAEvMvdLyZ9DZa/DxLWncBud7/Q3T9mZiuBpcAK4ELgEjO76ljHE5mMFh+UU9HbgGUZK4zWByuPzgD+zcyWkl69tyxjn8fdPfNaCM+6+34AM9tEeo2hn447zjCvLua4AXh7cP8KXr3Gwf3A/5skzqrguRcA20lfMwHSawz9dfDlnwq2z5lg/5XBbWPwuJZ0AnlykuOJTEkJQ05FEeBydx/KLDSzu4Afufu7gvGAJzI29497jnjG/SQT/18a8VcHCSerM5VBd78wWNL9MeDDwOdJX0ujGbjE3UfM7BWgcoL9Dfgbd//icR5XZELqkpJT0TrSq8UCYGajy1PP4NWlpN8X4vGfId0VBnDzsSq7+wDpS7r+iZnFSMd5JEgW1wCnBVV7gbqMXR8D3h+0njCzBWY2O0evQU5BShgy3VWb2f6M2x+T/vJtCQaCt5Felh7g74C/MbONhNv6/gjwx2b2AumL63Qfawd330h6ldv3kL6WRouZvQj8HumxF9y9HfhZMA33s+6+jnSX19NB3Qd5bUIROS6aViuSZ0EX06C7u5ndDLzH3W881n4ihaYxDJH8uwS4K5jZ1EWIl8QVySW1MEREJCsawxARkawoYYiISFaUMEREJCtKGCIikhUlDBERyYoShoiIZOX/A7O5J7R6Av93AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr_find(learn)\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 20:17 <p><table style='width:375px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "    <th>time</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>0</th>\n",
       "    <th>4.283346</th>\n",
       "    <th>4.043954</th>\n",
       "    <th>0.292471</th>\n",
       "    <th>20:17</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1,1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd8VuX9//HXh5AQSEJYYYklDNmbyBBliCKCxUWtqG3RKtVa9VtqLS4c1YrVuvprbdUK1Vaoo7gQRVDqZASZskdQhhACskIISa7fH+fkkJAdcudOyPv5eOTByVn354q39+c+13XO5zLnHCIiIgC1wh2AiIhUHUoKIiISUFIQEZGAkoKIiASUFEREJKCkICIiASUFEREJKCmIiEhASUFERAK1wx1AWTVp0sQlJiaGOwwRkWplyZIle5xzCSXtV+2SQmJiIsnJyeEOQ0SkWjGzraXZT91HIiISUFIQEZGAkoKIiASq3ZiCiJw6jh07xrZt28jIyAh3KKeM6OhoWrVqRWRkZLmOV1IQkbDZtm0bcXFxJCYmYmbhDqfac86RlpbGtm3baNOmTbnOoe4jEQmbjIwMGjdurIRQQcyMxo0bn9SVl5KCiISVEkLFOtm/Z41JCotT9vLo+2vR9KMiIkWrMUnhjSXbeHb+JnYdOBruUESkCkhLS6NXr1706tWL5s2bc9pppwW/Z2Zmluoc1157LevWrQtxpJWrxgw0t24cA8DBjGM0j48OczQiEm6NGzdm2bJlANx///3ExsZy++2359vHOYdzjlq1Cv/+PHXq1JDHWdlqzJVCq4Z1wx2CiFQDGzdupEuXLlx99dV07dqVnTt3MmHCBJKSkujatSsPPvhgsO/ZZ5/NsmXLyMrKokGDBkyaNImePXsycOBAdu/eHcZWlF+NuVKIqOUNvuRoSEGkSnrgna9ZveNAhZ6zS8v63PfDrmU+bu3atbz00kskJSUBMGXKFBo1akRWVhbDhg1j7NixdOnSJd8x+/fvZ8iQIUyZMoWJEyfy4osvMmnSpAppR2WqMVcKfk4gW1lBRErQrl27ICEATJ8+nT59+tCnTx/WrFnD6tWrCxxTt25dLrzwQgD69u1LSkpKZYVboWrMlUIty71SUFIQqYrK840+VGJiYoLlDRs28PTTT7No0SIaNGjANddcU+hzAFFRUcFyREQEWVlZlRJrRasxVwrHu4+UFESk9A4cOEBcXBz169dn586dfPDBB+EOKaRq3JWCuo9EpCz69OlDly5d6NSpE61bt2bQoEHhDimkrLo9zJWUlOTKM8nO/9an8rMXF/HGTQPp27pRCCITkbJas2YNnTt3DncYp5zC/q5mtsQ5l1TEIYGa030UXCmEORARkSqsxiSF3LuPNKYgIlK0mpMUcgeaNaYgIlKkGpMUcu8+ytaVgohIkWpMUjj+nEKYAxERqcJqUFLw/lX3kYhI0WpMUgi6j5QURMQ3bNiwAg+jPfXUU9x0001FHhMbGwvAjh07GDt2bKH7DB06lJJunX/qqadIT08Pfh81ahTff/99aUMPmRqTFFTmQkRONG7cOGbMmJFv3YwZMxg3blyJx7Zs2ZLXX3+93K99YlJ47733aNCgQbnPV1FqXFL4fOOeMEciIlXF2LFjmTVrVjCpTkpKCjt27KB3794MHz6cPn360L17d956660Cx6akpNCtWzcAjhw5wpVXXknnzp259NJLOXLkSLDfTTfdFJTdvu+++wB45pln2LFjB8OGDWPYsGEAJCYmsmeP9/n0xBNP0K1bN7p168ZTTz0VvF7nzp254YYb6Nq1KyNGjMj3OhUlpGUuzCwFOAhkA1knPk1n3mSiTwOjgHRgvHPuq1DEsuN774/3zy+38sDF3ULxEiJyMmZPgu9WVuw5m3eHC6cUublRo0b069eP2bNnc/HFFzNjxgyuuOIK6taty8yZM6lfvz579uxhwIABjBkzpsj5j5999lnq1avHmjVrWLFiBX369Am2PfzwwzRq1Ijs7GyGDx/OihUruPXWW3niiSf4+OOPadKkSb5zLVmyhKlTp7Jw4UKcc/Tv358hQ4bQsGFDNmzYwPTp03n++ee54ooreOONN7jmmmsq5m/lq4wrhWHOuV5FPF59IXCG/zMBeDZUQTSJqxOqU4tINZa3Cym368g5x1133UWPHj0477zz2L59O7t27SryHJ988knw4dyjRw969OgRbHv11Vfp06cPvXv35uuvvy607HZen332GZdeeikxMTHExsZy2WWX8emnnwLQpk0bevXqBYSuPHe4C+JdDLzkvAJMC8ysgZm1cM7trOgX6nW611c3tm+rij61iFSEYr7Rh9LFF1/Mr3/9a7766ivS09Pp27cv06ZNIzU1lSVLlhAZGUliYmKh5bJLsmXLFh5//HEWL15Mw4YNGT9+fLnOk6tOneNfbiMiIkLSfRTqKwUHzDGzJWY2oZDtpwHf5vl9m78uJJrXj+b1Jdt0W6qIBGJjYxk2bBjXXXddMMC8f/9+mjZtSmRkJB9//DFbt24t9hyDBw/mlVdeAWDVqlWsWLEC8Mpux8TEEB8fz65du5g9e3ZwTFxcHAcPHixwrnPOOYc333yT9PR0Dh8+zMyZMznnnHMqqrklCnVSONs51wevm+hmMxtcnpOY2QQzSzaz5NTU1HIH890BL0M/Nmdduc8hIqeecePGsXz58iApXH311SQnJ9O9e3deeuklOnXqVOzxN910E4cOHaJz585MnjyZvn37AtCzZ0969+5Np06duOqqq/KV3Z4wYQIjR44MBppz9enTh/Hjx9OvXz/69+/P9ddfT+/evSu4xUWrtNLZZnY/cMg593iedX8H5jvnpvu/rwOGFtd9VN7S2QCJk2YFyylTRpfrHCJScVQ6OzSqZOlsM4sxs7jcZWAEsOqE3d4GfmqeAcD+UIwnnOiaAT8I9UuIiFRLoew+agZ8ZmbLgUXALOfc+2Z2o5nd6O/zHrAZ2Ag8D/wyhPEwtGMCAJt2Hw7ly4iIVFshu/vIObcZ6FnI+r/lWXbAzaGK4UR//0lfOt7zPl9uTuPzjXsY1L5JyQeJSEg554q8/1/K7mSHBGrME80AdWpHBMsPzVoTxkhEBCA6Opq0tLST/iATj3OOtLQ0oqOjy32OcD+nUOma14/muwMZrNl5INyhiNR4rVq1Ytu2bZzMXYWSX3R0NK1alf95rBqXFD773TCunbaYr3coKYiEW2RkJG3atAl3GJJHjeo+AqgdUYsBbRuz93AmiZNmsVrJQUQkUOOSAkC7hJhgefaqkN8BKyJSbdTIpNC+aVywvHN/+euQiIicampoUohl+g0D6NKiPrsOKCmIiOSqkUkBYGC7xpzeqG4wz4KIiNTgpADQuUV9Nu85zOGjWeEORUSkSqjRSaFNkxicg880RaeICFDDk0LbJrEA/OLlJRzNyg5zNCIi4Vejk0K30+oHyyqSJyJSw5OCmTH9hgEArNulh9hERGp0UgDo/QNv7uZf/2d5mCMREQm/Gp8UoiMjSt5JRKSGqPFJAeCKJK+i4P70Y2GOREQkvJQUgBFdmgOwYEtamCMREQkvJQXgzMRGADz/yeYwRyIiEl5KCkB8vUgAkrfuI+OYnlcQkZpLScHXpolXTvthTdMpIjWYkoJv7sQhALy1bHuYIxERCR8lBV9ELQPgcKa6j0Sk5lJSyOPq/j8gJioC51y4QxERCQslhTzaJsRyICOLtMOZ4Q5FRCQslBTyaOvP3bxy+/4wRyIiEh5KCnl0bu5VTZ3z9a4wRyIiEh5KCnk0j4+mYb1I9qn7SERqKCWFE/Rt3ZCUNM2tICI1U8iTgplFmNlSM3u3kG0/MLOP/e0rzGxUqOMpyemN6vHN3nTdgSQiNVJlXCncBhT1mPA9wKvOud7AlcBfKyGeYrVuVI/0zGz2HFIXkojUPCFNCmbWChgNvFDELg7InRMzHtgRynhKo3Vj7w6kr3foDiQRqXlCfaXwFHAHkFPE9vuBa8xsG/AecEuI4ylR+6axAIyfupjNqYfCHI2ISOUKWVIws4uA3c65JcXsNg6Y5pxrBYwCXjazAjGZ2QQzSzaz5NTU1BBF7Dm9Ub1g+a6ZK0P6WiIiVU0orxQGAWPMLAWYAZxrZv86YZ+fA68COOe+BKKBJieeyDn3nHMuyTmXlJCQEMKQPSlTRgOwYPNedh3ICPnriYhUFSFLCs65O51zrZxziXiDyB855645YbdvgOEAZtYZLymE9lKgjOas1oNsIlJzVPpzCmb2oJmN8X/9DXCDmS0HpgPjXRW5F/TdW84G4N43V4U5EhGRylO7Ml7EOTcfmO8vT86zfjVeN1OV0+20+HCHICJS6fREczFuHtYOgJ37j4Q5EhGRyqGkUIykxEYADHzkozBHIiJSOZQUijG0w/E7nQ4fzQpjJCIilUNJoRhmxhA/MWzSg2wiUgMoKZTg3ou6ADB/XZW6U1ZEJCSUFErQurH3hPMTH64PcyQiIqGnpFCCyIjjf6LsnCrxCIWISMgoKZTCI5d1B2D7Pt2aKiKnNiWFUujQLA6ADbsPhjkSEZHQUlIohdxy2j//ZzJHs7LDHI2ISOgoKZRCfN3IYHnmV9vDGImISGgpKZTSvN8MAeCTDbo1VUROXUoKpdQuwetCem/ld2GOREQkdJQUyuCKpFYAHMw4FuZIRERCQ0mhDEZ2aw7Amp26C0lETk1KCmXQtaU3x8LXO/aHORIRkdBQUiiDpnF1aBIbxdc7DoQ7FBGRkFBSKAMzo11CLFvTDoc7FBGRkFBSKKP4upEsTtlHVnZOuEMREalwSgpltGKbN56g5xVE5FSkpFBGf7m6NwBphzLDHImISMVTUiijHq0aAPDmMpW7EJFTj5JCGeXOr/D5xjQOad5mETnFKCmUQ2SEAXDPzJVhjkREpGIpKZTDqgcuAODNZTtIz9TVgoicOpQUyqFO7QhioiIA+HD1rjBHIyJScUqVFMysnZnV8ZeHmtmtZtYgtKFVbV9NPp/ICGO1nm4WkVNIaa8U3gCyzaw98BxwOvBKyKKqBurUjqBdQiwbdx8KdygiIhWmtEkhxzmXBVwK/Nk591ugRWkONLMIM1tqZu8Wsf0KM1ttZl+bWbVKNO2bxrL2O1VMFZFTR2mTwjEzGwf8DMj9cI8sZv+8bgPWFLbBzM4A7gQGOee6Av9XynNWCfuPHGP790e4dfrScIciIlIhSpsUrgUGAg8757aYWRvg5ZIOMrNWwGjghSJ2uQH4i3NuH4Bzbncp46kSrhnQGoC3l+8IcyQiIhWjVEnBObfaOXerc266mTUE4pxzj5bi0KeAO4Ciqsd1ADqY2edmtsDMRha2k5lNMLNkM0tOTa06NYcu6No8WF6csjeMkYiIVIzS3n0038zqm1kj4CvgeTN7ooRjLgJ2O+eWFLNbbeAMYCgwzj9vgbuanHPPOeeSnHNJCQkJpQm50jwzzquF9NKXW8MciYjIyStt91G8c+4AcBnwknOuP3BeCccMAsaYWQowAzjXzP51wj7bgLedc8ecc1uA9XhJotoY07MlAO8s36Fy2iJS7ZU2KdQ2sxbAFRwfaC6Wc+5O51wr51wicCXwkXPumhN2exPvKgEza4LXnbS5lDFVOe3vns2mVN2iKiLVV2mTwoPAB8Am59xiM2sLbCjPC5rZg2Y2xv/1AyDNzFYDHwO/dc6llee84fTi+KRg+Zl55fqziIhUCeacC3cMZZKUlOSSk5PDHUYBRzKz6Tz5fQBSpowOczQiIvmZ2RLnXFJJ+5V2oLmVmc00s93+zxv+7abiqxsVwVntGgOQmaWxBRGpnkrbfTQVeBto6f+846+TPNomxAAw7YstYY5ERKR8SpsUEpxzU51zWf7PNKBq3RtaBfz2gk6ApuoUkeqrtEkhzcyu8esYRZjZNUC1GxAOtfi6XuWP5z6ttjdQiUgNV9qkcB3e7ajfATuBscD4EMVUrXU/LR7n4Pt0XS2ISPVT2jIXW51zY5xzCc65ps65S4DLQxxbtTSkg9er9pN/LApzJCIiZXcyM69NrLAoTiETz+8AwNa0w2GORESk7E4mKViFRXEKqVXL6PODBhzIyCJx0ix2fH8k3CGJiJTaySSF6vXUWyWadGHnYPn9Vd+FMRIRkbIpNimY2UEzO1DIz0G85xWkEP3aNKJfYiMAlmzdF+ZoRERKr9ik4JyLc87VL+QnzjlXu7KCrI7+84sBAMxauZPESbPIztGFlYhUfSfTfSTFMMs/5NLurveY87W6kkSkalNSCKHXbxxIndrH/8QTXi5uviERkfBTUgihpMRGrHvoQhLi6gTrnp2/KVg+lp3DkczscIQmIlIolc6uBKkHj/LFpj3cNmNZsG7ShZ34YlMan6xPValtEQm5Ci2dLScnIa4OP+yR/2atKbPX8sn6VAAOH80KR1giIgUoKVSSWrWMV27oX+i2b/amV3I0IiKFU1KoRGe1a8KWR0ax7qGR+dYrKYhIVaGkUMnMjDq1I0iZMpplk88H4FslBRGpIpQUwii+biR1IyPYdSAj3KGIiABKCmFlZuQ4x/OfbiFx0iyOZWtuZxEJLyWFMDuvc7Ng+fk8M7aNePJ/TP1ccz2LSOVSUgizX/vzLwD88f11JE6axb8XbmX9rkM88M5qXk3+NozRiUhNo6QQZu2bxjJ34uB86+6euSpYvuP1FZUdUrWWnePYf+RYuMMQqbaUFKqA9k3jmDC4bZHbD2Sc2h9yS7bu490VO076PEcys2l313v0fGAOew4drYDIRGoeJYUqon60V4n8txd05CcDWjP+rEReu3EgAF9sTAtnaCGRnnn8Ke7Ln/2CX72ylPKWXHl3xQ5SDx6l8+T3g3Uzv9p+0jGK1ESaE6GKuP6ctnRtGc/QjglB2e3cYnnTF33DyG7NwxlehXp5wVbufXMVC+4cTuPYqGD95j2HaZcQW6Zzbdx9iF+9srTA+offW8OQjgl0aBZ30vGK1CS6UqgioiMjGNapab55GOpGRQDwv/Wp7E8vexfS0axsXln4DVlV7FbXe9/0xkwGPDKPM+6eHaxfuHlvqc/xxaY9JE6axXlP/C/f+mnXnhksj3jyE+at2VXqcx7MOMYXG/eUen+RU1HIk4KZRZjZUjN7t5h9LjczZ2YlVvCraVrERwPQ88E5HCzF2MITH65n/rrdfLs3nY73vM9dM1cyc2nBrpRv96bzxIfrKyxhHM3KPuky4Iu2FN9NNu3zLXy3P4MnP1zPVc8vLHSfoR2bsubB42VESjuHRdqho3S/fw5XvbCQ1TsOlD5okVNMZVwp3AasKWqjmcX5+xT+f3kN98ZNZwXL3e+fw+6DRT/9nJmVwzPzNjB+6mIefX9tsP7bfUcK7Pv0vA08M28DryZvy7f+QMYx/jp/Y5HJIis7h6tfWMDCzcc/wPenH6PjPe/TefL7pB7MP8D7+3dXkzhpFv9Z/A2Jk2aROGkWAOPPSuTyPq0A6HZafS7o2ow3lxU92Pz6km3c/85qBjwyj6fnbSh0n7kThwDeFdbyySMA726kkU99UuR5d3x/hL/O30hynrm0Rz3zab4xD5GaJKRjCmbWChgNPAxMLGK33wOPAr8NZSzVVcsGdZnz68GMeNL7YFv6zfdc0PX4+MKuAxkYMH99Kj1bNQjWv7tiZ7D8zLwN4BwNY6KoXctYnLKPt5d7H8B3zVzJVf1/EOz76Oy1/HvhN6QePMp9P+xaIJ5VOw7w+cY0Pt+YFswD8Zf5G4Ptf5qzjimX9wDgzaXb+cdn3gN4v3tjZb7z/GZEB+KiI/nTFT0BmPCSN0fG3/63iesGtSEqz4x1CzencftrywvE8qth7bn9go6k7DnMh6t30S4hJtgWXy+Sjs3iWLfrIGu/O8j+I8eIrxtZ4BxnTfmowDqALpM/YN1DI6lTO6LQ7SKnqlBfKTwF3AEU+rXTzPoApzvnZhV3EjObYGbJZpacmpoagjCrtg7N4vjbNX0A78PeOUdOjmPh5jT6/2Ee/f4wjzteX8EFJ3wj7pfYiJ6ne4nimY828sA7q7n3ra+DhJDraJbX7ZOZlRNUbH1jiXcFkZPj6PfwXP40Zx1Ltu7jgXe+Do5LPXiU+et289wnx5/EnrHYe9ju273p/N9/jk8qlOuZcb1Zef8I4qLzf0DfP8ZLQFNmr+XO/3oJJDvHMXvlTn783IIC53nyxz2Z6D/4l9gkhhsGty0wL/aMCQOI8cdlej4wJ9825xx3zcyfqE7U8Z73+c/ib/J1iy3Zuo9rpy4i45i3Lj0zq9x3TYlURSG7UjCzi4DdzrklZja0kO21gCeA8SWdyzn3HPAceDOvVWyk1cPIbi1oEluH2St30sb/UP9x0unFHvOfXwzAzHhm3gae+HB9ge2/Pq8DT85dz4Zdh8jMzuGyv34RbGscWwfnHMlb97H74FH+/NFG/vzRxnzHn/nw3Hy/t28ay8bdh3hr2fZ8s8zlmnRhJ8b0bFlgPXhXRLne+GobaYePMn9d/i8Avxjclr/7CWh095bUqpU/CZyoYUwUS+49n073ereqPjxrNXeP7kLKnsN89c0+Xln4DQCX9GrJm8t2MKp7c+4a1ZmzH/04OMfv3ljJmp0Hue+HXbj3rVX8a4F3TKd732f8WYlM+yIFgMV3n5dv2lWR6ipk03Ga2SPAT4AsIBqoD/zXOXeNvz0e2AQc8g9pDuwFxjjnipxvszpOx1lRrpu2mI/W7g5+H9m1Oe9//V2B/dY9NJLMrJx838aHPPYx+w5ncknv03hgTFfMjO/2ZzDgkXnFvmZcdG0OZpTcv27mPWPxx/fX5Vv/yvX92XM4k+ycHC7t3arYc9zx+vICYxy5po4/k2GdmpKd46hlFLgqKM7dM1fybz8BzJ04mPOeyH9FtfkPo8jKcfm6rIBg/KNVw7psK2RcJq/rBrVh8g+7FFj/9NwNPDnXS8g9T2/AWzcPKnXcIhWptNNxVsoczf6Vwu3OuYuK2We+v0+xn/g1OSls2HWQ858sOGi64M7hHDqaxUdrd/HDni1pEV+3kKMLl/vBl2toxwR+Mbgd454v2GWTa9UDF9Dtvg+C32Pr1GbuxCHUjYrI103z7i1n0+20+FLHAvnb2CI+moFtG7P9+yO89PN+5e7f/z49k14PfgjAHy/vwR1vHC8dUtz82LsPZtDv4YJJ8+z2TfiskFtXR3RpxjPjepN2OJPTGtQlPTOLLpM/yLdPeefjds6VKREW5fDRLCIjahVIgHLqK21SqPSH18zsQSDZOfd2Zb92dXdGs7igiybXxb1a0ty/bbV907I9+AXe3U2XP3u822jatf0KlPC+pFdLImrVoklsFP3bNiK2Tm2mXXsmf/5oI0u27uPPV/UOYjivc1PmrtnN8vtGFDqwW5o2/jjpdDKysnn6yt5lPr4wDepFMXX8mVw7bXG+QfHf5ClGWJimcdHUqV2Lo1ne3+PiXi1JTtnHQ5d0Y+jj8wH49I5hnPNHr7tpzupd9Ht4LgeKubI6mHGMupER1I4o/YfyrgMZXP7sF0y+qAuDOyTw8drdXNi9RamPz3X4aBZd8yTzwsZ2RCrlSqEi1eQrBfAGfrNyHB3umc2kCztx45B2FXLexEmzuCKpFX8c2zNYd/hoFr9/dzUTR3SgaVx0kfGU1LdfFRzIOEaP+49fxZT2G/ux7By2ph2mRXxdYuoc/w711Tf7aNM4hoYxUTjnaHPne2WK5/qz23DPRQW7mwqT+wQ4wGW9T+O/S7cX+G9VGnf+dwXTF+Wvunte56ZcN6gNXVvGM2vlTi7p3ZJ6USp0cCqqUt1HFammJwUpv6ueX8AXm9IY27cVj/+obB+oJcnJcbS9q/DEsPy+EWxNO8yY//d5vvUpU0Zz6GgWUSV05+Tt4msZH82O/d6zKp9POpfTGpS/q7AoWx4ZVSFdVVK1VNnuI5FweeWGASE7d61axj+v68eanQe4rM9pbNx9iFteWUqrhnWJrxtJjzzPkORKz8zi3Mfnc2abRkw4py09WsUX+DA+fDR/V1RuQgAYNOUj2jeN5ZZz23PbjGWM6NKMhy7pRtP6+a/qDh3NYmvaYQAa1ovkyn4/4Nn5m4psy5eb0jirfZMy/w3k1KArBZFKdtfMlcHtsHn9OOl0Hh3bI/h9/a6DwUOL1w1qw4ulnIlv9YMXBF1AJ3Zt3T6iA7869wzW7zrI5xv3MP6sRK5+YSFfbMpfYqS8A+JSdZX2SkG3IIhUst9d0KnQ9d8dyF/CZESeO81uGNyG+/1bXmfdenax51+0xSssmJPjeHNZ/rpXuXeDdWgWx7WD2njPsYzrTXRkLRbcOTzY7/Ulhd8aLKc+dR+JVLL4epE0qBfJ9+nHmHRhJ/7x2RZSDx6lXtTxW243pR6/w6xT8zhaxNdl/KA2jB/UBoDnftKX7BzHnsOZ3PvmKmbdejZtmsTQZfIHjJ+6mJioCA4XUqBwaMemBdY1ia3D2t9fCMC9F3Xh9++u5vbXlnNZ79OqxU0EUrHUfSQSBvuPHGPV9v0M8vvuf/KPhRw4coy3fuVdBeQOCv95XG/O7dQ0351PxSlqMHnNgyOJjqxV4gBydo6jnT9gPqZnS54ZV/JtwQcyjpGRmV1gLEOqFnUfiVRh8XUjg4QA3jMRy7ft56UvUzg/zxwRo7u3KHVCAPjXz/sXur5uVESp7iiKqGXcem57AN5evoPl335P4qRZ3DK94ERGuc59/H/0+8Pxh/wys3Kq3BweUnpKCiJVQFu/wuvkt75mg/9w4gNjupa5++bsM5qQMmU0K+4fQcqU0ZzfpRm3Dj+jTOeYOKIjUf7DdRf/xbuN9p3lO4os/Jc7H/Y/v0jh273pdLhnNu3vno1zjsysnKB4oFQP6j4SqQJOLIkRGWGsfnAkkWV48rmiFdYV9fqNA2nVsF7wBPueQ0dJemhugf1O9Mr1/Xlr2Q6uPTuRTs3rV3isUjJ1H4lUI/WiavP6jQOD3x8b2zOsCQG8AocAwzsdH5we+7cvmfDy8S9lK7ftL9W5rnphIf9J/paRT31Kdk7RX0T/tz6VuatLP4WqVDwlBZEqIimxEYvuHs6L45O4uFfhJcYr0y+HtuMfP0viL1f34Z1fHb8NdsW2/eT4H+x1PI0dAAAQuElEQVTJW/diBk/9uFewfe7EwcWet+M9swtdn3Esm5+9uIjrX0rmrEfmkXrwaIGH9zalHuLjdbsLPV4qhpKCSBXSNC6aczs1qxJlJsyM4Z2bER0ZQfdW8ay8fwTndW4GwJzV37H7YAaLU/bRtklMvkHz9k3jSGrdkNHdW5AyZTSzbzsn33mzchx7D2cGv2dm5ZCcsper8lTm3bE/gzMfnkvX+z4IpnHdnHqI4X/6H9dOXRwki4xj2SROmhVMzFRe+9OPkThpFqOe/vSkzlNaH6/dzZqdB+h23wfc/O+vKuU1S0tjCiJSakcys+k8+f3g9+b1oxnUvgmP/6gH7e56j58OTAxm0ctr6GMfM7JbCzbuPsjcNd43/bduHhQMZJfVyz/vxzlnJBQY93j//86hU/P6vJr8LQ3rRbF6xwE6No9lZLfCq8ruOXSUTbsP8fynW5i7xuu2eu3GgZyZ2IjXl2zj9teWl7nGVElunb60wOyHPxnQmt9f0q3CXqMwKognIiFx4gfxby/oyM3DvNtYS5r3Yef+Iwx8pPB5sXNt/sMoFmxO46oXFha739rfjwxm1csrZcroAjEWVrajuEHyLY+MCsqDjD+r8ERXkv3px4ivV7A0eVHPkoS6tIgK4olIpRjSISFYLqnbq0V8XW45t32+qV3H9TudjGM5DGzbmEv8p6jPat+ED/5vMDnOEVunNoeOZnHhCV07hSUEKPxDN3HSLMb0bBl8Q68XFUF6IU985/pyUxpxdWpz8GgW075IYdoXKXz468Gc0Swu3365z2Pkzo9x18yVdGlRn/i6kdwyfWkwYyB4Vwjf7ksPjr33oi50Py2eK/7+JQD7DmfSMCYq2L5oy15mLPqGawa2xvCeNamMO7eUFESkTHInLMpV1tn1fjOiIw3rRfHgu6tpFBPFI5f1KHS/js3zfwDnfpPOOJadLyHcM7ozZzSLI/XgUW5/bXmRr5u3y6awhLBs8vn847Mt/PmjjWxJO8yRE56vOP/JT7igazM++NrrZnpmXG9u9R/q2/yHUfx+1uqg0OHoHl53Ve7fadHdw/O9/vBOTfn52V7Jkr//pC+/eHkJq3ceYOf+DPq3aUR6ZjY/fXEhGcdy+O/S4/Wr7hndmevPaVtkGyuCuo9EpMyOZedw5sNzuW34GVzr12Mqi+Xffs/Ff/mc6MhaQd2lspj0xgpmLPYmDMpNFidWhI2Lrs1jY3vyzd7D/OG9tYWeJ2/XF3jf/NvfPZuOzeJYt+sgpzWoy/bvi5+fuzzeuGkgfVs3AkrXpZbryzvPLdN0u3mp+0hEQiYyohbLJo8o9/FdW9anUUxU8CxEWd0y/AxmLP6WJrF1gnVmxn9/eRZREbVolxBL3TwFBls3juEXLy9h6rVncu1U79t73hLjuXK7gdbtOgjAlMu7s//IMQZ3SMg3c1953T2qM1G1awUJAbzB+qLUi4qgdi3jQEYWbRNiyp0QykJXCiJSLa3avp/TG9Ur81zgq7bvJ6ZObdo0iSl0+6V//Zyl33wPeIUE8yaXBZvTqBcVwfZ9R3hszjpeuq4fj3+wjjeXeV1D834zhOF/8mpXTTy/AznO0al5HDf+6ytq1zLWP3RhoaVLihp8/ure82kUE0V2jsPgpKrW6u4jEZFyWLg5jR8/t4CPfjOEtgmxJe5/6GgWvR+cQ2yd2iTfcz7f7k3ncGYWXVseH2v5Ji2dlg2igyuREx3JzCY9M4u6URHc9d+VQZKpyDuSlBRERKqpR99fS/P60fzsrMQKO6fGFEREqqnfjSx8dr7KoDIXIiISUFIQEZGAkoKIiASUFEREJKCkICIiASUFEREJhDwpmFmEmS01s3cL2TbRzFab2Qozm2dmrUMdj4iIFK0yrhRuA9YUsW0pkOSc6wG8DvyxEuIREZEihDQpmFkrYDTwQmHbnXMfO+dyC4wvAFqFMh4RESleqK8UngLuAHJKse/PgUJn9DazCWaWbGbJqampFRmfiIjkEbKkYGYXAbudc0tKse81QBLwWGHbnXPPOeeSnHNJCQkJhe0iIiIVIJS1jwYBY8xsFBAN1Dezfznnrsm7k5mdB9wNDHHOHQ1hPCIiUoKQXSk45+50zrVyziUCVwIfFZIQegN/B8Y453aHKhYRESmdSn9OwcweNLMx/q+PAbHAa2a2zMzerux4RETkuEopne2cmw/M95cn51l/XmW8voiIlI6eaBYRkYCSgoiIBJQUREQkoKQgIiIBJQUREQkoKYiISEBJQUREAkoKIiISUFIQEZGAkoKIiASUFEREJKCkICIiASUFEREJKCmIiEhASUFERAJKCiIiElBSEBGRgJKCiIgElBRERCSgpCAiIgElBRERCSgpiIhIQElBREQCSgoiIhJQUhARkYCSgoiIBJQUREQkEPKkYGYRZrbUzN4tZFsdM/uPmW00s4VmlhjqeEREpGiVcaVwG7CmiG0/B/Y559oDTwKPVkI8IiJShJAmBTNrBYwGXihil4uBf/rLrwPDzcxCGZOIiBQt1FcKTwF3ADlFbD8N+BbAOZcF7AcahzgmEREpQsiSgpldBOx2zi2pgHNNMLNkM0tOTU2tgOhERKQwobxSGASMMbMUYAZwrpn964R9tgOnA5hZbSAeSDvxRM6555xzSc65pISEhBCGLCJSs5lzLvQvYjYUuN05d9EJ628GujvnbjSzK4HLnHNXlHCuVGBrOUNpAuwp57FVzanSFrWj6jlV2nKqtAMqpi2tnXMlfquufZIvUmZm9iCQ7Jx7G/gH8LKZbQT2AleWdHxpGlXMayc755LKe3xVcqq0Re2oek6Vtpwq7YDKbUulJAXn3Hxgvr88Oc/6DOBHlRGDiIiUTE80i4hIoKYlhefCHUAFOlXaonZUPadKW06VdkAltqVSBppFRKR6qGlXCiIiUowakxTMbKSZrfOL700KdzwnMrMXzWy3ma3Ks66RmX1oZhv8fxv6683MnvHbssLM+uQ55mf+/hvM7GdhaMfpZvaxma02s6/N7LZq3JZoM1tkZsv9tjzgr2/jF3Dc6Bd0jPLXF1ng0czu9NevM7MLKrstfgz5ilNW43akmNlKM1tmZsn+uur4/mpgZq+b2VozW2NmA6tEO5xzp/wPEAFsAtoCUcByoEu44zohxsFAH2BVnnV/BCb5y5OAR/3lUcBswIABwEJ/fSNgs/9vQ3+5YSW3owXQx1+OA9YDXappWwyI9ZcjgYV+jK8CV/rr/wbc5C//Evibv3wl8B9/uYv/nqsDtPHfixFheI9NBF4B3vV/r67tSAGanLCuOr6//glc7y9HAQ2qQjsq9T9muH6AgcAHeX6/E7gz3HEVEmci+ZPCOqCFv9wCWOcv/x0Yd+J+wDjg73nW59svTG16Czi/urcFqAd8BfTHe4io9onvLeADYKC/XNvfz058v+XdrxLjbwXMA84F3vXjqnbt8F83hYJJoVq9v/CqN2zBH9etSu2oKd1HQeE93zZ/XVXXzDm301/+DmjmLxfVnirVTr/boTfeN+xq2Ra/y2UZsBv4EO/b8ffOK+B4YlxFFXisCm05sThlY6pnOwAcMMfMlpjZBH9ddXt/tQFSgal+l94LZhZDFWhHTUkK1Z7zvgZUm1vFzCwWeAP4P+fcgbzbqlNbnHPZzrleeN+0+wGdwhxSmVkFFqesIs52zvUBLgRuNrPBeTdWk/dXbbzu4medc72Bw3jdRYFwtaOmJIWg8J6vlb+uqttlZi0A/H93++uLak+VaKeZReIlhH875/7rr66WbcnlnPse+Bivm6WBeQUcT4yrqAKP4W5LgeKUwNNUv3YA4Jzb7v+7G5iJl6yr2/trG7DNObfQ//11vCQR9nbUlKSwGDjDv9siCm/w7O0wx1QabwO5dxP8DK9/Pnf9T/07EgYA+/1Lzg+AEWbW0L9rYYS/rtKYmeHVtFrjnHsiz6bq2JYEM2vgL9fFGxtZg5ccxvq7ndiW3DaOBT7yv+29DVzp39XTBjgDWFQ5rQDn3J3OuVbOuUS89/5HzrmrqWbtADCzGDOLy13Ge1+sopq9v5xz3wHfmllHf9VwYHWVaEdlDxKF6wdv9H49Xp/w3eGOp5D4pgM7gWN43yJ+jtePOw/YAMwFGvn7GvAXvy0rgaQ857kO2Oj/XBuGdpyNd8m7Aljm/4yqpm3pASz127IKmOyvb4v3YbgReA2o46+P9n/f6G9vm+dcd/ttXAdcGMb32VCO331U7drhx7zc//k69//lavr+6gUk+++vN/HuHgp7O/REs4iIBGpK95GIiJSCkoKIiASUFEREJKCkICIiASUFEREJKClIlWNm2X4FzOVm9pWZnVXC/g3M7JelOO98Mzsl5uytKGY2zczGlryn1BRKClIVHXHO9XLO9cQrwvZICfs3wKvsWSXleWpYpMpTUpCqrj6wD7x6SmY2z796WGlmF/v7TAHa+VcXj/n7/s7fZ7mZTclzvh+ZN0fCejM7x983wsweM7PFfq36X/jrW5jZJ/55V+Xun5d5tf3/6L/WIjNr76+fZmZ/M7OFwB/Nq5P/pn/+BWbWI0+bpvrHrzCzy/31I8zsS7+tr/m1pDCzKebNVbHCzB731/3Ij2+5mX1SQpvMzP6fefMhzAWaVuR/LKn+9A1GqqK65lUmjcYrD3yuvz4DuNQ5d8DMmgALzOxtvEJi3ZxXuA4zuxC4GOjvnEs3s0Z5zl3bOdfPzEYB9wHn4T09vt85d6aZ1QE+N7M5wGV45aQfNrMIvPLZhdnvnOtuZj/Fq0Z6kb++FXCWcy7bzP4MLHXOXWJm5wIv4T3Rem/u8X7sDf223QOc55w7bGa/Ayaa2V+AS4FOzjmXW4IDmAxc4JzbnmddUW3qDXTEmxuhGV5phRdL9V9FagQlBamKjuT5gB8IvGRm3fAe9f+DeVUxc/BKBDcr5PjzgKnOuXQA59zePNtyC/QtwZu/Arx6MT3y9K3H49X1WQy8aF6Bvzedc8uKiHd6nn+fzLP+Nedctr98NnC5H89HZtbYzOr7sV6Ze4Bzbp95VU274H2QgzcBy5d4JawzgH+YN3vau/5hnwPTzOzVPO0rqk2Dgel+XDvM7KMi2iQ1lJKCVGnOuS/9b84JeDWUEoC+zrlj5lX9jC7jKY/6/2Zz/P1vwC3OuQKFxPwENBrvQ/cJ59xLhYVZxPLhMsYWvCzwoXNuXCHx9MMrnjYW+BVwrnPuRjPr78e5xMz6FtUm/wpJpEgaU5Aqzcw64U2nmob3bXe3nxCGAa393Q7iTf2Z60PgWjOr558jb/dRYT4AbvKvCDCzDuZV42wN7HLOPQ+8gFfauDA/zvPvl0Xs8ylwtX/+ocAe580z8SFwc572NgQWAIPyjE/E+DHFAvHOufeAXwM9/e3tnHMLnXOT8SZuOb2oNgGfAD/2xxxaAMNK+NtIDaMrBamKcscUwPvG+zO/X/7fwDtmthKvuuRaAOdcmpl9bmargNnOud+aWS8g2cwygfeAu4p5vRfwupK+Mq+/JhW4BK+i6G/N7BhwCPhpEcc3NLMVeFchBb7d++7H64paAaRzvDzyQ8Bf/NizgQecc/81s/HAdH88ALwxhoPAW2YW7f9dJvrbHjOzM/x18/AqiK4ook0z8cZoVgPfUHQSkxpKVVJFToLfhZXknNsT7lhEKoK6j0REJKArBRERCehKQUREAkoKIiISUFIQEZGAkoKIiASUFEREJKCkICIigf8PiVWsDEFXd+gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot_losses()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7 fasta.ai1 DEV",
   "language": "python",
   "name": "fastai1_dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.text import *\n",
    "import re\n",
    "import sentencepiece as spm #https://github.com/google/sentencepiece\n",
    "\n",
    "np.random.seed(20180311)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentencepiece tokenizer & LM Build"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data processing to DataBunch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get IMDB Data and build DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/home/farzin/.fastai/data/imdb')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "IMDB_PATH = untar_data(URLs.IMDB)\n",
    "IMDB_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75000, 25000)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## raw data does not have the unsup in the train directory.  Just move it there\n",
    "#  ~/.fastai/data/imdb$ mv unsup/ ./train/\n",
    "CLASSES = ['neg', 'pos', 'unsup']\n",
    "\n",
    "def get_texts(path):\n",
    "    texts,labels = [],[]\n",
    "    for idx,label in enumerate(CLASSES):\n",
    "        for fname in (path/label).glob('*.*'):\n",
    "            texts.append(fname.open('r', encoding='utf8').read())\n",
    "            labels.append([0]) ## all unsupervised case\n",
    "\n",
    "    return np.array(texts),np.array(labels)\n",
    "\n",
    "train_texts,train_labels = get_texts(IMDB_PATH/'train')\n",
    "valid_texts,valid_labels = get_texts(IMDB_PATH/'test')\n",
    "\n",
    "len(train_texts),len(valid_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## we can combine the \"train\" and \"valid\" for the LM step\n",
    "all_texts = np.concatenate([train_texts,valid_texts])\n",
    "all_texts_df = pd.DataFrame({'text':all_texts, 'labels':[0]*len(all_texts)}, columns=['labels','text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Default Rules:\n",
      " ['fix_html', 'replace_rep', 'replace_wrep', 'spec_add_spaces', 'rm_useless_spaces'] \n",
      "\n",
      "\n",
      "<function fix_html at 0x7f1058e22ea0>\n",
      "<function replace_rep at 0x7f1058e22d90>\n",
      "<function replace_wrep at 0x7f1058e22e18>\n",
      "<function spec_add_spaces at 0x7f105dbc2730>\n",
      "<function rm_useless_spaces at 0x7f1058e22d08>\n"
     ]
    }
   ],
   "source": [
    "raw_text = all_texts_df.loc[:,'text']\n",
    "\n",
    "print(\"Default Rules:\\n\",[x.__name__ for x in defaults.text_pre_rules],\"\\n\\n\")\n",
    "\n",
    "for rule in defaults.text_pre_rules:\n",
    "    print(rule)\n",
    "    raw_text = raw_text.apply(lambda x: rule(str(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    <s>\"Fred Claus\" somehow avoids becoming this y...\n",
       "1    <s>Firstly I would like to point out that I on...\n",
       "2    <s>I finally got to have a look at this experi...\n",
       "3    <s>After having red the overwhelming reviews t...\n",
       "4    <s>About three minutes into this thing I start...\n",
       "Name: new_text, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## SentencePiece expects <s> ... </s> so we add that here:\n",
    "all_texts_df['new_text'] = '<s>' + raw_text + '</s>'\n",
    "all_texts_df['new_text'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "## save as a file for SP\n",
    "formatted_text_file = 'tmp_IMDB_SP_example'\n",
    "all_texts_df['new_text'].to_frame().to_csv(formatted_text_file, header=False,index=False,quotechar=' ')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SP Tokenizer SWIG wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# m.model and m.vocab created in the pwd\n",
    "vocab_size = 500\n",
    "model_prefix = 'wk103m_example'\n",
    "\n",
    "spm.SentencePieceTrainer.Train(f'--input={formatted_text_file}'\\\n",
    "                               f' --model_prefix={model_prefix}'\\\n",
    "                               f' --vocab_size={vocab_size}')\n",
    "#                               f'--unk_piece={UNK} --bos_piece={BOS} --eos_id=-1 --pad_piece={PAD}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     1\t<unk>\t0\r\n",
      "     2\t<s>\t0\r\n",
      "     3\t</s>\t0\r\n",
      "     4\t▁\t-2.84395\r\n",
      "     5\ts\t-2.97\r\n",
      "     6\tt\t-3.53742\r\n",
      "     7\t▁the\t-3.7748\r\n",
      "     8\t.\t-3.86454\r\n",
      "     9\t,\t-3.91818\r\n",
      "    10\t▁a\t-4.07048\r\n",
      "    11\tn\t-4.18876\r\n",
      "    12\ta\t-4.24687\r\n",
      "    13\te\t-4.25059\r\n",
      "    14\to\t-4.27973\r\n",
      "    15\ty\t-4.44583\r\n",
      "    16\t▁to\t-4.48563\r\n",
      "    17\ting\t-4.50046\r\n",
      "    18\t▁and\t-4.50961\r\n",
      "    19\t▁of\t-4.5516\r\n",
      "    20\t'\t-4.64682\r\n",
      "    21\td\t-4.69638\r\n",
      "    22\t▁in\t-4.76258\r\n",
      "    23\ti\t-4.77796\r\n",
      "    24\ted\t-4.78844\r\n",
      "    25\tc\t-4.81265\r\n",
      "    26\tr\t-4.84313\r\n",
      "    27\t▁is\t-4.84642\r\n",
      "    28\tre\t-4.896\r\n",
      "    29\tp\t-4.94282\r\n",
      "    30\t▁I\t-4.97998\r\n",
      "    31\ter\t-4.9895\r\n",
      "    32\tu\t-4.99602\r\n",
      "    33\t▁it\t-5.11542\r\n",
      "    34\tor\t-5.14418\r\n",
      "    35\tb\t-5.17535\r\n",
      "    36\tar\t-5.22629\r\n",
      "    37\tm\t-5.2283\r\n",
      "    38\tle\t-5.23756\r\n",
      "    39\t▁f\t-5.23919\r\n",
      "    40\tw\t-5.28417\r\n",
      "    41\tf\t-5.28488\r\n",
      "    42\tly\t-5.29174\r\n",
      "    43\t▁that\t-5.29407\r\n",
      "    44\tal\t-5.30172\r\n",
      "    45\tg\t-5.30389\r\n",
      "    46\tl\t-5.37982\r\n",
      "    47\tin\t-5.40889\r\n",
      "    48\t▁this\t-5.4378\r\n",
      "    49\t-\t-5.48409\r\n",
      "    50\tk\t-5.5797\r\n"
     ]
    }
   ],
   "source": [
    "## Vocab in order of frequency\n",
    "!head -n50 {model_prefix}.vocab | nl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "## load up the Processor\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.load(f'{model_prefix}.model')\n",
    "\n",
    "## itos from m.vocab file: just read directly and populate the dictionary\n",
    "itos = {}\n",
    "with open(f'{model_prefix}.vocab','r') as f:\n",
    "    for line_num,line in enumerate(f):\n",
    "        itos[line_num] = line.split(\"\\t\")[0]\n",
    "        \n",
    "stoi = {v:k for k,v in itos.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Based on Tokenizer() class: https://github.com/fastai/fastai/blob/master/fastai/text/transform.py#L87\n",
    "class CustomTokenizer():\n",
    "    '''Wrapper for SentencePiece toeknizer to fit into Fast.ai V1'''\n",
    "    def __init__(self,sp_processor,pre_rules:ListRules=None,post_rules:ListRules=None):\n",
    "        self.sp = sp_processor\n",
    "        self.pre_rules  = ifnone(pre_rules,  defaults.text_pre_rules )\n",
    "        \n",
    "    def __repr__(self) -> str:\n",
    "        return \"Custom Tokenizer\"\n",
    "\n",
    "    def process_text(self, t:str) -> List[str]:\n",
    "        \"Processe one text `t` with tokenizer `tok`.\"\n",
    "        for rule in self.pre_rules: t = rule(t)                                                                                 \n",
    "        toks = sp.EncodeAsIds(t)\n",
    "        \n",
    "        return toks \n",
    "    \n",
    "    def _process_all_1(self,texts:Collection[str]) -> List[List[str]]:\n",
    "        'Process a list of `texts` in one process'\n",
    "        return [self.process_text(t) for t in texts]\n",
    "                                                                     \n",
    "    def process_all(self, texts:Collection[str]) -> List[List[str]]: \n",
    "        \"Process a list of `texts`.\"                                 \n",
    "        return self._process_all_1(texts)\n",
    "    \n",
    "mycust_tok = CustomTokenizer(sp)\n",
    "#setup Vocab object for use in LM  \n",
    "# Vocab source: https://github.com/fastai/fastai/blob/master/fastai/text/transform.py#L122\n",
    "sp_vocab = Vocab(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labels</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Detective Frank Keller(Al Pacino, in a perfect...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>This is a VERY good movie. I give it a 10.&lt;br ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>I can't say I enjoyed this as much as \"The Big...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>I sat last night to see this film being played...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>First of all, I agree that the plot left somet...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   labels                                               text\n",
       "0       0  Detective Frank Keller(Al Pacino, in a perfect...\n",
       "1       0  This is a VERY good movie. I give it a 10.<br ...\n",
       "2       0  I can't say I enjoyed this as much as \"The Big...\n",
       "3       0  I sat last night to see this film being played...\n",
       "4       0  First of all, I agree that the plot left somet..."
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = np.random.permutation(len(all_texts))\n",
    "cut = int(0.1 * len(idx))\n",
    "LM_train_df = pd.DataFrame({'text':all_texts[idx[cut:]], 'labels':[0] * (len(all_texts)-cut)}, columns=['labels','text'])\n",
    "LM_valid_df = pd.DataFrame({'text':all_texts[idx[:cut]], 'labels':[0] * cut}, columns=['labels','text'])\n",
    "LM_train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path(\"./tmp_IMDB_example\")\n",
    "\n",
    "data = TextLMDataBunch.from_df(PATH, LM_train_df, LM_valid_df, \n",
    "                               tokenizer=mycust_tok, vocab=sp_vocab, \n",
    "                               text_cols='text', label_cols='labels')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  LM Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(data, AWD_LSTM, drop_mult=0.5)\n",
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  LM Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VOXZ//HPlY1ACCRA2PdFQFEWI+5brVRtK1ZtrV3cail1a/WprbXtz1a7WG0fq49WRVvbWrVWK60raF2q1gUDhEUExQCyJ6xhy379/pgTHWMCA+TMmUm+79drXpk55z5zvgkTrtxnuW9zd0RERPYkI+oAIiKSHlQwREQkISoYIiKSEBUMERFJiAqGiIgkRAVDREQSooIhIiIJUcEQEZGEqGCIiEhCsqIO0Jp69OjhgwcPjjqGiEjamD179gZ3L0qkbZsqGIMHD6akpCTqGCIiacPMViTaVoekREQkISoYIiKSEBUMERFJiAqGiIgkRAVDREQSooIhIiIJUcEQEZGEtKn7MPbVbc+/R119w8cXmn38Zdwiw5pt+rE2TbeP29bso7aN7xW/nQEZFjwP2hIsb9wmtj7WJsNirzOC9pkZHy0zMzLNyMiIrc/MaHxtZGXEfTUjK9PIysggOzPWLiczg6zM2OvszAxyMjPIyPj49yUi7YcKBnDXf95nV239h681zXnLGgtJTlbs0SH4mpuVSW52BrnZmXTMziQ3J/a1Y3YmnXIy6ZiTSV5OFp06ZJKfm01+bhZdcrPp2jGbgk6xr9mZ6vCKpDIVDGDR9afssY0HVaRpMfG49R89b1z3yW3cY8tjXz++rXvsDR2nwT++3GMrcKDBP9q+oSH2Lg0e26a+wfHgeYN78BrqP3zu1DU4DQ2xr/Ue97zBqa1voK7eqWtooLbeqauPfa2pb6A2eNTUBY/6BqrrGqiubaC6rp6q2gaqauvZuquWqtrY6501deysqae6rkkPrhn5HbLo1jmH7nk5dO/cgR6dc+jVJZdeXXLp3SWX3l1z6du1I106Zn2iByci4VPBSFDjf1At/z+l/8B2p77B2VVbz47qOrZV1VJZVUflrlq2Bo8tO2vZvLOGTTtq2Li9hpWbdjJnxWY27qj5xHt1zM6kT0Eu/Qo60q+gI30LOjKwWyeGFuUxtKgznTvoYy0SBv1mSVJkZhidO2TRuUMWvbrkJrxdTV0D5duqWF9Zxbqt1azduou1W6tYu3UXq7dU8c475WzYXv2xbXp3yWVk73zG9OvCQX27MqZvVwZ066heich+CrVgmFkBcC8whtgRlIvc/fW49V8FfkDsz/NtwLfdfV6w7hTgViATuNfdbwwzq6SmnKwM+hd2on9hpxbbVNXWs3LTTt6v2MH7FdtZWr6dd9ZW8urSDdQHh+y6dszmkP5dObhfV8YNKGDikG4UdMpJ1rch0iaE3cO4FZjh7mebWQ7Q9Ld+GXC8u282s1OBacDhZpYJ3AGcDKwC3jKzx919Uch5JQ3lZmcyolc+I3rlf2x5VW09767fxoLVW1m4eivzV21l2stl1DU4ZjCyVz5HDO3OsSN6cOSw7nTKUYdbZHdC+w0xs67AccAFAO5eA3zsgLS7vxb38g2gf/B8IrDU3cuC9/obMBlQwZCE5WZnckj/Ag7pX/Dhsqraeuav2sqbZRt5c9kmHn5rJX96bTk5WRkcPqQbJ47sySljetO3oGOEyUVSU5h/Ug0BKoD7zGwsMBv4jrvvaKH9N4Bnguf9gJVx61YBhze3kZlNAaYADBw4sBViS1uWm53JxCHdmDikG5cD1XX1vLVsMy8tKeeldyu4/slFXP/kIsYPLOC0MX343Ng+9Omq4iECYB7STQdmVkys13C0u79pZrcCle7+k2bangj8HjjG3Tea2dnAKe5+cbD+68Dh7n7Z7vZZXFzsmkBJ9seyDTt4ZuFanlmwjgWrt5JhcNLoXnzl8IEcN6KITN24KG2Mmc129+JE2obZw1gFrHL3N4PXjwLXNG1kZocQOzF+qrtvDBavBgbENesfLBMJ1ZAeeVxywnAuOWE4Kzbu4OG3VvL3kpU8t2g9/Qs78rUjBnFO8QAK83TCXNqf0HoYAGb2CnCxuy8xs58Cee5+ddz6gcALwHnx5zPMLAt4FziJWKF4C/iKu7+9u/2phyFhqKlr4NlF6/jrGyt4o2wTHbIymDyuLxcePYTRfbpEHU9kv+xNDyPsgjGOWO8hBygDLgTOAXD3u8zsXuAsoHFO2brG4GZ2GvA7YpfV/tHdf7Gn/algSNiWrNvGn19fzvQ5q9lVW8/nx/blqpMPYEiPvKijieyTlCkYyaaCIcmydWct97xSxh9eXUZNfQNfKh7AlZ8eQc+9uClRJBWoYIgkSfm2Ku54YSkPzvqA7MwMLj1xON84Zgi52ZlRRxNJyN4UDA0PKrIfeubn8rPJY/j3Vcdz7Ige3DxzCSf99j88vWAtbemPMRFQwRBpFYO653H314t58OLDyc/N4pIH5jD1r7M/Mc6VSDpTwRBpRUcN78FTVxzLD08dxYuLK5h0y8s8s2Bt1LFEWoUKhkgry8wwvnX8MJ684hj6FXTk2w/M4X/+Po+quEm6RNKRCoZISA7olc9jlxzFFZ8azj/mrOKcu19n3daqqGOJ7DMVDJEQZWdmcNWkkdz99UNZWr6d029/lbkfbI46lsg+UcEQSYLPHNSbxy45mg7ZGZxz9xvMWLgu6kgie00FQyRJRvbO5/FLj+Ggfl249ME5PK2T4ZJmVDBEkqgwL4e/XDSRcQMKuPyhuTwxb03UkUQSpoIhkmT5udn8+aKJTBhYwHf+Npd/lWogZkkPKhgiEejcIYs/XTiR4sHduPLhUp6ar8NTkvpUMEQiktchi/suOIwJAwv5zt/m8uzbOhEuqU0FQyRCeR2yuO/CwxjTryuXPjiHFxeXRx1JpEUqGCIRazynMbJ3Pt/662z+u3RD1JFEmhVqwTCzAjN71MwWm9k7ZnZkk/WjzOx1M6s2s+81WbfczBaYWamZacxyadO6dszm/osOZ0j3PL51/2zeXrM16kginxB2D+NWYIa7jwLGAu80Wb8JuAL4TQvbn+ju4xIdq10knRXm5fDniybSJTeLC+57i5WbdkYdSeRjQisYZtYVOA74A4C717j7lvg27l7u7m8BtWHlEEknvbvm8qeLJlJdW88F981iy86aqCOJfCjMHsYQoAK4z8zmmtm9ZrY3Ex878KyZzTazKeFEFEk9B/TK557zilm5aRcX/7lEo9xKygizYGQBE4A73X08sAO4Zi+2P8bdJwCnApea2XHNNTKzKWZWYmYlFRUV+x1aJBUcPrQ7t5wzjpIVm/nFU02P5IpEI8yCsQpY5e5vBq8fJVZAEuLuq4Ov5cB0YGIL7aa5e7G7FxcVFe1nZJHU8dlD+jDluKHc/8YKjTslKSG0guHu64CVZjYyWHQSsCiRbc0sz8zyG58Dk4CFoQQVSWHfmzSSsQMK+MGj8/lgo06CS7TCvkrqcuABM5sPjAN+aWZTzWwqgJn1NrNVwFXAj81slZl1AXoBr5rZPGAW8JS7zwg5q0jKycnK4PZzx4PB5Q/NoaauIepI0o6Zu0edodUUFxd7SYlu2ZC255kFa/n2A3O4+Jgh/PhzB0YdR9oQM5ud6K0LutNbJA2cenAfvnbEQO59dRlvlG2MOo60UyoYImni2tNGM7BbJ77/6Hx21tRFHUfaIRUMkTTRKSeLm84+hA827eSmGUuijiPtkAqGSBo5Ymh3LjhqMH96bbkOTUnSqWCIpJnvnzJSh6YkEioYImmmU04WNweHpv732XejjiPtiAqGSBo6fGh3zp04gPteW87idZVRx5F2QgVDJE19/zOj6JKbxU/+uZC2dD+VpC4VDJE0VZiXww9PHc1byzfzjzmro44j7YAKhkgaO/vQ/kwYWMCvnn5Hc2dI6FQwRNJYRobx8zMOZvPOGm6eqXszJFwqGCJp7sC+XbjgqCE8OOsDFq3RCXAJjwqGSBvwnZNGkN8hi5tnLo46irRhKhgibUDXTtlceuJwXlxSoTvAJTQqGCJtxPlHDaZP11xufGaxLrOVUKhgiLQRudmZXPnpAyhduYWZb6+LOo60QaEWDDMrMLNHzWyxmb1jZkc2WT/KzF43s2oz+16TdaeY2RIzW2pm14SZU6StOHNCP0b07MxNM5dQV6/Z+aR1hd3DuBWY4e6jgLHAO03WbwKuAH4Tv9DMMoE7gFOBA4FzzUzTjInsQVZmBld/ZiRlFTt4ZPaqqONIGxNawTCzrsBxwB8A3L3G3bfEt3H3cnd/C6htsvlEYKm7l7l7DfA3YHJYWUXakpMP7MWhgwr53b/fZVdNfdRxpA0Js4cxBKgA7jOzuWZ2r5nlJbhtP2Bl3OtVwbJPMLMpZlZiZiUVFRX7l1ikDTAzrjl1FOsrq7nvtWVRx5E2JMyCkQVMAO509/HADqDVz0W4+zR3L3b34qKiotZ+e5G0dNjgbnx6dE/ufOl9DRkirSbMgrEKWOXubwavHyVWQBKxGhgQ97p/sExEEnT1Z0axo7qO37/0ftRRpI0IrWC4+zpgpZmNDBadBCxKcPO3gBFmNsTMcoAvA4+HEFOkzRrZO58zJ/TnT68tZ/WWXVHHkTYg7KukLgceMLP5wDjgl2Y21cymAphZbzNbBVwF/NjMVplZF3evAy4DZhK7surv7v52yFlF2pwrTz4AgFue08x8sv+ywnxzdy8Fipssvitu/Tpih5ua2/Zp4Onw0om0ff0KOnLBUYO555UyvnnsUEb2zo86kqQx3ekt0sZdcsIw8nKyuO3596KOImlOBUOkjSvolMMFRw3m6YVreXf9tqjjSBpTwRBpB75xzBA6ZWeqlyH7RQVDpB0ozMvh/KMG89SCtbynXobsIxUMkXbi4mOH0jE7k9teWBp1FElTKhgi7US3vBzOO3IwT85fw9Jy9TJk76lgiLQj3zx2SKyX8bx6GbL3VDBE2pHunTvw9SMH8cT8NbxfsT3qOJJmVDBE2plvHjuUDlkZ3PGiehmyd1QwRNqZHp078NXDB/Gv0jWs2Lgj6jiSRlQwRNqhbx03lMwM4/cvaiRbSZwKhkg71LNLLl+ZOJB/zFnFyk07o44jaUIFQ6Sd+tbxQ8kw467/qJchiVHBEGmn+nTtyBeL+/NIySrWbtV8GbJnKhgi7di3TxhGgzt3aVY+SUCoBcPMCszsUTNbbGbvmNmRTdabmd1mZkvNbL6ZTYhbV29mpcFDs+2JhKB/YSfOPrQ/D721kvWVVVHHkRQXdg/jVmCGu48CxhKbPS/eqcCI4DEFuDNu3S53Hxc8Tg85p0i7dckJw6lvcO7+T1nUUSTFhVYwzKwrcBzwBwB3r3H3LU2aTQb+4jFvAAVm1iesTCLySQO7d+LM8f144M0VlG9TL0NaFmYPYwhQAdxnZnPN7F4zy2vSph+wMu71qmAZQK6ZlZjZG2Z2Rog5Rdq9S08cTl2DM029DNmNMAtGFjABuNPdxwM7gGv2YvtB7l4MfAX4nZkNa66RmU0JCktJRUXFfocWaY8G98hj8ri+/PXNFWzYXh11HElRYRaMVcAqd38zeP0osQISbzUwIO51/2AZ7t74tQx4CRjf3E7cfZq7F7t7cVFRUeulF2lnLjtxODV1DdzzsnoZ0rzQCoa7rwNWmtnIYNFJwKImzR4HzguuljoC2Orua82s0Mw6AJhZD+DoZrYVkVY0tKgzp4/ty19eX0HFNvUy5JPCvkrqcuABM5sPjAN+aWZTzWxqsP5poAxYCtwDXBIsHw2UmNk84EXgRndXwRAJ2RUnjaCuoYFfz1gcdRRJQVlhvrm7lwLFTRbfFbfegUub2e414OAws4nIJw0t6szFxw7lzpfe50vFA5g4pFvUkSSF6E5vEfmYyz81nH4FHfnJPxdSW98QdRxJISoYIvIxnXKyuO7zB7Jk/Tb+9N/lUceRFKKCISKfcPKBvThpVE9u+fe7GphQPqSCISKfYGb89PSDqG9wrn9C15tITEIFw8yGxV3meoKZXWFmBeFGE5EoDejWiStOGsEzC9cx8+11UceRFJBoD+MfQL2ZDQemEbvZ7sHQUolISphy3FBG9c7nJ/9cyNZdtVHHkYglWjAa3L0O+ALwf+5+NaBBAkXauOzMDG46+xA2bK/mxmeaDjYt7U2iBaPWzM4FzgeeDJZlhxNJRFLJIf0L+OaxQ3lo1kpee39D1HEkQokWjAuBI4FfuPsyMxsC3B9eLBFJJd/99AEM7t6JHz62gF019VHHkYgkVDDcfZG7X+HuD5lZIZDv7r8OOZuIpIiOOZn86sxDWLFxJzfN1LAh7VWiV0m9ZGZdzKwbMAe4x8z+N9xoIpJKjhzWnfOPHMR9/13Of5fq0FR7lOghqa7uXgmcSWyGvMOBT4cXS0RS0TWnjmZoUR7fe2SerppqhxItGFnB1Klf4qOT3iLSznTMyeSWL42jfFs1P3387ajjSJIlWjCuB2YC77v7W2Y2FHgvvFgikqrGDijg8k8NZ/rc1Tw1f23UcSSJEj3p/Yi7H+Lu3w5el7n7WeFGE5FUdemJwxnbvyvXTl/Ayk07o44jSZLoSe/+ZjbdzMqDxz/MrH8C2xWY2aNmttjM3jGzI5usNzO7zcyWmtl8M5sQt+58M3sveJy/99+aiIQlOzOD3315PO7ON/9SwvbquqgjSRIkekjqPmLTqfYNHk8Ey/bkVmCGu48CxgJNbxU9FRgRPKYAdwIEV2NdBxwOTASuCy7nFZEUMaRHHrd/ZQLvrt/GlQ+X0tDgUUeSkCVaMIrc/T53rwsefwKKdreBmXUFjgP+AODuNe6+pUmzycSuunJ3fwMoCE6ufwZ4zt03uftm4DnglMS/LRFJhuMOKOLHnz2Q5xat53+fezfqOBKyRAvGRjP7mpllBo+vARv3sM0QoAK4z8zmmtm9ZpbXpE0/YGXc61XBspaWi0iKufDowZxTPIDbX1zKE/PWRB1HQpRowbiI2CW164C1wNnABXvYJguYANzp7uOBHcA1+xazZWY2xcxKzKykoqKitd9eRPbAzLjhjDFMGFjAtdMXsL6yKupI7crba7by6nvJuZEy0aukVrj76e5e5O493f0MYE9XSa0CVrn7m8HrR4kVkHiriQ2V3qh/sKyl5c1lm+buxe5eXFS026NkIhKSnKwMfvulcdTUNfDjfy7EXeczkuX+11fw3YdLk7Kv/Zlx76rdrXT3dcBKMxsZLDoJaDp11+PAecHVUkcAW919LbF7PiaZWWFwsntSsExEUtSQHnlcdfIBPLdoPU8v0IRLyVK+rZpeXTokZV9Z+7GtJdDmcuABM8sByoALzWwqgLvfBTwNnAYsBXYSGxUXd99kZjcAbwXvc727b9qPrCKSBN84ZghPzl/LdY8v5Khh3SnMy4k6UptXvq2KnvnJKRj708PYY5/T3UuDw0WHuPsZ7r7Z3e8KigXB1VGXuvswdz/Y3Uvitv2juw8PHolcwisiEcsKJlzasrOWG57UXODJsL6ymp75uUnZ124LhpltM7PKZh7biN2PISLyMaP7dOGSE4bx2NzVvLikPOo4bVp9g7NxezU9k3RIarcFw93z3b1LM498d9+fw1ki0oZd+qnhjOjZmR89toBtVRrVNiwbt1fT4NCzSwr0MERE9kWHrEx+ffYhrK2s4tczNOFSWMq3VQOkxTkMEZEWTRhYyEVHD+Gvb3zAG2V7us9X9kXjPS8qGCKS9r43aSQDu3Ximn/M11zgIfiwh6FDUiKS7jrmZHLjWQezfONObvm3xppqbeWVsYJR1Fk9DBFpA44a1oNzJw7knlfKmLVMt1O1pvJtVXTLyyEnKzn/latgiEjofvzZ0Qzs1okrHy6lUldNtZrYPRjJ6V2ACoaIJEFehyxuOWcc6yqr+Om/NBd4a6nYVpW08xeggiEiSTJhYCGXf2o4j81drWHQW0n5NvUwRKSNuuzE4YwfWMCPpi9gzZZdUcdJaw0NToUKhoi0VVmZGfzunHHUNzhXPlxKvaZ13WebdtZQ1+AqGCLSdg3qnsfPJo/hzWWb+P2LS6OOk7YaL6ntpXMYItKWnTWhH5PH9eV3z7/H7BW61HZflG8L7vJO0sCDoIIhIhEwM35+xhj6FuRyxUOlbN2lS233VmMPI1lDm4MKhohEJD83m9u+PJ71lVVcO32BpnXdS409jKK2cg7DzJab2QIzKzWzkmbWF5rZdDObb2azzGxMotuKSPobP7CQK08+gKfmr+VxXWq7V8q3VdO1Yza52ZlJ22cyehgnuvs4dy9uZt21QKm7HwKcB9y6F9uKSBsw9fhhjO3fleufWMTmHTVRx0kb5Um+yxuiPyR1IPACgLsvBgabWa9oI4lIMmVmGDeedQhbd9Xy86feiTpO2li/rSqpJ7wh/ILhwLNmNtvMpjSzfh5wJoCZTQQGAf0T3FZE2ojRfbrwreOH8o85q3j1vQ1Rx0kL5Umcy7tR2AXjGHefAJwKXGpmxzVZfyNQYGalwOXAXKA+wW0BMLMpZlZiZiUVFRXhfBciErrLPzWCoT3yuHb6As2dsQfuwV3ebamH4e6rg6/lwHRgYpP1le5+obuPI3YOowgoS2TbuPeY5u7F7l5cVFQU2vciIuHKzc7kl2cezAebNHfGnmzdVUtNfUPb6WGYWZ6Z5Tc+ByYBC5u0KTCznODlxcDL7l6ZyLYi0vYcMbQ7Xz5sAH94dRmL1lRGHSdlra9M7lzejcLsYfQCXjWzecAs4Cl3n2FmU81satBmNLDQzJYQO/T0nd1tG2JWEUkR15w6ioKO2Vw7fYHGmmrBh3d5J7lgZIX1xu5eBoxtZvldcc9fBw5IdFsRafsKOuXwk88dyHcfLuXBWR/w9SMGRR0p5UQxjhREf1mtiMgnTB7Xl2OG9+CmZxZTXlkVdZyUU74tOCTVlk56i4jsCzPjhjPGUF3fwPVPLoo6TspZX1lF5w5ZdMoJ7SBRs1QwRCQlDemRx2UnDufJ+Wt5aUl51HFSSrInTmqkgiEiKetbxw9lWFEeP/nXQt2bEac8gru8QQVDRFJYh6xMfvGFg1m5aRe3vfBe1HFSRmwu7+Se8AYVDBFJcUcM7c4XD+3PPS+XsWTdtqjjRM7dWV9ZpUNSIiLNufa00XQJ7s1oaOf3ZmyrrqOqtiHpl9SCCoaIpIHCvByuPW00s1ds5m9vrYw6TqQ+nGlP5zBERJp31oR+HDG0G7965h1WbtoZdZzIrN6yC4De6mGIiDTPzLjprLEYMOX+2e32qqmyiu0ADC3qnPR9q2CISNoY2L0Tt547nsXrKvnhY/Pb5TzgZRU7yM/NokfnnD03bmUqGCKSVk4c2ZP/OfkA/lm6hj/+d3nUcZKubMN2hvbIw8ySvm8VDBFJO5ecMJxJB/bil0+/w+vvb4w6TlItq9gRyeEoUMEQkTSUkWH89ktjGdy9E5c/NJeKYDC+tm5nTR1rtlYxtEdeJPtXwRCRtJSfm80dX53Atqparvp7abu4P2PZhh1ANCe8IeSCYWbLzWyBmZWaWUkz6wvNbLqZzTezWWY2Jm7dKWa2xMyWmtk1YeYUkfQ0qncXrvv8Qbzy3gbuevn9qOOErqyisWC03R7Gie4+zt2Lm1l3LVDq7ocQm9P7VgAzywTuIDYL34HAuWZ2YBKyikiaOXfiAD57SB9+++y7zF6xKeo4oWosGIO7t92CsTsHAi8AuPtiYLCZ9QImAkvdvczda4C/AZOjiykiqcrM+NWZB9OvoCNXPFTKlp01UUcKzbIN2+lX0JGOOZmR7D/sguHAs2Y228ymNLN+HnAmgJlNBAYB/YF+QPz9/6uCZSIin9AlN5v/O3c86yur+NH0hW32/oyyDTsiOxwF4ReMY9x9ArFDS5ea2XFN1t8IFJhZKXA5MBfYq9s3zWyKmZWYWUlFRUWrhBaR9DN2QAFXnnwATy1Yy2NzVkcdp9W5O2UVOyK7QgpCLhjuvjr4Wg5MJ3aoKX59pbtf6O7jiJ3DKALKgNXAgLim/YNlze1jmrsXu3txUVFRCN+FiKSLqccPY+KQblz3+Nttbrypim3VbK+uY0hbLBhmlmdm+Y3PgUnAwiZtCsys8f72i4GX3b0SeAsYYWZDgvVfBh4PK6uItA2ZGcb/fmksZvDdh0upq2+IOlKreb8i2ktqIdweRi/gVTObB8wCnnL3GWY21cymBm1GAwvNbAmxw1bfAXD3OuAyYCbwDvB3d387xKwi0kb0L+zEz88Yw+wVm/n9S23nUtuP7sGIroeRFdYbu3sZMLaZ5XfFPX8dOKCF7Z8Gng4rn4i0XZPH9eOFxeX87t/vMnFIN44Y2j3qSPutrGI7HbIy6Nu1Y2QZor6sVkQkFD8/YwyDu+dx2YNzKa+sijrOfivbsIMhPfLIyEj+oIONVDBEpE3Kz83mzq8dyo7qOi57aG7an88oq9ge6eEoUMEQkTZsZO98fnXmwcxatombZy6JOs4+q6lrYOXmXQztEd0Jb1DBEJE27ozx/fjaEQO5++UyZr69Luo4++SDTTupb3D1MEREwvaTzx3IIf27cvUj81i1Of3uz4hyWtZ4Khgi0uZ1yMrk/84dT4PDFQ/NpTbNzmeUBZfURnnTHqhgiEg7Mah7Hr8682DmfLCFW557N+o4e6WsYjs9OufQtWN2pDlUMESk3fj82L58+bAB3Pmf93nlvfQZe27Zhh2Rn/AGFQwRaWeu+/xBjOjZmSsfLk2L+zPq6htYtKaSA3qrYIiIJFXHnExu/8oEdlTXp8X9GYvXbWNHTT3Fg7pFHUUFQ0TanwN6pc/9GbNXbAbg0EGFESdRwRCRduqM8f34+hGDuPvlMmYsXBt1nBaVrNhMry4d6F8Y3RhSjVQwRKTd+vHnRjN2QAFXPzL/w9FgU83s5ZsoHtQNs+jGkGqkgiEi7VaHrEzu+Mp4MjONb/91NlW1ezXhZ+jWbNnFmq1VFA+O/nAUqGCISDvXv7ATv/3iWBav28ZNM1LrfEZJcP4iFU54gwqGiAgnje7F+UcO4o//XcZLS8qjjvOh2cs30Sknk9F98qOOAoRcMMxsuZktMLNSMytpZn1XM3vCzOaZ2dtmdmGymVKYAAAOZ0lEQVTcuvpgu1Iz0/SsIhKqH542mpG98vneI/PZsL066jhArIcxbkABWZmp8bd9MlKc6O7j3L24mXWXAovcfSxwAvDbuDm+dwXbjXP305OQU0TasdzsTG49dxyVVbX84NH5uHukebZX1/HO2kqKU+By2kZRly0H8i12+r8zsAmoizaSiLRXo3p34YenjuL5xeXc8eLSSLOUfrCFBodDB6fG+QsIv2A48KyZzTazKc2svx0YDawBFgDfcffG2y5zzazEzN4wszNa2oGZTQnalVRUpM/YMCKSmi44ajBnjOvLb559l7+8vjyyHCUrNmEG4wcWRJahqayQ3/8Yd19tZj2B58xssbu/HLf+M0Ap8ClgWNDmFXevBAYF2w4FXjCzBe7+ftMduPs0YBpAcXFxtH1IEUl7ZsbNXxzLjpp6/t+/3qZTThZnH9o/6Tlmr9jMyF75dMmNdoTaeKH2MNx9dfC1HJgOTGzS5ELgMY9ZCiwDRjXZtgx4CRgfZlYRkUbZmRn837njOXp4d77/6DyeWZDcO8HrG5y5H2zhsBQ6HAUhFgwzyzOz/MbnwCRgYZNmHwAnBW16ASOBMjMrNLMOwfIewNHAorCyiog0lZudybSvFzNuQAFX/G0ur763IWn7Xryuku3VdSlzw16jMHsYvYBXzWweMAt4yt1nmNlUM5satLkBOMrMFgDPAz9w9w3EzmuUBNu+CNzo7ioYIpJUeR2yuO+CiQwr6syU+0soXbklKfudtWwTkBoDDsazqC8da03FxcVeUvKJ2z1ERPZLeWUVZ931Gtur6nhk6pEM7xnujXRn3/kalVW1zPzucaGPIWVms1u47eETor6sVkQk5fXsksv9Fx1OZkYGX//DLNZs2RXavlZu2knJis1MHtcvJQYcjKeCISKSgME98vjzRYexvaqO8/44i607a0PZz+Pz1gAweVzfUN5/f6hgiIgk6KC+Xbn7vENZsXEHU+4vobqudUe3dXf+OXc1hw0upH9hp1Z979aggiEisheOGtaDm88ey5vLNnH1I/NpaGi988CL1lbyXvl2Jo/r12rv2ZrCvnFPRKTNOWN8P1Zv2cXNM5fQr7AjPzhlVKu8779K15CVYXz24D6t8n6tTQVDRGQfXHLCMFZt3sWdL71Pl9xsvn3CsP16v/oG5/HSNZwwsojCvJw9bxABFQwRkX1gZtww+SB2VNfx6xmL2VVbz5WfHrHPVza9uWwj6yqr+NFnR7dy0tajgiEiso+yMjO45ZxxdMjK4Lbn36Oqtp4fnjpqn4rGv+auIS8nk0+P7hVC0tahgiEish8yM4xfn3UIHbIzmPZyGTtr6vjZ6WPIzEi8aGzdWcvTC9fymTG96ZiTGWLa/aOCISKynzIyjBsmj6FTThbTXi5j1eZd3Hbu+IRGmq1vcK7421yqauu58KghSUi773RZrYhIKzAzrj1tND8/YwyvvLeBs37/Gh9s3LnH7W6euYT/vFvBz04fw8H9uyYh6b5TwRARaUVfO2IQ9180kfJt1Uy+41Wemr+2xXs1npi3hrv+8z5fPXwgXzl8YJKT7j0VDBGRVnbU8B7889Kj6Zmfy6UPzmHS715m+txV1NU30NDglFdW8cLi9Vz96DyKBxVy3ecPijpyQjRarYhISOobnKcWrOWOF5ayZP02Cjpls7O6npr62EzUvbvk8vjlscISlb0ZrVYnvUVEQpKZYZw+ti+fO7gP/35nPTPfXk+Pzjn0L+xIv8KOjB9QmLI36TUn1IJhZsuBbUA9UNe0iplZV+CvwMAgy2/c/b5g3fnAj4OmP3f3P4eZVUQkLBkZxqSDejPpoN5RR9kvyehhnBjMotecS4FF7v55MysClpjZA0Bn4DqgGHBgtpk97u6bk5BXRESaEfVJbwfyLXZbZGdgE1AHfAZ4zt03BUXiOeCU6GKKiEjYBcOBZ81stplNaWb97cTm714DLAC+4+4NQD9gZVy7VcEyERGJSNiHpI5x99Vm1hN4zswWu/vLces/A5QCnwKGBW1e2ZsdBIVoCsDAgal/HbOISLoKtYfh7quDr+XAdGBikyYXAo95zFJgGTAKWA0MiGvXP1jW3D6muXuxuxcXFRW19rcgIiKB0AqGmeWZWX7jc2ASsLBJsw+Ak4I2vYCRQBkwE5hkZoVmVhhsOzOsrCIismdhHpLqBUwPhvnNAh509xlmNhXA3e8CbgD+ZGYLAAN+0HhFlZndALwVvNf17r4pxKwiIrIHutNbRKQd25s7vdtUwTCzrcB7zazqCmxN8HXj8+aW9QBauqekJU33lej65pY3l6ml5/uTeXe5Es2XLpmbW56On49EMsc/1+cj8fVt/fMxwt0TGybX3dvMA5iWyPLdvW583sKyktbKtLeZW8q0p/z7knlfc6dj5rby+Ugkc9Q/a30+Uv/zsadH1DfutbYnEly+u9dP7GZZa2ba0/rmlreUaU/598W+5E7HzM0tT8fPRyKZ45/r85H4+vb0+ditNnVIKmxmVuIJHutLFcqcPOmYW5mTJ11zx2trPYywTYs6wD5Q5uRJx9zKnDzpmvtD6mGIiEhC1MMQEZGEtNuCYWZ/NLNyM2t693ki2x5qZgvMbKmZ3RaMttu47nIzW2xmb5vZTame2cx+amarzaw0eJyW6pnj1v+PmbmZ9Wi9xB++dxg/6xvMbH7wc37WzPqmQeabg8/zfDObbmYFaZD5i8HvX4OZtdo5g/3J2sL7nW9m7wWP8+OW7/ZzH6l9uTytLTyA44AJwMJ92HYWcASxu9OfAU4Nlp8I/BvoELzumQaZfwp8L51+zsG6AcSGi1kB9EiH3ECXuDZXAHelQeZJQFbw/NfAr9Mg82hiwwy9BBRHnTXIMbjJsm7EhkHqBhQGzwt3932lwqPd9jA8Nmrux4YbMbNhZjYjGI79FTMb1XQ7M+tD7Bf/DY/96/4FOCNY/W3gRnevDvZRngaZQxVi5luA7xMbQj8tcrt7ZVzTvNbOHlLmZ929Lmj6BrGBQFM98zvuvqQ1c+5P1hY0O+dPlL+riWi3BaMF04DL3f1Q4HvA75tp04/Y/ByN4ufqOAA41szeNLP/mNlhoaaN2d/MAJcFhxz+aLHBHsO2X5nNbDKw2t3nhR20if3+WZvZL8xsJfBV4P+FmLVRa3w+Gl1E7C/esLVm5rAlkrU5Lc35kyrfV7OSMUVrWjCzzsBRwCNxhww77OXbZBHrYh4BHAb83cyGBn8ptLpWynwnsUEgPfj6W2L/MYRifzObWSfgWmKHSpKmlX7WuPuPgB+Z2Q+By4hNRRyK1socvNePiM2G+UDrpGtxP62WOWy7y2pmFwLfCZYNB542sxpgmbt/IdlZW4sKxkcygC3uPi5+oZllArODl48T+w82vlseP1fHKoL5PYBZZtZAbPyYilTN7O7r47a7B3gypKyN9jfzMGAIMC/4Je0PzDGzie6+LoVzN/UA8DQhFgxaKbOZXQB8DjgprD9+4rT2zzlMzWYFcPf7gPsAzOwl4AJ3Xx7XZDVwQtzr/sTOdawm+u+rZVGfRInyAQwm7gQW8BrwxeC5AWNb2K7pSanTguVTiQ3FDrHDUysJ7nVJ4cx94tpcCfwt1X/OTdosJ4ST3iH9rEfEtbkceDQNMp8CLAKKwvgZh/n5oJVPeu9rVlo+6b2M2AnvwuB5t0Q/91E9Ig8Q2TcODwFrgVpiPYNvEPvLdQYwL/gl+X8tbFtMbDKo94nNS954A2QO8Ndg3RzgU2mQ+X5i86nPJ/aXW59Uz9ykzXLCuUoqjJ/1P4Ll84mN39MvDTIvJfaHT2nwaO0ru8LI/IXgvaqB9cDMKLPSTMEIll8U/HyXAhfuzec+qofu9BYRkYToKikREUmICoaIiCREBUNERBKigiEiIglRwRARkYSoYEibZmbbk7y/e83swFZ6r3qLjWy70Mye2NNIsWZWYGaXtMa+RZqjy2qlTTOz7e7euRXfL8s/GowvVPHZzezPwLvu/ovdtB8MPOnuY5KRT9of9TCk3TGzIjP7h5m9FTyODpZPNLPXzWyumb1mZiOD5ReY2eNm9gLwvJmdYGYvmdmjFpsr4oHGOQuC5cXB8+3BYIPzzOwNM+sVLB8WvF5gZj9PsBf0Oh8NvtjZzJ43sznBe0wO2twIDAt6JTcHba8Ovsf5ZvazVvwxSjukgiHt0a3ALe5+GHAWcG+wfDFwrLuPJzaS7C/jtpkAnO3uxwevxwPfBQ4EhgJHN7OfPOANdx8LvAx8M27/t7r7wXx8ZNJmBeMonUTsTnyAKuAL7j6B2Bwsvw0K1jXA++4+zt2vNrNJwAhgIjAOONTMjtvT/kRaosEHpT36NHBg3AijXYKRR7sCfzazEcRG782O2+Y5d4+fC2GWu68CMLNSYmMMvdpkPzV8NJjjbODk4PmRfDTHwYPAb1rI2TF4737AO8TmTIDYGEO/DP7zbwjW92pm+0nBY27wujOxAvJyC/sT2S0VDGmPMoAj3L0qfqGZ3Q686O5fCM4HvBS3ekeT96iOe15P879Ltf7RScKW2uzOLncfFwzpPhO4FLiN2FwaRcCh7l5rZsuB3Ga2N+BX7n73Xu5XpFk6JCXt0bPERosFwMwah6fuykdDSV8Q4v7fIHYoDODLe2rs7juJTen6P2aWRSxneVAsTgQGBU23Aflxm84ELgp6T5hZPzPr2Urfg7RDKhjS1nUys1Vxj6uI/edbHJwIXkRsWHqAm4Bfmdlcwu19fxe4yszmE5tcZ+ueNnD3ucRGuT2X2FwaxWa2ADiP2LkX3H0j8N/gMtyb3f1ZYoe8Xg/aPsrHC4rIXtFltSJJFhxi2uXubmZfBs5198l72k4kajqHIZJ8hwK3B1c2bSHEKXFFWpN6GCIikhCdwxARkYSoYIiISEJUMEREJCEqGCIikhAVDBERSYgKhoiIJOT/AybiM+FdBW+3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr_find(learn)\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 14:24 <p><table style='width:375px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "    <th>time</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>0</th>\n",
       "    <th>3.747901</th>\n",
       "    <th>3.465941</th>\n",
       "    <th>0.260205</th>\n",
       "    <th>14:24</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1,1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4VPXZ//H3nY3sCUtYAwZBZF9T3BfctS1WRSs/fSq2lmpr61O7aTet3Wxrfay1jz5q1ap1b9211CpurYpBFlkFBSQQIIQtEpYs9++POYxDSEgImcxM5vO6rrly5pzvOblPDsxnzvY95u6IiIgApMS6ABERiR8KBRERCVMoiIhImEJBRETCFAoiIhKmUBARkTCFgoiIhCkUREQkTKEgIiJhabEu4ED16NHDS0pKYl2GiEhCmT179kZ3L2qpXcKFQklJCWVlZbEuQ0QkoZjZqta00+EjEREJUyiIiEiYQkFERMIS7pyCiHQetbW1lJeXs3PnzliX0mlkZmZSXFxMenp6m+ZXKIhIzJSXl5OXl0dJSQlmFutyEp67U1VVRXl5OQMHDmzTMnT4SERiZufOnXTv3l2B0E7MjO7dux/UnpdCQURiSoHQvg7275k0obBk3TZumrGUTdt3x7oUEZG4lTShsHLjdm6buZx1W3VCS0RCqqqqGDt2LGPHjqV3797069cv/H737tZ9gbz00ktZunRplCvtOElzojk/M3QmftvO2hhXIiLxonv37sydOxeA66+/ntzcXL773e/u1cbdcXdSUpr+Dn3vvfdGvc6OlDR7CvlZQSjsUCiIyP4tX76c4cOHc9FFFzFixAgqKiqYPn06paWljBgxghtuuCHc9thjj2Xu3LnU1dVRWFjINddcw5gxYzjqqKPYsGFDDNeibZJwT6EuxpWISFN+9uxCFq3d1q7LHN43n+s+P6JN8y5ZsoT777+f0tJSAG688Ua6detGXV0dkyZNYsqUKQwfPnyvebZu3coJJ5zAjTfeyNVXX80999zDNddcc9Dr0ZGSaE8hlH/aUxCR1hg0aFA4EAAefvhhxo8fz/jx41m8eDGLFi3aZ56srCzOPPNMACZMmMDKlSs7qtx2kzR7CrldglDQOQWRuNTWb/TRkpOTEx5etmwZf/jDH5g1axaFhYVcfPHFTd4LkJGRER5OTU2lri7xjkwkzZ5CWmoKuV3S2LYj8TaSiMTWtm3byMvLIz8/n4qKCmbMmBHrkqImafYUAPIz07SnICIHbPz48QwfPpyhQ4dyyCGHcMwxx8S6pKgxd491DQektLTU2/qQnTNueZ0B3bK580ulLTcWkahbvHgxw4YNi3UZnU5Tf1czm+3uLX74Jc3hIwhdgaQ9BRGR5iVXKGTpnIKIyP4kVyhoT0FEZL+iGgpmVmhmT5jZEjNbbGZHNZpuZnarmS03s/lmNj6a9eRnpes+BRGR/Yj21Ud/AP7h7lPMLAPIbjT9TOCw4HUEcHvwMyryM9Oo3lVHQ4OTkqLuekVEGovanoKZFQDHA38GcPfd7r6lUbOzgfs95G2g0Mz6RKum/Kx03KF6l84riIg0JZqHjwYClcC9ZjbHzO42s5xGbfoBqyPelwfjoqIwO3S34dYaHUISEZg0adI+N6LdcsstXHHFFc3Ok5ubC8DatWuZMmVKk21OPPFEWrp0/pZbbqGmpib8/qyzzmLLlsbfmzteNEMhDRgP3O7u44DtQJt6hjKz6WZWZmZllZWVbS6oa3aoU7xNNXrQjojA1KlTeeSRR/Ya98gjjzB16tQW5+3bty9PPPFEm39341B44YUXKCwsbPPy2ks0Q6EcKHf3d4L3TxAKiUhrgP4R74uDcXtx9zvdvdTdS4uKitpc0J49hc0KBREBpkyZwvPPPx9+oM7KlStZu3Yt48aN4+STT2b8+PGMGjWKp59+ep95V65cyciRIwHYsWMHF154IcOGDeOcc85hx44d4XZXXHFFuMvt6667DoBbb72VtWvXMmnSJCZNmgRASUkJGzduBODmm29m5MiRjBw5kltuuSX8+4YNG8ZXv/pVRowYwWmnnbbX72kvUTvR7O7rzGy1mR3u7kuBk4HG3Qo+A1xpZo8QOsG81d0rolXTnj2FLQoFkfjz4jWw7v32XWbvUXDmjc1O7tatGxMnTuTFF1/k7LPP5pFHHuGCCy4gKyuLJ598kvz8fDZu3MiRRx7J5MmTm33+8e233052djaLFy9m/vz5jB//6fffX/7yl3Tr1o36+npOPvlk5s+fz7e+9S1uvvlmZs6cSY8ePfZa1uzZs7n33nt55513cHeOOOIITjjhBLp27cqyZct4+OGHueuuu7jgggv429/+xsUXX9w+f6tAtO9T+CbwVzObD4wFfmVml5vZ5cH0F4CPgOXAXcDXo1lM1z17Ctt1TkFEQiIPIe05dOTu/PCHP2T06NGccsoprFmzhvXr1ze7jNdffz384Tx69GhGjx4dnvbYY48xfvx4xo0bx8KFC5vscjvSm2++yTnnnENOTg65ubmce+65vPHGGwAMHDiQsWPHAtHrmjuql6S6+1ygcV8bd0RMd+Ab0awhUn5WOimmPQWRuLSfb/TRdPbZZ/Ptb3+b9957j5qaGiZMmMB9991HZWUls2fPJj09nZKSkia7ym7JihUruOmmm3j33Xfp2rUr06ZNa9Ny9ujSpUt4ODU1NSqHj5LqjubUFKMgK10nmkUkLDc3l0mTJvHlL385fIJ569at9OzZk/T0dGbOnMmqVav2u4zjjz+ehx56CIAFCxYwf/58INTldk5ODgUFBaxfv54XX3wxPE9eXh7V1dX7LOu4447jqaeeoqamhu3bt/Pkk09y3HHHtdfqtiipus6G0CGkzbokVUQiTJ06lXPOOSd8GOmiiy7i85//PKNGjaK0tJShQ4fud/4rrriCSy+9lGHDhjFs2DAmTJgAwJgxYxg3bhxDhw6lf//+e3W5PX36dM444wz69u3LzJkzw+PHjx/PtGnTmDhxIgCXXXYZ48aN67CnuCVV19kA5/7vv8nKSOWvlx3ZjlWJSFuo6+zoUNfZB6BrdoZONIuINCP5QiEnQ/cpiIg0I/lCITtdoSASRxLtEHa8O9i/Z9KFQmF2BjtrG9hZWx/rUkSSXmZmJlVVVQqGduLuVFVVkZmZ2eZlJOXVRxDq6qJPQVaMqxFJbsXFxZSXl3MwfZrJ3jIzMykuLm7z/EkYCkGneNsVCiKxlp6ezsCBA2NdhkRIusNHXXNCewpbdK+CiMg+ki4UugehULVdJ5tFRBpLulAoygv1HVJZvSvGlYiIxJ+kC4WCrHTSU40N1W3vlEpEpLNKulAwM3rlZ7J+q0JBRKSxpAsFgL6FWazdolAQEWksKUOhX2EWa7a0fz/kIiKJLilDoW9hJuu37aS+QXdRiohESspQ6FOQRV2D6wokEZFGkjIU+hWG7mRes6UmxpWIiMSXpAyFQUW5AHyw/pMYVyIiEl+SMhSKu2aR2yWNxRXbYl2KiEhcScpQSEkxhvbOUyiIiDSSlKEAMLRPHksqqtWPu4hIhKQNhWF98qneVUf5Zt2vICKyR1KHAqBDSCIiEZI2FIb2zsMMFldUx7oUEZG4kbShkJ2RRkn3HO0piIhEiOrjOM1sJVAN1AN17l7aaPqJwNPAimDU3939hmjWFGlYnzwWrlUoiIjs0RHPaJ7k7hv3M/0Nd/9cB9SxjxF9C3jh/XWs37aTXvmZsShBRCSuJO3hI4CzRvUB4G/vlce4EhGR+BDtUHDgn2Y228ymN9PmKDObZ2YvmtmIKNezl4E9cphY0o3Hy8p1v4KICNEPhWPdfTxwJvANMzu+0fT3gEPcfQzwR+CpphZiZtPNrMzMyiorK9u1wAs+058VG7dTtmpzuy5XRCQRRTUU3H1N8HMD8CQwsdH0be7+STD8ApBuZj2aWM6d7l7q7qVFRUXtWuNZo3pjBlc/NrddlysikoiiFgpmlmNmeXuGgdOABY3a9DYzC4YnBvVURaumpmRnpDFlfDGrN+3gvx+ZQ119Q0f+ehGRuBLNq496AU8Gn/lpwEPu/g8zuxzA3e8ApgBXmFkdsAO40GNwcP/nXxjJhupdPDV3LWWrNvPiVceRl5ne0WWIiMScJdoJ1tLSUi8rK4vKsr//xDweKwtdiTSkVy6/P38so4oLovK7REQ6kpnNbnyvWFOS+pLUxn47ZQx/vqSUMcUFfLD+Ez5/25vc8+aKlmcUEekkFAqNnDysF09feSx/vewIAG54bhG//ccS6hsSa49KRKQtFArNOGZwD+b+9FQO75XH/776IZc/OJsGBYOIdHIKhf0ozM5gxreP5wdnDOWlReu5/bUPY12SiEhUKRRa4fITDuX0Eb343YylfFT5SazLERGJGoVCK5gZPzhjKAAn/f41HUYSkU5LodBKhxbl8o1JgwB4Z8WmGFcjIhIdCoUDcOWkw8jtksbf1auqiHRSCoUDkJWRypkje/OPBevYXafuMESk81EoHKDTRvSmelcds3QISUQ6IYXCATpmcHcyUlN4demGWJciItLuFAoHKDsjjSMO7cZMhYKIdEIKhTY48fCefFi5nYqtO2JdiohIu1IotMGEQ7oCMOfjLTGuRESkfSkU2mB4n3y6pKXwnh7hKSKdjEKhDTLSUhjZr4A5q7WnICKdi0KhjUYXF7Bo7TZ1qS0inYpCoY2G9clnR209H2+qiXUpIiLtRqHQRkN75wGwpGJbjCsREWk/CoU2GtIrDzNYsq461qWIiLQbhUIbZaanMqBbNss2KBREpPNQKByEw3rm8cF6PXRHRDoPhcJBGNIrl5Ubt6vHVBHpNBQKB2FIrzzqGpwVG7fHuhQRkXahUDgIh/XKBeCD9TqvICKdg0LhIAwqyiXFYNkGnVcQkc5BoXAQMtNTOaR7Dsu0pyAinURUQ8HMVprZ+2Y218zKmphuZnarmS03s/lmNj6a9UTDYT1zdfhIRDqNjthTmOTuY929tIlpZwKHBa/pwO0dUE+7GtIrj5VVNeyqq491KSIiBy3Wh4/OBu73kLeBQjPrE+OaDshhvXKp1xVIItJJRDsUHPinmc02s+lNTO8HrI54Xx6MSxhDeoX6QNJNbCLSGaRFefnHuvsaM+sJvGRmS9z99QNdSBAo0wEGDBjQ3jUelIE9ckJXIOm8goh0AlHdU3D3NcHPDcCTwMRGTdYA/SPeFwfjGi/nTncvdffSoqKiaJXbJpnpqZR0z9HJZhHpFFoVCmY2yMy6BMMnmtm3zKywhXlyzCxvzzBwGrCgUbNngC8FVyEdCWx194oDXosYO6xXLst0+EhEOoHW7in8Dag3s8HAnYS+3T/Uwjy9gDfNbB4wC3je3f9hZpeb2eVBmxeAj4DlwF3A1w90BeLBkF55rNpUw85aXYEkIomttecUGty9zszOAf7o7n80szn7m8HdPwLGNDH+johhB75xIAXHo2F98qlvcJauq2ZM//3uQImIxLXW7inUmtlU4BLguWBcenRKSjxjgyCY8/HmGFciInJwWhsKlwJHAb909xVmNhB4IHplJZa+hVn0yO3CgrV6NKeIJLZWHT5y90XAtwDMrCuQ5+6/iWZhiWZE33wWKRREJMG19uqjV80s38y6Ae8Bd5nZzdEtLbEM75vPsg3VeuCOiCS01h4+KnD3bcC5hLqlOAI4JXplJZ4RffOprXc9s1lEElprQyEt6JPoAj490SwRhvfJB2ChDiGJSAJrbSjcAMwAPnT3d83sUGBZ9MpKPCXdc8jOSNV5BRFJaK090fw48HjE+4+A86JVVCJKSTGG9dHJZhFJbK090VxsZk+a2Ybg9TczK452cYlmeJ98FlVso6HBY12KiEibtPbw0b2E+inqG7yeDcZJhBF98/lkVx2rN9fEuhQRkTZpbSgUufu97l4XvO4D4qu70jgwvG/oZLMOIYlIomptKFSZ2cVmlhq8LgaqollYIhrSK4/UFNMVSCKSsFobCl8mdDnqOqACmAJMi1JNCSszPZXBRbksqlAoiEhialUouPsqd5/s7kXu3tPdv4CuPmrSiL75LFy7NdZliIi0ycE8ee3qdquiExneN5/123axoXpnrEsRETlgBxMK1m5VdCLjBoS60X5v1ZYYVyIicuAOJhR0MX4TRvYrICM1hff0bAURSUD7vaPZzKpp+sPfgKyoVJTguqSlMqq4gNmrFAoiknj2GwruntdRhXQmY/sX8uDbq6itbyA99WB2xkREOpY+saJgbP9CdtU1sKRC3WiLSGJRKETBnmc2z12tQ0giklgUClFQ3DWLHrkZzFmtK5BEJLEoFKLAzBjbv5C5CgURSTAKhSgZ27+Qjyq3s7WmNtaliIi0mkIhSsb27wrAvHLtLYhI4lAoRMno/gWYoUNIIpJQFApRkp+ZzuCiXObozmYRSSBRD4Xg+QtzzOy5JqZNM7NKM5sbvC6Ldj0dac/JZnf1CCIiiaEj9hSuAhbvZ/qj7j42eN3dAfV0mLEDCtlcU8vHm/R4ThFJDFENBTMrBj4LdKoP+9YaUxy6iW1euZ6vICKJIdp7CrcA3wca9tPmPDObb2ZPmFn/KNfToQ7vnUdmeorOK4hIwohaKJjZ54AN7j57P82eBUrcfTTwEvCXZpY13czKzKyssrIyCtVGR3pqCmP7F/Luyk2xLkVEpFWiuadwDDDZzFYCjwAnmdmDkQ3cvcrddwVv7wYmNLUgd7/T3UvdvbSoqCiKJbe/McWFLFizjc3bd8e6FBGRFkUtFNz9WncvdvcS4ELgFXe/OLKNmfWJeDuZ/Z+QTkgj+hUA8PKSDTGuRESkZft9nkI0mNkNQJm7PwN8y8wmA3XAJmBaR9cTbaeP6EVGWgpL122LdSkiIi3qkFBw91eBV4Phn0aMvxa4tiNqiJUuaamM7V/Ifz6sinUpIiIt0h3NHeC4wT1YVLGNTTqvICJxTqHQAY49rAfu8MayxLlySkSSk0KhA4wuLqRrdjqvLVUoiEh8Uyh0gNQU4/ghRbz2QSUNDeoHSUTil0Khg0w6vCdV23fz/hp1eSEi8Uuh0EGOH1KEGbyqQ0giEscUCh2kW04Go4sLmblUN7GJSPxSKHSgE4YUMb98C1t36LnNIhKfFAod6OhB3WlweHeFOsgTkfikUOhAY/sXkpGWwmsf6LyCiMQnhUIHykxP5fQRvXlm3lpqdtfFuhwRkX0oFDrYRUcMYOuOWp6bVxHrUkRE9qFQ6GBHDOzGoUU5/PWdVbjrRjYRiS8KhQ5mZlx6dAnzyrcySyecRSTOKBRiYMqE/uR2SePqx+bFuhQRkb0oFGIgKyOVU4f3Ys2WHazeVBPrckREwhQKMXL1qUNIMfjjK8tiXYqISJhCIUb6d8vmpKE9eaysnPnlW2JdjogIoFCIqZ+dPRKAybf9m+27dN+CiMSeQiGG+hVmMe3oEgDueO3D2BYjIoJCIeaunzyCowd15+m5a6mtb4h1OSKS5BQKceDLxwzk4001nH7L63oym4jElEIhDpw8rCcj++XzUeV2fjNjSazLEZEkplCIA2bGs1cey+Ceufzfax/x9kdVsS5JRJKUQiFOmBk3TB4BwIV3vs0Nzy6KcUUikowUCnHk6ME9+PkXQpep3vPvFby7Un0jiUjHUijEmYuPGMDUif0BOP+Ot3h23toYVyQiySTqoWBmqWY2x8yea2JaFzN71MyWm9k7ZlYS7XrinZnx63NH851ThwDwzYfn8PTcNTGuSkSSRUfsKVwFLG5m2leAze4+GPgf4DcdUE9C+ObJh3Hb/xsHwFWPzOXDyk+o1+WqIhJlUQ0FMysGPgvc3UyTs4G/BMNPACebmUWzpkTyudF9uePiCQCc/PvXGPTDF1i6rjrGVYlIZxbtPYVbgO8Dzd2q2w9YDeDudcBWoHuUa0ooZ4zszah+BeH3p9/yOp+onyQRiZK0aC3YzD4HbHD32WZ24kEuazowHWDAgAHtUF1iefabxwJQcs3zAIy8bgbdcjLomp3OtKNLOHd8MTldorYpRSSJWLSeE2xmvwb+C6gDMoF84O/ufnFEmxnA9e7+lpmlAeuAIt9PUaWlpV5WVhaVmuPdtp21jL7+n01OS00xfjZ5BD9+agGj+hXwxBVH0SUttYMrFJF4ZWaz3b20xXYd8fD4YE/hu+7+uUbjvwGMcvfLzexC4Fx3v2B/y0rmUADYuqOWLmkpbNi2iz++sozHZ5c32e6Igd149GtHdXB1IhKv4jYUzOwGoMzdnzGzTOABYBywCbjQ3T/a37KSPRSasrO2niN//TJbamo5elB3/vPh3t1kTB7Tl+nHH8qNLy5hdHEBX580mFwdbhJJKnEVCu1JodCyhWu38tlb39xvm+F98hnQLZs/TB2rw0wiSUChIPxn+Uaee7+Ctz6sYsXG7c22++pxA/nhWcPQ1cAinZdCQZq1YdtOvvfEfF77oLLFtmbgDlnpqTz2taMo6ZFNXmY69Q1Oaoqxs7aeNVt2MKgotwMqF5G2UihIq7z2QSWX3DOr3ZY3pn8hAPd/eSIFWekA3PryMm5+6QOeufIYuud24fGy1Uw7uoTC7AwAduyu5/f/XMr3zjhch7JEokShIAesvsFZuq6aqu27mH7/bK48aTBjigt5Y3kldfXOn99cEfUafv6FkXx+dB+em1/Bj59awAlDijjusB78/p8f8MJVx/GzZxdy9alDGF1cSEODc+7t/+F7px/O0YO6U7O7nuyMVB0GE2mCQkGiwt2pb3BSzPjRUwv40WeHkZ5qPPruasYUF/LignX8a/F6lm/4ZK/5BhXl8GFl8+c12tMh3bO5+tQhjB/Qlf7dsptss2HbTib+6mVe+c4JzC/fyglDihj385cAePW7J1LSI2efedyd6l115GemR7V+kWhQKEjMVWzdwXn/+x/uuqSUEX0L9pq2taaWnz27kEO653DpsSXkZ6bz8KyPufbv74fbpKUYde3YCeDz3zqWtz6sCt3o14qHGH3zpMH88ZXlzU5//XuTGNA9FDrujjukpIT2UnbW1mOGDodJ3FAoSKfj7pgZS9dVs2n7bo4atHc3WVt31PLg26vISk9l7uotPBPlZ1Fkpqcw/7rTmXzbmyyJ6Kjw82P6hp+DcdP5YzhvfD/MjBkL1/G1B2Zz69RxTB7Tt1W/o2LrDnbWNjCwiT0XkQOhUBABlq6rZuuOWj7ZVcuX79v73827PzqForwu+8yzfVcdVz82l1krNlHSI4c5H2/hjBG9+cfCdTz81SPpW5jJY2Wr+dPMD9tc16wfncx/llcxaWhPHn33Y86f0J81W3aws7YeB4b2zmPz9lqO/93M8DwXlBYD8FhZ6C72Z688lpv+uRQPppUe0o3eBZlsqdnN2x9t4jMlXdm+K3R12J6LCQ4tyuGm88fQuyCT7bvqOKR702GzJ4Bbo3pnLW9/tIlThvXU+Zw4plAQiaLddQ1Mf6CMV5eGLuuddnQJ3zxpMFc8+B4De+Rw43mjqKzexcRfvRzjSlvn1OG9eGnR+r3GTZ04gF+fO6rFeSf+8l9sqN7FlAnF/Pa80VTvrKMgO50X36/gn4tC55feX7OVi48cwC++0PLyJDoUCiIdoLa+gfTU1vVAv+fb9wNvreQnTy9s1TwnHl7E788fw+Ozy/nzmytwd646+bBWz7/Hjz87jF8839yzrlr2nVOHcOVJg7n+mYX85a1VDOyRw7+uPoHHylbvdR6oJb8/fwyrNtWQn5nG6SN6s6O2nm8+NIf+3bL46nGHcsShzfec7+5UVu+iZ37mPtMaGpz11TvpU5B1QOu1fEM1/btlx9W5nysfeo/n5lcw87sntuthQ4WCSBxzdxocHnx7FV/8TH8y01P3md7SoZgdu+vJyvh0vvoG5/n3K+idn8lnSrq2+lDO3W98xC+eX8wDX5nI0YN68IeXl1Gzq467D+AS5P2FTlsvGPjVOaP47Kg+3P/WSpasr+b5+RUAdM1OZ3NNLRcfOYAH3/54n/le/e6JzFqxifGHdOW2V5bx1NxPzy1NOKQr4/oXcvebK7jzvyYw/YHZe83772tOol/hp8FSV99AWhD6X7pnFhmpxp3/VRq+oCBS4+3xxrJKnpwTepTu788fw4yF63hqzlqmTCjmsvs//Qw7ZVhPjh7Ug8lj+7K7roGjb3xlr+VectQhXPf5EU3+zgOhUBCRg/LC+xWs2LidvoWZfPvReeHxJwwp2utu+HPG9eN/vjiW5+dX8NsZS/jGpMF0SUth8pi+ewXT03PXcNUjc5v9fb3yu7B+267orMxB2nNnf3PyM9PYtrP9Hn7VPSeDqu279xpX0j2bf/z38ft8gWgthYKItKvddQ1kpIW+Ne+srWfoT/7BoUU5vHjVca0+/LKrrp70lJRmv/VWfbKLWSs2ccVf39tn2lUnH8awPvlc/uBsbvt/47jyoTlAaC/lmME9OLxXHtMfKONfizfsNd+PPzuMv7y1kvTUFD5qdK9MZnoKs350Cqs31XDVI3P3ub8mmt77yanM+Xgz33l8HltqasPjRxcX8NTXj+HY37xCQXYGiyu2haedOrwXd32pxc/1JikURCQp1dY3kGqGE3r4VKSGBueZeWs5bUQvsjP27T7+zWUbcZzyzTu49u/v8+6PTuHVpRt4YnY5A3vkcP3kESyu2Eb55h1s3VHLxIHdAHh58QYuPaaEOR9v4bUPKkM3Qw4oDH+r/6jyEzLTU+lbmMWqqu0M6Ja9z+G9/3y4kfzMdEb22/ueHndnZVUNZSs3ccKQoibPqbSGQkFERMJaGwqtu2xCRESSgkJBRETCFAoiIhKmUBARkTCFgoiIhCkUREQkTKEgIiJhCgUREQlTKIiISJhCQUREwhQKIiISplAQEZGwqIWCmWWa2Swzm2dmC83sZ020mWZmlWY2N3hdFq16RESkZfv2Hdt+dgEnufsnZpYOvGlmL7r7243aPeruV0axDhERaaWohYKH+uTe88SK9OCVWP10i4gkmaieUzCzVDObC2wAXnL3d5podp6ZzTezJ8ysfzPLmW5mZWZWVllZ2VQTERFpB1ENBXevd/exQDEw0cxGNmryLFDi7qOBl4C/NLOcO9291N1Li4qKolmyiEhS65Crj9x9CzA02ZI5AAAJaUlEQVQTOKPR+Cp33/Ok7ruBCR1Rj4iINC2aVx8VmVlhMJwFnAosadSmT8TbycDiaNUjIiIti+bVR32Av5hZKqHweczdnzOzG4Ayd38G+JaZTQbqgE3AtCjWIyIiLbDQRUKJo7S01MvKymJdhohIQjGz2e5e2lI73dEsIiJhCgUREQlTKIiISJhCQUREwhQKIiISplAQEZEwhYKIiIQl3H0KZlYJrGrj7D2Aje1YTjzRuiWezrpeoHWLR4e4e4udxyVcKBwMMytrzc0biUjrlng663qB1i2R6fCRiIiEKRRERCQs2ULhzlgXEEVat8TTWdcLtG4JK6nOKYiIyP4l256CiIjsR9KEgpmdYWZLzWy5mV0T63paYmb9zWymmS0ys4VmdlUwvpuZvWRmy4KfXYPxZma3Bus338zGRyzrkqD9MjO7JFbr1FjwDO85ZvZc8H6gmb0TrMOjZpYRjO8SvF8eTC+JWMa1wfilZnZ6bNbkU2ZWGDxvfImZLTazozrLNjOzbwf/FheY2cNmlpmo28zM7jGzDWa2IGJcu20nM5tgZu8H89xqZtaxa3gQ3L3Tv4BU4EPgUCADmAcMj3VdLdTcBxgfDOcBHwDDgd8C1wTjrwF+EwyfBbwIGHAk8E4wvhvwUfCzazDcNdbrF9R2NfAQ8Fzw/jHgwmD4DuCKYPjrwB3B8IXAo8Hw8GBbdgEGBts4Ncbr9BfgsmA4AyjsDNsM6AesALIittW0RN1mwPHAeGBBxLh2207ArKCtBfOeGcvtd0B/m1gX0EH/AI4CZkS8vxa4NtZ1HeA6PE3okaZLgT7BuD7A0mD4/4CpEe2XBtOnAv8XMX6vdjFcn2LgZeAk4LngP89GIK3xNgNmAEcFw2lBO2u8HSPbxWidCoIPTms0PuG3WRAKq4MPwLRgm52eyNsMKGkUCu2ynYJpSyLG79Uu3l/Jcvhozz/oPcqDcQkh2PUeB7wD9HL3imDSOqBXMNzcOsbrut8CfB9oCN53B7a4e13wPrLO8DoE07cG7eNt3QYClcC9wWGxu80sh06wzdx9DXAT8DFQQWgbzCbxt1mk9tpO/YLhxuMTQrKEQsIys1zgb8B/u/u2yGke+hqScJePmdnngA3uPjvWtbSzNEKHJG5393HAdkKHIcISeJt1Bc4mFHx9gRzgjJgWFUWJup3aQ7KEwhqgf8T74mBcXDOzdEKB8Fd3/3swer2Z9Qmm9wE2BOObW8d4XPdjgMlmthJ4hNAhpD8AhWaWFrSJrDO8DsH0AqCK+Fu3cqDc3d8J3j9BKCQ6wzY7BVjh7pXuXgv8ndB2TPRtFqm9ttOaYLjx+ISQLKHwLnBYcKVEBqETX8/EuKb9Cq5W+DOw2N1vjpj0DLDnKodLCJ1r2DP+S8GVEkcCW4Nd4RnAaWbWNfi2d1owLmbc/Vp3L3b3EkLb4hV3vwiYCUwJmjVetz3rPCVo78H4C4MrXQYChxE6wRcT7r4OWG1mhwejTgYW0Qm2GaHDRkeaWXbwb3PPuiX0NmukXbZTMG2bmR0Z/K2+FLGs+Bfrkxod9SJ0BcEHhK52+FGs62lFvccS2n2dD8wNXmcROi77MrAM+BfQLWhvwJ+C9XsfKI1Y1peB5cHr0livW6P1PJFPrz46lNAHxHLgcaBLMD4zeL88mH5oxPw/CtZ5KXFwhQcwFigLtttThK5K6RTbDPgZsARYADxA6AqihNxmwMOEzo3UEtrD+0p7biegNPg7fQjcRqOLD+L5pTuaRUQkLFkOH4mISCsoFEREJEyhICIiYQoFEREJUyiIiEiYQkHijpnVm9lcM5tnZu+Z2dEttC80s6+3YrmvmlmnfbZuW5jZfWY2peWWkiwUChKPdrj7WHcfQ6gDtV+30L6QUK+ccSnijl+RuKdQkHiXD2yGUD9QZvZysPfwvpmdHbS5ERgU7F38Lmj7g6DNPDO7MWJ555vZLDP7wMyOC9qmmtnvzOzdoL/8rwXj+5jZ68FyF+xpH8nMVprZb4PfNcvMBgfj7zOzO8zsHeC3QV/9TwXLf9vMRkes073B/PPN7Lxg/Glm9lawro8HfWBhZjda6Bkb883spmDc+UF988zs9RbWyczsNgs9y+BfQM/23FiS+PQNRuJRlpnNJXRXbB9CfSMB7ATOcfdtZtYDeNvMniHU6dxIdx8LYGZnEuq87Qh3rzGzbhHLTnP3iWZ2FnAdoT59vkKo64LPmFkX4N9m9k/gXELdFvzSzFKB7Gbq3eruo8zsS4R6f/1cML4YONrd683sj8Acd/+CmZ0E3E/o7uef7Jk/qL1rsG4/Bk5x9+1m9gPgajP7E3AOMNTd3cwKg9/zU+B0d18TMa65dRoHHE7ouQa9CHVVcU+rtookBYWCxKMdER/wRwH3m9lIQt0N/MrMjifU5XY/Pu3eONIpwL3uXgPg7psipu3pWHA2of70IdRnzeiIY+sFhPrkeRe4x0IdEz7l7nObqffhiJ//EzH+cXevD4aPBc4L6nnFzLqbWX5Q64V7ZnD3zRbqRXY4oQ9yCD2s5y1C3U/vBP5soafVPRfM9m/gPjN7LGL9mlun44GHg7rWmtkrzayTJCmFgsQ1d38r+OZcRKjvpyJggrvXWqiX1cwDXOSu4Gc9n/77N+Cb7r5Pp3NBAH2W0Ifuze5+f1NlNjO8/QBrC/9a4CV3n9pEPRMJdUY3BbgSOMndLzezI4I6Z5vZhObWKdhDEmmWzilIXDOzoYQep1pF6NvuhiAQJgGHBM2qCT2ydI+XgEvNLDtYRuTho6bMAK4I9ggwsyFmlmNmhwDr3f0u4G5C3WA35YsRP99qps0bwEXB8k8ENnro+RgvAd+IWN+uwNvAMRHnJ3KCmnKBAnd/Afg2MCaYPsjd33H3nxJ6yE//5tYJeB34YnDOoQ8wqYW/jSQZ7SlIPNpzTgFC33gvCY7L/xV41szeJ9QT6RIAd68ys39b6CHsL7r798xsLFBmZruBF4Af7uf33U3oUNJ7FjpeUwl8gVAPrt8zs1rgE0JdIDelq5nNJ7QXss+3+8D1hA5FzQdq+LSL5l8Afwpqrwd+5u5/N7NpwMPB+QAInWOoBp42s8zg73J1MO13ZnZYMO5lQs9Ant/MOj1J6BzNIkLdYTcXYpKk1EuqyEEIDmGVuvvGWNci0h50+EhERMK0pyAiImHaUxARkTCFgoiIhCkUREQkTKEgIiJhCgUREQlTKIiISNj/ByeoZfppHxwhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot_losses()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7 fasta.ai1 DEV",
   "language": "python",
   "name": "fastai1_dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.text import *\n",
    "import re\n",
    "import sentencepiece as spm #https://github.com/google/sentencepiece\n",
    "\n",
    "np.random.seed(20180311)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentencepiece tokenizer & LM Build"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data processing to DataBunch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get IMDB Data and build DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/home/farzin/.fastai/data/imdb')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "IMDB_PATH = untar_data(URLs.IMDB)\n",
    "IMDB_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75000, 25000)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## raw data does not have the unsup in the train directory.  Just move it there\n",
    "#  ~/.fastai/data/imdb$ mv unsup/ ./train/\n",
    "CLASSES = ['neg', 'pos', 'unsup']\n",
    "\n",
    "def get_texts(path):\n",
    "    texts,labels = [],[]\n",
    "    for idx,label in enumerate(CLASSES):\n",
    "        for fname in (path/label).glob('*.*'):\n",
    "            texts.append(fname.open('r', encoding='utf8').read())\n",
    "            labels.append([0]) ## all unsupervised case\n",
    "\n",
    "    return np.array(texts),np.array(labels)\n",
    "\n",
    "train_texts,train_labels = get_texts(IMDB_PATH/'train')\n",
    "valid_texts,valid_labels = get_texts(IMDB_PATH/'test')\n",
    "\n",
    "len(train_texts),len(valid_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## we can combine the \"train\" and \"valid\" for the LM step\n",
    "all_texts = np.concatenate([train_texts,valid_texts])\n",
    "all_texts_df = pd.DataFrame({'text':all_texts, 'labels':[0]*len(all_texts)}, columns=['labels','text'])"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "raw_text = all_texts_df.loc[:,'text']\n",
    "\n",
    "print(\"Default Rules:\\n\",[x.__name__ for x in defaults.text_pre_rules],\"\\n\\n\")\n",
    "\n",
    "for rule in defaults.text_pre_rules:\n",
    "    print(rule)\n",
    "    raw_text = raw_text.apply(lambda x: rule(str(x)))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "## SentencePiece expects <s> ... </s> so we add that here:\n",
    "all_texts_df['new_text'] = '<s>' + raw_text + '</s>'\n",
    "all_texts_df['new_text'].head()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "## save as a file for SP\n",
    "formatted_text_file = 'tmp_IMDB_SP_example'\n",
    "all_texts_df['new_text'].to_frame().to_csv(formatted_text_file, header=False,index=False,quotechar=' ')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SP Tokenizer SWIG wrapper"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# m.model and m.vocab created in the pwd\n",
    "vocab_size = 500\n",
    "model_prefix = 'wk103m_example'\n",
    "\n",
    "spm.SentencePieceTrainer.Train(f'--input={formatted_text_file}'\\\n",
    "                               f' --model_prefix={model_prefix}'\\\n",
    "                               f' --vocab_size={vocab_size}')\n",
    "#                               f'--unk_piece={UNK} --bos_piece={BOS} --eos_id=-1 --pad_piece={PAD}')"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "## Vocab in order of frequency\n",
    "!head -n50 {model_prefix}.vocab | nl"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "## load up the Processor\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.load(f'{model_prefix}.model')\n",
    "\n",
    "## itos from m.vocab file: just read directly and populate the dictionary\n",
    "itos = {}\n",
    "with open(f'{model_prefix}.vocab','r') as f:\n",
    "    for line_num,line in enumerate(f):\n",
    "        itos[line_num] = line.split(\"\\t\")[0]\n",
    "        \n",
    "stoi = {v:k for k,v in itos.items()}"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "#Based on Tokenizer() class: https://github.com/fastai/fastai/blob/master/fastai/text/transform.py#L87\n",
    "class CustomTokenizer():\n",
    "    '''Wrapper for SentencePiece toeknizer to fit into Fast.ai V1'''\n",
    "    def __init__(self,sp_processor,pre_rules:ListRules=None,post_rules:ListRules=None):\n",
    "        self.sp = sp_processor\n",
    "        self.pre_rules  = ifnone(pre_rules,  defaults.text_pre_rules )\n",
    "        \n",
    "    def __repr__(self) -> str:\n",
    "        return \"Custom Tokenizer\"\n",
    "\n",
    "    def process_text(self, t:str) -> List[str]:\n",
    "        \"Processe one text `t` with tokenizer `tok`.\"\n",
    "        for rule in self.pre_rules: t = rule(t)                                                                                 \n",
    "        toks = sp.EncodeAsIds(t)\n",
    "        \n",
    "        return toks \n",
    "    \n",
    "    def _process_all_1(self,texts:Collection[str]) -> List[List[str]]:\n",
    "        'Process a list of `texts` in one process'\n",
    "        return [self.process_text(t) for t in texts]\n",
    "                                                                     \n",
    "    def process_all(self, texts:Collection[str]) -> List[List[str]]: \n",
    "        \"Process a list of `texts`.\"                                 \n",
    "        return self._process_all_1(texts)\n",
    "    \n",
    "mycust_tok = CustomTokenizer(sp)\n",
    "#setup Vocab object for use in LM  \n",
    "# Vocab source: https://github.com/fastai/fastai/blob/master/fastai/text/transform.py#L122\n",
    "sp_vocab = Vocab(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labels</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Detective Frank Keller(Al Pacino, in a perfect...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>This is a VERY good movie. I give it a 10.&lt;br ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>I can't say I enjoyed this as much as \"The Big...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>I sat last night to see this film being played...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>First of all, I agree that the plot left somet...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   labels                                               text\n",
       "0       0  Detective Frank Keller(Al Pacino, in a perfect...\n",
       "1       0  This is a VERY good movie. I give it a 10.<br ...\n",
       "2       0  I can't say I enjoyed this as much as \"The Big...\n",
       "3       0  I sat last night to see this film being played...\n",
       "4       0  First of all, I agree that the plot left somet..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = np.random.permutation(len(all_texts))\n",
    "cut = int(0.1 * len(idx))\n",
    "LM_train_df = pd.DataFrame({'text':all_texts[idx[cut:]], 'labels':[0] * (len(all_texts)-cut)}, columns=['labels','text'])\n",
    "LM_valid_df = pd.DataFrame({'text':all_texts[idx[:cut]], 'labels':[0] * cut}, columns=['labels','text'])\n",
    "LM_train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path(\"./tmp_IMDB_example\")\n",
    "\n",
    "data = TextLMDataBunch.from_df(PATH, LM_train_df, LM_valid_df, \n",
    "#                                tokenizer=mycust_tok, vocab=sp_vocab, \n",
    "                               text_cols='text', label_cols='labels')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60003"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data.vocab.itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['xxunk',\n",
       " 'xxpad',\n",
       " 'xxbos',\n",
       " 'xxfld',\n",
       " 'xxmaj',\n",
       " 'xxup',\n",
       " 'xxrep',\n",
       " 'xxwrep',\n",
       " 'the',\n",
       " '.',\n",
       " ',',\n",
       " 'and',\n",
       " 'a',\n",
       " 'of',\n",
       " 'to',\n",
       " 'is',\n",
       " 'it',\n",
       " 'in',\n",
       " 'i',\n",
       " 'this',\n",
       " 'that',\n",
       " '\"',\n",
       " \"'s\",\n",
       " '-',\n",
       " '\\n \\n ',\n",
       " 'was',\n",
       " 'as',\n",
       " 'with',\n",
       " 'for',\n",
       " 'movie',\n",
       " 'but',\n",
       " 'film',\n",
       " 'you',\n",
       " ')',\n",
       " 'on',\n",
       " \"n't\",\n",
       " '(',\n",
       " 'not',\n",
       " 'are',\n",
       " 'he',\n",
       " 'his',\n",
       " 'have',\n",
       " 'one',\n",
       " 'be',\n",
       " 'all',\n",
       " 'at',\n",
       " 'they',\n",
       " 'by',\n",
       " 'an',\n",
       " 'who']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.vocab.itos[:50]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  LM Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(data, AWD_LSTM, drop_mult=0.5)\n",
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  LM Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xuc3HV97/HXZ2b2fsnuJpt7QrhEMFzCZQlgKYiXFCjKsaUWj20R9aR4hNa2eoqn59hW7cV6etFSiSmK1QK2IigqhqAVUQEhIQnkJrmASTa3zd6vszszn/PH/HYZlt3NJJnfXDbv5+Mxj535/r6/+X1mMpnPfC+/78/cHRERkWOJFDoAEREpDUoYIiKSFSUMERHJihKGiIhkRQlDRESyooQhIiJZUcIQEZGsKGGIiEhWlDBERCQrsUIHkEuzZs3yJUuWFDoMEZGSsWHDhqPu3pxN3WmVMJYsWcL69esLHYaISMkws19mW1ddUiIikhUlDBERyYoShoiIZEUJQ0REsqKEISIiWVHCEBGRrChhiIhIVkJNGGbWYGYPmtkOM9tuZleM2/4xM9sU3LaYWdLMmoJtr5jZi8E2nVwhIjKBH2w7zOof787LscJuYXwOWOvu5wDLge2ZG939s+5+obtfCHwc+LG7d2RUuSbY3hJynCIiJemxrYf4ys9eycuxQjvT28xmAFcB7wNw92FgeIpd3gM8EFY8IiLTUdfgCA3VZXk5VpgtjNOBNuBeM9toZveYWc1EFc2sGrgW+GZGsQPrzGyDma0KMU4RkZLVNTA8LRJGDLgYuNvdLwL6gTsnqfsO4GfjuqOudPeLgeuAD5vZVRPtaGarzGy9ma1va2vLYfgiIsWvc2CExuryvBwrzISxH9jv7j8PHj9IOoFM5GbGdUe5e2vw9wjwMLBioh3dfY27t7h7S3NzVgsuiohMG+kWRoknDHc/BOwzs7ODorcC28bXC8Y6rga+nVFWY2Z1o/eBlcCWsGIVESlF7k7XwAiNeeqSCnt58zuA+8ysHNgD3GpmtwG4++qgzruAde7en7HfHOBhMxuN8X53XxtyrCIiJaUvniCR8ryNYYSaMNx9EzB+SuzqcXW+AnxlXNke0tNwRURkEl0DIwCl3yUlIiLh6hxIn6kwHQa9RUQkRJ1BCyNfYxhKGCIiJaoraGFMh/MwREQkRBrDEBGRrIyOYTRUqYUhIiJT6BoYoa4yRiyan69yJQwRkRKVz3WkQAlDRKRk5XMdKVDCEBEpWflcRwqUMERESlZnHteRAiUMEZGS1TUwnLcZUqCEISJSkhLJFD1DCXVJiYjI1LoH87ssCChhiIiUpLF1pGrUwhARkSl0D6bP8p6hMQwREZlKZ/9ol5RaGCIiMoV8XwsDlDBERErS2Eq1NdOkS8rMGszsQTPbYWbbzeyKcdvfbGbdZrYpuH0iY9u1ZvYLM9tlZneGGaeISKnpGhwmGjHqKkK90vZrhH2kzwFr3f0mMysHqieo8xN3vyGzwMyiwL8Abwf2A8+Z2SPuvi3keEVESkLnwAgNVWWYWd6OGVoLw8xmAFcBXwJw92F378py9xXALnff4+7DwNeBG8OJVESk9OR7pVoIt0vqdKANuNfMNprZPWZWM0G9K8xss5l938zODcoWAPsy6uwPykREhPQsqXwOeEO4CSMGXAzc7e4XAf3A+LGI54HT3H058M/At473IGa2yszWm9n6tra2k41ZRKQkdA2OTKsWxn5gv7v/PHj8IOkEMsbde9y9L7j/KFBmZrOAVmBRRtWFQdnruPsad29x95bm5uZcvwYRkaKU76XNIcSE4e6HgH1mdnZQ9FbgNYPWZjbXghEbM1sRxNMOPAcsNbPTg8Hym4FHwopVRKTUdA4M53UdKQh/ltQdwH3Bl/4e4FYzuw3A3VcDNwEfMrMEMAjc7O4OJMzsduAxIAp82d23hhyriEhJGBpJMjSSynsLI9SE4e6bgJZxxasztt8F3DXJvo8Cj4YXnYhIaRo7aW8ajWGIiEgICrEsCChhiIiUnNGEoRaGiIhMabRLSi0MERGZksYwREQkKxrDEBGRrHQNDFNZFqGyLJrX4yphiIiUmM6B/K8jBUoYIiIlp2tgOK/X8h6lhCEiUmK61MIQEZFsdA4M05jHS7OOUsIQESkxXQMjeV9HCpQwRERKirunr4WhMQwREZlKbzxBMuUawxARkal19RfmLG9QwhARKSmFOssblDBEREpKe38cgJm1ShgiIjKFtt50wmiuq8j7sZUwRERKyGjCmFU7zRKGmTWY2YNmtsPMtpvZFeO2v9fMXjCzF83sKTNbnrHtlaB8k5mtDzNOEZFS0dYbp74ylveFByHka3oDnwPWuvtNZlYOVI/b/jJwtbt3mtl1wBrgsozt17j70ZBjFBEpGW19cWYVoDsKQkwYZjYDuAp4H4C7DwPDmXXc/amMh88AC8OKR0RkOjjaO0xzAbqjINwuqdOBNuBeM9toZveYWc0U9T8AfD/jsQPrzGyDma2abCczW2Vm681sfVtbW24iFxEpUm198YIMeEO4CSMGXAzc7e4XAf3AnRNVNLNrSCeMP80ovtLdLwauAz5sZldNtK+7r3H3FndvaW5uzukLEBEpNm290zNh7Af2u/vPg8cPkk4gr2FmFwD3ADe6e/toubu3Bn+PAA8DK0KMVUSk6A0OJ+mLJwoyQwpCTBjufgjYZ2ZnB0VvBbZl1jGzxcBDwO+6+0sZ5TVmVjd6H1gJbAkrVhGRUnC0r3DnYED4s6TuAO4LZkjtAW41s9sA3H018AlgJvAFMwNIuHsLMAd4OCiLAfe7+9qQYxURKWpHCnjSHoScMNx9E9Ayrnh1xvYPAh+cYL89wPLx5SIip7Kxs7ynW5eUiIjkVluBu6SUMERESsTR3jhm0FST/4UHQQlDRKRktPXFaaoupyxamK9uJQwRkRJRyHMwQAlDRKRktPXGC3YOBihhiIiUjKMFXBYElDBEREqCu6tLSkREjq03niCeSBXsHAxQwhARKQljV9qrK8yUWlDCEBEpCUfHzvKuLFgMShgiIiWg0Gd5gxKGiEhJaCvwwoOghCEiUhLaeuNEI0ZDVVnBYlDCEBEpAUf74syqLScSsYLFoIQhIlICCn0OBihhiIiUhLa+eEHPwQAlDBGRklDodaRACUNEpOilUk573/D07pIyswYze9DMdpjZdjO7Ytx2M7PPm9kuM3vBzC7O2HaLme0MbreEGaeISDHrGhwhkfKCJ4xQr+kNfA5Y6+43mVk5UD1u+3XA0uB2GXA3cJmZNQF/Tvp64A5sMLNH3L0z5HhFRIpOMZyDASG2MMxsBnAV8CUAdx92965x1W4EvuppzwANZjYP+DXgcXfvCJLE48C1YcUqIlLMxtaRmsZjGKcDbcC9ZrbRzO4xs5pxdRYA+zIe7w/KJit/HTNbZWbrzWx9W1tb7qIXESkSR4tgWRAIN2HEgIuBu939IqAfuDPXB3H3Ne7e4u4tzc3NuX56EZGCm/ZdUqRbBfvd/efB4wdJJ5BMrcCijMcLg7LJykVETjltfXEqYhHqKsIedp5aaAnD3Q8B+8zs7KDorcC2cdUeAX4vmC11OdDt7geBx4CVZtZoZo3AyqBMROSUM3qWt1nhlgWB8GdJ3QHcF8yQ2gPcama3Abj7auBR4HpgFzAA3Bps6zCzTwHPBc/zSXfvCDlWEZGiVAwn7UHICcPdN5GeGptpdcZ2Bz48yb5fBr4cXnQiIqWhrTfOoqbxZyXkn870FhEpYu5Oa9cgCxurCh1KdgnDzM40s4rg/pvN7A/MrCHc0EREpHNghL54gsUl1ML4JpA0s7OANaRnMN0fWlQiIgLA3o4BgJLqkkq5ewJ4F/DP7v4xYF54YYmICLyaMEqphTFiZu8BbgG+G5QV7jqBIiKniH1jLYwSGcMgPd31CuCv3P1lMzsd+Fp4YYmICMDe9gFm1VZQXV7Yk/Ygy2m17r4N+AOA4ES6Onf/TJiBiYhIuktqcRG0LiD7WVJPmFl9sOz488C/mtk/hBuaiIikE0bhxy8g+y6pGe7eA/wG6eXILwPeFl5YIiIynEhxsHuw5BJGLLhOxbt5ddBbRERCdKBrkJQXx5RayD5hfJL04n+73f05MzsD2BleWCIiUkxTaiH7Qe9vAN/IeLwH+M2wghIRkYyEMbM4Eka2g94LzexhMzsS3L5pZgvDDk5E5FS2r2OA8miEOXWVhQ4FyL5L6l7S166YH9y+E5SJiEhI9nYMsLCpikiksNfBGJVtwmh293vdPRHcvgLoeqgiIiEqpim1kH3CaDez3zGzaHD7HaA9zMBERE5l7s7e9tJMGO8nPaX2EHAQuAl4X0gxiYic8roHR+gtkmXNR2U7S+qXwDszy8zsI8A/TbWfmb0C9AJJIOHuLeO2fwx4b0YsbyTd/dVxrH1FRKazYlrWfNTJrGb1xxwjYQSucfejE21w988CnwUws3cAfzTu2t2T7isiMp0V2zkYcHKXaM31sP17gAdy/JwiIiWpGFsYJ5MwPMs668xsg5mtmqySmVUD15K+st9x7SsiMh3t6xhkZk05tRWFX9Z81JSRmFkvEycGA7JZb/dKd281s9nA42a2w92fnKDeO4CfjeuOymrfIJmsAli8eHEWIYmIFL99HQNF1bqAY7Qw3L3O3esnuNW5+zHTnru3Bn+PAA8DKyapejPjuqOy3dfd17h7i7u3NDfr1BARmR6K7RwMOLkuqSmZWY2Z1Y3eB1YCWyaoNwO4Gvj28e4rIjIdJZIpWruKZ1nzUWF2js0BHjaz0ePc7+5rzew2AHdfHdR7F7DO3fuPtW+IsYqIFI2D3UMkU37qJIxgRdvlE5SvHvf4K8BXstlXRORUUIwzpCDELikRETkxxbas+SglDBGRIvPS4V6qyqLMrS+OZc1HKWGIiBSZrQd6OGdeHdEiWdZ8lBKGiEgRcXe2H+jh3Pn1hQ7ldZQwRESKyL6OQXrjCc6dP6PQobyOEoaISBHZeqAbgGXz1MIQEZEpbD3QQzRinD23rtChvI4ShohIEdl2sIezmmupLIsWOpTXUcIQESkiWw90F+WANyhhiIgUjaN9cQ73xFmmhCEiIlPZeqAHQAlDRESmNjpD6tx5xTelFpQwRESKxrYDPSxsrGJGdVmhQ5mQEoaISJHYVqRneI9SwhARKQL98QQvt/ezrEi7o0AJQ0SkKGw/2IM7amGIiMjURmdInbtACUNERKaw7UAPTTXlRXcNjEyhJgwze8XMXjSzTWa2foLtbzaz7mD7JjP7RMa2a83sF2a2y8zuDDNOEZFC23owfYa3WXFdAyNTaNf0znCNux+dYvtP3P2GzAIziwL/Arwd2A88Z2aPuPu2EOMUESmIkWSKlw71ceuvLCl0KFMq1i6pFcAud9/j7sPA14EbCxyTiEgoXmztZjiZ4oKFDYUOZUphJwwH1pnZBjNbNUmdK8xss5l938zODcoWAPsy6uwPykREpp2nd7cDcPkZTQWOZGphd0ld6e6tZjYbeNzMdrj7kxnbnwdOc/c+M7se+Baw9HgOECSiVQCLFy/OVdwiInnz9O52zplbx8zaikKHMqVQWxju3hr8PQI8TLqrKXN7j7v3BfcfBcrMbBbQCizKqLowKJvoGGvcvcXdW5qbm0N4FSIi4Yknkjz3SgdXnDmz0KEcU2gJw8xqzKxu9D6wEtgyrs5cC6YEmNmKIJ524DlgqZmdbmblwM3AI2HFKiJSKBv3dhFPpHjTmbMKHcoxhdklNQd4OMgHMeB+d19rZrcBuPtq4CbgQ2aWAAaBm93dgYSZ3Q48BkSBL7v71hBjFREpiKd3txMxWHF6cY9fQIgJw933AMsnKF+dcf8u4K5J9n8UeDSs+EREisHTu9s5b8EMZlQV5wq1mYp1Wq2IyLQ3OJxk475Orjij+McvQAlDRKRg1v+yg5Gkl8SANyhhiIgUzFO724lFjEuXFP/4BShhiIgUzNO721m+qIGainys0nTylDBERAqgd2iEF1u7eVOJdEeBEoaISEE890oHyZSXzIA3KGGIiBTEU7vaKY9FuPi0xkKHkjUlDBGRAvjxS220nNZIZVm00KFkTQlDRCTP9rT1sfNIHyuXzSl0KMdFCUNEJM/WbTsMwNvPnVvgSI6PEoaISJ6t23qI8xbUs6ChqtChHBclDBGRPDrSM8TGfV2sXFZarQtQwhARyasfbD+CO6w8t7TGL0AJQ0Qkr9ZtO8TipmrOnlNX6FCOmxKGiEie9A6N8NSudlYum0NwraCSooQhIpInP36pjeFkipUlNjtqlBKGiEierNt6mJk15VxSQmd3Z1LCEBHJg+FEih/tOMLb3jiHaKT0uqMg3Gt6Y2avAL1AEki4e8u47e8F/hSwoN6H3H1zNvuKiJSSn+0+Sm88UZKzo0blYxH2a9z96CTbXgaudvdOM7sOWANcluW+IiIl44Gf72VmTTlXLp1V6FBOWEG7pNz9KXfvDB4+AywsZDwiImE40DXID7Yf5t2XLqIiVjqLDY4XdsJwYJ2ZbTCzVceo+wHg+ye4r4hI0Xrg2b048N9XLC50KCcl7C6pK9291cxmA4+b2Q53f3J8JTO7hnTCuPIE9l0FrAJYvLi0/zFEZPoZTqR44Nl9vOXs2Sxqqi50OCcl1BaGu7cGf48ADwMrxtcxswuAe4Ab3b39ePYNtq9x9xZ3b2lubs79ixAROQmPbT3E0b44v3PFaYUO5aSFljDMrMbM6kbvAyuBLePqLAYeAn7X3V86nn1FRErB1575JYuaqrh6aen/oA2zS2oO8HBw+nsMuN/d15rZbQDuvhr4BDAT+EJQb3T67IT7hhiriEjOvXS4l2df7uDj151DpETPvcgUWsJw9z3A8gnKV2fc/yDwwWz3FREpJf/+zC8pj0X4rZZFhQ4lJ3Smt4hICI72xfnmhv3ccME8mmrKCx1OTihhiIiE4B8ff4l4IsXt15xV6FByRglDRCTHdh7u5YFn9/LeyxZzRnNtocPJGSUMEZEc++tHt1NTEeMP3/aGQoeSU0oYIiI59NOdR/nRL9q4/Zqzps3YxSglDBGRHEmmnE9/bxsLG6u45U1LCh1OzilhiIjkyDfW72PHoV7+9NpzqCwr3UUGJ6OEISKSA3vbB/j097azYkkTN1wwr9DhhEIJQ0TkJCWSKf7wPzZiBv/w28sJVqmYdvJxASURkWnt8z/cyca9XXz+PRexsLG0V6SdiloYIiIn4dmXO7jrR7v4jYsX8M7l8wsdTqiUMERETlD3wAh/9B+bWNhYzSdvPK/Q4YROXVIiIicgkUxx+wPPc6R3iP/8/SuorZj+X6fT/xWKiITgM2t38JOdR/nb3zifixY3FjqcvFCXlIjIcXro+f38609e5pYrTuPmEr9O9/FQwhAROQ6b93Vx50MvcvkZTfyfG5YVOpy8UsIQEcnS83s7ufUrzzG7roIvvPcSyqKn1lfoqfVqRURO0NotB3nPmmeoq4zxtQ9cNu0WFsxGqAnDzF4xsxfNbJOZrZ9gu5nZ581sl5m9YGYXZ2y7xcx2BrdbwoxTRGQqX/rpy3zovudZNr+ehz70Jk6fVVPokAoiH7OkrnH3o5Nsuw5YGtwuA+4GLjOzJuDPgRbAgQ1m9oi7d4YR4Hc2HyAWMarKo1SXx2isLuOs2bXT9vT+TO19cXYd6WN3Wz9DI0lm1pbTVFPOzJoKzpxdQ0Xs+BZQc3eAon3vkimne3CE/niC3qEEA8MJHIiYETGIRoxYJEJZ1IhFI0TNGH0pZpBKQdKdlDt9QwkOdg9xqHuQQz1xhkaSJFNOIuXEIsa8hkoWNVazsLGKBY1VzKqpIBIpzvdFJtbZP8xffmcr39p0gGvPncs/3XzhtFxUMFuFnlZ7I/BVT3/LPGNmDWY2D3gz8Li7dwCY2ePAtcADYQTx0W9sJp5IvaZsYWMVv37BPG44fz7nLagv2i/A49XeF+eJX7TxXzuO8PSedjr6hyetWxY1ls2r58JFDZw1u5aRpDOUSDI0kqKzf5jDPUMc7o1ztDfO4EiSoeAWi0aYN6MyuFVRFjUSSWck5SSSqaBeiqFEkspYlDn1Fcypr2ROfSXzG6pY0JD+gq2rjNHWG08fpyfOcDJFxNJf7u7QH0/QMzRC71CCvniCgeEkA8PpvwaUxSKURyOk3DnYNURr1yCHeoZIpjzn72tZ1KgqixKLRoiYkUil6BoYeV2dOfWVzJ9RxcKmKk5rquG0mdUsaKyisbqchuoyGqrKiGXRL55KOcPJFBWxyLT5bBabtVsO8n++tZWugWE+8ral3PGWpURP8YQfdsJwYJ2ZOfBFd18zbvsCYF/G4/1B2WTlr2Nmq4BVAIsXn9j0tsc+chUDw0kGR9JfNge6Bvn+lkN86Scv88Uf72FRUxXXnzeP68+fxwULZ+T8P2gy5fQMjlBVHs36C8Dd6RlK0NY7RPdggqGRJIPDSQZGkvTHE/QFX6I9QyN09A/T0T9MW2+cXxzuxR1m11XwlnNm88Z59ZzZXMNZs2upKY/RHtQ90jvEi63dbNrbxTc27GdgOPma4zdUlzGnrpLZ9RWcOauG6ooolbEolWVRhpMpDnYPcbBrkGdf7iDlTjRilEUjxCJGZVmUyrIItRUxBoeTbNjbmU4I45L28agpj1JdEaOmPDr2C3AkmWIk6TjOvPoqVpzexPyGSmbVVlBbEaOuMkZ1eSzdcnBIuZNMOolUer9EKkUylX6vHcAhEjGikXTSqimPMTdIjE015a/7d+uLJ9jfOcC+jkEOdg9yoCvdGjnQNcRTu9p5qKd1wtdSEYtQHotQEYtQFiS8ZCod30gyRTyRGnuvqsujzG+oGkvQjdXl1FeV0VBdRlN1Oc11FcyqraC5roLq8qiSSxYOdg/y6e9u53svHuTc+fV89f0rWDa/vtBhFYWwE8aV7t5qZrOBx81sh7s/mcsDBEloDUBLS8sJ/XRcMkF/5G9fupiugWHWbT3Mo1sO8qWfvswXn9zD/BmVNNWWk0g6yVT6i7C5roJ5MyqZW19JLBqhfzjBQDzJSDLFufPrufyMmWNdXK1dg/xs51Ge2n2UV9oHONwzxJHe+Niv3mjEqCmPUh6LjH2JpVJOWTT95ZEud9p6469rFU2ktiJGU026m2l+QxXXnTePt75xNsvm1U/YPdKYMZB3wwXpdXESyRQd/cNUxKJUlEVC+VXr7nQOjHCga5D9nYO0dg3SN5Rgdn0Fc+ormF1XSWVZFHdntIFQUxGlrrKM2opYUf7yq62Icc7ces6ZO/GXzdBIkv2dA7R2DdE1MEzXwAidA8MMDifTSSGZYiSRSneXBYkqFolQWRYNkonR0Z9+zw50D/KLQ710DY5MmngrYpGxz0JzXcVYa25hYxXzZqSTzuz6iuPuhpwuhkaSrHlyD3c/sZukOx9d+QZ+/+ozT7mZUFMJNWG4e2vw94iZPQysADITRiuwKOPxwqCslXS3VGb5E2HGOpGG6nLefeki3n3pIroHRnh8+2H+a8dhhkZSwS9mYyTpHOkZ4qXDvbT1xkk5lMciY8sEfP25dENpZk36l9/LR/sBmFVbwTlz6zjrrFljvwyHEunWQX88yXAyRTToVzez4Ndy+pelWTpJNddWMLu+gvqqMqrLolSVR6kqi1JbGaOmIkZNeW6+SGPRCLPrK0/6eaZiZmNfZuctmBHqsYpFZVmUs2bXcdbsupw+79BIku7BEdr7hmnrS3cZtvXF6egfpr1vmI7+OEd642ze10XnuG4zYOzfIT2WVU5FLBK0wNNdjouaqrlgwQzOX9jAufPrS75Pf2A4wXc3H+RzP9xJa9cg158/l49f90YWNU3fVWdPlI0OUub8ic1qgIi79wb3Hwc+6e5rM+r8OnA7cD3pQe/Pu/uKYNB7AzA6a+p54JLRMY3JtLS0+Pr1r5uMlTeJZAqHsV8k7s6+jkGe2dPO03va6R0a4fIzZvKrS5t5w5xTY1Bdilt/PMGBYGznYPcQh7uHONQzNNaN2dE/TDyRoro8/YOkLBphT1s/R/viAEQMFjVVs3R2LWfOrmXJzJqxbrp59VXUV8WK9nO+pbWbrz+3l29vPEBvPMGyefX83xuWccWZMwsdWl6Z2QZ3b8mmbpgtjDnAw8GHJQbc7+5rzew2AHdfDTxKOlnsAgaAW4NtHWb2KeC54Lk+eaxkUQzGD1aaGYtnVrN4ZjXvvnTRJHuJFE5NRYylc+pYOif7Vo67c7gnzub9XWw90MPuI33sPNLLj19qYyT52h+gZVFjZk0FM2vLmVtfydlz6zh7bh3nzK1nyazqvHd/7W0f4JHNrTyy+QAvHe6jIhbh18+fx80rFnPpksaiTW7FIrQWRiEUuoUhcipLJFMc7o1zqHswmG48RHv/MO19cdr7htnfOcjutj4SqdGp1zCnrpKFjVUsnlnN8oUNXLS4gXPm1lMey924QVtvnO+9cIBvbz7Axr1dAFy6pJF3Lp/PO5cvYEZ1Wc6OVYqKpYUhIqeQWDSSnhLdUDVpneFEij1H+9hxsJdftg+wr3OAfR0D/GTnUR56Pj1rrCIW4Q1z6lg8s5rTmqo5bWY1c2dUjU1+aKwum7Il0BdPsLW1mxdbu3ly51F+tusoyZTzxnn13HndObxj+fwpY5TJKWGISN6UxyITzhxzdw52D7Fxbxcb93by0pE+trZ289iWQ2MtklERI5iaHaUyFqEslj73xSw9lXp/5yCjHSeLm6q57eozuPHCBbzhOLrdZGJKGCJScGbG/IYq5jekT5gdlQjO6Rmdfn64Z4j2vuH0yZ/BSaSJZGpsCrqZcdPFizh/YT3nLZjB7LpwZ/edapQwRKRoxaIRFjVVa4prkdAZKSIikhUlDBERyYoShoiIZEUJQ0REsqKEISIiWVHCEBGRrChhiIhIVpQwREQkK9Nq8UEz6wZ2TrBpBtCd5ePR+xOVzQImuz75ZMYfK9vtE5VPFNNk908m5qniyja+Uol5ovJS/HxkE3PmfX0+st8+3T8fS909u4vQuPu0uQFrsimf6vHo/UnK1ucqpuONebKYjhX/icR8onGXYszT5fORTcyFfq/1+Sj+z8exbtOtS+o7WZZP9fg7U5TiKMGOAAAHWUlEQVTlMqZjbZ+ofLKYjhX/iTiRuEsx5onKS/HzkU3Mmff1+ch++6n0+ZjStOqSCpuZrfcs140vFoo5f0oxbsWcP6Uad6bp1sII25pCB3ACFHP+lGLcijl/SjXuMWphiIhIVtTCEBGRrJyyCcPMvmxmR8xsywnse4mZvWhmu8zs85ZxvUgzu8PMdpjZVjP7u2KP2cz+wsxazWxTcLu+2GPO2P4nZuZmNit3EY89dxjv9afM7IXgfV5nZvNLIObPBp/nF8zsYTNrKIGYfyv4/5cys5yNGZxMrJM83y1mtjO43ZJRPuXnvqBOZHradLgBVwEXA1tOYN9ngcsBA74PXBeUXwP8AKgIHs8ugZj/AvhoKb3PwbZFwGPAL4FZpRA3UJ9R5w+A1SUQ80ogFtz/DPCZEoj5jcDZwBNAS6FjDeJYMq6sCdgT/G0M7jdO9bqK4XbKtjDc/UmgI7PMzM40s7VmtsHMfmJm54zfz8zmkf6P/4yn/3W/Cvy3YPOHgL9193hwjCMlEHOoQoz5H4H/BYQyCBdG3O7ek1G1JtexhxTzOndPBFWfARaWQMzb3f0XuYzzZGKdxK8Bj7t7h7t3Ao8D1xby/2o2TtmEMYk1wB3ufgnwUeALE9RZAOzPeLw/KAN4A/CrZvZzM/uxmV0aarRpJxszwO1Bl8OXzawxvFDHnFTMZnYj0Orum8MOdJyTfq/N7K/MbB/wXuATIcY6Khefj1HvJ/2LN2y5jDls2cQ6kQXAvozHo/EXy+uakK7pHTCzWuBNwDcyugwrjvNpYqSbmJcDlwL/aWZnBL8Uci5HMd8NfIr0r91PAX9P+oshFCcbs5lVA/+bdFdJ3uTovcbd/wz4MzP7OHA78Oc5C3KcXMUcPNefAQngvtxEN+lxchZz2KaK1cxuBf4wKDsLeNTMhoGX3f1d+Y41V5QwXhUButz9wsxCM4sCG4KHj5D+gs1sli8EWoP7+4GHggTxrJmlSK8f01asMbv74Yz9/hX4bkixjjrZmM8ETgc2B/9JFwLPm9kKdz9UxHGPdx/wKCEmDHIUs5m9D7gBeGtYP34y5Pp9DtOEsQK4+73AvQBm9gTwPnd/JaNKK/DmjMcLSY91tFL41zW5Qg+iFPIGLCFjAAt4Cvit4L4ByyfZb/yg1PVB+W3AJ4P7byDd5LQij3leRp0/Ar5e7O/zuDqvEMKgd0jv9dKMOncAD5ZAzNcC24DmMN7jMD8f5HjQ+0RjZfJB75dJD3g3Bvebsv3cF+pW8AAK9sLhAeAgMEK6ZfAB0r9c1wKbg/8kn5hk3xZgC7AbuItXT4AsB/492PY88JYSiPlrwIvAC6R/uc0r9pjH1XmFcGZJhfFefzMof4H0+j0LSiDmXaR/+GwKbrme2RVGzO8KnisOHAYeK2SsTJAwgvL3B+/vLuDW4/ncF+qmM71FRCQrmiUlIiJZUcIQEZGsKGGIiEhWlDBERCQrShgiIpIVJQyZ1sysL8/Hu8fMluXouZKWXtl2i5l951grxZpZg5n9z1wcW2QimlYr05qZ9bl7bQ6fL+avLsYXqszYzezfgJfc/a+mqL8E+K67n5eP+OTUoxaGnHLMrNnMvmlmzwW3XwnKV5jZ02a20cyeMrOzg/L3mdkjZvZfwA/N7M1m9oSZPWjpa0XcN3rNgqC8JbjfFyw2uNnMnjGzOUH5mcHjF83s01m2gp7m1cUXa83sh2b2fPAcNwZ1/hY4M2iVfDao+7HgNb5gZn+Zw7dRTkFKGHIq+hzwj+5+KfCbwD1B+Q7gV939ItIryf51xj4XAze5+9XB44uAjwDLgDOAX5ngODXAM+6+HHgS+B8Zx/+cu5/Pa1cmnVCwjtJbSZ+JDzAEvMvdLyZ9DZa/DxLWncBud7/Q3T9mZiuBpcAK4ELgEjO76ljHE5mMFh+UU9HbgGUZK4zWByuPzgD+zcyWkl69tyxjn8fdPfNaCM+6+34AM9tEeo2hn447zjCvLua4AXh7cP8KXr3Gwf3A/5skzqrguRcA20lfMwHSawz9dfDlnwq2z5lg/5XBbWPwuJZ0AnlykuOJTEkJQ05FEeBydx/KLDSzu4Afufu7gvGAJzI29497jnjG/SQT/18a8VcHCSerM5VBd78wWNL9MeDDwOdJX0ujGbjE3UfM7BWgcoL9Dfgbd//icR5XZELqkpJT0TrSq8UCYGajy1PP4NWlpN8X4vGfId0VBnDzsSq7+wDpS7r+iZnFSMd5JEgW1wCnBVV7gbqMXR8D3h+0njCzBWY2O0evQU5BShgy3VWb2f6M2x+T/vJtCQaCt5Felh7g74C/MbONhNv6/gjwx2b2AumL63Qfawd330h6ldv3kL6WRouZvQj8HumxF9y9HfhZMA33s+6+jnSX19NB3Qd5bUIROS6aViuSZ0EX06C7u5ndDLzH3W881n4ihaYxDJH8uwS4K5jZ1EWIl8QVySW1MEREJCsawxARkawoYYiISFaUMEREJCtKGCIikhUlDBERyYoShoiIZOX/A7O5J7R6Av93AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr_find(learn)\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 20:17 <p><table style='width:375px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "    <th>time</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>0</th>\n",
       "    <th>4.283346</th>\n",
       "    <th>4.043954</th>\n",
       "    <th>0.292471</th>\n",
       "    <th>20:17</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1,1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd8VuX9//HXh5AQSEJYYYklDNmbyBBliCKCxUWtqG3RKtVa9VtqLS4c1YrVuvprbdUK1Vaoo7gQRVDqZASZskdQhhACskIISa7fH+fkkJAdcudOyPv5eOTByVn354q39+c+13XO5zLnHCIiIgC1wh2AiIhUHUoKIiISUFIQEZGAkoKIiASUFEREJKCkICIiASUFEREJKCmIiEhASUFERAK1wx1AWTVp0sQlJiaGOwwRkWplyZIle5xzCSXtV+2SQmJiIsnJyeEOQ0SkWjGzraXZT91HIiISUFIQEZGAkoKIiASq3ZiCiJw6jh07xrZt28jIyAh3KKeM6OhoWrVqRWRkZLmOV1IQkbDZtm0bcXFxJCYmYmbhDqfac86RlpbGtm3baNOmTbnOoe4jEQmbjIwMGjdurIRQQcyMxo0bn9SVl5KCiISVEkLFOtm/Z41JCotT9vLo+2vR9KMiIkWrMUnhjSXbeHb+JnYdOBruUESkCkhLS6NXr1706tWL5s2bc9pppwW/Z2Zmluoc1157LevWrQtxpJWrxgw0t24cA8DBjGM0j48OczQiEm6NGzdm2bJlANx///3ExsZy++2359vHOYdzjlq1Cv/+PHXq1JDHWdlqzJVCq4Z1wx2CiFQDGzdupEuXLlx99dV07dqVnTt3MmHCBJKSkujatSsPPvhgsO/ZZ5/NsmXLyMrKokGDBkyaNImePXsycOBAdu/eHcZWlF+NuVKIqOUNvuRoSEGkSnrgna9ZveNAhZ6zS8v63PfDrmU+bu3atbz00kskJSUBMGXKFBo1akRWVhbDhg1j7NixdOnSJd8x+/fvZ8iQIUyZMoWJEyfy4osvMmnSpAppR2WqMVcKfk4gW1lBRErQrl27ICEATJ8+nT59+tCnTx/WrFnD6tWrCxxTt25dLrzwQgD69u1LSkpKZYVboWrMlUIty71SUFIQqYrK840+VGJiYoLlDRs28PTTT7No0SIaNGjANddcU+hzAFFRUcFyREQEWVlZlRJrRasxVwrHu4+UFESk9A4cOEBcXBz169dn586dfPDBB+EOKaRq3JWCuo9EpCz69OlDly5d6NSpE61bt2bQoEHhDimkrLo9zJWUlOTKM8nO/9an8rMXF/HGTQPp27pRCCITkbJas2YNnTt3DncYp5zC/q5mtsQ5l1TEIYGa030UXCmEORARkSqsxiSF3LuPNKYgIlK0mpMUcgeaNaYgIlKkGpMUcu8+ytaVgohIkWpMUjj+nEKYAxERqcJqUFLw/lX3kYhI0WpMUgi6j5QURMQ3bNiwAg+jPfXUU9x0001FHhMbGwvAjh07GDt2bKH7DB06lJJunX/qqadIT08Pfh81ahTff/99aUMPmRqTFFTmQkRONG7cOGbMmJFv3YwZMxg3blyJx7Zs2ZLXX3+93K99YlJ47733aNCgQbnPV1FqXFL4fOOeMEciIlXF2LFjmTVrVjCpTkpKCjt27KB3794MHz6cPn360L17d956660Cx6akpNCtWzcAjhw5wpVXXknnzp259NJLOXLkSLDfTTfdFJTdvu+++wB45pln2LFjB8OGDWPYsGEAJCYmsmeP9/n0xBNP0K1bN7p168ZTTz0VvF7nzp254YYb6Nq1KyNGjMj3OhUlpGUuzCwFOAhkA1knPk1n3mSiTwOjgHRgvHPuq1DEsuN774/3zy+38sDF3ULxEiJyMmZPgu9WVuw5m3eHC6cUublRo0b069eP2bNnc/HFFzNjxgyuuOIK6taty8yZM6lfvz579uxhwIABjBkzpsj5j5999lnq1avHmjVrWLFiBX369Am2PfzwwzRq1Ijs7GyGDx/OihUruPXWW3niiSf4+OOPadKkSb5zLVmyhKlTp7Jw4UKcc/Tv358hQ4bQsGFDNmzYwPTp03n++ee54ooreOONN7jmmmsq5m/lq4wrhWHOuV5FPF59IXCG/zMBeDZUQTSJqxOqU4tINZa3Cym368g5x1133UWPHj0477zz2L59O7t27SryHJ988knw4dyjRw969OgRbHv11Vfp06cPvXv35uuvvy607HZen332GZdeeikxMTHExsZy2WWX8emnnwLQpk0bevXqBYSuPHe4C+JdDLzkvAJMC8ysgZm1cM7trOgX6nW611c3tm+rij61iFSEYr7Rh9LFF1/Mr3/9a7766ivS09Pp27cv06ZNIzU1lSVLlhAZGUliYmKh5bJLsmXLFh5//HEWL15Mw4YNGT9+fLnOk6tOneNfbiMiIkLSfRTqKwUHzDGzJWY2oZDtpwHf5vl9m78uJJrXj+b1Jdt0W6qIBGJjYxk2bBjXXXddMMC8f/9+mjZtSmRkJB9//DFbt24t9hyDBw/mlVdeAWDVqlWsWLEC8Mpux8TEEB8fz65du5g9e3ZwTFxcHAcPHixwrnPOOYc333yT9PR0Dh8+zMyZMznnnHMqqrklCnVSONs51wevm+hmMxtcnpOY2QQzSzaz5NTU1HIH890BL0M/Nmdduc8hIqeecePGsXz58iApXH311SQnJ9O9e3deeuklOnXqVOzxN910E4cOHaJz585MnjyZvn37AtCzZ0969+5Np06duOqqq/KV3Z4wYQIjR44MBppz9enTh/Hjx9OvXz/69+/P9ddfT+/evSu4xUWrtNLZZnY/cMg593iedX8H5jvnpvu/rwOGFtd9VN7S2QCJk2YFyylTRpfrHCJScVQ6OzSqZOlsM4sxs7jcZWAEsOqE3d4GfmqeAcD+UIwnnOiaAT8I9UuIiFRLoew+agZ8ZmbLgUXALOfc+2Z2o5nd6O/zHrAZ2Ag8D/wyhPEwtGMCAJt2Hw7ly4iIVFshu/vIObcZ6FnI+r/lWXbAzaGK4UR//0lfOt7zPl9uTuPzjXsY1L5JyQeJSEg554q8/1/K7mSHBGrME80AdWpHBMsPzVoTxkhEBCA6Opq0tLST/iATj3OOtLQ0oqOjy32OcD+nUOma14/muwMZrNl5INyhiNR4rVq1Ytu2bZzMXYWSX3R0NK1alf95rBqXFD773TCunbaYr3coKYiEW2RkJG3atAl3GJJHjeo+AqgdUYsBbRuz93AmiZNmsVrJQUQkUOOSAkC7hJhgefaqkN8BKyJSbdTIpNC+aVywvHN/+euQiIicampoUohl+g0D6NKiPrsOKCmIiOSqkUkBYGC7xpzeqG4wz4KIiNTgpADQuUV9Nu85zOGjWeEORUSkSqjRSaFNkxicg880RaeICFDDk0LbJrEA/OLlJRzNyg5zNCIi4Vejk0K30+oHyyqSJyJSw5OCmTH9hgEArNulh9hERGp0UgDo/QNv7uZf/2d5mCMREQm/Gp8UoiMjSt5JRKSGqPFJAeCKJK+i4P70Y2GOREQkvJQUgBFdmgOwYEtamCMREQkvJQXgzMRGADz/yeYwRyIiEl5KCkB8vUgAkrfuI+OYnlcQkZpLScHXpolXTvthTdMpIjWYkoJv7sQhALy1bHuYIxERCR8lBV9ELQPgcKa6j0Sk5lJSyOPq/j8gJioC51y4QxERCQslhTzaJsRyICOLtMOZ4Q5FRCQslBTyaOvP3bxy+/4wRyIiEh5KCnl0bu5VTZ3z9a4wRyIiEh5KCnk0j4+mYb1I9qn7SERqKCWFE/Rt3ZCUNM2tICI1U8iTgplFmNlSM3u3kG0/MLOP/e0rzGxUqOMpyemN6vHN3nTdgSQiNVJlXCncBhT1mPA9wKvOud7AlcBfKyGeYrVuVI/0zGz2HFIXkojUPCFNCmbWChgNvFDELg7InRMzHtgRynhKo3Vj7w6kr3foDiQRqXlCfaXwFHAHkFPE9vuBa8xsG/AecEuI4ylR+6axAIyfupjNqYfCHI2ISOUKWVIws4uA3c65JcXsNg6Y5pxrBYwCXjazAjGZ2QQzSzaz5NTU1BBF7Dm9Ub1g+a6ZK0P6WiIiVU0orxQGAWPMLAWYAZxrZv86YZ+fA68COOe+BKKBJieeyDn3nHMuyTmXlJCQEMKQPSlTRgOwYPNedh3ICPnriYhUFSFLCs65O51zrZxziXiDyB855645YbdvgOEAZtYZLymE9lKgjOas1oNsIlJzVPpzCmb2oJmN8X/9DXCDmS0HpgPjXRW5F/TdW84G4N43V4U5EhGRylO7Ml7EOTcfmO8vT86zfjVeN1OV0+20+HCHICJS6fREczFuHtYOgJ37j4Q5EhGRyqGkUIykxEYADHzkozBHIiJSOZQUijG0w/E7nQ4fzQpjJCIilUNJoRhmxhA/MWzSg2wiUgMoKZTg3ou6ADB/XZW6U1ZEJCSUFErQurH3hPMTH64PcyQiIqGnpFCCyIjjf6LsnCrxCIWISMgoKZTCI5d1B2D7Pt2aKiKnNiWFUujQLA6ADbsPhjkSEZHQUlIohdxy2j//ZzJHs7LDHI2ISOgoKZRCfN3IYHnmV9vDGImISGgpKZTSvN8MAeCTDbo1VUROXUoKpdQuwetCem/ld2GOREQkdJQUyuCKpFYAHMw4FuZIRERCQ0mhDEZ2aw7Amp26C0lETk1KCmXQtaU3x8LXO/aHORIRkdBQUiiDpnF1aBIbxdc7DoQ7FBGRkFBSKAMzo11CLFvTDoc7FBGRkFBSKKP4upEsTtlHVnZOuEMREalwSgpltGKbN56g5xVE5FSkpFBGf7m6NwBphzLDHImISMVTUiijHq0aAPDmMpW7EJFTj5JCGeXOr/D5xjQOad5mETnFKCmUQ2SEAXDPzJVhjkREpGIpKZTDqgcuAODNZTtIz9TVgoicOpQUyqFO7QhioiIA+HD1rjBHIyJScUqVFMysnZnV8ZeHmtmtZtYgtKFVbV9NPp/ICGO1nm4WkVNIaa8U3gCyzaw98BxwOvBKyKKqBurUjqBdQiwbdx8KdygiIhWmtEkhxzmXBVwK/Nk591ugRWkONLMIM1tqZu8Wsf0KM1ttZl+bWbVKNO2bxrL2O1VMFZFTR2mTwjEzGwf8DMj9cI8sZv+8bgPWFLbBzM4A7gQGOee6Av9XynNWCfuPHGP790e4dfrScIciIlIhSpsUrgUGAg8757aYWRvg5ZIOMrNWwGjghSJ2uQH4i3NuH4Bzbncp46kSrhnQGoC3l+8IcyQiIhWjVEnBObfaOXerc266mTUE4pxzj5bi0KeAO4Ciqsd1ADqY2edmtsDMRha2k5lNMLNkM0tOTa06NYcu6No8WF6csjeMkYiIVIzS3n0038zqm1kj4CvgeTN7ooRjLgJ2O+eWFLNbbeAMYCgwzj9vgbuanHPPOeeSnHNJCQkJpQm50jwzzquF9NKXW8MciYjIyStt91G8c+4AcBnwknOuP3BeCccMAsaYWQowAzjXzP51wj7bgLedc8ecc1uA9XhJotoY07MlAO8s36Fy2iJS7ZU2KdQ2sxbAFRwfaC6Wc+5O51wr51wicCXwkXPumhN2exPvKgEza4LXnbS5lDFVOe3vns2mVN2iKiLVV2mTwoPAB8Am59xiM2sLbCjPC5rZg2Y2xv/1AyDNzFYDHwO/dc6llee84fTi+KRg+Zl55fqziIhUCeacC3cMZZKUlOSSk5PDHUYBRzKz6Tz5fQBSpowOczQiIvmZ2RLnXFJJ+5V2oLmVmc00s93+zxv+7abiqxsVwVntGgOQmaWxBRGpnkrbfTQVeBto6f+846+TPNomxAAw7YstYY5ERKR8SpsUEpxzU51zWf7PNKBq3RtaBfz2gk6ApuoUkeqrtEkhzcyu8esYRZjZNUC1GxAOtfi6XuWP5z6ttjdQiUgNV9qkcB3e7ajfATuBscD4EMVUrXU/LR7n4Pt0XS2ISPVT2jIXW51zY5xzCc65ps65S4DLQxxbtTSkg9er9pN/LApzJCIiZXcyM69NrLAoTiETz+8AwNa0w2GORESk7E4mKViFRXEKqVXL6PODBhzIyCJx0ix2fH8k3CGJiJTaySSF6vXUWyWadGHnYPn9Vd+FMRIRkbIpNimY2UEzO1DIz0G85xWkEP3aNKJfYiMAlmzdF+ZoRERKr9ik4JyLc87VL+QnzjlXu7KCrI7+84sBAMxauZPESbPIztGFlYhUfSfTfSTFMMs/5NLurveY87W6kkSkalNSCKHXbxxIndrH/8QTXi5uviERkfBTUgihpMRGrHvoQhLi6gTrnp2/KVg+lp3DkczscIQmIlIolc6uBKkHj/LFpj3cNmNZsG7ShZ34YlMan6xPValtEQm5Ci2dLScnIa4OP+yR/2atKbPX8sn6VAAOH80KR1giIgUoKVSSWrWMV27oX+i2b/amV3I0IiKFU1KoRGe1a8KWR0ax7qGR+dYrKYhIVaGkUMnMjDq1I0iZMpplk88H4FslBRGpIpQUwii+biR1IyPYdSAj3KGIiABKCmFlZuQ4x/OfbiFx0iyOZWtuZxEJLyWFMDuvc7Ng+fk8M7aNePJ/TP1ccz2LSOVSUgizX/vzLwD88f11JE6axb8XbmX9rkM88M5qXk3+NozRiUhNo6QQZu2bxjJ34uB86+6euSpYvuP1FZUdUrWWnePYf+RYuMMQqbaUFKqA9k3jmDC4bZHbD2Sc2h9yS7bu490VO076PEcys2l313v0fGAOew4drYDIRGoeJYUqon60V4n8txd05CcDWjP+rEReu3EgAF9sTAtnaCGRnnn8Ke7Ln/2CX72ylPKWXHl3xQ5SDx6l8+T3g3Uzv9p+0jGK1ESaE6GKuP6ctnRtGc/QjglB2e3cYnnTF33DyG7NwxlehXp5wVbufXMVC+4cTuPYqGD95j2HaZcQW6Zzbdx9iF+9srTA+offW8OQjgl0aBZ30vGK1CS6UqgioiMjGNapab55GOpGRQDwv/Wp7E8vexfS0axsXln4DVlV7FbXe9/0xkwGPDKPM+6eHaxfuHlvqc/xxaY9JE6axXlP/C/f+mnXnhksj3jyE+at2VXqcx7MOMYXG/eUen+RU1HIk4KZRZjZUjN7t5h9LjczZ2YlVvCraVrERwPQ88E5HCzF2MITH65n/rrdfLs3nY73vM9dM1cyc2nBrpRv96bzxIfrKyxhHM3KPuky4Iu2FN9NNu3zLXy3P4MnP1zPVc8vLHSfoR2bsubB42VESjuHRdqho3S/fw5XvbCQ1TsOlD5okVNMZVwp3AasKWqjmcX5+xT+f3kN98ZNZwXL3e+fw+6DRT/9nJmVwzPzNjB+6mIefX9tsP7bfUcK7Pv0vA08M28DryZvy7f+QMYx/jp/Y5HJIis7h6tfWMDCzcc/wPenH6PjPe/TefL7pB7MP8D7+3dXkzhpFv9Z/A2Jk2aROGkWAOPPSuTyPq0A6HZafS7o2ow3lxU92Pz6km3c/85qBjwyj6fnbSh0n7kThwDeFdbyySMA726kkU99UuR5d3x/hL/O30hynrm0Rz3zab4xD5GaJKRjCmbWChgNPAxMLGK33wOPAr8NZSzVVcsGdZnz68GMeNL7YFv6zfdc0PX4+MKuAxkYMH99Kj1bNQjWv7tiZ7D8zLwN4BwNY6KoXctYnLKPt5d7H8B3zVzJVf1/EOz76Oy1/HvhN6QePMp9P+xaIJ5VOw7w+cY0Pt+YFswD8Zf5G4Ptf5qzjimX9wDgzaXb+cdn3gN4v3tjZb7z/GZEB+KiI/nTFT0BmPCSN0fG3/63iesGtSEqz4x1CzencftrywvE8qth7bn9go6k7DnMh6t30S4hJtgWXy+Sjs3iWLfrIGu/O8j+I8eIrxtZ4BxnTfmowDqALpM/YN1DI6lTO6LQ7SKnqlBfKTwF3AEU+rXTzPoApzvnZhV3EjObYGbJZpacmpoagjCrtg7N4vjbNX0A78PeOUdOjmPh5jT6/2Ee/f4wjzteX8EFJ3wj7pfYiJ6ne4nimY828sA7q7n3ra+DhJDraJbX7ZOZlRNUbH1jiXcFkZPj6PfwXP40Zx1Ltu7jgXe+Do5LPXiU+et289wnx5/EnrHYe9ju273p/N9/jk8qlOuZcb1Zef8I4qLzf0DfP8ZLQFNmr+XO/3oJJDvHMXvlTn783IIC53nyxz2Z6D/4l9gkhhsGty0wL/aMCQOI8cdlej4wJ9825xx3zcyfqE7U8Z73+c/ib/J1iy3Zuo9rpy4i45i3Lj0zq9x3TYlURSG7UjCzi4DdzrklZja0kO21gCeA8SWdyzn3HPAceDOvVWyk1cPIbi1oEluH2St30sb/UP9x0unFHvOfXwzAzHhm3gae+HB9ge2/Pq8DT85dz4Zdh8jMzuGyv34RbGscWwfnHMlb97H74FH+/NFG/vzRxnzHn/nw3Hy/t28ay8bdh3hr2fZ8s8zlmnRhJ8b0bFlgPXhXRLne+GobaYePMn9d/i8Avxjclr/7CWh095bUqpU/CZyoYUwUS+49n073ereqPjxrNXeP7kLKnsN89c0+Xln4DQCX9GrJm8t2MKp7c+4a1ZmzH/04OMfv3ljJmp0Hue+HXbj3rVX8a4F3TKd732f8WYlM+yIFgMV3n5dv2lWR6ipk03Ga2SPAT4AsIBqoD/zXOXeNvz0e2AQc8g9pDuwFxjjnipxvszpOx1lRrpu2mI/W7g5+H9m1Oe9//V2B/dY9NJLMrJx838aHPPYx+w5ncknv03hgTFfMjO/2ZzDgkXnFvmZcdG0OZpTcv27mPWPxx/fX5Vv/yvX92XM4k+ycHC7t3arYc9zx+vICYxy5po4/k2GdmpKd46hlFLgqKM7dM1fybz8BzJ04mPOeyH9FtfkPo8jKcfm6rIBg/KNVw7psK2RcJq/rBrVh8g+7FFj/9NwNPDnXS8g9T2/AWzcPKnXcIhWptNNxVsoczf6Vwu3OuYuK2We+v0+xn/g1OSls2HWQ858sOGi64M7hHDqaxUdrd/HDni1pEV+3kKMLl/vBl2toxwR+Mbgd454v2GWTa9UDF9Dtvg+C32Pr1GbuxCHUjYrI103z7i1n0+20+FLHAvnb2CI+moFtG7P9+yO89PN+5e7f/z49k14PfgjAHy/vwR1vHC8dUtz82LsPZtDv4YJJ8+z2TfiskFtXR3RpxjPjepN2OJPTGtQlPTOLLpM/yLdPeefjds6VKREW5fDRLCIjahVIgHLqK21SqPSH18zsQSDZOfd2Zb92dXdGs7igiybXxb1a0ty/bbV907I9+AXe3U2XP3u822jatf0KlPC+pFdLImrVoklsFP3bNiK2Tm2mXXsmf/5oI0u27uPPV/UOYjivc1PmrtnN8vtGFDqwW5o2/jjpdDKysnn6yt5lPr4wDepFMXX8mVw7bXG+QfHf5ClGWJimcdHUqV2Lo1ne3+PiXi1JTtnHQ5d0Y+jj8wH49I5hnPNHr7tpzupd9Ht4LgeKubI6mHGMupER1I4o/YfyrgMZXP7sF0y+qAuDOyTw8drdXNi9RamPz3X4aBZd8yTzwsZ2RCrlSqEi1eQrBfAGfrNyHB3umc2kCztx45B2FXLexEmzuCKpFX8c2zNYd/hoFr9/dzUTR3SgaVx0kfGU1LdfFRzIOEaP+49fxZT2G/ux7By2ph2mRXxdYuoc/w711Tf7aNM4hoYxUTjnaHPne2WK5/qz23DPRQW7mwqT+wQ4wGW9T+O/S7cX+G9VGnf+dwXTF+Wvunte56ZcN6gNXVvGM2vlTi7p3ZJ6USp0cCqqUt1HFammJwUpv6ueX8AXm9IY27cVj/+obB+oJcnJcbS9q/DEsPy+EWxNO8yY//d5vvUpU0Zz6GgWUSV05+Tt4msZH82O/d6zKp9POpfTGpS/q7AoWx4ZVSFdVVK1VNnuI5FweeWGASE7d61axj+v68eanQe4rM9pbNx9iFteWUqrhnWJrxtJjzzPkORKz8zi3Mfnc2abRkw4py09WsUX+DA+fDR/V1RuQgAYNOUj2jeN5ZZz23PbjGWM6NKMhy7pRtP6+a/qDh3NYmvaYQAa1ovkyn4/4Nn5m4psy5eb0jirfZMy/w3k1KArBZFKdtfMlcHtsHn9OOl0Hh3bI/h9/a6DwUOL1w1qw4ulnIlv9YMXBF1AJ3Zt3T6iA7869wzW7zrI5xv3MP6sRK5+YSFfbMpfYqS8A+JSdZX2SkG3IIhUst9d0KnQ9d8dyF/CZESeO81uGNyG+/1bXmfdenax51+0xSssmJPjeHNZ/rpXuXeDdWgWx7WD2njPsYzrTXRkLRbcOTzY7/Ulhd8aLKc+dR+JVLL4epE0qBfJ9+nHmHRhJ/7x2RZSDx6lXtTxW243pR6/w6xT8zhaxNdl/KA2jB/UBoDnftKX7BzHnsOZ3PvmKmbdejZtmsTQZfIHjJ+6mJioCA4XUqBwaMemBdY1ia3D2t9fCMC9F3Xh9++u5vbXlnNZ79OqxU0EUrHUfSQSBvuPHGPV9v0M8vvuf/KPhRw4coy3fuVdBeQOCv95XG/O7dQ0351PxSlqMHnNgyOJjqxV4gBydo6jnT9gPqZnS54ZV/JtwQcyjpGRmV1gLEOqFnUfiVRh8XUjg4QA3jMRy7ft56UvUzg/zxwRo7u3KHVCAPjXz/sXur5uVESp7iiKqGXcem57AN5evoPl335P4qRZ3DK94ERGuc59/H/0+8Pxh/wys3Kq3BweUnpKCiJVQFu/wuvkt75mg/9w4gNjupa5++bsM5qQMmU0K+4fQcqU0ZzfpRm3Dj+jTOeYOKIjUf7DdRf/xbuN9p3lO4os/Jc7H/Y/v0jh273pdLhnNu3vno1zjsysnKB4oFQP6j4SqQJOLIkRGWGsfnAkkWV48rmiFdYV9fqNA2nVsF7wBPueQ0dJemhugf1O9Mr1/Xlr2Q6uPTuRTs3rV3isUjJ1H4lUI/WiavP6jQOD3x8b2zOsCQG8AocAwzsdH5we+7cvmfDy8S9lK7ftL9W5rnphIf9J/paRT31Kdk7RX0T/tz6VuatLP4WqVDwlBZEqIimxEYvuHs6L45O4uFfhJcYr0y+HtuMfP0viL1f34Z1fHb8NdsW2/eT4H+x1PI0dAAAQuElEQVTJW/diBk/9uFewfe7EwcWet+M9swtdn3Esm5+9uIjrX0rmrEfmkXrwaIGH9zalHuLjdbsLPV4qhpKCSBXSNC6aczs1qxJlJsyM4Z2bER0ZQfdW8ay8fwTndW4GwJzV37H7YAaLU/bRtklMvkHz9k3jSGrdkNHdW5AyZTSzbzsn33mzchx7D2cGv2dm5ZCcsper8lTm3bE/gzMfnkvX+z4IpnHdnHqI4X/6H9dOXRwki4xj2SROmhVMzFRe+9OPkThpFqOe/vSkzlNaH6/dzZqdB+h23wfc/O+vKuU1S0tjCiJSakcys+k8+f3g9+b1oxnUvgmP/6gH7e56j58OTAxm0ctr6GMfM7JbCzbuPsjcNd43/bduHhQMZJfVyz/vxzlnJBQY93j//86hU/P6vJr8LQ3rRbF6xwE6No9lZLfCq8ruOXSUTbsP8fynW5i7xuu2eu3GgZyZ2IjXl2zj9teWl7nGVElunb60wOyHPxnQmt9f0q3CXqMwKognIiFx4gfxby/oyM3DvNtYS5r3Yef+Iwx8pPB5sXNt/sMoFmxO46oXFha739rfjwxm1csrZcroAjEWVrajuEHyLY+MCsqDjD+r8ERXkv3px4ivV7A0eVHPkoS6tIgK4olIpRjSISFYLqnbq0V8XW45t32+qV3H9TudjGM5DGzbmEv8p6jPat+ED/5vMDnOEVunNoeOZnHhCV07hSUEKPxDN3HSLMb0bBl8Q68XFUF6IU985/pyUxpxdWpz8GgW075IYdoXKXz468Gc0Swu3365z2Pkzo9x18yVdGlRn/i6kdwyfWkwYyB4Vwjf7ksPjr33oi50Py2eK/7+JQD7DmfSMCYq2L5oy15mLPqGawa2xvCeNamMO7eUFESkTHInLMpV1tn1fjOiIw3rRfHgu6tpFBPFI5f1KHS/js3zfwDnfpPOOJadLyHcM7ozZzSLI/XgUW5/bXmRr5u3y6awhLBs8vn847Mt/PmjjWxJO8yRE56vOP/JT7igazM++NrrZnpmXG9u9R/q2/yHUfx+1uqg0OHoHl53Ve7fadHdw/O9/vBOTfn52V7Jkr//pC+/eHkJq3ceYOf+DPq3aUR6ZjY/fXEhGcdy+O/S4/Wr7hndmevPaVtkGyuCuo9EpMyOZedw5sNzuW34GVzr12Mqi+Xffs/Ff/mc6MhaQd2lspj0xgpmLPYmDMpNFidWhI2Lrs1jY3vyzd7D/OG9tYWeJ2/XF3jf/NvfPZuOzeJYt+sgpzWoy/bvi5+fuzzeuGkgfVs3AkrXpZbryzvPLdN0u3mp+0hEQiYyohbLJo8o9/FdW9anUUxU8CxEWd0y/AxmLP6WJrF1gnVmxn9/eRZREbVolxBL3TwFBls3juEXLy9h6rVncu1U79t73hLjuXK7gdbtOgjAlMu7s//IMQZ3SMg3c1953T2qM1G1awUJAbzB+qLUi4qgdi3jQEYWbRNiyp0QykJXCiJSLa3avp/TG9Ur81zgq7bvJ6ZObdo0iSl0+6V//Zyl33wPeIUE8yaXBZvTqBcVwfZ9R3hszjpeuq4fj3+wjjeXeV1D834zhOF/8mpXTTy/AznO0al5HDf+6ytq1zLWP3RhoaVLihp8/ure82kUE0V2jsPgpKrW6u4jEZFyWLg5jR8/t4CPfjOEtgmxJe5/6GgWvR+cQ2yd2iTfcz7f7k3ncGYWXVseH2v5Ji2dlg2igyuREx3JzCY9M4u6URHc9d+VQZKpyDuSlBRERKqpR99fS/P60fzsrMQKO6fGFEREqqnfjSx8dr7KoDIXIiISUFIQEZGAkoKIiASUFEREJKCkICIiASUFEREJhDwpmFmEmS01s3cL2TbRzFab2Qozm2dmrUMdj4iIFK0yrhRuA9YUsW0pkOSc6wG8DvyxEuIREZEihDQpmFkrYDTwQmHbnXMfO+dyC4wvAFqFMh4RESleqK8UngLuAHJKse/PgUJn9DazCWaWbGbJqampFRmfiIjkEbKkYGYXAbudc0tKse81QBLwWGHbnXPPOeeSnHNJCQkJhe0iIiIVIJS1jwYBY8xsFBAN1Dezfznnrsm7k5mdB9wNDHHOHQ1hPCIiUoKQXSk45+50zrVyziUCVwIfFZIQegN/B8Y453aHKhYRESmdSn9OwcweNLMx/q+PAbHAa2a2zMzerux4RETkuEopne2cmw/M95cn51l/XmW8voiIlI6eaBYRkYCSgoiIBJQUREQkoKQgIiIBJQUREQkoKYiISEBJQUREAkoKIiISUFIQEZGAkoKIiASUFEREJKCkICIiASUFEREJKCmIiEhASUFERAJKCiIiElBSEBGRgJKCiIgElBRERCSgpCAiIgElBRERCSgpiIhIQElBREQCSgoiIhJQUhARkYCSgoiIBJQUREQkEPKkYGYRZrbUzN4tZFsdM/uPmW00s4VmlhjqeEREpGiVcaVwG7CmiG0/B/Y559oDTwKPVkI8IiJShJAmBTNrBYwGXihil4uBf/rLrwPDzcxCGZOIiBQt1FcKTwF3ADlFbD8N+BbAOZcF7AcahzgmEREpQsiSgpldBOx2zi2pgHNNMLNkM0tOTU2tgOhERKQwobxSGASMMbMUYAZwrpn964R9tgOnA5hZbSAeSDvxRM6555xzSc65pISEhBCGLCJSs5lzLvQvYjYUuN05d9EJ628GujvnbjSzK4HLnHNXlHCuVGBrOUNpAuwp57FVzanSFrWj6jlV2nKqtAMqpi2tnXMlfquufZIvUmZm9iCQ7Jx7G/gH8LKZbQT2AleWdHxpGlXMayc755LKe3xVcqq0Re2oek6Vtpwq7YDKbUulJAXn3Hxgvr88Oc/6DOBHlRGDiIiUTE80i4hIoKYlhefCHUAFOlXaonZUPadKW06VdkAltqVSBppFRKR6qGlXCiIiUowakxTMbKSZrfOL700KdzwnMrMXzWy3ma3Ks66RmX1oZhv8fxv6683MnvHbssLM+uQ55mf+/hvM7GdhaMfpZvaxma02s6/N7LZq3JZoM1tkZsv9tjzgr2/jF3Dc6Bd0jPLXF1ng0czu9NevM7MLKrstfgz5ilNW43akmNlKM1tmZsn+uur4/mpgZq+b2VozW2NmA6tEO5xzp/wPEAFsAtoCUcByoEu44zohxsFAH2BVnnV/BCb5y5OAR/3lUcBswIABwEJ/fSNgs/9vQ3+5YSW3owXQx1+OA9YDXappWwyI9ZcjgYV+jK8CV/rr/wbc5C//Evibv3wl8B9/uYv/nqsDtPHfixFheI9NBF4B3vV/r67tSAGanLCuOr6//glc7y9HAQ2qQjsq9T9muH6AgcAHeX6/E7gz3HEVEmci+ZPCOqCFv9wCWOcv/x0Yd+J+wDjg73nW59svTG16Czi/urcFqAd8BfTHe4io9onvLeADYKC/XNvfz058v+XdrxLjbwXMA84F3vXjqnbt8F83hYJJoVq9v/CqN2zBH9etSu2oKd1HQeE93zZ/XVXXzDm301/+DmjmLxfVnirVTr/boTfeN+xq2Ra/y2UZsBv4EO/b8ffOK+B4YlxFFXisCm05sThlY6pnOwAcMMfMlpjZBH9ddXt/tQFSgal+l94LZhZDFWhHTUkK1Z7zvgZUm1vFzCwWeAP4P+fcgbzbqlNbnHPZzrleeN+0+wGdwhxSmVkFFqesIs52zvUBLgRuNrPBeTdWk/dXbbzu4medc72Bw3jdRYFwtaOmJIWg8J6vlb+uqttlZi0A/H93++uLak+VaKeZReIlhH875/7rr66WbcnlnPse+Bivm6WBeQUcT4yrqAKP4W5LgeKUwNNUv3YA4Jzb7v+7G5iJl6yr2/trG7DNObfQ//11vCQR9nbUlKSwGDjDv9siCm/w7O0wx1QabwO5dxP8DK9/Pnf9T/07EgYA+/1Lzg+AEWbW0L9rYYS/rtKYmeHVtFrjnHsiz6bq2JYEM2vgL9fFGxtZg5ccxvq7ndiW3DaOBT7yv+29DVzp39XTBjgDWFQ5rQDn3J3OuVbOuUS89/5HzrmrqWbtADCzGDOLy13Ge1+sopq9v5xz3wHfmllHf9VwYHWVaEdlDxKF6wdv9H49Xp/w3eGOp5D4pgM7gWN43yJ+jtePOw/YAMwFGvn7GvAXvy0rgaQ857kO2Oj/XBuGdpyNd8m7Aljm/4yqpm3pASz127IKmOyvb4v3YbgReA2o46+P9n/f6G9vm+dcd/ttXAdcGMb32VCO331U7drhx7zc//k69//lavr+6gUk+++vN/HuHgp7O/REs4iIBGpK95GIiJSCkoKIiASUFEREJKCkICIiASUFEREJKClIlWNm2X4FzOVm9pWZnVXC/g3M7JelOO98Mzsl5uytKGY2zczGlryn1BRKClIVHXHO9XLO9cQrwvZICfs3wKvsWSXleWpYpMpTUpCqrj6wD7x6SmY2z796WGlmF/v7TAHa+VcXj/n7/s7fZ7mZTclzvh+ZN0fCejM7x983wsweM7PFfq36X/jrW5jZJ/55V+Xun5d5tf3/6L/WIjNr76+fZmZ/M7OFwB/Nq5P/pn/+BWbWI0+bpvrHrzCzy/31I8zsS7+tr/m1pDCzKebNVbHCzB731/3Ij2+5mX1SQpvMzP6fefMhzAWaVuR/LKn+9A1GqqK65lUmjcYrD3yuvz4DuNQ5d8DMmgALzOxtvEJi3ZxXuA4zuxC4GOjvnEs3s0Z5zl3bOdfPzEYB9wHn4T09vt85d6aZ1QE+N7M5wGV45aQfNrMIvPLZhdnvnOtuZj/Fq0Z6kb++FXCWcy7bzP4MLHXOXWJm5wIv4T3Rem/u8X7sDf223QOc55w7bGa/Ayaa2V+AS4FOzjmXW4IDmAxc4JzbnmddUW3qDXTEmxuhGV5phRdL9V9FagQlBamKjuT5gB8IvGRm3fAe9f+DeVUxc/BKBDcr5PjzgKnOuXQA59zePNtyC/QtwZu/Arx6MT3y9K3H49X1WQy8aF6Bvzedc8uKiHd6nn+fzLP+Nedctr98NnC5H89HZtbYzOr7sV6Ze4Bzbp95VU274H2QgzcBy5d4JawzgH+YN3vau/5hnwPTzOzVPO0rqk2Dgel+XDvM7KMi2iQ1lJKCVGnOuS/9b84JeDWUEoC+zrlj5lX9jC7jKY/6/2Zz/P1vwC3OuQKFxPwENBrvQ/cJ59xLhYVZxPLhMsYWvCzwoXNuXCHx9MMrnjYW+BVwrnPuRjPr78e5xMz6FtUm/wpJpEgaU5Aqzcw64U2nmob3bXe3nxCGAa393Q7iTf2Z60PgWjOr558jb/dRYT4AbvKvCDCzDuZV42wN7HLOPQ+8gFfauDA/zvPvl0Xs8ylwtX/+ocAe580z8SFwc572NgQWAIPyjE/E+DHFAvHOufeAXwM9/e3tnHMLnXOT8SZuOb2oNgGfAD/2xxxaAMNK+NtIDaMrBamKcscUwPvG+zO/X/7fwDtmthKvuuRaAOdcmpl9bmargNnOud+aWS8g2cwygfeAu4p5vRfwupK+Mq+/JhW4BK+i6G/N7BhwCPhpEcc3NLMVeFchBb7d++7H64paAaRzvDzyQ8Bf/NizgQecc/81s/HAdH88ALwxhoPAW2YW7f9dJvrbHjOzM/x18/AqiK4ook0z8cZoVgPfUHQSkxpKVVJFToLfhZXknNsT7lhEKoK6j0REJKArBRERCehKQUREAkoKIiISUFIQEZGAkoKIiASUFEREJKCkICIigf8PiVWsDEFXd+gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot_losses()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7 fasta.ai1 DEV",
   "language": "python",
   "name": "fastai1_dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

IMDB_data tokenize LM build.ipynb