Wiki103 (backward) from scratch to IMDB classifier pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.text import *\n",
    "\n",
    "import re\n",
    "import sentencepiece as spm #https://github.com/google/sentencepiece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(20190123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path('/home/farzin/rnn_python_code/wiki103_from_download')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pull and prep IMDB data for supervised and unsupervised step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/home/farzin/.fastai/data/imdb')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "IMDB_PATH = untar_data(URLs.IMDB)\n",
    "IMDB_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLAS_PATH = PATH/'clas_bwd'\n",
    "LM_PATH = PATH/'lm_bwd'\n",
    "os.makedirs(CLAS_PATH, exist_ok=True)\n",
    "os.makedirs(LM_PATH, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "## raw data does not have the unsup in the train directory.  Just move it there\n",
    "#  ~/.fastai/data/imdb$ mv unsup/ ./train/\n",
    "CLASSES = ['neg', 'pos', 'unsup']\n",
    "\n",
    "def get_texts(path):\n",
    "    texts,labels,scores = [],[],[]\n",
    "    for idx,label in enumerate(CLASSES):\n",
    "        for fname in (path/label).glob('*.*'):\n",
    "            texts.append(fname.open('r', encoding='utf8').read())\n",
    "            labels.append(idx)\n",
    "            review_name = listify(fname.parts)[-1].split('.')[0]\n",
    "            scores.append( int(review_name.split(\"_\")[1]) )\n",
    "\n",
    "    return np.array(texts),np.array(labels),np.array(scores)\n",
    "\n",
    "train_texts,train_labels,train_scr = get_texts(IMDB_PATH/'train')\n",
    "valid_texts,valid_labels,valid_scr = get_texts(IMDB_PATH/'test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75000, 25000)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_texts),len(valid_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_idx = np.random.permutation(len(train_texts))\n",
    "valid_idx = np.random.permutation(len(valid_texts))\n",
    "\n",
    "train_texts,train_labels,train_scr = train_texts[train_idx],train_labels[train_idx],train_scr[train_idx]\n",
    "valid_texts,valid_labels,valid_scr = valid_texts[valid_idx],valid_labels[valid_idx],valid_scr[valid_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.DataFrame({'text':train_texts, 'labels':train_labels, 'rating':train_scr}, columns=['labels','text','rating'])\n",
    "valid_df = pd.DataFrame({'text':valid_texts, 'labels':valid_labels, 'rating':valid_scr}, columns=['labels','text','rating'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    5100\n",
      "4    2696\n",
      "3    2420\n",
      "2    2284\n",
      "Name: rating, dtype: int64\n",
      "10    4732\n",
      "8     3009\n",
      "7     2496\n",
      "9     2263\n",
      "Name: rating, dtype: int64\n",
      "0    50000\n",
      "Name: rating, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "for this_cls in range(3):\n",
    "    print(train_df[train_df.labels == this_cls]['rating'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_texts = np.concatenate([train_texts,valid_texts])\n",
    "idx = np.random.permutation(len(all_texts))\n",
    "cut = int(0.1 * len(idx))\n",
    "LM_train_df = pd.DataFrame({'text':all_texts[idx[cut:]], 'labels':[0] * (len(all_texts)-cut)}, columns=['labels','text'])\n",
    "LM_valid_df = pd.DataFrame({'text':all_texts[idx[:cut]], 'labels':[0] * cut}, columns=['labels','text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 90000, 10000)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_texts),len(LM_train_df),len(LM_valid_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labels</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Simply boring! This is one of the worst movies...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>After you've watched The Love Bug, you will no...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>The third Muppet movie is perhaps the most rel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>A movie like the life - complex, with unexpect...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>&lt;br /&gt;&lt;br /&gt;I didn´t enjoy the film at all. Th...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   labels                                               text\n",
       "0       0  Simply boring! This is one of the worst movies...\n",
       "1       0  After you've watched The Love Bug, you will no...\n",
       "2       0  The third Muppet movie is perhaps the most rel...\n",
       "3       0  A movie like the life - complex, with unexpect...\n",
       "4       0  <br /><br />I didn´t enjoy the film at all. Th..."
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LM_train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labels</th>\n",
       "      <th>text</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Another trashy Grade Z quickie from the prolif...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>This show stinks. For parents, they usually wa...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>Just two comments....SEVEN years apart? Hardly...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>SOME NOT-SO-SPOILY SPOILERS AHEAD&lt;br /&gt;&lt;br /&gt;W...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>This is a very funny movie, easy to watch, tha...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    labels                                               text  rating\n",
       "0        0  Another trashy Grade Z quickie from the prolif...       1\n",
       "1        0  This show stinks. For parents, they usually wa...       2\n",
       "5        1  Just two comments....SEVEN years apart? Hardly...       7\n",
       "8        0  SOME NOT-SO-SPOILY SPOILERS AHEAD<br /><br />W...       4\n",
       "10       1  This is a very funny movie, easy to watch, tha...       7"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CLS_train_df = train_df[train_df.labels != 2]\n",
    "CLS_valid_df = valid_df[valid_df.labels != 2]\n",
    "CLS_train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tokenizer SpacyTokenizer in en with the following rules:\n",
       " - fixup\n",
       " - replace_rep\n",
       " - replace_wrep\n",
       " - deal_caps\n",
       " - spec_add_spaces\n",
       " - rm_useless_spaces\n",
       " - sub_br\n",
       " - replace_all_caps\n",
       " - deal_caps"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def sub_br(t:str) -> str:\n",
    "    \"Replaces the <br /> by \\n\"\n",
    "    re_br = re.compile(r'<\\s*br\\s*/?>', re.IGNORECASE)\n",
    "    return re_br.sub(\"\\n\", t)\n",
    "\n",
    "def spec_add_spaces(t:str) -> str:\n",
    "    \"Add spaces between special characters\"\n",
    "    return re.sub(r'([/#])', r' \\1 ', t)\n",
    "\n",
    "def rm_useless_spaces(t:str) -> str:\n",
    "    \"Remove multiple spaces\"\n",
    "    return re.sub(' {2,}', ' ', t)\n",
    "\n",
    "def replace_rep(t:str) -> str:\n",
    "    \"Replace repetitions at the character level\"\n",
    "    def _replace_rep(m:Collection[str]) -> str:\n",
    "        c,cc = m.groups()\n",
    "        return f' {TK_REP} {len(cc)+1} {c} '\n",
    "    re_rep = re.compile(r'(\\S)(\\1{3,})')\n",
    "    return re_rep.sub(_replace_rep, t)\n",
    "    \n",
    "def replace_wrep(t:str) -> str:\n",
    "    \"Replace word repetitions\"\n",
    "    def _replace_wrep(m:Collection[str]) -> str:\n",
    "        c,cc = m.groups()\n",
    "        return f' {TK_WREP} {len(cc.split())+1} {c} '\n",
    "    re_wrep = re.compile(r'(\\b\\w+\\W+)(\\1{3,})')\n",
    "    return re_wrep.sub(_replace_wrep, t)\n",
    "\n",
    "def deal_caps(t:str) -> str:\n",
    "    \"Replace words in all caps\"\n",
    "    res = []\n",
    "    for s in re.findall(r'\\w+|\\W+', t):\n",
    "        res += ([f' {TK_UP} ',s.lower()] if (s.isupper() and (len(s)>2)) else [s.lower()])\n",
    "    return ''.join(res)\n",
    "\n",
    "def fixup(x:str) -> str:\n",
    "    \"List of replacements from html strings\"\n",
    "    re1 = re.compile(r'  +')\n",
    "    x = x.replace('#39;', \"'\").replace('amp;', '&').replace('#146;', \"'\").replace(\n",
    "        'nbsp;', ' ').replace('#36;', '$').replace('\\\\n', \"\\n\").replace('quot;', \"'\").replace(\n",
    "        '<br />', \"\\n\").replace('\\\\\"', '\"').replace('<unk>',UNK).replace(' @.@ ','.').replace(\n",
    "        ' @-@ ','-').replace('\\\\', ' \\\\ ')\n",
    "    return re1.sub(' ', html.unescape(x))\n",
    "\n",
    "default_rules = [fixup, replace_rep, replace_wrep, deal_caps, spec_add_spaces, rm_useless_spaces, sub_br]\n",
    "default_spec_tok = [BOS, FLD, UNK, PAD]\n",
    "\n",
    "## this is setup for BWD case, with custom Tokenizer() class\n",
    "tokenizer = Tokenizer(pre_rules=default_rules, special_cases=[BOS, FLD, 'xxunk', 'xxpad'], n_cpus=4,bwd_toks=True)\n",
    "tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build LM DataBunch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_vocab = 60000 #can we make this the full possible vocab at 260k?\n",
    "\n",
    "keyword_args = {'max_vocab':max_vocab}\n",
    "imdb_data_lm = TextLMDataBunch.from_df(LM_PATH, LM_train_df, LM_valid_df, tokenizer=tokenizer,\n",
    "                               text_cols='text', label_cols='labels',**keyword_args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "imdb_data_lm.save('lm_data_bwd')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### finetune LM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "bptt = 140\n",
    "emb_sz,nh,nl = 400,1111,4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(imdb_data_lm,bptt,emb_sz,nh,nl,drop_mult=0.5,qrnn=True, \n",
    "                               pretrained_fnames=['wiki103_raw_articles_spacy_BWD_20190123',\n",
    "                                                  'itos_raw_spacy'])\n",
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8nGW5//HPNTPZlzZt0oW2tKUtlH0LUEBQ2QSPshxAi6KiIooLAud4zo+f5wcePHpQXHBjqRsuiGABxY1FZRFogbTQ0oJI9zZtaZq0SbNnkuv3xzxphzRJp22eWZLv+/WaFzP3cz/zXBmmuXIvz32buyMiIrInkUwHICIiuUEJQ0REUqKEISIiKVHCEBGRlChhiIhISpQwREQkJUoYIiKSEiUMERFJiRKGiIikJJbpAIZSZWWlT5s2LdNhiIjkjEWLFm1196pU6g6rhDFt2jRqamoyHYaISM4ws7Wp1lWXlIiIpEQJQ0REUhJql5SZXQdcCTjwCvBRd29POv5t4J3By2JgnLuPDo51B+cArHP388OMVUREBhdawjCzScA1wGHu3mZm9wNzgbt767j7dUn1Pwccm/QWbe5+TFjxiYjI3gm7SyoGFJlZjEQLYuMgdS8D7g05HhER2UehJQx3rwW+AawDNgGN7v5Yf3XNbCowHfhbUnGhmdWY2UIzuzCsOEVEJDWhJQwzqwAuIJEIDgBKzOzyAarPBea7e3dS2VR3rwY+ANxmZjMGuM5VQWKpqaurG8KfQEREkoXZJXUWsNrd69y9C3gQOGWAunPp0x0VtFBw91XAk7x1fCO53jx3r3b36qqqlO49EREZNh5/9U3ufGplWq4VZsJYB8wxs2IzM+BM4LW+lcxsNlABLEgqqzCzguB5JXAq8GqIsYqI5KTHX93M3c+uScu1whzDeB6YDywmMT02Aswzs5vNLHmK7Fzg1+7uSWWHAjVmtgR4ArjF3ZUwRET62NEep6wwPYt2hHoVd78JuKlP8Y196nypn/OeA44MLzIRkeEhnQlDd3qLiOSwHe1dlBXmpeVaShgiIjlMLQwREUlJk1oYIiKSiqb2OOVqYYiIyGA64t10xnsoL1ILQ0REBrGjPQ6gMQwRERmcEoaIiKRkR3sXAGUF6pISEZFBqIUhIiIpaWoLWhiaVisiIoNRC0NERFLSFIxhaFqtiIgMqreFUVqgFoaIiAxiR3uc0oIY0Yil5XpKGCIiOSqxUm16WheghCEikrPSuVItKGGIiOSsdK5UC0oYIiI5Sy0MERFJyY72LsrVwhARkT0ZVi0MM7vOzJab2TIzu9fMCvscv8LM6szs5eBxZdKxj5jZG8HjI2HGKSKSixIJI30tjNBSk5lNAq4BDnP3NjO7H5gL3N2n6n3u/tk+544BbgKqAQcWmdnD7r4trHhFRHJJe1c3nd09w6eFQSIhFZlZDCgGNqZ43ruAx929IUgSjwPnhhSjiEjO6b3LO13bs0KICcPda4FvAOuATUCjuz/WT9WLzWypmc03sylB2SRgfVKdDUGZiIiwax2pYTGt1swqgAuA6cABQImZXd6n2u+Bae5+FIlWxM/24TpXmVmNmdXU1dXtb9giIjkh3SvVQrhdUmcBq929zt27gAeBU5IruHu9u3cEL38EHB88rwWmJFWdHJTtxt3nuXu1u1dXVVUN6Q8gIpKtdqR5pVoIN2GsA+aYWbGZGXAm8FpyBTObmPTy/KTjjwLnmFlF0FI5JygTEREy08II7Uru/ryZzQcWA3HgJWCemd0M1Lj7w8A1ZnZ+cLwBuCI4t8HMvgy8GLzdze7eEFasIiK5ZkcGxjBCTU3ufhOJ6bHJbkw6fgNwwwDn/gT4SXjRiYjkruE2hiEiIiFpao9jBqX5ShgiIjKIprYuSvNjRNK0eRIoYYiI5KR0ryMFShgiIjlpR3tXWqfUghKGiEhOUgtDRERSsqMjvbvtgRKGiEhOUgtDRERS0tTWpYQhIiKDc/e0b54EShgiIjmnvauHeI+rhSEiIoPbuVKtWhgiIjKYpgysIwVKGCIiOUctDBERSUkmVqoFJQwRkZyTif28QQlDRCTnqIUhIiIp2bXbnhKGiIgMYkd7nIhBSRo3TwIlDBGRnLOjPU5pQXo3TwIlDBGRnNPUnv6VakEJQ0Qk52RipVoIOWGY2XVmttzMlpnZvWZW2Of49Wb2qpktNbO/mtnUpGPdZvZy8Hg4zDhFRHJJU1tX2m/agxAThplNAq4Bqt39CCAKzO1T7aXg+FHAfODrScfa3P2Y4HF+WHGKiOSaYdnCAGJAkZnFgGJgY/JBd3/C3VuDlwuBySHHIyKS8xK77Q2jhOHutcA3gHXAJqDR3R8b5JSPA39Oel1oZjVmttDMLhzoJDO7KqhXU1dXNySxi4hksx3tccqLhleXVAVwATAdOAAoMbPLB6h7OVAN3JpUPNXdq4EPALeZ2Yz+znX3ee5e7e7VVVVVQ/oziIhkm12bJw2jFgZwFrDa3evcvQt4EDilbyUzOwv4InC+u3f0lgctFNx9FfAkcGyIsYqI5IS2rm66e3zYTatdB8wxs2IzM+BM4LXkCmZ2LHAXiWSxJam8wswKgueVwKnAqyHGKiKSEzK1jhQkBqVD4e7Pm9l8YDEQJzEjap6Z3QzUuPvDJLqgSoHfJHIK64IZUYcCd5lZD4mkdou7K2GIyIjX1JaZlWohxIQB4O43ATf1Kb4x6fhZA5z3HHBkiKGJiOSk3t32yofZGIaIiAyxhpZOAMaWFKT92koYIiI5pL45MTdoTGl+2q+thCEikkPqd7YwlDBERGQQ9c2dlORHKcyLpv3aShgiIjmkoaWDsaXpH78AJQwRkZxS39LJmAx0R4EShohITqlv7qQyAwPeoIQhIpJT6ls61MIQEZHBuTsNLZ0awxARkcE1tcfp6vaMTKkFJQwRkZyx8y5vjWGIiMhgdt7lnYFlQUAJQ0QkZ2TyLm9QwhARyRn1zeqSEhGRFDS09HZJKWGIiMggtjZ3UlYQoyCW/nWkQAlDRCRnJO7ByEzrApQwRERyRibv8gYlDBGRnFHfnLm7vEEJQ0QkZ9S3dGZsSi2EnDDM7DozW25my8zsXjMr7HO8wMzuM7MVZva8mU1LOnZDUP66mb0rzDhFRLJdT4+zbbiOYZjZJOAaoNrdjwCiwNw+1T4ObHP3mcC3ga8F5x4W1D0cOBe43cwyMy1ARCQLNLV3Ee/xjN3lDeF3ScWAIjOLAcXAxj7HLwB+FjyfD5xpZhaU/9rdO9x9NbACODHkWEVEstbW4Ka9TO2FASEmDHevBb4BrAM2AY3u/lifapOA9UH9ONAIjE0uD2wIykRERqTehQeH5SwpM6sg0VKYDhwAlJjZ5SFc5yozqzGzmrq6uqF+exGRrNC78ODYYdoldRaw2t3r3L0LeBA4pU+dWmAKQNBtNQqoTy4PTA7KduPu89y92t2rq6qqhvhHEBHJDvUZXtocwk0Y64A5ZlYcjEucCbzWp87DwEeC55cAf3N3D8rnBrOopgOzgBdCjFVEJKv1LjxYUZy5hBEL643d/Xkzmw8sBuLAS8A8M7sZqHH3h4EfA78wsxVAA8EsKndfbmb3A68G537G3bvDilVEJNs1tHRQXhgjP5a52+dSShhmNgPY4O4dZvYO4Cjg5+6+fbDz3P0m4KY+xTcmHW8HLh3g3K8AX0klPhGR4W5rSyeVGbzLG1LvknoA6DazmcA8EuMLvwotKhEReYuG5s6MzpCC1BNGTzDt9SLge+7+BWBieGGJiEiy+paOjA54Q+oJo8vMLiMxQP2HoCwvnJBERKSvhpbOjN7lDaknjI8CJwNfcffVwcylX4QXloiI9OrpcRpaOjN6lzekOOjt7q+SWBeq94a8Mnf/WpiBiYhIwva2Lno8s3d5Q4otDDN70szKzWwMiWmyPzSzb4UbmoiIQNJd3jkyS2qUuzcB/0piOu1JJO7kFhGRkO28yzsXWhhAzMwmAu9j16C3iIikQe9d3rkyS+pm4FFgpbu/aGYHAW+EF5aIiPRqaEl0SWV6DCPVQe/fAL9Jer0KuDisoEREZJfevTDGZHAdKUh90HuymT1kZluCxwNmNjns4EREJHEPxujiPGLRzK0jBal3Sf2UxAqyBwSP3wdlIiISsvqWjowPeEPqCaPK3X/q7vHgcTegzSdERNKgvrkzoxsn9Uo1YdSb2eVmFg0el5PY6EhEREK2tTnz60hB6gnjYySm1G4msT/3JcAVIcUkIiIBd2dTYzsTRxVlOpTUEoa7r3X38929yt3HufuFaJaUiEjotrd20drZzQGjCzMdyn5t0Xr9kEUhIiL9qt3eBsDkihxpYQzAhiwKERHpV2/CmDS6OMOR7F/C8CGLQkRE+lW7LUgYWdDCGPRObzPbQf+JwYDMRy8iMszVbm+jMC9CRXHm96wbNGG4e9m+vrGZHQLcl1R0EHCju9+WVOcLwAeTYjmUxD0fDWa2BtgBdANxd6/e11hERHLVxu1tTBpdhFnmRwFSWktqX7j768AxAGYWBWqBh/rUuRW4NajzXuA6d29IqvJOd98aVowiItmudnsbkyoyP34B+zeGsTfOJLHS7dpB6lwG3JumeEREckLttjYmZcGUWkhfwpjLIMnAzIqBc4EHkoodeMzMFpnZVSHHJyKSddo6u6lv6WTS6OwYMg6tS6qXmeUD5wM3DFLtvcCzfbqj3ubutWY2DnjczP7h7k/38/5XAVcBHHjggUMYuYhIZm1szJ4ZUpCeFsZ5wGJ3f3OQOru1QNy9NvjvFhJjHyf2d6K7z3P3anevrqrSeogiMnzsnFKbBfdgQHoSxqBjE2Y2Cng78LukshIzK+t9DpwDLAs5ThGRrNJ70142LAsCIXdJBb/szwY+mVT2KQB3vzMough4zN1bkk4dDzwUTCOLAb9y90fCjFVEJNvUbmsjGjEmlI+AhBEkgbF9yu7s8/pu4O4+ZauAo8OMTUQk223c3saE8sKM77TXKzuiEBGR3WzY3pY13VGghCEikrUS92BkxwwpUMIQEclK8e4eNje1Z82UWlDCEBHJSlt2dNDd41kzpRaUMEREslK2TakFJQwRkazUe9NeNuy010sJQ0QkC+1qYShhiIjIIGq3t1FRnEdxfuhL/qVMCUNEJAvVbmvLqhlSoIQhIpKVardn1z0YoIQhIpJ13D24aS97ptSCEoaISNbZ3tpFW1d3Vk2pBSUMEZGs0ztDKpum1IIShohI1tmQZRsn9cqe+VoCJNaPWVnXwoZtrdQ3d1Lf0sn21k7iPY4BZpAfizCmpIDK0nyqSgs4ZEIZY0sLMh26iAyRbLzLG5QwMq61M87f39jK39+oY1ltE69taqIj3vOWOgWxCLGI4YA7dHb30N3jO4+bwdGTR3PG7HGcOrOSorwo8Z4euroT71OUF6MoP0pJfpTK0gIiEUvnj7hPeoKfLxdiFRlqq7c2U1YYY0xJfqZDeQsljH3U0NLJK7WNbN3RwdFTRjOjqoRgh8C3aO6Is76hlXUNiRZDd0/il31HvIcXVjfwzIqtdMR7KC2IccSkci6fM5UjJpUzvbKUsSX5jC3N3+3GnZ4ep7Gti63NHWzZ0UHNmm387fUtfPsv/+Rbj/9z0LhL8qMcdkA5R0waxcxxpYkEFO+hM0guBbEIBbEoBbEI0YhhBmZGvLuHba1dbG/tpKGlk1jEGF2cT0VxHqOL88mLRohGIBqJUBCLUF6UR3lhjLLCPFo64mxt7mBrcwf1LZ20dMRp7uimtSNOfUsnmxrb2NTYzptN7XTGe0jKhRTlRSnOj1JcEKUwFqUgLxFffjRC8sddEIswtrSAytJEyysWMXocetyJRozx5YVMHFXIAaOLqCjOJy9qb/n/1dXdQ3N7HAcqivP6/X8pki4rt7Qwo6o0676HShgp2tLUzpOv1/HUP+t4ef32nU3GXhXFeRw/dQzlhTHqmjuo25H4Zd7Q0jnge04aXcRlJx7IOYeN54TpY8hLcVetSMSoKMmnoiSfWePLOHVmJZ8/axZbmxPJA5y8aIS8aAQH2jrjtHZ209IRZ8WWZpZtbOLXL6ynrat7rz+HiMHo4nzi3T00tcf3+vxkxflRKorzmTiqkKMmj2Z8WQGFeVEiESNi0OPQ3pWIu7Wzm/aubjriPXTEu+ns0wprau/iH5t3sLW5g65uH+CKu0QjRmEsQkFelNbOOO1du96vOD/KlIpipowpIi8aoSX47No6uzFLnBsxIz8WobQgRnF+lNKCGPmxCLFIhLyYUZQX3ZmgDhhdxKTRRRTmRffr85KRY2VdM6cfXJXpMHajhDGIDdtaeWhxLY8s38zyjU0ATCgvpHpaBR8+eSpHThrF2NICXl6/jZo121i0dhsd8R6qygqYMqaY46ZWMLmiiAPHFHPgmGKqygqIRRLdS9GoUVYQG9K/ICpLCzj3iAkp1e3ucbbsaCcaMQqiUfJiiTg64z10xHto7+re+Re6e+KXZEVxHuWFeTu7ieLdPTS2ddHY1kW8x4l3O909Tnu8mx3tXTS1xWlq76I4P0ZVWeIv/8rSAkoLYhQFiWGouTtN7XG6e5yoGRaBeLezubGdTY1tbGxsp7G1k/auHtq6uumId1OcH6O0IEZZYQz3xIDjuoZWNmxrpbvHKS6IUVoQpaI4D3fodt/ZSqzb0UFLR5yWzjid8R66up2u7p7duhXN4IBRRUwdW8z0yhJmjitl1rgyDh5fSlVZQdb9JSmZ09TexZYdHcyoKs10KLtRwgA2N7bT0hmntaObls446+pbefClDSxc1QDA8VMr+MK7DuGM2eOYPaFst3/ch0wo4/0nHJiJ0PdZNGJMHLX7lL3ivegyjUUT3UDZNOBuZowqytutfExJPocdUJ62OLq6e9jc2M7G7W3Ubk8koDVbW1hd38rvl2x8S+tsTEk+xx1YQfW0CqqnVnDoxHJKCvRPc6RaVdcCwIyqkgxHsjt9K4HTb31ity6OqWOLuf7sg7no2ElMGZNdU9sk++VFI0wZU9zvd8fdqWvu4I03m3njzR0s29jEorXb+Mtrb+6sM66sgGmVJcyoKuGYKaM5fmpFVvZpy9BbsaUZgBnjRlALw8wOAe5LKjoIuNHdb0uq8w7gd8DqoOhBd785OHYu8B0gCvzI3W8JK9avXnQkeVGjOD9GSUGUMSX5HDJ+95aEyFAwM8aVFTKurJBTZ1buLN/a3MHitdt4Y0szq7e2sGZrC396ZTP3vrAegNHFeVRPHcPJM8Zy8kFjmT2hTLPIhqGVdc3kRY0Ds/AP1dAShru/DhwDYGZRoBZ4qJ+qf3f39yQXBPV/AJwNbABeNLOH3f3VMGK95PjJYbytyF6pLC3gnMMncM7hu8rcnVVbW1i0Zhs1axt4YXXDzpZIYqJFBcdMGc0xUyo4asooygt3746T3LJySzNTx5akPAkmndLVJXUmsNLd16ZY/0RghbuvAjCzXwMXAKEkDJFsZWbMqCplRlUp7zthCgAbt7exYGU9C1bVs3jdNv7y2pad9ceVFXBQVQkHVZVyyPgyjpw8isMmlmuGVg5ZWdfMzCzsjoL0JYy5wL0DHDvZzJYAG4F/d/flwCRgfVKdDcBJ/Z1sZlcBVwEceGBuDTyL7IsDRhdx8fGTuThoGTe2drFkw3aWbWxkVV0Lq+qa+dMrm/jV8+uAxASHWeNKOWn6GE6ZWcmcg8b2OzFAMq+ru4e19a286/DUZjumW+gJw8zygfOBG/o5vBiY6u7NZvZu4LfArL15f3efB8wDqK6u3vMEfJFhZlRxHqcfXPWWefvuzuamdpZuaGRZbSMvr9/O/TUb+NmCtUQsMfPvY6dO55zDJxDVOEjWWFvfSrzHR3QL4zxgsbu/2feAuzclPf+Tmd1uZpUkxjumJFWdHJSJSArMEtOmJ44q2vnXame8h5fXb+eZFVv57Uu1XH3PYqaNLeYTpx/ExcdNVrdVFlhZF8yQysJ7MCA9q9VexgDdUWY2wYKpSGZ2YhBPPfAiMMvMpgctlLnAw2mIVWTYyo9FOHH6GK4/+2Ce+Pd38IMPHEd5UR5ffGgZb/vaE9z51Ep2tHdlOswRrTdhHJSF92BAyC0MMyshMdPpk0llnwJw9zuBS4CrzSwOtAFz3d2BuJl9FniUxLTanwRjGyIyBKIR41+Omsi7j5zAglX13PHkSm758z+4/YkVXHHKND7+toMYVaxxjnRbsaWZ8eUFlGXpbDdL/H4eHqqrq72mpibTYYjkpKUbtnP7Eyt5ZPlmygtjfPLtM/joqdN2W/xSwnPBD56ltCDKPVfOSds1zWyRu1enUjf7JvqKSEYcNXk0d37oeP78+dM4YdoYbn30dU7/+pP8YsEa4t09ezxf9o+7s2pLc9aOX4AShoj0cejEcn58xQk8cPXJzKgq4f/9bjnv+d4zLFhZn+nQhrW6HR3s6IgrYYhI7jl+6hh+fdUc7rz8OHa0x7nshwv5zK8Ws7mxPdOhDUsrsnyGFChhiMggzIxzj5jIX65/O9eeNYu/vPomZ3/7Ke57cR3DafwzG6wMFh3M1nswQAlDRFJQlB/l2rMO5tFrT+ewieX85wOv8KEfv8D6htZMhzZsrKxroSQ/yvjy7NkuoC8lDBFJ2bTKEu79xBz+58IjeGndNs697WkeemlDpsMaFlbWNTNjXHYvYa+EISJ7JRIxLp8zlceufzuHHzCK6+5bwhd+s4TWzv3bsnekW7mlmZlZPH4BShgiso8mjS7iV584iWvOmMn8xRt47/ee4bVNTXs+UXbT3BFnY2N7Vm6alEwJQ0T2WSwa4fpzDuGej59EU3ucC77/LHc+tZLuHg2I740l67cDcHgatxHeF0oYIrLfTplZySOfP40zZo/jlj//g/fftYC19S2ZDitnvLC6YecqwtlMCUNEhsTY0gLuuPw4vv3+o3n9zR2ce9vfeXjJxkyHlRNeXNPA7AnlWbuGVC8lDBEZMmbGRcdO5rHrTueISeVcc+9LfP2Rf9CjLqoBdXX38NK67Zw4fUymQ9kjJQwRGXITRxVxz5VzuOzEKdz+5Equ+kWNlk4fwPKNTbR1dXPCNCUMERmh8mMRvnrRkdx8weE88XodF9/xHJsa2zIdVtZ5cXUDACdMy+7xC1DCEJEQmRkfPnkaP//YiWzc3s4ldyzYuUmQJLywpoGpY4sZV16Y6VD2SAlDREJ36sxKfn3VHNq7urn0zgUs3bA90yFlBXenZk1DTnRHgRKGiKTJEZNGMf/qUyjKi3LZvIU8u2JrpkPKuJV1zWxr7eJEJQwRkbeaXlnCg58+hckVxXz0py/yh6Uje9rtC6u3AXBCDsyQAiUMEUmz8eWF3P/Jkzlq8ig+d+9L/HzBmkyHlDEvrmmgsjSfaWOLMx1KSpQwRCTtRhXn8csrT+LM2eO58XfL+eZjr4/I/TVeDMYvsnmF2mShJQwzO8TMXk56NJnZtX3qfNDMlprZK2b2nJkdnXRsTVD+spnVhBWniGRGYV6UOy8/jvdXT+F7f1vBF+YvpWsE7R2+qbGNDdvacmbAGyAW1hu7++vAMQBmFgVqgYf6VFsNvN3dt5nZecA84KSk4+90d42MiQxTsWiEWy4+kgmjCvnOX9/gzaZ2bv/gcVm/RMZQeCG4/yIX7vDula4uqTOBle6+NrnQ3Z9z923By4XA5DTFIyJZwsy47uyD+folR7FgZT2X3rlgRNzg9+KaBkoLYsyeUJbpUFKWroQxF7h3D3U+Dvw56bUDj5nZIjO7KrTIRCQrvK96Cj+54gTWN7Ry4Q+e3bnk93Dk7jz1zzpOnD6GWDR3hpJDj9TM8oHzgd8MUuedJBLGfyYVv83djwPOAz5jZqcPcO5VZlZjZjV1dXVDGLmIpNvpB1cx/+pTiEUiXHrXgmG7/eurm5pY39DGuw4fn+lQ9ko6Utt5wGJ3f7O/g2Z2FPAj4AJ3r+8td/fa4L9bSIx9nNjf+e4+z92r3b26qqpqyIMXkfQ6dGI5D3/2VI6dMprr7lvC//7ptWG3IdOjyzYTMTjrUCWMvi5jgO4oMzsQeBD4kLv/M6m8xMzKep8D5wDL0hCriGSBsaUF/PLKk/jQnKnc9fQqPvHz4bXa7aPL3+SEaWMYW1qQ6VD2SqgJI/hlfzaJpNBb9ikz+1Tw8kZgLHB7n+mz44FnzGwJ8ALwR3d/JMxYRSS75EUjfPnCI/jyhUfw1D/ruOSOBaxvaM10WPttVV1zYoOpIyZkOpS9Ftq0WgB3byGREJLL7kx6fiVwZT/nrQKO7lsuIiPPh+ZMZfrYEj59zyIu/MGzzPvw8Rw/NXemovb16PJE7/y7Ds+9hJE7w/MiMmK9bVYlD33mVMoKY8ydt5A7n1qZs+MajyzfzFGTR3HA6KJMh7LXlDBEJCfMqCrlt585lbMOHc8tf/4Hl81bmHNdVJsa21iyfntOti5ACUNEcsjo4nxu/+BxfOt9R/PapibOve1pHlycO1NvHwu6o3Jx/AKUMEQkx5gZ/3rcZP587WkcPmkU19+/hBsefIX2ru5Mh7ZHjyzbzMxxpcyoKs10KPtECUNEctLkimJ+deVJXP2OGdz7wjouufO5rO6iamjp5IU1DZybo91RoIQhIjksFo3wn+fO5ocfrmZtfSvv+d4z/OmVTZkOq18PLt5Ad4/nbHcUKGGIyDBw9mHj+ePnTmPa2GI+fc9irrvvZRrbsudGv5aOOHc8uZLTZlVyxKRRmQ5nnylhiMiwcODYYuZffQrXnjWLh5ds5Lzbns6afcPvfm4N9S2dXH/2wZkOZb8oYYjIsJEXjXDtWQfz4NWnUJgf5YM/ep7//v3yjA6IN7Z1cddTKznr0HEce2BFxuIYCkoYIjLsHD1lNH/83Glccco0fvrsGv7lu39n6YbMLJf+47+voqk9znU53roAJQwRGaaK8qN86fzD+eXHT6K1s5uLbn+O7/71jbTeIV7f3MGPn1nNvxw5kcMPyN2xi15KGCIyrL1tViWPXHs67zlqIt96/J984IcL07aj311Pr6Ktq5vrzp6VluuFTQlDRIa9UUV53Pb+Y/jmpUfzSm0j533n7zy6fHOo1/zD0o385JnVXHjMJGaOy51tWAejhCEiI4KZcfHxk/nS2IgDAAAKwElEQVTjNacxpaKYT/5iEf/121do6xz6AfH7X1zPNfe+xLEHjuZLFxw+5O+fKUoYIjKiTK8s4YGrT+GTpx/ELxeu473ff4ZXNzYN2fv/+JnV/McDSzl1ZiU//9hJlBfmDdl7Z5oShoiMOPmxCDe8+1B++fGTaGrr4sIfPMsPnlixX9Nv65s7+OJDr/DlP7zKuYdP4EcfqaYoPzqEUWeeuefmmvL9qa6u9pqamj1XFBEJNLR0csODS3l0+ZtMKC/kmjNncWn1ZPKiqf093d7Vzd3PreEHf1tBa1c3Hz55Kl9896HEUjw/08xskbtXp1RXCUNEBBauqufWR19n0dptTB1bzBmzxzFzXCmzxpUxrbKYiuL8nUmktTPO86saePqNOh5dtpmNje2cMXscN5w3m1njc2uAWwlDRGQfuDtPvL6FO55cybLaJtr6dFGVFcQYVZzHm03tdHU7BbEIJx00lk+efhCnzqzMUNT7Z28SRqh7eouI5BIz44zZ4zlj9nh6epyNjW2s2NLMuoZWtrV0sb2tk+2tXYwrL+C0mVVUT6ugMG94jVMMJrSEYWaHAPclFR0E3OjutyXVMeA7wLuBVuAKd18cHPsI8F9B1f9x95+FFauISF+RiDG5opjJFcWZDiVrhJYw3P114BgAM4sCtcBDfaqdB8wKHicBdwAnmdkY4CagGnBgkZk97O7bwopXREQGl65h/DOBle6+tk/5BcDPPWEhMNrMJgLvAh5394YgSTwOnJumWEVEpB/pShhzgXv7KZ8ErE96vSEoG6hcREQyJPSEYWb5wPnAb0J6/6vMrMbMaurq6sK4hIiIkJ4WxnnAYnd/s59jtcCUpNeTg7KBynfj7vPcvdrdq6uqqoYoZBER6SsdCeMy+u+OAngY+LAlzAEa3X0T8ChwjplVmFkFcE5QJiIiGRLqfRhmVgKcDXwyqexTAO5+J/AnElNqV5CYVvvR4FiDmX0ZeDE47WZ3bwgzVhERGZzu9BYRGcFG7NIgZtYIvNHPoVFAY4qve5/3V1YJbN3LsPpeK9Xj/ZX3F9NAz/cn5sHiSjW+XIm5v/Jc/H6kEnPyc30/Uj8+3L8fs9w9tf1j3X3YPIB5qZQP9rr3+QBlNUMV097GPFBMe4p/X2Le17hzMebh8v1IJeZMf9b6fmT/92NPj9xYfzd1v0+xfLDXvx+kbChj2tPx/soHimlP8e+LfYk7F2PurzwXvx+pxJz8XN+P1I+PpO/HoIZVl1TYzKzGU+zryxaKOX1yMW7FnD65Gney4dbCCNu8TAewDxRz+uRi3Io5fXI17p3UwhARkZSohSEiIikZsQnDzH5iZlvMbNk+nHu8mb1iZivM7LvBvh69xz5nZv8ws+Vm9vVsj9nMvmRmtWb2cvB4d7bHnHT838zMzWzItzoL6bP+spktDT7nx8zsgByI+dbg+7zUzB4ys9E5EPOlwb+/HjMbsjGD/Yl1gPf7iJm9ETw+klQ+6Pc+o/ZletpweACnA8cBy/bh3BeAOYABfwbOC8rfCfwFKAhej8uBmL8E/Hsufc7BsSkklotZC1TmQtxAeVKda4A7cyDmc4BY8PxrwNdyIOZDgUOAJ4HqTMcaxDGtT9kYYFXw34rgecVgP1c2PEZsC8PdnwbestyImc0ws0fMbJGZ/d3MZvc9L9ivo9zdF3ri/+7PgQuDw1cDt7h7R3CNLTkQc6hCjPnbwH+Q2GArJ+J296akqiVDHXtIMT/m7vGg6kISC4Fme8yveWIDtyG1r7EOoN89fzL5bzUVIzZhDGAe8Dl3Px74d+D2fupMIrE/R6/kvToOBk4zs+fN7CkzOyHUaBP2N2aAzwZdDj+xxGKPYduvmM3sAqDW3ZeEHWgf+/1Zm9lXzGw98EHgxhBj7TUU349eHyPxF2/YhjLmsKUSa38G2wsoG36ufoW6+GAuMbNS4BTgN0ldhgV7+TYxEk3MOcAJwP1mdlDwl8KQG6KY7wC+TOKv3S8D3yTxiyEU+xuzmRUD/5dEV0naDNFnjbt/Efiimd0AfJbEVsShGKqYg/f6IhAH7hma6Aa8zpDFHLbBYjWzjwKfD8pmAn8ys05gtbtflO5Yh4oSxi4RYLu7H5NcaIn9yBcFLx8m8Qs2uVmevFfHBuDBIEG8YGY9JNaPCWtnp/2O2ZP2KTGzHwJ/CCnWXvsb8wxgOrAk+Ec6GVhsZie6++Ysjruve0is1hxawmCIYjazK4D3AGeG9cdPkqH+nMPUb6wA7v5T4KcAZvYkcIW7r0mqUgu8I+n1ZBJjHbVk/ucaWKYHUTL5AKaRNIAFPAdcGjw34OgBzus7KPXuoPxTJJZih0T31HqCe12yOOaJSXWuA36d7Z9znzprCGHQO6TPelZSnc8B83Mg5nOBV4GqMD7jML8fDPGg977GysCD3qtJDHhXBM/HpPq9z9Qj4wFk7AdPbOq0Cegi0TL4OIm/XB8BlgT/SG4c4NxqYBmwEvg+u26AzAd+GRxbDJyRAzH/AngFWEriL7eJ2R5znzprCGeWVBif9QNB+VIS6/dMyoGYV5D4w+fl4DHUM7vCiPmi4L06gDeBRzMZK/0kjKD8Y8HnuwL46N587zP10J3eIiKSEs2SEhGRlChhiIhISpQwREQkJUoYIiKSEiUMERFJiRKGDGtm1pzm6/3IzA4bovfqtsTKtsvM7Pd7WinWzEab2aeH4toi/dG0WhnWzKzZ3UuH8P1ivmsxvlAlx25mPwP+6e5fGaT+NOAP7n5EOuKTkUctDBlxzKzKzB4wsxeDx6lB+YlmtsDMXjKz58zskKD8CjN72Mz+BvzVzN5hZk+a2XxL7BVxT++eBUF5dfC8OVhscImZLTSz8UH5jOD1K2b2Pym2ghawa/HFUjP7q5ktDt7jgqDOLcCMoFVya1D3C8HPuNTM/nsIP0YZgZQwZCT6DvBtdz8BuBj4UVD+D+A0dz+WxEqyX0065zjgEnd/e/D6WOBa4DDgIODUfq5TAix096OBp4FPJF3/O+5+JG9dmbRfwTpKZ5K4Ex+gHbjI3Y8jsQfLN4OE9X+Ale5+jLt/wczOAWYBJwLHAMeb2el7up7IQLT4oIxEZwGHJa0wWh6sPDoK+JmZzSKxem9e0jmPu3vyXggvuPsGADN7mcQaQ8/0uU4nuxZzXAScHTw/mV17HPwK+MYAcRYF7z0JeI3EngmQWGPoq8Ev/57g+Ph+zj8neLwUvC4lkUCeHuB6IoNSwpCRKALMcff25EIz+z7whLtfFIwHPJl0uKXPe3QkPe+m/39LXb5rkHCgOoNpc/djgiXdHwU+A3yXxF4aVcDx7t5lZmuAwn7ON+B/3f2uvbyuSL/UJSUj0WMkVosFwMx6l6cexa6lpK8I8foLSXSFAczdU2V3byWxpeu/mVmMRJxbgmTxTmBqUHUHUJZ06qPAx4LWE2Y2yczGDdHPICOQEoYMd8VmtiHpcT2JX77VwUDwqySWpQf4OvC/ZvYS4ba+rwWuN7OlJDbXadzTCe7+EolVbi8jsZdGtZm9AnyYxNgL7l4PPBtMw73V3R8j0eW1IKg7n7cmFJG9omm1ImkWdDG1ubub2VzgMne/YE/niWSaxjBE0u944PvBzKbthLglrshQUgtDRERSojEMERFJiRKGiIikRAlDRERSooQhIiIpUcIQEZGUKGGIiEhK/j9oac6TTwE9tAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr_find(learn)\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 16:13 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>4.910686</th>\n",
       "    <th>4.731900</th>\n",
       "    <th>0.244333</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-1, moms=(0.8,0.7), wd=0.03, pct_start=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('fit_head_bwd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 2:55:25 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>4.631639</th>\n",
       "    <th>4.496713</th>\n",
       "    <th>0.267463</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>4.468457</th>\n",
       "    <th>4.336883</th>\n",
       "    <th>0.283023</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>4.374462</th>\n",
       "    <th>4.249422</th>\n",
       "    <th>0.291475</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>4.335640</th>\n",
       "    <th>4.195748</th>\n",
       "    <th>0.296655</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>5</th>\n",
       "    <th>4.281790</th>\n",
       "    <th>4.154370</th>\n",
       "    <th>0.300653</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>6</th>\n",
       "    <th>4.244815</th>\n",
       "    <th>4.126259</th>\n",
       "    <th>0.303676</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>7</th>\n",
       "    <th>4.218197</th>\n",
       "    <th>4.104016</th>\n",
       "    <th>0.306006</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>8</th>\n",
       "    <th>4.174376</th>\n",
       "    <th>4.088624</th>\n",
       "    <th>0.307699</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>9</th>\n",
       "    <th>4.144981</th>\n",
       "    <th>4.081542</th>\n",
       "    <th>0.308560</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>10</th>\n",
       "    <th>4.124908</th>\n",
       "    <th>4.079950</th>\n",
       "    <th>0.308787</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(10, 1e-3, moms=(0.9,0.75), wd=0.01, pct_start=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd4VGX2wPHvSYckBEhCLwGk9xABpQsqxcWGSNO177L4s+vGslh2dVkLYte1Y8PCYgMpCogivXeIEJAeegmQ9v7+uDfDTDKTOiXlfJ5nnszcNufiOGfu+973vGKMQSmllAIICnQASimlyg5NCkoppRw0KSillHLQpKCUUspBk4JSSikHTQpKKaUcNCkopZRy0KSglFLKQZOCUkoph5BAB1BccXFxJiEhIdBhKKVUubJixYpDxpj4wrYrd0khISGB5cuXBzoMpZQqV0RkZ1G282nzkYikisg6EVktIvm+ycXysoikiMhaEUn0ZTxKKaUK5o8rhX7GmEMe1g0CmtuPbsAb9l+llFIBEOiO5iuBycayGKguInUDHJNSSlVavr5SMMBsETHAW8aY/+ZZXx/4w+n1bnvZPh/HpZQqAzIzM9m9ezdnz54NdCgVRkREBA0aNCA0NLRE+/s6KfQ0xuwRkVrAHBHZbIxZUNyDiMgdwB0AjRo18naMSqkA2b17N9HR0SQkJCAigQ6n3DPGcPjwYXbv3k2TJk1KdAyfNh8ZY/bYfw8C04CueTbZAzR0et3AXpb3OP81xiQZY5Li4wu9o0opVU6cPXuW2NhYTQheIiLExsaW6srLZ0lBRCJFJDr3OXAZsD7PZt8CN9p3IXUHjhtjtOlIqUpEE4J3lfbf05fNR7WBaXaAIcCnxpiZIvJXAGPMm8AMYDCQAqQDN/sqmCOnM1iy/TCD2ms/tlJKeeKzpGCM2Q50dLP8TafnBhjnqxic3TF5Oct3HmX5YwOIiwr3x1sqpcqww4cP079/fwD2799PcHAwuc3TS5cuJSwsrNBj3HzzzSQnJ9OyZUufxupP5W5Ec0ntOXYGgHNZOQGORClVFsTGxrJ69WoAnnjiCaKionjggQdctjHGYIwhKMh9S/v777/v8zj9LdDjFPwmyG5ny8kxAY5EKVWWpaSk0KZNG0aPHk3btm3Zt28fd9xxB0lJSbRt25annnrKsW3Pnj1ZvXo1WVlZVK9eneTkZDp27MhFF13EwYMHA3gWJVdprhRyE73RnKBUmfTkdxvYuPeEV4/Zpl41Hv9T22Lvt3nzZiZPnkxSUhIAEyZMoGbNmmRlZdGvXz+GDRtGmzZtXPY5fvw4ffr0YcKECdx333289957JCcne+U8/KnSXSlka1ZQShWiWbNmjoQA8Nlnn5GYmEhiYiKbNm1i48aN+fapUqUKgwYNAqBLly6kpqb6K1yvqjRXCsG5zUeaFJQqk0ryi95XIiMjHc+3bdvGSy+9xNKlS6levTpjxoxxOw7AuWM6ODiYrKwsv8TqbZXmSgH71l2jSUEpVQwnTpwgOjqaatWqsW/fPmbNmhXokHyq0lwpODqaNScopYohMTGRNm3a0KpVKxo3bkyPHj0CHZJPSXn75ZyUlGRKMslOQvJ0AL4Z14OODat7OyylVAls2rSJ1q1bBzqMCsfdv6uIrDDGJHnYxaHyNB/ZsnJ0nIJSSnlS6ZJCtuYEpZTyqBImhfLVXKaUUv5U6ZKC3pKqlFKeVZqk0L5+DABVwoIDHIlSSpVdlSYp3NW/OQChHgpbKaWUqkRJIdg+Uy1zoZTK1a9fv3yD0SZNmsTYsWM97hMVFQXA3r17GTZsmNtt+vbtS2G3zk+aNIn09HTH68GDB3Ps2LGihu4zlSYpOGofaUezUso2cuRIpkyZ4rJsypQpjBw5stB969Wrx1dffVXi986bFGbMmEH16oEfQ1VpkkKI3WykHc1KqVzDhg1j+vTpZGRkAJCamsrevXvp3Lkz/fv3JzExkfbt2/PNN9/k2zc1NZV27doBcObMGUaMGEHr1q25+uqrOXPmjGO7sWPHOspuP/744wC8/PLL7N27l379+tGvXz8AEhISOHToEAATJ06kXbt2tGvXjkmTJjner3Xr1tx+++20bduWyy67zOV9vKXylLmw019WtiYFpcqkH5Jh/zrvHrNOexg0wePqmjVr0rVrV3744QeuvPJKpkyZwvDhw6lSpQrTpk2jWrVqHDp0iO7duzN06FCP8x+/8cYbVK1alU2bNrF27VoSExMd655++mlq1qxJdnY2/fv3Z+3atdx1111MnDiRefPmERcX53KsFStW8P7777NkyRKMMXTr1o0+ffpQo0YNtm3bxmeffcbbb7/N8OHDmTp1KmPGjPHOv5Wt0lwpaJVUpZQ7zk1IuU1HxhgeeeQROnTowIABA9izZw8HDhzweIwFCxY4vpw7dOhAhw4dHOu++OILEhMT6dy5Mxs2bHBbdtvZr7/+ytVXX01kZCRRUVFcc801/PLLLwA0adKETp06Ab4rz11prhSCg6ykcOJMZoAjUUq5VcAvel+68soruffee1m5ciXp6el06dKFDz74gLS0NFasWEFoaCgJCQluy2UXZseOHTz//PMsW7aMGjVqcNNNN5XoOLnCw8/PLx8cHOyT5qPKc6VgJ4Wxn6wMcCRKqbIkKiqKfv36ccsttzg6mI8fP06tWrUIDQ1l3rx57Ny5s8Bj9O7dm08//RSA9evXs3btWsAqux0ZGUlMTAwHDhzghx9+cOwTHR3NyZMn8x2rV69efP3116Snp3P69GmmTZtGr169vHW6hao0ScHZx4sL/g+slKpcRo4cyZo1axxJYfTo0Sxfvpz27dszefJkWrVqVeD+Y8eO5dSpU7Ru3Zrx48fTpUsXADp27Ejnzp1p1aoVo0aNcim7fccddzBw4EBHR3OuxMREbrrpJrp27Uq3bt247bbb6Ny5s5fP2LNKUzp7xc4jXPvGIsfr1AlDvBmWUqoEtHS2b2jp7CIIynPXQJaWS1VKqXwqTVI4mp7h8vp0RnaAIlFKqbKr0iSFRjWrurzu+ORsvVpQqgwob03YZV1p/z0rTVKoX71qvmV/n+rlgTJKqWKJiIjg8OHDmhi8xBjD4cOHiYiIKPExKk1HM8Cuw+n0fm6eyzLtcFYqcDIzM9m9e3ep7t1XriIiImjQoAGhoaEuy4va0VxpBq8BxEeHF76RUspvQkNDadKkSaDDUE4qTfMR6AQ7SilVmEqVFPKqX70KP23yXM9EKaUqm0qdFPYcO8OtHy5n/Z7jgQ5FKaXKhEqXFL4e14Ppd/V0WbZ2tyYFpZSCSpgUOjWsTtt6MS7LHpmmt6YqpRRUwqTgSULy9ECHoJRSAefzpCAiwSKySkS+d7OukYjMs9evFZHBvo6nIOVtzIZSSnmbP64U7gY2eVj3GPCFMaYzMAJ43Q/xALDisQH5lmXYZS/OZmZrCQylVKXk06QgIg2AIcA7HjYxQDX7eQyw15fxOIuNyj+Q7cSZLLJzDK3+MZMb31vqr1CUUqrM8PWI5knAQ0C0h/VPALNF5P+ASCD/z3c/uvDpH6lR1Roa/tvvhwMZilJKBYTPrhRE5ArgoDFmRQGbjQQ+MMY0AAYDH4lIvphE5A4RWS4iy9PS0nwUseVous7hrJSqvHzZfNQDGCoiqcAU4BIR+TjPNrcCXwAYYxYBEUBc3gMZY/5rjEkyxiTFx8d7LcCV/7iU56/r6LXjKaVUeeezpGCMedgY08AYk4DViTzXGDMmz2a7gP4AItIaKyn49lLASc3IMIZ1acDDgwqef1UppSoLv49TEJGnRGSo/fJ+4HYRWQN8BtxkAnBf6J8vTnC7fO3uY3qbqlKqUvFL6WxjzHxgvv18vNPyjVjNTAEVEeq+eurQVxfy6ODW3N67qZ8jUkqpwNARzYX4z8zNgQ5BKaX8RpNCIbJyDAnJ00k7eS7QoSillM9pUiiicZ+sDHQISinlc5oUbE3jIwtcvzT1CC/9uM1P0SilVGBoUrB9fGs3JlzTnpt7JHDfpS3cbvPij1v9HJVSSvmXX+4+Kg/qVa/CiK6NHK8nztEEoJSqfPRKoZhSDp4KdAhKKeUzmhSK6YZ3lwQ6BKWU8hlNCoX48JauLq/3HT8boEiUUsr3NCkUok+L/AX4Ug6e5OTZTKau2M3uo+kBiEoppXxDO5o9+OWhfuw5dsbtugETFziex0eHs+zRARw/k0lMlVB/haeUUj6hVwoeNKxZle5NYwvdLu3kOZZsP0zHJ2fz48YDfohMKaV8R5OCFyxMOQTAuj3HAxyJUkqVjiYFL/jgt1QAftqsVwpKqfJN+xSK4Pv/68nR9Axa1anGhU//mG/9ibNZAGzed9LfoSmllFdpUiiCdvVjirRdXFS4jyNRSinf0uYjL+rcqHqgQ1BKqVLRpFBMVcPcz9IG0LZeNT9GopRS3qdJoZjmP9gXgHkP9M237vnZW8nMzvFvQEop5UWaFIqpVnQEqROG0CQukoXJl+RbP+yN3wIQlVJKeYcmhVKoX71KvmVrdutYBaVU+aVJwQcOntCieUqp8kmTgg90feYnbvlgGcfSMwIdilJKFYsmBR+Zu/kgf5+6NtBhKKVUsWhSKKWpYy8GIEjyr5u1QcteKKXKF00KpdSlcQ1SJwxh3ROXBzoUpZQqNU0KXhIZ7r5iyF4PczIopVRZpEnBx576biMJydPZsl+L5Smlyj5NCj42c8N+AC6ftIDtaacCHI1SShVMk4IfDXtzUaBDUEqpAmlS8IEhHeq6Xa7jFpRSZZ0mBS9qWNMqe/HS9Z0Yf0WbfOtzjL8jUkqp4tGk4EXfjOvJ9Lt6EhIcxMiujQIdjlJKFZvOvOZFNSPDqBkZBkBosJvRbEopVcbplYKPhAQH0bxWVL7l6RlZJCRP58PfUv0flFJKFcLnSUFEgkVklYh872H9cBHZKCIbRORTX8fjT9PG9ci3rO3jswB4/NsNAJw+l8VvKYf8GpdSSnnijyuFu4FN7laISHPgYaCHMaYtcI8f4vEb5yakuKhwAEyezuarXlvIqHeWcPjUOX+GppRSbvk0KYhIA2AI8I6HTW4HXjPGHAUwxhz0ZTz+FhZs/fP2ah5H50bV860/fS6LbQetAW1Tlv3h19iUUsodX18pTAIeAjxNXNwCaCEiC0VksYgM9HE8fiUirHn8Mt6/6ULmbMxfMTW3KQnguVlb/BmaUkq55bOkICJXAAeNMSsK2CwEaA70BUYCb4tIvp/UInKHiCwXkeVpaWk+iddXYqqEEhKs/flKqfLBl99WPYChIpIKTAEuEZGP82yzG/jWGJNpjNkBbMVKEi6MMf81xiQZY5Li4+N9GHJgfbx4J8YYTp7NDHQoSqlKymdJwRjzsDGmgTEmARgBzDXGjMmz2ddYVwmISBxWc9J2X8VU1j329XpemZtC+ydms/toeqDDUUpVQn5v1xCRp0RkqP1yFnBYRDYC84AHjTGH/R2TP3x0a9cibTdxzlYA/jii8zAopfzPL0nBGDPfGHOF/Xy8MeZb+7kxxtxnjGljjGlvjJnij3gCoVfzeJY+0r/I2+fkvXdVKaX8QHtA/SgiLBgAKUIFjE+W7PRxNEoplV+RkoKINBORcPt5XxG5y91dQqpgkWEhNK8VxYvDO7ksb1CjSr5tZ6zbj9GrBaWUnxX1SmEqkC0iFwD/BRoCFaokhT8EBwlz7uvD0I71XJYnxEa63b7JwzP4ZvUef4SmlFJA0ZNCjjEmC7gaeMUY8yDgfiYZVaigICF1whDH6zdv6OJx27unrPZHSEopBRQ9KWSKyEjgz0BuYbtQ34RU+USFF1zBfFnqETKyPA0KV0op7ylqUrgZuAh42hizQ0SaAB/5LqzK4af7+7Aw+ZJCt7vuzUU8M2MTx9MzeX/hDkdfw7u/7uDp6Rt9HaZSqhKR4nZmikgNoKExZq1vQipYUlKSWb58eSDe2qcSkqcXeduYKqEcP3N+1LNzU5RSSrkjIiuMMUmFbVfUu4/mi0g1EakJrMSqUTSxtEGqknFOCEop5U1FbT6KMcacAK4BJhtjugEDfBdW5XNZm9qANfL53T8XmsxdTJytFVaVUt5R1KQQIiJ1geGc72hWXvTa6ERWj7+UXs3j6d+6NlPHXlzkfV+em+LDyJRSlUlRk8JTWHWKfjfGLBORpsA234VV+YQGB1G9apjjdZfGNQIYjVKqsipSUjDGfGmM6WCMGWu/3m6Muda3oamiFtEDaxY3pZQqraJ2NDcQkWkictB+TLWn2lQ+1Kt5PN/d2bNI2x7SOZ6VUl5Q1Oaj94FvgXr24zt7mfKx9g1iWPfEZYVu1+e5+Uycs5U7Jlu3657LyiYzWwe8KaWKp+ChtOfFG2Ock8AHInKPLwJS+UVH5B883ji2KjsPu07E8/JPVjdP7piHVnWimXlPb98HqJSqMIp6pXBYRMaISLD9GANUyMlwyqp3bkzi6avbOV5PvsXqb7i4WazHfTbvP+nzuJRSFUtRk8ItWLej7gf2AcOAm3wUU8ViDPw6CY6Wbn6EAW1qM6prI8frxrGRpE4YwlWd6pc2QqWUcihS85ExZicw1HmZ3Xw0yRdBVSgn9sKvE2H1p3DrbKhS8mkoRIRtTw9y6SuoExPhjSiVUgoo3cxr93ktioospj5c/wkc2Q5f3ABZGaU6XGhwEFXDzufyXs3jShuhUko5lCYpFGFSSQVAk15w5WuwYwF8d7fVpOQlUsjcnnuPnfHaeymlKr7SJAWdK7I4Ol4PfR+BNZ/Cz8/67W0vnjDXb++llCr/CkwKInJSRE64eZzEGq+giqPPQ9BxFMx/BtZM8dphFzzYr8D1CcnTmbJ0F4u3H+aLZX947X2VUhVPgR3NxphofwVSKYjAn16CE7vhmzuhWn2raamUGsVWLXSb52Zt4fBpqz9j+IUNS/2eSqmKqTTNR6okQsJg+EcQ2ww+Hw1p3il7veVfA/kt+RJSJwzh41u75Vuv03kqpYpCk0IgVKkOo76A4HD4ZBicOljqQ4aHBFOvehUAerq5I+mkU8G81+enkJA8nYJm3cvJMZw4q5P5KFXZaFIIlBqNYdTncPoQfHo9ZKQXvk8xvDyys8d1z860rk5yy2RsO3AyX0G9l+duo8MTszlyunS30CqlyhdNCoFUPxGufRf2roL/3Q452V479NCO9ejWpGaB2xw6dY7LX1zApS8uIOlfP5KTY8jKzmHWhv18v3afYxulVOVR1IJ4yldaDYaBE2Dm32H2P2DgM1479JIdRwpcP+zNRS6vmz4yg86NqrNq1zHHsuwcvfNYqcpErxTKgu5/hW5jYfFrsOStgIbinBAAXpi9xVF1Na9dh9P5ceMBf4SllPITvVIoKy5/Go7tgpnJENPQuoIoA37cZHWCG2Mco6d/TzvF379ay/KdRwFY/HB/rcGkVAWhVwplRVAwXPs21O0EU2+1+hlK6c0xXQAYnlT6SfLW7D7ueP78rC2OhADQ/d8/lfr4SqmyQZNCWRIWad2RVDXOuiPp2K5SHW5guzr8/GBf/nNth1KHdtVrC9l5+DQnz2byw/r9pT6eUqps0qRQ1kTVgtFfQuZZ+GQ4nDlW+D4FaBwbWWjRvKJ6dNp62j8x2yvHUkqVTZoUyqJareD6j+BwilfKbQNEh5e+++jXlEMe1+07rtVYlaoINCmUVU37wNBXrHLb399T6nLbtX3cEXziTFbhGymlyjyfJwV7TudVIvJ9AdtcKyJGRJJ8HU+50mkk9EmG1Z/AgudKdagWtaMcz8NC8v9n/+n+PqU6/mNfr6Pt+JmcOqfJQanyzB9XCncDmzytFJFoe5slfoil/OmbDB1GwLynYc3nJT7Ms8M6MqB1bQCu6Zx/Xudm8VG8MTrR8frytrWLdfxlqUc5nZFNu8dnccO7S0hIns6ZDO+N0FZK+YdPk4KINACGAO8UsNk/gf8AZ30ZS7klYjUjJfSCb8ZB6q8lOkxUeAiXtqkFWKOUnW9T/eUhaz6GQe3rOpa9dUMSreoUXjnd3XSgv2yz+h7SM/SqQanyxteD1yYBDwFuv11EJBFoaIyZLiIP+jiW8iskzOp4fvdymDIKbp0D8S2Lf5gg6zdAVo7hxes78cBlLTmXlUPDmufnY1j5j0sd1VNzitCPsWnfSY/rgrx015NSyn98dqUgIlcAB40xKzysDwImAvcX4Vh3iMhyEVmelpbm5UjLiSo1rFtVg8Pgk+vgVPH/HS5pVYumcZGM7dsMgFrVIlwSAkDNyDBio8IB2HrgFAB9W8Z7POa4fs08rsvM1jkclCpvfNl81AMYKiKpwBTgEhH52Gl9NNAOmG9v0x341l1nszHmv8aYJGNMUny85y+oCq9GYxj5uTX/wmcjil1uu0ZkGHMf6EuL2sWbUO+2nk09ruvTwvN/j/0ntEVQqfLGZ0nBGPOwMaaBMSYBGAHMNcaMcVp/3BgTZ4xJsLdZDAw1xiz3VUwVQoMucO07sGeF18tt55Xbp1AzMszjNgmxkdSuFu523Tb7SkMpVX74fZyCiDwlIkP9/b4VSusrYOC/YfP3MGe8z97m9dGJ3NC9MS3rRPPlXy9yu01QkPDBzV3drsudCU4pVX74pUqqMWY+MN9+7vZbzBjT1x+xVBjdx8KRHbDoVaiRAF1v9/pbNI2P4p9XtQPgwoTzE/Z8d2dPmteO4uRZ6+6i1nWrud3/i+V/cFGzWDKycnj7l+3c1qsJ4SHBLtuczcwmMzuH6IhQABKSpzOuXzNa1I5maMd6XivRoZQqGi2dXZ4N/LdVNO+Hh6xy2y0H+vTtnrm6PVXDgmnfIAaAiNDzX/Cvj07kb5+sdNl+2qo9vHh9J6Ys28Vzs7Zw6lwWfx/YyrF++FuLWGpPBJQ6YYjjrqfX5v0OWPNOD2xXhz+OpHP4dAadGlZ3Of7qP46RYwyJjWp4/2SVqqS0zEV5FhQMw96FOh3gq5u9Um67IKO6NeIqNwPfAAa3r0vqhCEsfrg//7m2PWDdmWSMYfw3GwB4Y/7vju2zsnMcCSFXRp67lXJHR/d6dh5XvbYw33te9dpCrnn9t5KfkFIqH00K5V1YJIz6AqrG2uW2/whoOHViIriklTUaukbVMN5bmOpYFyQ4rgae/G5jvn0zslyTwgNfrinSe+46XLy7sJRSnmlSqAiia9vlts9YYxjOHi98Hx8KDrL6Af41fRP//P78l3+OgSYPz+DUuSw+WrzTZR9jDKfPFf1OqoMnz9/u2vu5eazbHdhzVqqi0KRQUdRqbZfb3gZf3AjZmQELpbCu4XaPz8q3rMnDMwqdwe34mfPn1PVp123/9OqvJCRP51yW1ltSqjQ0KVQkTfvCn16G7fO9Um67pKIivHf/QkLydMfzjk/OZsv+k0xbtdvj9v9bucdr761UZaR3H1U0nUfD0VRY8Kx1q2pv/5eUCgny3W2kl09aUOD6vcd0sh+lSkOvFCqifo9Ah+th7r9gzRS/v30gxxa8MjeFjXtPBOz9lSrvNClURM7ltqf9xSq5nX6k8P0qiMEv/xLoEJQqtzQpVFQh4datqj3uhtWfwWtdYd1XAetnKI5hXRrw4vUdAx2GUpWSJoWKLKwqXPoU3DHfGvE89Vb4ZJjV5+BjQzqcn7DnpRGdeHPM+VndOjWszogLGzom98lrwjXtC5ynQSnlO9rRXBnU7QC3/QhL34a5/4TXL4K+D0P3v0Gwbz4CLw7vxA3dGyNAt6axbN5/vp3/63E9HM+vT2rI58tdB9yFBAcxdYXnO4xK6nh6JodOn+NMRjb7j59lQJvCpxzNzjGs23M8X4kNpSoqvVKoLIKCoftfYdwSaNIH5vwD3u4He1YWvm8JhIUE0b1pLN2axgJwQXwUdapF8NqoRJftmteOcrv/7Ht7ezx20/hIAAa1q+Nxm+yc/M1k/V6YT/8XfuaKV37ltsnLXbY9dOqc2+O8/NM2rnptIWv+OObxvZSqSDQpVDYxDWDkZzB8sjVZzzv9YebDcM63cx+EBAex+JH+Ls1KADf3aALAhQmuRe1yZ3/LlVut9baeTZh7f19+eagfk0Z0olGemeNyHXbzJX/kdIbL6+NnMsnJMTw7azNJ//qRY+mu60+czWTu5oMAbNx3gm0HTpKQPJ0vlwe2lIhSviSmHHQ8OktKSjLLl+s8PF5x9jj8+CQsf9fqcxjyArS43O9hnMvKJiQoiE37TnA2M5skpzLdC1MOcWFCTcJC3P9+uX3ycuZsPJBv+ff/15N29WNcljkPhHNnwYP9aBRblW0HTnLpiwWPh0idMKTA9UqVNSKywhiTb2bLvPRKoTKLiIErJsIts6zCep8Ohy9vgpP5v2R9KTwkmOAgoV39GJeEANDjgjiPCQGgo13G+6aLExjS/vxVyJb9J7n2jd/IsiuvFpYQnBWWEHK1fOwHEpKn8/aC7RzUqUdVBaEdzQoadYe//AK/vQQ/Pwcpc+HSJyHxzxBUtn83jOt3ARGhwdzcownBQcJ0+8v/frvC6vq9J4rcSfzR4lT2HS/al/unS3Zxzq7q+vSMTTw9YxM7/j1YJwVS5V7Z/j9e+U9ImFUSY+xv1t1K398D7w+Cg5sDHVmBRITbejV1VGZ9eFArl/VLth8u8rHe/mUH36/dV6RtH5m2Lt+ywgr6KVUeaFJQruIugD9/B1e+Boe2wJs9Yd4zkFk+mkda1I52eV27WgQfLNzhl/c+cML9HUxKlSeaFFR+ItB5DIxbBm2vhp//YyWH1F8DHVmh8vY/3PP5ap5wM6GPr/jyxo29x85wwSMz2LBX545QvqNJQXkWFQ/Xvg1jpkJ2BnwwBL65s0zXUQoNDuxH+ts1e0u1/4mzmfyWcsjtulkb9pOVY5jwQ9lu0lPlmyYFVbgLBsDfFtt1lD4t03WUsnJyCt/Ih+ZvSSvV/qPfXsKod5bkGzMBOAbQ/bLNfdJQyhs0Kaii8VhHaWdhe/pXEfLUpOs7eeWtqlcNZWjHei7L9hwtfD6HzOwcfk87xa7D6SzdcYQDTrezrttjNQ3NWLc/334RocGljFipwmlSUMWTW0dp4H9g5yJ4vTv89gpkZwU6MsCaB7owV3Wu73Hdp7d1I6ZKqON1RKjn/0XqxVTJ11x0gYeyHc4d6EOiAAAdb0lEQVSaP/oD/V/4md7PzWP4W4vo9kz+u5YmL0olIyuHaat2M2OddUdUlTD3SSEjK7BXR6pi0aSgis+ljlJvmP2YVUdp76pAR0aDGlWKtN1Vnc7/wv/qrxc5nsdUDSU0+PxYgyZxUWx/ZjCbnhqY7xi9msdRu5prOY68LWppJ88xcNIC/jiSDuAYTOfOn99b6ni+ef9J2j8xi3s/X8PfPrHqU8VHn3+vPfYMc9+u2UuLx37go8Vl7IpNlVuaFFTJVW8II6fAdR9adZTevgRmPuLzOkoFSYiLLHD9d3f2BOCGixIA+GZcD5dR1LWiI6gZGQbAE39qw+RbuhIUJFQJC6Z2tXD+eWVbx7bXJTVwuaoAaFOvmsvrb1bvYfP+k7xn3xb7xHcb3Mb12rwUft7q2h9xzukKYOPeEzw7c4vjdY8Jc/l12yHu+sxKxP/4en2B561UUWntI+UdZ47BT0/C8vcCWkcJzpe06NK4Bit2HmXN+MvINsbxZe9Obg2ltU9cxsmzWUxfu5c7ejcr8PjbnxlM12d+yldhtWODGN676UKycgyvz0vhw0Wl/xX/p471+K6QO5u0HpMqSFFrH2lSUN61azF8dzekbYa6nSDxBmg3DKr4bz6C3C/t4nxJnsnIJuXgKdo3iCl026krdvPholS+vbMnv6UcYtQ7S2gWH8nvaacd29zQvbHfm3SKc76HTp2jalgwVcO00k1loUlBBU5WBqz8EFZ8CAfWQUgEtLkSEm+Exj2swXE+9OmSXXyyZCfT7+rl0/fJNWvDftrXj+HiCXP98n6eFCcpJCRPJz46nGWPDijx+x1Pz+RURhb1qxetH0cFVlGTgv5MUN4XEgZdb4cLb4N9q2HlZGtcw9rPoWZTa7R0x1FQrW7hxyqBUd0aMapbI58c253L29Yhpyi3PZUxaSdLV5aj7/PzOJqeqc1WFYx2NCvfEYF6neGKF+H+LXD1WxBdD356Cl5sC5+OgM3TITsz0JGWWlBQYKujdvTQ7PXerzto9sgMj0nr6OkMvlm9h91H01ntYXa5c1nZzN9y0PF6/paDHDmdwdH08v/fTeWnVwrKP8KqQscR1uPw77DqI2t09NYfILIWdBoJnW+0CvKpYsvMdv+l/9T3Vt2njOwcIoLyj3Po/M85Lq/d/eqfOHsrby3Yzld/vYh29WO46f1lXohYlVV6paD8L7YZDHgC7t1o3dLaIAl+exVe7QLvDbKSRcbpwo5S4Xx2e3dCinDFcWe//Ikz0834h5SDJx3Pj5+xftXn5Bi+WrG7wON/tWI3v2yzbo/de+yM41bZz5f94fZ9VMWiVwoqcIJDoOUg63FyP6z5DFZ+BF+PhRkPQfth1t1L9RJ93jnta8FBQnaOoXrVUPq1rMW0VXtc1keFh3BRs1hSnhnsMktc/1a1OJuVzcKU8/NCtKzjWh68Ze1oMpy+rH/ZlkZocBB3frrSseyTJbtYtesocVHh+d7b2ceLd/KYPebhqk71+Hr1+dtgv1yxmy/dJJSTZzOJjgjNt7wgxhgys02Bs+qpwNCkoMqG6DrQ817ocQ/s/M1qXlozBVa8D7XaWsmhw/VQtWbhxwoQT2MJXh3VmSHt6/J72ikuqGV9ob94fSfW/HGMGev38dbP211Kbl+b2ICpK60v39fHJPKXj1Y41n3114scSSEuKpxf/96PR6atY+mO85Vrb3j3/MjoXC//tK1I5/CY0yA454RQkF+2HWJw++LdNPCv6Zt499cd/P7MYMcESaps8HlSEJFgYDmwxxhzRZ519wG3AVlAGnCLMUbH61dmIpDQw3oM+g+sn2rdvTQzGeaMh1ZXWAmiSd8yN1XoPQOau00KfVvWQkQcCSFXx4bVaV8/hiXbj7g0CT1/XQf+cUVrqle1BtsN7ViP+VvS+OWhfjSsWRVwbfs/ejqD3UfP8Pr8FG6+uIkvTq1AsQUMCvRk8qJUwGr2CnbT16ECxx9XCncDm4BqbtatApKMMekiMhZ4FrjeDzGp8iAiBpJusR7715+/etjwP4hpBJ1HQ6fRVrmNMqBZfBT/ubY9f5/qOlVnlQKqmwYFCV+P6+GyTEQcCQHgmsQGDO1YjxAPc0XMs8t1PztzC6t2ub+DyJdK0gSU2zGeVQ5v5a3ofPpTS0QaAEOAd9ytN8bMM8ak2y8XAw18GY8qx+q0s64c7t8C174LsU1h/r9hUnv46BrYMA2yAj8d5vUXNuLRwa0JCwnix/t68+L1Hb3SPOIpIeQ1Z+OBUr9XcWVk5ZCdY0hInp6vE3vvsTOczcz2uO+RU/nnjfDEGMOzMzdrVVgf8/WVwiTgISC6sA2BW4EffBuOKvdCI6wO6Pb2XA6rP4FVH8OXN0HVWOgwAjpeD3U6BKxz+vbeTbm9d1OAfE1GZcnobo34ZMmuUh/nu7V7OWgPhHvgyzW0qhNNu/oxHDhx1jHKe/szgzmTmU1kuOtXTkGTIp04m0lGVg5xUVZ12OFvLWJZ6lHmbj7IzHt6lzpu5Z7PylyIyBXAYGPM30SkL/BA3j4Fp23HAHcCfYwx+X7uicgdwB0AjRo16rJzp3Y7KCc52fD7PFg1GTbPgJxMqFbfKsjXYhA06QWhFbcUw7MzN/P6/N+Lvd9fejflrQXbAasMuLdndKtfvYqjxLezZ65uzyPTzjexrR5/Kb2encfJs1kufSV5a1g535V1c48EHv/T+Yq1qnBFLXPhy+ajHsBQEUkFpgCXiMjHeTcSkQHAo8BQdwkBwBjzX2NMkjEmKT4+3ochq3IpKBiaD4Dhk+H+zTD0VWsk9ZrP4dPr4Nmm8NlIWPEBnNgX6Gi97i8eqrkWx8sjOnshElfuEgLgkhAAOj01h5NnrUma3vz5d9IzXCdscvfD9f2Fqd4JshCrdh0lIXk6t31Yeeqt+az5yBjzMPAwgNOVwhjnbUSkM/AWMNAYczDfQZQqrsg46+6kxBsg8yzs/BW2zIStM2HLDGubup2ssREtLreel/MxENERxf/fuHXdagxuX9dxpVBWxgtM+GEzP248wPKdRx3LXpmbQsOa+a/0Ug+dLnT+jNI4m5nN1a//BsCPm/zfVxMofh+nICJPAcuNMd8CzwFRwJdi/Y+5yxgz1N8xqQoqNAIuGGA9Bj8HBzdayWHrLJg/weqojq4LzS+zkkSTPlY5jnKmOHWXHry8JdWrhnJlp/pEObXvOyeFT27rxuh3lgDw+R3daRwbSUyVUFqPnwlAqzrRvHVDF/o8N987J5CHc0IAmDhnq9vtnp21mddHd3G8Pptp1Wga2M47hRZPnysbU8z6m1+SgjFmPjDffj7eaXnJ6/YqVRwiULut9eh1P5w+BNvmWLWX1v/PKvUdEmElhhaXQ4uBEON5LueyJnXCEJc297oxEew7fhaAqzvXp1fzOAa2q5Nv/oQx3RvRtUksoU53N9WNiXA879Y01vG8Td1qbNx3gieGtqVe9Sq0rVeNDXtP+OqUCjVj3X6X15N+3MabP//OezclcXGzOCIKuBXY2aFT55i5fj9D2telhtOYi+wS9rcaY5g4ZyuD2tV1mYlvxc4jpJ3MYGC7OiU6rr/oiGZVOUXGWUX4Oo205n/YudC6gtj6A2ybBdPvgzrtrY7qFgOtPooyNliuICO7NnL8wr6rf3OaeGhm+ddV7fMt8/Rl+vlfujN97T66NamJiDD9rl4uiSgQJi9KJToihKs7N2D3Uevu9ls+ON/+/+N9fagWEULXZ35i6tiLSGxUgxyD4zbhtbuPMfTVhYA1mnvN+MuIqWqV7Nh77GyJYsrIzuGVuSm8MjfFpeP82jcWAUWf9yIzO4e5mw9yWZvaiB+bOMvPp1wpXwkJg2b9YNAEuGs1jFsKA56EsGj45Xl45xJ4oSV8Mw42fR/QOagLkjvZzbJHB1A1zPpiH9u3mceE4ElkeAgtakfx6ijXzufoiFBGdG3k8gV1cbPYvLt7VJK+j8KM/2YD936+BoDdR/N3bA+Y+DMvzLaS47u/7uDLFbtp9sgM/jhiJZDchJCr41OzmbneuhlhT57jZWXnuC1Bfiw9w2UsRm6neVFsTzvF/C0HSUiezhPfbuCtn8/fRfbyT9v4y0craPWPmUxZWvpbh4tKrxSUciYC8S2tR897IP0IpPwIW36Ajd9ZYyKCw63bXFsMtJqaqvtvQp+CTL61K9+s2kNcVBhjujdmz7EzjHNTUbUwYcFBzL63T5G2jbXHEBTFLT2a8FIRazAVV0FXLJ8v/wOwmptym5ze+Pl3+rWs5Xb7v368krVPXMY4p4KCABc8ag2juqF7Y/55VTvH8k5PzSEuKpx5D/QhOiKUoa/8WmCsP29No0+L+Hwxf/BbKgDnsnK4pFUtdh5Od7xO/t86RnT1z+dMp+NUqqiyM2HXIquZacsPcMT+VVerLbQcCM0vh3qdIKToX5RlSe6XVMrTg4o8gnrIy78U2q/w6ODWzNywnxEXNuTBr9aWOk5v6Ncy3lEepCSWPzbAMaiuoIT04S1daRYfSYMaVV222/KvgbR8bGax3rO0M9zpHM1K+dqhFKsPYussq7KryYagUKjdxuqDyH3UagPBxSstHQhfLPuD1+an8POD/Yq8T/LUtUxZ9ofH9Q9e3tJxtXImI9txBxPAoocvYfWuY2w9cIorO9Wj7/PzXfatXS2cAydchy5d1qY21yU15PbJgf0OeGhgSwa3q0twkNDr2XnF3j82MozDp4te4gNw6e8oCU0KSvnTmaOwYwHsWQl7V8He1XDuuLUuONyq3eScKOJaWvNJlHNHTmfwxvwU7r+sJa3+kf+X7+rxl7oU90vPyKLN+FlA/l++L8zewitzUxyvP7u9OyPfXux43b1pTT67vTsiEvAO7kB4+8YkLm1Tu8T7a1JQKpCMgSPb7QSxCvatsRJFhj0bWkgVqNvBNVHEXmCNzi6nbp+8PF9Bvm1PD3K53bUgGVk5JP9vLf9baU0ClPc2W+cksuPQafrlubIojYnDO3LfF2u8djxfKU0TUlGTQvn/qaJUWSRiTTsa28wq3geQk2P1Q+Qmir2rrLkilrxprQ+LgrodXRNFjSbl5lbYewY0z5cUipoQwBpAd++AFo6k4KxVntnmGtf07iDDujFVePDyljw3a4tXj1seaVJQyl+CgiCuufXoMNxalpMNh7a6Jopl70CWfY98eAzUy5Moqjcuk6U52taLIXXCEFb/cYyrXltY+A5uNKxZlUnXd2LdnuMuy7//v54ur4syivuaxPocT8/kp82FV9ARgXH9Lig0KUSGBXM6w3Mp8IpAk4JSgRQUDLVaW49Oo6xl2ZmQttk1USx+A7LtjskqNVyTRL3OVlXYMpIoOjWszu/PDCa7hBPoXNW5Pld1tkaTX9elAV+u2O32bqhljw5g474T/Pm9/NOPVg0LZsI1Hdh9NL1IScHTnBdTx15Em7oxTPppK83iomjfIIZBL/1SzDPynrST54iP9u3dbdqnoFR5kJVh1W5yThQHN0KOPVAqoro1tiKuhf23JcS3sGaoKyfNT+4YY8jOMQXeIrtgaxo3vreUujERVAkLZnvaaa7oUJdXRyU6jnH4dAbp57KpVS08X4f4NZ3r8/x1HQkKEtJOnmPjvhMs3n6Yvw9sle+9Nu8/wcBJgUsK/VvV4t2bLizRvtqnoFRFEhJmjYGo1wm42VqWeRYObIC9K60EkbbVKvi36iOn/apYzVXOiSKuJdRsah2zjBMRQoILvgLq3SKeH+/rQ7P4SH5NOcQN7y51jO7OPUZcVLhVetONBy5v6WiOio8Op090PH1auC/RXzs6wu3yvP0RnuaSeHVUZ+78dFW+5Ze1qc1sD7PmjevXjNfmWWNifDEqPC9NCkqVV6ER0KCL9XCWfsTqp0jbbCWKQ1tg1xJY9+X5bYJCrE7s3NHbjoTRAsJ8V47aVy6oZX3j97wgjjdGJ3JJa/ejlQE+va0bo95ZwjWJ9dm49wS1itEc41wwz1naSdfxFFPHXkz3f/+Ub7tqEaGM7NqQz5a6ju0Y2bWRx6Rwd/8WjqTQso67qe69S5OCUhVN1ZrQqLv1cJZxGg5tc0oYW6znW2eeb4YCiGno1Axl/41vZR23jBMRBrUvuHT2xRfElerWzs3/HMjHi3dyRYd6XPLCfNIzsqkTE0HqhCFkZOXwx9F06jhVmt38z4GOJqvF2w/z72s68JfezVwG6zWJi+Sn+/vQ/4WfAWv60qaPWPN/OJc1/+NoOr6mSUGpyiIs0qkJyklWBhzdYSWJtC3WlUXaFmuUdpZTE0jVuDz9Fi2spqnoehViIF5RRYQGc1svaw7ut27owg3vLqVbEythhoUE0Szeump5blgHnpu1hXCnL/V29WMAaBxblXsHtKBvy3g27TtBQlwkm/adLxcSFCT0uCCWGvbAv/duSuKWD5bz8KD8/Rzeph3NSin3cnLg+B/5ryzStsDZY+e3kyCIqgPV6llzUFTLfdSz/sbUt9ZX0MRxLiub8JCCBx3mDsIb2bUh/76mg9ttMrNz+NsnK7m7f3NH8vAm7WhWSpVOUBDUaGw9ml96frkxcDrNSg6HU+DEXjixx3oc2GhNXpSZp5mjoMQR08D6W04TR2EJAawSHYu3H+GaxAYetwkNDuLtGwv9zva58vdfQCkVWCIQVct6NOmVf70x1pXEib1wfM/5hJGbPCph4vjvjUl8u3ovSY1rBDqUQpW/f12lVNkmYg2wq1LDmv7UndImjpj61vHDoiA8ypoQKTzKw+to65G7LrSq3wf6VYsIZUz3xn59z5LSpKCU8j9vJI5TByFjuzUTXob9KNJ7B1kJwpFA7L/h1fIvc5dswqqCBFuj0SXYamaTYOu4jmXuXjstLyOjz93RpKCUKpuKkjic5eScTw7nTlkVac95en0y/7LTO1xfZxdvvoPinVuQ+2SRN4lI0PmkExQMfZOh3bW+iwtNCkqpiiIoCCKqWQ9vyMqwE8gJ1+SSmW5NqJSTDSbH/pt9/q/JsRKU8zLHX3fL3Rwjd3nedVV83yehSUEppdwJCYOQmuVi0J43ld9KWUoppbxOk4JSSikHTQpKKaUcNCkopZRy0KSglFLKQZOCUkopB00KSimlHDQpKKWUcih38ymISBqws4S7xwGHvBhOWaLnVn5V5PPTcys7Ghtj3E8+7aTcJYXSEJHlRZlkojzScyu/KvL56bmVP9p8pJRSykGTglJKKYfKlhT+G+gAfEjPrfyqyOen51bOVKo+BaWUUgWrbFcKSimlClBpkoKIDBSRLSKSIiLJgY7HExF5T0QOish6p2U1RWSOiGyz/9awl4uIvGyf01oRSXTa58/29ttE5M9Oy7uIyDp7n5dF/DcvoIg0FJF5IrJRRDaIyN0V5fxEJEJElorIGvvcnrSXNxGRJXY8n4tImL083H6dYq9PcDrWw/byLSJyudPygH2GRSRYRFaJyPcV6bzs90+1PzOrRWS5vazcfyZLzBhT4R9AMPA70BQIA9YAbQIdl4dYewOJwHqnZc8CyfbzZOA/9vPBwA+AAN2BJfbymsB2+28N+3kNe91Se1ux9x3kx3OrCyTaz6OBrUCbinB+9vtF2c9DgSV2HF8AI+zlbwJj7ed/A960n48APreft7E/n+FAE/tzGxzozzBwH/Ap8L39ukKclx1bKhCXZ1m5/0yW9FFZrhS6AinGmO3GmAxgCnBlgGNyyxizADiSZ/GVwIf28w+Bq5yWTzaWxUB1EakLXA7MMcYcMcYcBeYAA+111Ywxi431aZ3sdCyfM8bsM8astJ+fBDYB9SvC+dkx5s4cH2o/DHAJ8JWHc8s956+A/vYvyCuBKcaYc8aYHUAK1uc3YJ9hEWkADAHesV8LFeC8ClHuP5MlVVmSQn3gD6fXu+1l5UVtY8w++/l+oLb93NN5FbR8t5vlfmc3K3TG+kVdIc7PbmJZDRzE+lL4HThmjMlyE4/jHOz1x4FYin/O/jAJeAjIsV/HUjHOK5cBZovIChG5w15WIT6TJaFzNJczxhgjIuX6ljERiQKmAvcYY044N7GW5/MzxmQDnUSkOjANaBXgkEpNRK4ADhpjVohI30DH4yM9jTF7RKQWMEdENjuvLM+fyZKoLFcKe4CGTq8b2MvKiwP2ZSj234P2ck/nVdDyBm6W+42IhGIlhE+MMf+zF1eY8wMwxhwD5gEXYTUv5P74co7HcQ72+hjgMMU/Z1/rAQwVkVSspp1LgJco/+flYIzZY/89iJXMu1LBPpPFEuhODX88sK6ItmN1cOV2ZrUNdFwFxJuAa0fzc7h2ej1rPx+Ca6fXUnt5TWAHVodXDft5TXtd3k6vwX48L8FqU52UZ3m5Pz8gHqhuP68C/AJcAXyJa4fs3+zn43DtkP3Cft4W1w7Z7VidsQH/DAN9Od/RXCHOC4gEop2e/wYMrAifyRL/mwQ6AD/+xx+MdbfL78CjgY6ngDg/A/YBmVjtj7ditcn+BGwDfnT6sAnwmn1O64Akp+PcgtWZlwLc7LQ8CVhv7/Mq9gBGP51bT6z227XAavsxuCKcH9ABWGWf23pgvL28qf2lkGJ/kYbbyyPs1yn2+qZOx3rUjn8LTneqBPozjGtSqBDnZZ/HGvuxIff9K8JnsqQPHdGslFLKobL0KSillCoCTQpKKaUcNCkopZRy0KSglFLKQZOCUkopB00KqswRkWy7YuUaEVkpIhcXsn11EflbEY47X0Qq3Jy6pSEiH4jIsEDHocoOTQqqLDpjjOlkjOkIPAz8u5Dtq2NV5yyTnEb+KlXmaVJQZV014ChYNZNE5Cf76mGdiORW05wANLOvLp6zt/27vc0aEZngdLzrxJr3YKuI9LK3DRaR50RkmV0j/y/28roissA+7vrc7Z3Ztfiftd9rqYhcYC//QETeFJElwLN2ff6v7eMvFpEOTuf0vr3/WhG51l5+mYgsss/1S7teFCIyQaz5KNaKyPP2suvs+NaIyIJCzklE5FWx5i/4Eajlzf9YqvzTXzCqLKpiVxuNwJqD4RJ7+VngamMV0YsDFovIt1hlCNoZYzoBiMggrBLH3Ywx6SJS0+nYIcaYriIyGHgcGIA1avy4MeZCEQkHForIbOAaYJYx5mkRCQaqeoj3uDGmvYjciFVR9Ap7eQPgYmNMtoi8AqwyxlwlIpdglfvoBPwjd3879hr2uT0GDDDGnBaRvwP3ichrwNVAK2OMsQvvAYwHLjdWUbfcZZ7OqTPQEmt+g9rARuC9Iv1XUZWCJgVVFp1x+oK/CJgsIu2wSgw8IyK9sco41+d8SWNnA4D3jTHpAMYY5/kpcovwrcCqMQVwGdDBqW09BmgOLAPes4v4fW2MWe0h3s+c/r7otPxLY1VOBavEx7V2PHNFJFZEqtmxjsjdwRhz1K5M2gbrixysmkCLsMpQnwXeFWsGtO/t3RYCH4jIF07n5+mcegOf2XHtFZG5Hs5JVVKaFFSZZoxZZP9yjseqkRMPdDHGZNqVOyOKechz9t9szn/+Bfg/Y8ysvBvbCWgI1pfuRGPMZHdhenh+upixOd4Wa8KWkW7i6Qr0B4YBdwKXGGP+KiLd7DhXiEgXT+dkXyEp5ZH2KagyTURaYVXTPIz1a/egnRD6AY3tzU5iTe+Zaw5ws4hUtY/h3HzkzixgrH1FgIi0EJFIEWkMHDDGvI0161iih/2vd/q7yMM2vwCj7eP3BQ4ZY07YsY5zOt8awGKgh1P/RKQdUxQQY4yZAdwLdLTXNzPGLDHGjAfSsEo4uz0nYAFwvd3nUBfoV8i/japk9EpBlUW5fQpg/eL9s90u/wnwnYisA5YDmwGMMYdFZKGIrAd+MMY8KCKdgOUikgHMAB4p4P3ewWpKWilWe00a1pSJfYEHRSQTOAXc6GH/GiKyFusqJN+ve9sTWE1Ra4F0IHdi938Br9mxZwNPGmP+JyI3AZ/Z/QFg9TGcBL4RkQj73+U+e91zItLcXvYTVsXPtR7OaRpWH81GYBeek5iqpLRKqlKlYDdhJRljDgU6FqW8QZuPlFJKOeiVglJKKQe9UlBKKeWgSUEppZSDJgWllFIOmhSUUko5aFJQSinloElBKaWUw/8DhNqnR/gKHUoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtkAAAD8CAYAAACis7gyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xl4VOXZx/HvnckesicsScgChCUBwhJARAWlCq64oeBSbd1qpdYWrdjXulVc6r5Rq7XVurBqKypKVdC6ICQEEghrCAHCkp0A2TPzvH/M4Js3ZQmQ5MxM7s91eTE5c87J76g53DlzP88jxhiUUkoppZRS7cfH6gBKKaWUUkp5Gy2ylVJKKaWUamdaZCullFJKKdXOtMhWSimllFKqnWmRrZRSSimlVDvTIlsppZRSSql2pkW2UkoppZRS7UyLbKWUUkoppdqZFtlKKaWUUkq1M1+rA7SHmJgYk5ycbHUMpZQ6YatXry43xsRanaMz6T1bKeWpTuSe7RVFdnJyMtnZ2VbHUEqpEyYiO6zO0Nn0nq2U8lQncs/WdhGllFJKKaXamRbZSimllFJKtTMtspVSSimllGpnWmQrpZRSSinVzrTIVkoppZRSqp21qcgWkckisllECkRk1hHeDxCR+a73V4pIcov37nNt3ywik1ps/5uIlIrI+lbnihKRz0Vkq+vPyJO/PKWUUm3Rhvt8koh8KSJ5IvKViCS0eO8G1z17q4jc0LnJlVLKPR23yBYRG/AKcD6QBkwXkbRWu90EVBlj+gHPAU+6jk0DpgHpwGRgjut8AG+6trU2C/jSGJMKfOn6WimlVAdp433+aeAfxpihwCPA465jo4AHgTHAaOBBfTiilFJtmyd7NFBgjCkEEJF5wBRgQ4t9pgAPuV4vAl4WEXFtn2eMaQC2i0iB63wrjDH/afnEu9W5JrhevwV8Bdzb5itSJ6y6ronvCsrZXl5Ds90Q7G8j0N9GWKAvPcMCiYsIokdYIP6+2l2klJdqy30+Dfit6/Vy4F+u15OAz40xla5jP8f5AGVuewZ849vtOByG9PgwRidH4WvT+5FS6tiMMVTXNVFcVUdxVR0H6po41NDMoYZmmu0OLhwax4CeoR32/dtSZMcDu1p8XYzzicUR9zHGNItINRDt2v5Dq2Pjj/P9ehhj9rpe7wN6HGknEbkVuBUgMTHx+Feh/kt9k50XvtzKm98VUddkP+a+Nh+hT0wIg3qFkRYXxqjkKIYmhOOnf9Ep5Q3acp/PBS4HXgAuA0JFJPoox/7Xff5U79nv/LCD7eU1APSOCuL35w/i/CG9Tvg8SinvVVxVy8rCStbvqSZ/9wE27j3AwYbmI+4rAqk9Qi0vsi1jjDEiYo7y3mvAawCZmZlH3Ecd3a7KWm5+K5vNJQeZMiyO609LYnB8OL4+Ql2TnbomOwfqmtizv5591fXsrKxl074DrN5RxeLcPQCE+NsYlRLFOQO7Mym9Jz3CAi2+KqVUB7ob56eUNwL/AXYDx/7tvIVTvWcvmzne9albBS8vL+D2d3P46dgkHrw4HZuPnOjplFJeoLHZwbcFZXy5sZTvCsopqqgFINDPh7ReYUwZHkdydAgJkcHERwQREexHaKAvIQG+nfKQsC1F9m6gd4uvE1zbjrRPsYj4AuFARRuPba1ERHoZY/aKSC+gtA0Z1QkoKq/hmtd/4FBDM2/+bBQTBnT/f++H2nwIDfSje2gg/br/9294lTWNrCys4PttFXxXUM4DH+bz4OJ8RiZGcsmwOKYMiyc8yK+zLkcpdeqOe682xuzB+SQbEekGXGGM2S8iu/m/Fr/Dx37V3gFFhIhgfy4c2otJ6T148rNNvP7Ndg7VN/PU1AwttJXqIhwOw4rCCj5cu5ul+SVU1zUR4m/jtD7R3HB6MmP7RpPaPdQt7gltKbKzgFQRScF5050GXNNqn8XADcAK4Epgmesp9GLgPRF5FogDUoFVx/l+h8/1hOvPD9t4LaoNqmub+PmbWdQ12Zl762mkx4Wf8DmiQvw5f0ivHz+q3VpykE/X72PJur088GE+jy3ZyIVD4rj2tERGJOr4J6U8wHHv8yISA1QaYxzAfcDfXG8tBR5rMdjxPNf7HcbX5sP/XJhGWKAfz3y+hfjIIGaeN6Ajv6VSymLVtU0sXL2Ld1fuZHt5Dd0CfDkvrQcXZfTijH6xbjlu7LhFtqvHegbOG6kN+JsxJl9EHgGyjTGLgTeAt10DGytx3qBx7bcA5+CZZuAOY4wdQETm4nz6ESMixcCDxpg3cBbXC0TkJmAHcFW7XnEX5nAY7py3hl1Vtbx3y8kV2EeS2iOU1B6h3DkxlXXF1czN2smHa3bzfk4xo5Ij+eWEfkwYEItzLKxSyt208T4/AXjc1cL3H+AO17GVIvJHnIU6wCOHB0F2tBnn9KO4qo6XlhUwJD6c89J7dsa3VUp1oj3763jtP4XMy9pJfZODkUmR/HpiKpMH9yTQz3b8E1hIjPH8dubMzEyTnZ1tdQy39/aKIv7wYT5/nJLO9WOTO/R71TQ0szB7F69/s53d++sY2DOU35zbn/PSemixrVQLIrLaGJNpdY7O1J737IZmO5fP+Z6SA/X8+zfjiQrxb5fzKqWstauyljlfFbBodTHGwGXD4/nZuBTS4sIszXUi92z3e7auOsSuyloe/3QTZ6bGcN1pSR3+/UICfLlxXApf3TOBZ6Zm0Gh3cNvbq7ny1RVkF3XKQy6lVBcQ4Gvj6akZVNc18fBH+VbHUUqdouq6Jh5bspGJz3zN+6t3M21UIl/dM4GnpmZYXmCfKLeeXUS1n4c/2oAAT1wxtFOfJPvZfLhiZAJThsWxILuY577YwpWvrmByek8euDiNuIigTsuilPJOg3qFcfuEfrz45VauOy2JUclRVkdSSp2gZruDd1fu5PkvtrC/rokrRiQw87z+9Ar33DpBn2R3Ad9vK+eLjSX88ux+xFtU1PrafLhmTCJf3zOB357bn6+2lPKTZ7/m9f8U0mR3WJJJKeU9fjG+Dz3DAnn0k404HJ7fBqlUV7J+dzVTXvmOBxfnM6hXGB/NOIOnp2Z4dIENWmR7PYfDMPuTjcRHBHHTGSlWxyHY35c7J6by+W/GM7ZPNLOXbOTil74lr3i/1dGUUh4s2N+XuycNIHfXfj5Zt/f4ByilLFfb2MzsTzZwycvfUnqwgVeuGcG7N49hcHz7TMxgNS2yvdxn+fvI33OAuyf1d6tRuL2jgvnrDZn85fqR7K9t4rI53/P8F1v0qbZS6qRdPjye1O7deHlZgT7NVsrNrd5RxeTnv+H1b7YzbXQiX/x2PBcO7eVVkyNoke3FjDG8sryAlJgQLsk43mr2nU9EmJTek6V3ncUlGXE8/8VWrvzz9xSUHrI6mlLKA/n4CL88uy+bSw7yxcYSq+MopY6g2e7guc+3cNVfVuAwhvm3nsZjlw3xykXstMj2Yl9tLiN/zwFuH9/XLVY+OprwYD+eu3oYc64dwc7KWi566RveX11sdSyllAe6eGgciVHBvLy8AG+YolYpb7Kzopapf1nBC19uZUpGHEt+fSZj+kRbHavDaJHtxV5ZXkBceCCXDne/p9hHcsGQXiy96yyG9Y5g5sJc7l2UR32T3epYSikP4mvz4bbxfcgrriZ7R5XVcZRSLss2lXDRS9+wrfQQL00fzrNXDyMs0PueXrekRbaXWuf6C+amM/u45VKjR9M9LJB3bhrDHWf3ZX72Li6b8z1F5TVWx1JKeZDLhscTFujLW98XWR1FqS7P7jA8+/kWfv5mNr2jgvnkzjO5OCPO6lidwnOqL3VC/rGiiCA/G1eOTLA6ygnztflwz6SB/P1no9hbXcclL3/LdwXlVsdSSnmIYH9frsrszWfr91FyoN7qOEp1WdW1Tfz8zSxe/HIrV45M4P3bT6d3VLDVsTqNFtleqKqmkcW5e7h0eLxHDyQ4e0B3PppxBr3Cg/jp31bx9ooiqyMppTzEdaclYTeG91butDqKUl3SjooaLpvzHd9vK2f2ZYN56sqhbjXLWWfQItsLLVy9i4ZmBz8d2/HLp3e03lHBLLp9LOP7x/KHD/N54MP1NOs0f0qp40iOCWF8/1jmZ+3CrtP5KdWpsosqufSV76iqbeS9W07j2jFJXjU1X1tpke1ljOvJzajkSAb1CrM6TrsIDfTj9Z9mcutZffjHih3c9FY2tY3NVsdSSrm5qzJ7s+9AvbabKdWJPly7m2teX0lEsD///OU4RiVHWR3JMlpke5nVO6ooqqjl6lGJVkdpVzYf4fcXDOLxy4fwzdYypr++ksqaRqtjKaXc2MRB3QkP8uP9HJ0SVKnO8OrX2/j1vLUMS4zgg9tPJzkmxOpIltIi28u8n7ObID8bkwf3tDpKh5g+OpE/XzeSTXsPcOWr31NcVWt1JKWUmwrwtXFxRi8+W7+PA/VNVsdRymsZY3j804088ekmLs6I4+2bRhMZ4m91LMtpke1F6pvsfJy3h/MH96RbgK/VcTrMpPSevH3TGMoONnDFn79n876DVkdSSrmpK0f2pqHZwZK8vVZHUcor2R2G3/9zHX/5upDrTkvkhauHEeDbtQY4Ho0W2V7ki40lHKxv5vIRnjdt34kanRLFwl+MBWDaayvI31NtcSKllDvKSAinb2wIH6zZbXUUpbxOY7ODO+euYe6qXdxxdl/+OGUwPm68wnRn0yLbi3yQs5ueYYGM7eu9S5S2NLBnGPNvHUuQn41rXl/JumIttJU6WSIyWUQ2i0iBiMw6wvuJIrJcRNaISJ6IXODa7icib4nIOhHZKCL3dX76oxMRLs6II6uoklKdM1updtPQbOcX76zmk3V7+f0FA7ln0sAuOYPIsWiR7SUqaxr5eksZlw6Px9aFfotMjglh/m1jCQ305Zq//sCanbqMslInSkRswCvA+UAaMF1E0lrtdj+wwBgzHJgGzHFtnwoEGGOGACOB20QkuTNyt9WFQ3phDHy6fp/VUZTyCg3Ndm5/J4dlm0qZfdlgbj2rr9WR3JIW2V7i8w37sDsMFw3tZXWUTtc7Kpj5t40lMtif699YRXZRpdWRlPI0o4ECY0yhMaYRmAdMabWPAQ7PCxoO7GmxPUREfIEgoBE40PGR2y61Ryj9e3Tjk3Xal63UqWpotvPLFgX2tWM8f02OjqJFtpdYsm4fiVHBpMd5x9zYJyo+Ioj5t51GbGgAN/49i9xd+62OpJQniQd2tfi62LWtpYeA60SkGFgC/Mq1fRFQA+wFdgJPG2Pc7jfdC4c4W0Z0mXWlTl5js4M73s3hy02lPHqpFtjHo0W2F9hf28h3BeWcP6Rnl+6H6hUexNxbTiMyxI+f/m0VG/e61cM0pTzddOBNY0wCcAHwtoj44HwKbgfigBRgpoj0aX2wiNwqItkikl1WVtaZuQG4cGhPZ8uIPs1W6qQ02R3c8V4OX2ws5Y9T0rnuNC2wj0eLbC/w+YYSmh2GCwZ3vVaR1nqGB/LezacR6OfD9W+spLDskNWRlPIEu4HeLb5OcG1r6SZgAYAxZgUQCMQA1wCfGWOajDGlwHdAZutvYIx5zRiTaYzJjI2N7YBLOLZ+3Z0tI0vWaV+2UifK4TDcszCXzzeU8PAl6Vw/NtnqSB5Bi2wv8On6fcRHBDE0IdzqKG6hd1Qw7958GsbAtX9dya5KXbBGqePIAlJFJEVE/HEObFzcap+dwEQAERmEs8guc20/x7U9BDgN2NRJuU/IpPSeZO+opEpXi1WqzYwxPPLxBv61dg/3TBrADacnWx3JY2iR7eEO1DfxzdYyLujirSKt9evejX/cNJqahmau/etKSg9qH6ZSR2OMaQZmAEuBjThnEckXkUdE5BLXbjOBW0QkF5gL3GiMMThnJekmIvk4i/W/G2PyOv8qju8ng3rgMLB8c6nVUZTyGC98uZU3vy/i5jNS+OUEnUXkRGiR7eGWbSylyW44f4i2irSWHhfOWz8fTdnBBn729ywO6rLKSh2VMWaJMaa/MaavMWa2a9sDxpjFrtcbjDHjjDEZxphhxph/u7YfMsZMNcakG2PSjDFPWXkdxzIkPpzY0AC+3KhFtlJt8eZ323n+i61MHZnA/1w4SB/mnSAtsj3cl5tKiekWwLCECKujuKXhiZHMuW4Em/Yd5PZ3cmhsdlgdSSllER8fYeLA7ny9pUzvBUodx4drd/PQRxs4L60Hj18+RAvsk6BFtgdrsjv4enMp5wyM1WVMj+HsAd15/PIhfFtQzr3v5+H8hFsp1RVNHNSDQw3NrNrudrMMKuU2vt9Wzt0LcxnbJ5oXpw/H16bl4snQf2sebPWOKg7UN3POwB5WR3F7V2X2Zua5/fnnmt38aelmq+MopSxyRr8YAnx9+GJjidVRlHJLW0sOctvbq0mJCeHV60cS6GezOpLH0iLbg325sQR/mw9npMZYHcUjzDinH9eOSeTPX23jre+LrI6jlLJAkL+Ncf1i+HJTiX6qpVQrpQfrufHvWQT62fjbjaMID/KzOpJHa1ORLSKTRWSziBSIyKwjvB8gIvNd768UkeQW793n2r5ZRCYd75wiMlFEckRkrYh8KyL9Tu0SvdeXm0oZ0yeKbgG+VkfxCCLCI1MGc25aDx7+KJ/lm3Twk1Jd0cRB3dlVWcc2nUdfqR/VNjZz05vZVNY08rcbRpEQGWx1JI933CJbRGw4p2g6H0gDpotIWqvdbgKqjDH9gOeAJ13HpuGcbzUdmAzMERHbcc75Z+BaY8ww4D3g/lO7RO+0vbyGwrIaJg7sbnUUj2LzEV6YNoy0uDB+NXcNm/cdtDqSUqqTnZXqXAzn6y3lFidRyj3YHYY7564hf081L18znCG67ka7aMuT7NFAgTGm0BjTCMwDprTaZwrwluv1ImCiOIehTgHmGWMajDHbgQLX+Y51TgOEuV6HA3tO7tK825eufkLtxz5xwf6+/PWnowj2t/HzN7MoP9RgdSSlVCfqHRVMn5gQvtna+cu7K+WOHv1kA19sLOXhS9KZOEjrivbSliI7HtjV4uti17Yj7uNa1KAaiD7Gscc6583AEhEpBq4HnmjLhXQ1yzaVktq9G4nR+nHOyegZHshfb8ikoqaB295eTX2T3epISqlOdFb/WH4orNCffdXlzc/ayd+/K+Ln41J0ufR25o4DH38DXGCMSQD+Djx7pJ1E5FYRyRaR7LKyrvU0oqahmayiSs7RVpFTMjQhgmevGsbqHVXc98E6HQSlVBdyVv8Y6pscrN5RZXUUpSyTXVTJ/f9az5mpMfz+goFWx/E6bSmydwO9W3yd4Np2xH1ExBdnm0fFMY494nYRiQUyjDErXdvnA6cfKZQx5jVjTKYxJjM2NrYNl+E9fiisoMluOKt/17rujnDBkF4/Tu0356ttVsdRSnWSMSnR+NmE/2zpWg9plDps9/46fvHOahIig3l5+gidC7sDtOXfaBaQKiIpIuKPcyDj4lb7LAZucL2+ElhmnI8FFwPTXLOPpACpwKpjnLMKCBeR/q5znQtsPPnL807fbC0n0M+HkUmRVkfxCjPO6ceUYXE8/e/NOuOIUl1ESIAvmUlRfK1FtuqC6hrt3PqPbOqbHLz+05GEB+tUfR3huEW2q8d6BrAUZ8G7wBiTLyKPiMglrt3eAKJFpAD4LTDLdWw+sADYAHwG3GGMsR/tnK7ttwDvi0guzp7se9rvcr3DN1vLGJ0SrRPEtxMR4YnLh5LWK4w7562hqLzG6khKqU5wZv8YNu07SOmBequjKNVpjDHcvSiXDXsP8OL0YfTrHmp1JK/Vps8GjDFLjDH9jTF9jTGzXdseMMYsdr2uN8ZMNcb0M8aMNsYUtjh2tuu4AcaYT491Ttf2fxpjhhhjMowxE1qeSzk/3tlWVsNZugBNuwryt/HqdSPx9RFufTubmoZmqyMppTrY4an8vtmqU/mpruOV5QV8kreX300aqDOUdTBtwPEw37qmnDozVfux21vvqGBevmYEBaWHuGdRrg6EVMrLpfUKI6abv07lp7qMrzaX8sznW7gkI45fjO9jdRyvp0W2h/lmazndQwPo36Ob1VG80rh+Mdx3/iCWrNvHq1/rhyhKeTMfH+G0PtGsKKzQX6qV19tVWctd89cyoEcoT14xFOdyJqojaZHtQewOw7cF5ZyRGqM/HB3o5jNTuDgjjj8t3aSDopTycqf3jaHkQAOFOhZDebH6Jju/fDcHu93w5+tGEuSvY7o6gxbZHiR/TzX7a5t+7CNUHUNEePKKIQzoEcqdc9ewq7LW6khKqQ4ytm80ACu2VVicRKmO8/BHG1i3u5pnrsogJSbE6jhdhhbZHuTw4Jxx/XTQY0cL9vflL9ePxGEMM97LobHZYXUkpTqUiEwWkc0iUiAis47wfqKILBeRNSKSJyIXtHhvqIisEJF8EVknIoGdm/7kJUcH0zMskBWFWmQr77QwexdzV+3kF+P7cl56T6vjdClaZHuQb7aWMahXGLGhAVZH6RKSokN46soMcoureWyJTteuvJeI2IBXgPOBNGC6iKS12u1+nNOtDse5tsEc17G+wDvAL4wx6cAEoKmTop8yEWFs32h+2KZ92cr75O+p5v5/rWdsn2juPq//8Q9Q7UqLbA9R32QnZ8d+zugXbXWULmXy4J78fFwKb35fxJJ1e62Oo1RHGQ0UGGMKjTGNwDxgSqt9DBDmeh0O7HG9Pg/IM8bkAhhjKowx9k7I3G7G9o2moqaRLSWHrI6iVLuprm3i9ndyiAj246VrhuuKjhbQf+MeYvWOKhrtDk7vq60inW3W+QMZ1juC3y3K04VqlLeKB3a1+LrYta2lh4DrRKQYWAL8yrW9P2BEZKmI5IjI7zo6bHsb2+dwX7bOl628w+EFZ/bsr2POtSOI6aafgFtBi2wP8UNhBTYfITNZl1LvbP6+Prxy7Qh8bcIv382hvsmjHtIp1V6mA28aYxKAC4C3RcQH8AXOAK51/XmZiExsfbCI3Coi2SKSXVbmXrP29I4KJiEySPuyldd48/siPt9QwqzzBzIyKcrqOF2WFtkeYsW2CgbHhxMa6Gd1lC4pPiKIZ6/KYMPeAzz8Ub7VcZRqb7uB3i2+TnBta+kmYAGAMWYFEAjE4Hzq/R9jTLkxphbnU+4Rrb+BMeY1Y0ymMSYzNtb9Zkga2yeaHworcTi0L1t5tvW7q3l8ySZ+Mqg7N52RYnWcLk2LbA9Q29hMbvF+Tuujv41a6ZyBPbh9Ql/mrtrFBznFVsdRqj1lAakikiIi/jgHNi5utc9OYCKAiAzCWWSXAUuBISIS7BoEOR7Y0GnJ28np/aKprmtiw94DVkdR6qQdamhmxns5RHfz56krM3RNDYtpke0BVu+ooslufuwbVNaZeW5/RqdEcf+/1lNYpoOklHcwxjQDM3AWzBtxziKSLyKPiMglrt1mAreISC4wF7jROFUBz+Is1NcCOcaYTzr/Kk7N2D7O8S4/aMuI8lDGGH7/wTp2VtbywrThRIb4Wx2py9Mi2wP8Xz+2Psm2mq/NhxemDcPf14c7562hoVn7s5V3MMYsMcb0N8b0NcbMdm17wBiz2PV6gzFmnDEmwxgzzBjz7xbHvmOMSTfGDDbGeNzAR4Ce4YEkRwfzQ2Gl1VGUOikLsnexOHcPv3U9DFLW0yLbA6zYVsHQhHC6BfhaHUUBvcKDePKKoazffYCnl262Oo5Sqp2MSo4ie4f2ZSvPs6XkIA8uzmdcv2hun9DP6jjKRYtsN1fT0ExecbW2iriZSek9uf60JF7/ZjtfbS61Oo5Sqh2MSolif20TBdoKpjxIXaOdGe/l0C3Al+euHobNR/uw3YUW2W4ue0cVzQ7DaVpku53/uXAQA3qEcvfCXEoP1lsdRyl1ika7WvJWbdeWEeU5Hvk4ny0lh3j2qmF0Dw20Oo5qQYtsN/dDYQW+Oj+2Wwr0s/HSNcM5WN/MzAW5+hGzUh4uKTqY2NAAsoq0yFae4dN1e5m7ahe3T+jLWf3db2rMrk6LbDe3YlsFGb0jCPbXfmx31L9HKH+4KI1vtpbzxrfbrY6jlDoFIsLo5Ciy9Em28gD7quuZ9cE6hiaE89tz+1sdRx2BFtlu7FBDM+t2az+2u7t2TCKT0nvwp6WbyCveb3UcpdQpGJUcyZ7qeoqraq2OotRRORyGmQvX0tjs4Pmrh+Fn03LOHel/FTeWVVSJXfux3Z6I8OQVQ4npFsCdc9dwqKHZ6khKqZM0yjX1mbaMKHf2t++2811BBQ9cnEaf2G5Wx1FHoUW2G/uhsAI/mzAySfux3V1EsD/PXz2MHZW1PPqxxy12p5RyGdgzjNAAX1Ztr7I6ilJHtGHPAf702WbOTevBtFG9rY6jjkGLbDeWtb2SoQkRBPnbrI6i2mBMn2h+Mb4v87J28fmGEqvjKKVOgs1HGJkcqU+ylVuqb7Lz63lrCA/248krhuqy6W5Oi2w3Vd9kZ93uap1VxMP85if9SesVxqz38yg72GB1HKXUSRiVHEVB6SEqaxqtjqLU//PEp5vYWnqIZ6ZmEKXLprs9LbLdVO6u/TTZDaOSdGlUT+Lv68Pz04ZxsKGZe9/Pwxid1k8pTzMqWfuylftZvrmUN78v4mfjknW6Pg+hRbabOnxz1yfZnqd/j1BmTR7Isk2lvLdqp9VxlFInaGhCOP42H53KT7mN8kMN3LMwjwE9Qrl38kCr46g20iLbTWUVVdG/RzcigvXjIE904+nJnNEvhkc/3kihLtGslEcJ9LOR0Ttcn2Qrt2CM4d5FeRyob+KF6cMI9NNxWp5Ci2w3ZHcYcnZUkZmsrSKeysdHeHpqBv6+PvxmQS5NdofVkZRSJ2BUchTr9xygtlGn5FTWmrtqF19uKuXeyQMZ2DPM6jjqBGiR7YY27TvAwYZmRmuR7dF6hgcy+7LB5O7az8vLCqyOo5Q6AaOSo7A7DLm7qq2OorqwXZW1PPrJBsb1i+ZnpydbHUedIC2y3VB2kXN+Vu3H9nwXDY3jsuHxvLy8gDU7dd5dpTzF8MQIAHL051ZZxOEw3LMoFx/Xgmc+Pjpdn6fRItsNZRVV0is8kPiIIKujqHbw8JR0eoYF8pv5a6nR1SCV8ggRwf70696N1Tu0yFbW+MeKIn4orOTR0ue+AAAgAElEQVT+CweREBlsdRx1ErTIdjPGGLKKKhmVHKWTzHuJsEA/np6awY7KWmYv2Wh1HKVUG41MjCRnZxUOh07FqTrX9vIanvhsExMGxHK1rurosdpUZIvIZBHZLCIFIjLrCO8HiMh81/srRSS5xXv3ubZvFpFJxzunOM0WkS0islFE7jy1S/QsxVV1lBxoYJS2iniVsX2jufmMFN5buZOvt5RZHUep/9KG+3yiiCwXkTUikiciFxzh/UMicnfnpe5YI5Mi2V/bRGF5jdVRVBdidxjuXpiLv82HJy7XVR092XGLbBGxAa8A5wNpwHQRSWu1201AlTGmH/Ac8KTr2DRgGpAOTAbmiIjtOOe8EegNDDTGDALmndIVepj/mx9bBz16m5nnDaBf927cuyiP6romq+Mo9aM23ufvBxYYY4bjvK/PafX+s8CnHZ21M41Icj7syNGWEdWJ3vi2kNU7qnjoknR6hgdaHUedgrY8yR4NFBhjCo0xjTiL3imt9pkCvOV6vQiYKM5fvaYA84wxDcaY7UCB63zHOuftwCPGGAeAMab05C/P82QVVRIa6MuAHqFWR1HtLNDPxjNTMyg71MAjH22wOo5SLbXlPm+Aw/OHhQN7Dr8hIpcC24H8TsjaafrEhBAR7Kd92arTbC05yNP/3sK5aT24bHi81XHUKWpLkR0P7GrxdbFr2xH3McY0A9VA9DGOPdY5+wJXi0i2iHwqIqlHCiUit7r2yS4r856P37OKqshMitRRxF4qo3cEv5zQl/dzivl8Q4nVcZQ6rC33+YeA60SkGFgC/ApARLoB9wIPH+sbeOI928dHGJEYyWqdYUR1gma7g7sX5hLib+Oxy4Zom4gXcMeBjwFAvTEmE3gd+NuRdjLGvGaMyTTGZMbGxnZqwI5SWdNIQekhRqVoq4g3+9U5qQzqFcZ9H6yjsqbR6jhKtdV04E1jTAJwAfC2iPjgLL6fM8Ycc2lTT71nj0yKpKD0EPtr9WdVday//KeQ3OJq/njpYGJDA6yOo9pBW4rs3Th7pA9LcG074j4i4ovzo8SKYxx7rHMWAx+4Xv8TGNqGjF4h29WPPUr7sb2av68Pz0zNoLqukT98uN7qOEpB2+7zNwELAIwxK4BAIAYYA/xJRIqAu4Dfi8iMjg7cWUYkOvuy1+zcb3ES5c027j3A819s4cIhvbhoaJzVcVQ7aUuRnQWkikiKiPjjHPCyuNU+i4EbXK+vBJYZY4xr+zTX7CMpQCqw6jjn/Bdwtuv1eGDLyV2a58neUYW/zYch8eFWR1EdLC0ujLt+0p9P8vbyUe6e4x+gVMdqy31+JzARQEQG4Syyy4wxZxpjko0xycDzwGPGmJc7L3rHyugdjs1HtC9bdZjGZgczF+QSHuTHHy8dbHUc1Y6OW2S7eqxnAEuBjThHl+eLyCMicolrtzeAaBEpAH4LzHIdm4/zyccG4DPgDmOM/WjndJ3rCeAKEVkHPA7c3D6X6v6yiirJ6B1OoJ/N6iiqE9x2Vh8yekfwhw/XU3qw3uo4qgtr431+JnCLiOQCc4EbXQ9TvFqwvy9pvcK0yFYd5uXlBWzYe4DZlw0hKsTf6jiqHfm2ZSdjzBKcA11abnugxet6YOpRjp0NzG7LOV3b9wMXtiWXN6lrtLOuuJpbzupjdRTVSXxtzraRC1/8ht9/sJ7XfzpSB7ooy7ThPr8BGHecczzUIeEsNjIpkvlZu2i2O/C1ueNQJuWp1hVX88ryAi4bHs+k9J5Wx1HtTO8WbmLtrv00O4wuQtPF9OvejXsmDeCLjSW8n9O6BVYp5Q5GJEVS12Rn076DVkdRXqSh2c7MhWuJ6ebPQxenWx1HdQAtst1EVlElIjAyUQc9djU/G5fC6OQoHv4onz3766yOo5RqZaRrURptGVHt6fkvtrKl5BBPXDGU8GA/q+OoDqBFtpvIKqpkQI9Q/UHrgmw+wlNTh2J3GO59P48u0OaqlEeJCw+kZ1igFtmq3eTsrOIvX2/j6szenD2gu9VxVAfRItsNNNsd5OyoIlNbRbqspOgQ7rtgEN9sLefdlTutjqOUakFEGJkUqUW2ahf1TXbuXphLz7BA7r9okNVxVAfSItsNbNp3kJpGu86P3cVdNyaRM1NjeGzJRnZW1FodRynVwoikSHbvr2Nftc4EpE7NU0s3U1hWw5+uzCA0UD+99mZaZLuBLF2ERuF8WvbkFUOxiXDPolwcDm0bUcpdHO7LztEl1tUpWLW9kr99t53rTkvkjNQYq+OoDqZFthvILqoiPiKIuIggq6Moi8VFBPGHi9NYub2SN78vsjqOUsolrVcYAb4+2jKiTlptYzN3L8yld2Qw952vbSJdgRbZFjPGsKqoUqfuUz+aOjKBiQO786elmygsO2R1HKUU4O/rQ0ZCBNlaZKuT9MSnm9hVVctTVw4lJKBNy5QoD6dFtsV2VtZSdrCBTG0VUS4iwmOXDyHA18bdC3Oxa9uIUm5hRFIkG/ZUU99ktzqK8jDfFZTzjxU7+NnpKYzpE211HNVJtMi2WFaR86mI9mOrlnqEBfLwJenk7NzPX78ptDqOUgpnX3aT3bBud7XVUZQHOVjfxO8W5dEnJoTfTR5gdRzVibTItljW9krCg/xI7d7N6ijKzUwZFsek9B488/kWtpboSnNKWW1EYgQAOdoyok7A7E82sre6jqevyiDQz2Z1HNWJtMi2WNaOSjKTIvHxEaujKDcjIsy+bAjdAnyZuTCXZrvD6khKdWnR3QJIiQnRwY+qzZZvLmVe1i5uOasPIxJ17FVXo0W2hSoONVBYVqP92OqoYroF8Oilg8krrubPX22zOo5SXd6IxEhydlbpyqzquKprm5j1fh6p3bvxm5/0tzqOsoAW2RY6PEp9dIr+dquO7oIhvbg4I44Xl21lw54DVsdRqksbmRRJ+aFGdlbqglHq2B7+OJ/yQ408e9UwbRPporTItlB2USX+vj4Mjg+3Oopyc49ckk54kD8zF+bS2KxtI0pZ5fCiNNoyoo7l8w0lfJCzmzsm9GVIgv4d31VpkW2hVUVVDEuIIMBXf8NVxxYZ4s/jlw9h494DvLxsq9VxlOqyUrt3IzTAV4tsdVRVNY3c98E60nqFMeOcVKvjKAtpkW2R2sZm8ndXk6mL0Kg2OjetB5ePiOeVr7aRV7zf6jhKdUk+PsKwxAgtstVR/eHD9VTXNfLMVRn4+2qZ1ZXpf32LrN21n2aH0fmx1Ql58OJ0YrsFMHNBri6IodqViEwWkc0iUiAis47wfqKILBeRNSKSJyIXuLafKyKrRWSd689zOj995xqZFMnmkoMcrG+yOopyM5/k7eXjvL38emIqg3qFWR1HWUyLbItkF1Uhgk7po05IeJAfT1wxhK2lh3juiy1Wx1FeQkRswCvA+UAaMF1E0lrtdj+wwBgzHJgGzHFtLwcuNsYMAW4A3u6c1NYZmRSJMc6HJUodVnawgfv/tY6hCeH8Ynxfq+MoN6BFtkWyiioZ0COU8GA/q6MoDzNhQHemjerN6/8p1I+sVXsZDRQYYwqNMY3APGBKq30McPjRXDiwB8AYs8YYs8e1PR8IEpGATshsmWG9IxDRwY/q/xhjuP9f66hptPPM1Ax8bVpeKS2yLdFsd5Czo0pbRdRJ+58LB9ErPIi7F+ZS16htI+qUxQO7Wnxd7NrW0kPAdSJSDCwBfnWE81wB5BhjGjoipLsIDfRjQI9QLbLVjz5cu4el+SXMPLc/qT1CrY6j3IQW2RbYtO8gNY12HfSoTlpooB9PXTmU7eU1PLV0s9VxVNcwHXjTGJMAXAC8LSI//h0iIunAk8BtRzpYRG4VkWwRyS4rK+uUwB1pZFIka3fux+HQRWm6upID9Tzw4XpGJkVy85l9rI6j3IgW2RbILqoE0CfZ6pSc3i+Gn45N4u/fb2dlYYXVcZRn2w30bvF1gmtbSzcBCwCMMSuAQCAGQEQSgH8CPzXGHHFpUmPMa8aYTGNMZmxsbDvH73wjkyI52NDM1tJDVkdRFjLGMOv9PBrtDp6emoHNR6yOpNyIFtkWyNpRRXxEEHERQVZHUR5u1vkD6R0ZzN2LcqlpaLY6jvJcWUCqiKSIiD/OgY2LW+2zE5gIICKDcBbZZSISAXwCzDLGfNeJmS2li9IogIXZxSzfXMa9kweSEhNidRzlZrTI7mTGGLK2V2qriGoXwf6+PD01g+KqOh7/dKPVcZSHMsY0AzOApcBGnLOI5IvIIyJyiWu3mcAtIpILzAVuNMYY13H9gAdEZK3rn+4WXEanSowKJqabvxbZXVhxVS2PfLyBMSlR3DA22eo4yg35Wh2gq9lVWUfpwQYytVVEtZPRKVH8fFwKb3y7ncnpvTgjNcbqSMoDGWOW4BzQ2HLbAy1ebwDGHeG4R4FHOzygmxERhidGkrNTi+yuyOEw/G5RHsYYnp6agY+2iagj0CfZnSzrx35sfZKt2s89kwbQJzaE3y3K1QUylOokI5Mi2V5eQ8Uhr55MRR3BOyt38P22Cu6/KI3eUcFWx1FuSovsTpa9o5KwQF/6d9cpflT7CfSz8czUDPYdqOfRj7VtRKnOcLgvO2enLkrTlRSV1/D4kk2M7x/LtFG9j3+A6rK0yO5kWUVVZCZH6UdLqt0NT4zktvF9mZ+9i+WbSq2Oo5TXGxIfjp9NtC+7C7E7DHcvzMXPJjx5xVBE9O9ydXRaZHeiyppGCkoP6aBH1WHu+kkq/Xt0Y9YHeVTXatuIUh0p0M9Gelw4OVpkdxlvfFtI9o4qHp6STs/wQKvjKDfXpiJbRCaLyGYRKRCRWUd4P0BE5rveXykiyS3eu8+1fbOITDqBc74oIl41Aenhpx06P7bqKAG+Np6ZOozyQ4089FG+1XGU8nojkyLJLd5PY7PD6iiqg20tOcjT/97CeWk9uHRY6wVRlfpvxy2yRcQGvAKcD6QB00UkrdVuNwFVxph+wHM4V/3Ctd80IB2YDMwREdvxzikimYDXPe7NLqrE3+bDkPhwq6MoLzYkIZw7zu7HP9fsZmn+PqvjKOXVRiZF0tDsYMPeA1ZHUR2oye5g5sJcugX4MvuyIdomotqkLU+yRwMFxphCY0wjMA+Y0mqfKcBbrteLgIni/D9wCjDPGNNgjNkOFLjOd9Rzugrwp4DfndqluZ+sokqGJoQT6GezOorycjPO7kdarzD+55/rqKxptDqOUl7rx8GP2jLi1f781Tbyiqt59NLBxIYGWB1HeYi2FNnxwK4WXxe7th1xH9eiBtVA9DGOPdY5ZwCLjTF723YJnqG+yc663dWMStFWEdXx/H19eOaqDKrrmvjDh+utjqOU1+oRFkh8RBCrdb5sr7V+dzUvfrmVSzLiuGBIL6vjKA/iVgMfRSQOmAq81IZ9bxWRbBHJLisr6/hwp2jNzv002Q2ZSV7XBaPc1KBeYfx6Yiqf5O3l47w9VsdRymuNTIrUJ9leqqHZzt0Lc4kM8eeRKelWx1Eepi1F9m6g5USQCa5tR9xHRHyBcKDiGMcebftwnMvzFohIERAsIgVHCmWMec0Yk2mMyYyNjW3DZVhr5fYKRNCVHlWn+sX4vmQkhPOHf62n7KAumKFURxiRGMHe6nr27K+zOopqZy98sZVN+w7y5BVDiAj2tzqO8jBtKbKzgFQRSRERf5wDGRe32mcxcIPr9ZXAMmOMcW2f5pp9JAVIBVYd7ZzGmE+MMT2NMcnGmGSg1jWY0uOtLKwkrVcY4UF+VkdRXYivzdk2UtNo5/f/XIfzx1Ip1Z5GJjkfnuh82d4lZ2cVr369jasyEzhnYA+r4ygPdNwi29VjPQNYCmwEFhhj8kXkERG5xLXbG0C066nzb4FZrmPzgQXABuAz4A5jjP1o52zfS3MfDc12cnZWMSYl2uooqgvq1z2Uu8/rz+cbSvjX2tYfQimlTtXAXqEE+dm0yPYitY3N3L0gl17hQfzhotYTqinVNr5t2ckYswRY0mrbAy1e1+PspT7SsbOB2W055xH26daWfO4ur7iahmYHY/poq4iyxk1n9GFpfgkPfpjP2D4xuoiCUu3Iz+ZDRu9wcnTwo9d4bMlGtlfU8O7NYwgN1E+g1clxq4GP3mplYQUAo7UfW1nE5iM8PTWDRruDWR/kaduIUu1sZFIk+XsOUNvYbHUUdYqWbyrlnR92cvMZKZzeN8bqOMqDaZHdCVZur2Rgz1AiQ3TQhLJOSkwI904eyFeby5iftev4Byil2mxkUiR2hyGvuNrqKOoUVBxq4J5FeQzsGcrdkwZYHUd5OC2yO1iT3UF2URVjdH5s5QZuGJvM2D7RPPLxBraX11gdRymvMby3c3pW7cv2XMYY7vtgHQfqmnh+2jACfHXhOHVqtMjuYHnF1dQ12RnTRwc9Kuv5+AjPXJWBr49w1/y1NNkdVkdSyitEhvjTNzZE58v2YAuzi/n3hhLumTSAgT3DrI6jvIAW2R1s5XZXP7Y+yVZuIi4iiMcuH0Lurv289OVWq+MoNyEik0Vks4gUiMisI7yfKCLLRWSNiOSJyAUt3rvPddxmEZnUucndx8ikSHJ2VumYBw+0s6KWhz/KZ2yfaG46I8XqOMpLaJHdwVYWVpLavRsx3QKsjqLUjy4aGsflI+J5eXkB2UWVVsdRFhMRG/AKcD6QBkwXkdbzlt2Pc7rV4TjXNpjjOjbN9XU6MBmY4zpflzMiMZKq2iZtxfIwzXYHv1mw9sdP+nx8xOpIyktokd2Bmu0Ososqdeo+5ZYeviSd+Mgg7pq/loP1TVbHUdYaDRQYYwqNMY3APGBKq30McPgz9HBgj+v1FGCeMabBGLMdKHCdr8sZmeTsy87WlhGP8urX21i9o4pHLx1MXESQ1XGUF9EiuwPl7zlATaNdF6FRbik00I/nrhrGnv11PLjYa9eCUm0TD7SccqbYta2lh4DrRKQY5xoHvzqBY7uEvrHdiAj200+HPEhe8X6e/2IrF2fEMWVYl/zfVnUgLbI70OF+bH2SrdxVZnIUM87uxwc5u/k4b8/xD1Bd2XTgTWNMAnAB8LaItPnvEBG5VUSyRSS7rKysw0JaycdHGJUcxcrtWmR7grpGO3fNX0tsaACPThlsdRzlhbTI7kArCyvpExNC91BdXU+5r19NTCWjdwS//2Ade/bXWR1HWWM30LvF1wmubS3dBCwAMMasAAKBmDYeizHmNWNMpjEmMzY2th2ju5cxKVHsqKhlX3W91VHUcRyeyvTpqRmEB+uqjqr9aZHdQZrtDlZur9Sp+5Tb87P58MLVw2h2GGYuyMXh0JkRuqAsIFVEUkTEH+dAxsWt9tkJTAQQkUE4i+wy137TRCRARFKAVGBVpyV3M4fbA1dpy4hb+3TdXuau2sltZ/VlXD9d1VF1DC2yO0hucTWHGpo5Q394lQdIjgnhoYvTWVFYwevfFFodR3UyY0wzMANYCmzEOYtIvog8IiKXuHabCdwiIrnAXOBG45SP8wn3BuAz4A5jjL3zr8I9DOoVSrcAX1a52gWV+9m9v457388jo3cEM8/rb3Uc5cV8rQ7grb4vKAdgbF99kq08w9TMBJZvLuXpf29mbN9ohiZEWB1JdSJjzBKcAxpbbnugxesNwLijHDsbmN2hAT2Er82HkUmRrNK+bLfUbHdw17w1OAy8OG0YfjZ91qg6jv7f1UG+21ZOelwYUSH+VkdRqk1EhMcvH0JstwB+NXeNTuun1EkanRLFlpJDVNY0Wh1FtfLSsgKyiqqYfdlgkqJDrI6jvJwW2R2grtFOzo792uelPE5EsD8vTh9OcVUdv//nel25TqmTcJprRil9mu1eVhZW8NKyrVwxIkGn61OdQovsDpBVVEmj3cHp2iqiPFBmchS/Pbc/H+XuYX7WruMfoJT6f4bERxDg66NFthvZX9vIXfPXkhQdwsNT0q2Oo7oILbI7wHcF5fjZhNEpOj+28ky3j+/LGf1ieOijfLaUHLQ6jlIexd/XhxGJkawq0sGP7sAYw73v51F+qIEXpw2nW4AOR1OdQ4vsDvDdtnKGJ0YS7K8/yMoz+fgIz16dQbcAX+54N4e6xi47WYRSJ2V0ShQb9hzggI5tsNw7K3eyNL+EeycPZEhCuNVxVBeiRXY7q6ppJH/PAcb11X5s5dm6hwby3NXD2Fp6iIc/0mXXlToRY1KicBhYXVRldZQubf3uav740QbG94/l5+NSrI6juhgtstvZisIKjIEzUrUfW3m+M1NjuX1CX+Zl7WJxri67rlRbDU+MxM8musS6harrmrj93dVEd/PnuauH4eMjVkdSXYwW2e3su4JyQvxtOsew8hq/Pbc/IxKdy64XlddYHUcpjxDk+ntAF6WxhjGGexbmsnd/PS9fM0Kn01WW0CK7HRlj+GZrOaf1idYJ7pXX8LP58OL04dh8hNvfzaG+SfuzlWqL0SlR5BVXU9vYbHWULueNb7fz7w0lzDp/ICOTIq2Oo7oorQTb0fbyGnZW1jJhQKzVUZRqVwmRwTx/9TA27TvA/f/S+bOVaovT+0bT7DA6lV8nW72jkic+3cSk9B7cdIb2YSvraJHdjr7aXAbA+P7dLU6iVPs7e2B3fnVOKotWFzNP589W6rgyk6Lwt/nw/TZtGekslTWNzHhvDXERQfzpygxEtA9bWUeL7Hb09ZYy+sSEkBgdbHUUpTrEryemcmZqDA9+mE9e8X6r4yjl1oL8bYxIiuDbreVWR+kSHA7Db+avpeJQI3OuHUF4kJ/VkVQXp0V2O6lvsvNDYQXjtVVEeTGbj/DCtOHEhgZw+zs5VNU0Wh1JKbd2Rr8YNuw9QKX+rHS4F5dt5estZTxwcRqD43U+bGU9LbLbyYrCChqaHUwYoK0iyrtFhfgz59oRlB1s4K75a3E4tD9bqaM5vZ9zzYQV2jLSob7cWMLzX2zl8hHxXDsm0eo4SgFaZLebrzeXEeDrwxhdSl11ARm9I3jg4jS+3lLGS8sKrI6jlNsaGh9OaIAv3xZoy0hHKSw7xF3z1jI4PozHLhuifdjKbWiR3U6+3lLG2L7RBPrZrI6iVKe4dkwilw+P5/kvt7BsU4nVcZRyS742H8b0ieb7bVpkd4RDDc3c+vZq/Hx9ePW6kfp3sHIrWmS3gx0VNWwvr2FCf+3HVl2HiDD7siGkx4Vx59y1FJQetDqSUm5pXL9odlTUUlxVa3UUr+JwGGYuWMv28hpevmY4CZE66YByL1pkt4Nlm0oBtB9bdTlB/jZeuz6TQD8fbn4rm+raJqsjKeV2xrn6sr8v0L7s9vTnr7exNL+E+84fyOl9Y6yOo9R/aVORLSKTRWSziBSIyKwjvB8gIvNd768UkeQW793n2r5ZRCYd75wi8q5r+3oR+ZuIuP0cPJ9vKCG1ezeSY0KsjqJUp4uLCOLV60aye38dM+bm0Gx3WB1JnYQ23OefE5G1rn+2iMj+Fu/9SUTyRWSjiLwo2hT7/6R270ZsaID2Zbej5ZtLefrfm5kyLE4XnFFu67hFtojYgFeA84E0YLqIpLXa7SagyhjTD3gOeNJ1bBowDUgHJgNzRMR2nHO+CwwEhgBBwM2ndIUdbH9tIyu3V3JuWg+royhlmczkKP44ZTDfbC3niU83WR1HnaC23OeNMb8xxgwzxgwDXgI+cB17OjAOGAoMBkYB4zsxvtsTEcb1dfZl62w8p25LyUHufG8Ng3qG8cTlQ3Wgo3JbbXmSPRooMMYUGmMagXnAlFb7TAHecr1eBEx0PcmYAswzxjQYY7YDBa7zHfWcxpglxgVYBSSc2iV2rGWbSrE7DOel97Q6ilKWmjY6kRvGJvHXb7fz/upiq+OoE9OW+3xL04G5rtcGCAT8gQDAD9CRsK2cmRpL+aFGNuw9YHUUj1ZxqIGfv5lFoL+Nv96QSZC/DnRU7qstRXY80HIN5WLXtiPuY4xpBqqB6GMce9xzutpErgc+O1IoEblVRLJFJLusrKwNl9ExPt9QQo+wAIbqxPdKcf9FaYztE819H6wjZ2eV1XFU27XlPg+AiCQBKcAyAGPMCmA5sNf1z1JjzMYjHOcW92yrHF6o7KvNpRYn8VwNzXZue3s1ZQcb+OtPM4mLCLI6klLH5M4DH+cA/zHGfHOkN40xrxljMo0xmbGx1szqUd9k5+stZfxkUA98fPTjKqX8bD7MuXYEvSICueWtbHZW6GwKXmgasMgYYwcQkX7AIJyfOsYD54jIma0Pcod7tpViugUwNCGc5Zu73i8Y7cEYw33vryN7RxXPXJVBRu8IqyMpdVxtKbJ3A71bfJ3g2nbEfUTEFwgHKo5x7DHPKSIPArHAb9tyEVb5fls5tY12bRVRqoXIEH/+fuMo7MZw45ur2F+ry0l7gLbc5w+bxv+1igBcBvxgjDlkjDkEfAqM7ZCUHm7CgO6s2VmlPxMnYc5X2/hgzW5mntufi4bGWR1HqTZpS5GdBaSKSIqI+OO8wS5utc9i4AbX6yuBZa6e6sXANNfsIylAKs4+66OeU0RuBiYB040xbj1NwdL1JYQG+DK2T7TVUZRyK31iu/Ha9ZkUV9Zx69uraWi2Wx1JHVtb7vOIyEAgEljRYvNOYLyI+Lra/MYD/9UuomDCgFgcBv6zVWcZORGf5O3lqaWbuXRYHDPO6Wd1HKXa7LhFtqvHegawFOeNc4ExJl9EHhGRS1y7vQFEi0gBzqfPs1zH5gMLgA04e6vvMMbYj3ZO17leBXoAK1xTRT3QTtfarhqbHSzdsI9zBnXH39edu26UssbolCiemjqUVdsr+d2iPJy/dyt31Mb7PDiL73nm///HXARsA9YBuUCuMeajToruUTISIogM9tO+7BOwsrCC3yxYy8ikSJ64QmcSUZ7Fty07GWOWAEtabXugxet6YOpRjp0NzG7LOV3b25TJat8VlLO/tolLMvRjK6WOZsqweIqr6nhq6WYSo4KZed4AqyOpozjefd719UNHOM4O3Nah4byEzXHaJVoAABUQSURBVEc4q38sX28uw+EwOpbnODbvO8jN/8imd2QQb/xve3ceHVV9/nH8/WSyQyBhTQhhlT0KYgRBUKs/BfelFaFqcUHc1/bX6tHTVmsX9fjTaq2g1aqIIgoqYBUXVKqyhZ0gS0SWEJAQFlkkkOT7+yPXdsAISZjkzvJ5nTMnd753mefJ3HzzzJ3vvXdknm6ZLhFHh2DraOriYpokxzO4S+ydwCNSGzed1pnhJ+bw5IxCXpmz3u9wRHz1k26tKN2zn2XFO/0OJawV7/iOkc/PJSUhwIvX9CM9NdHvkERqTUV2Hew7UMH7y79haG6mhoqIHIGZ8YeLcjmtW0vue2sp/1q6ye+QRHxzSteWmFXdY0Gqt3PvAa7651z2lJXzwtX9aJuR6ndIInWiCrEOPllZwu6ycs7XUBGRGkkIxPH05SfQt10Gd0xYxGc68UtiVLNGifRtl8EHy3W/nursO1DBdePy+XrrHsZeeQI92zTxOySROlORXQdTFxfTvFGirioiUgspiQGeu+pEOrVsxOhx+SzUzWokRg3tlUlB8bds2KbryAfbX17JTeMXMG/tNh4d1oeBx7TwOySRo6Iiu5Z27N3PB19+w3nHZREf0K9PpDaapiTw0jX9aNE4iatfmMfqb3b5HZJIgxvi3VthesFmnyMJH+UVldzx2kJmrNjCgxfl6qICEhVUJdbSlMXF7C+vZNiJOUdeWER+oFWTZF6+tj+JgTiueG4Oa7fu8TskkQbVrnkqPbKaqMj2VFY6fj1pCf9aupn7zu3B5f3b+x2SSEioyK6lifkb6NWmCb3aNPU7FJGI1a55KuOu7c+BCseIZ2ezrlSFtsSWIb1ak79uOyW7yvwOxVfOOX47ZRmTF2zkrjO7MmpwJ79DEgkZFdm1UFC8k2Ubv2VYno5iixytbplpvHxtf/YdqGDEM7NZX6rxqRI7hvTKxDli+gRI5xwPvvMlL89ez/WnduJW3c1RooyK7Fp4Pb+IxEAcF/bRWDGRUOjZpgkvj+rP3gMVjHh2tk4Ek5jRPTON9s1TeS9Gh4w45/j9lAKe++xrrhrYgbuHdtfdHCXqqMiuoT1l5UxaUMTQ3ExdFF8khHq1acrL1/Znd1k5w59RoS2xwcwYmpvJF4Vb2bZnv9/hNKjKSse9by3jxVnrGDWoI787v6cKbIlKKrJraNKCInbtK+eqkzv4HYpI1MnNbsr4UVWF9qVjZlG4RVcdkeh3Ye9syisd7ywp9juUBlNR6fjNpCW8Mmc9N57WmXvP7aECW6KWiuwaqKx0vPD5WnrnpNO3XYbf4YhEpdzspkwYfRLllY5hY2eztEi3nZbo1iMrja6tG/PWotgosg9UVPLLiYt4fX4Rt53RhV8P6aYCW6KaiuwamLm6hDVb93CNjmKL1KseWU1444YBpCQEGPHsbOasKfU7JJF6Y2ZcdHw289dtj/phUnvKyrn2xXzeWlTM/w7pxl1ndlWBLVFPRXYN/P2Tr8hskszZuVl+hyIS9Tq0aMSkGweS2TSZXzw/lw9j+OoLEv2+v+nK24s2+hxJ/SndXcbPn53NZ6tL+Mslx3LzT3QVEYkNKrKPYNZXpcz9ehs3nNqJxHj9ukQaQmbTZCZeP4BumWmMHpfPi1+s9TskkXrRNiOVfh2a8ebCjTjn/A4n5DZs28vPxsxixeZdjL0yj+H92vkdkkiDUdV4BE98tJpWaUnqGEQaWLNGiUwYfRJn9GjN76YUcP/UAioqo68IEbmkbzZflexhwfrtfocSUgvWb+fiv39B6e4yxo/qz5k9W/sdkkiDUpF9GJ+t3sqsNaVcf2pnkhMCfocjEnNSE+MZc8UJXHNyR/75+VquH5fPnrJyv8MSCanze7ehcVI84+es9zuUkHlzYRHDn5lNamKASTcOJK9DM79DEmlwKrJ/RHlFJQ9MKyCnWQqX99dRbBG/BOKM357fk/sv6MWMFVv46dNf6O6QIWZmQ81spZkVmtnd1cx/zMwWeY9VZrYjaF47M3vfzL40s+Vm1qEhY48GjZLiuej4NkxbsokdeyP7mtmVlY6H31vBna8tpm+7dN6++WS6tE7zOywRX6jI/hGvzl3Pqm92c+85PXQUWyQMjBzYgeevOpFNO/dx3pP/5uMVW/wOKSqYWQB4Cjgb6AmMMLOewcs45+50zvVxzvUBngQmB81+CXjEOdcD6AfojamDn/drz/7ySiYtiNwTIHfs3c91L+Xz90++YkS/HF66pj8ZjXTzNoldKrKrUbR9Lw+9t5KBnZszpFem3+GIiOe0bq2YessgsjNSuebFeTz+4SoqNU77aPUDCp1za5xz+4EJwIWHWX4E8CqAV4zHO+c+AHDO7XbO6WuGOujZpgnHt0tn/Jx1EblPL9qwg3Of+IyZq0u4/4Je/OniY3WxAIl5+gs4RKV3NyrnHA/99Dhdx1MkzLRrnsrkGwdycZ9sHv9wNSP/OZct3+7zO6xIlg1sCHpe5LX9gJm1BzoCM7ymrsAOM5tsZgvN7BHvyLjUwVUDO7CmZA8fRdC3NM45Xvj8ay4d8wUAr98wkJEDO+h/pwgqsn/gsQ9X8XlhKfed15OcZql+hyMi1UhJDPDosN788eJc5q3dxpDHZ/J+wWa/w4oFw4E3nHMV3vN4YDDwK+BEoBNwVXUrmtloM8s3s/ySkpKGiDXinHtsFm0zUnj6k8KIuJzfll37uO6lfH4/dTmndGnJO7cNok9Out9hiYQNFdlB3phfxJMzCrksL4fhJ+b4HY6IHIaZcXn/9ky7dTBt0lMYPW4+90xeqquP1N5GILjDa+u1VWc43lARTxGwyBtqUg68BfStbkXn3DPOuTznXF7Lli1DEHb0iQ/Ecd3gTixYv4N5a8P7cn7vLt3EkMdmMnP1Vu47twfP/iKP9FSNvxYJpiKbqq+7XvxiLf/7xmIGHdOCBy7qpa+6RCLEMa0a8+ZNJ3P9qZ2YMG89Zz02UydF1s48oIuZdTSzRKoK6SmHLmRm3YEMYNYh66ab2fdV8+nA8nqON6oNy8uhWaNEnpyx2u9QqlW6u4w7JizkxvELaJuRyju3DmLU4E7Exel/psihYr7ILtyym+vHzed3Uwo4vVsr/jEyj6R4DSkUiSSJ8XHcc3YP3rhhAKmJAa5+YR63vLKAkl1lfocW9rwj0LcA04EvgYnOuQIze8DMLghadDgwwQWNY/CGjfwK+MjMlgIGPNtw0UeflMQAN57amX+v3srnhVv9Duc/Kisdr8xZz+mPfsq0JZu4/YwuTL5poC7PJ3IYFgnjvo4kLy/P5efn13j5kl1l3PnaIoq272Vt6V5SEwPcdkYXRuvTuEjEKyuvYMwna3jq40KSEuK49fRjGDmwQ9h+eDaz+c65PL/jaEi17bNjzb4DFZzx6KdkNEpgys2DfP+/tLRoJ7+dsoyF63fQv2MzHrwoV8W1xKza9Nnx9R1MOEpNDLC7rJzumU244qT2XHR8Ni0aJ/kdloiEQFJ8gNv/pwvnHpfFH99Zzp/+tYJxs9dxz9k9ODs3U0PBJOwlJwS468yu/PL1xUxaUMSlef6cI7SudA+PTF/JtCWbaN4okf8b1puLj8/W35BIDcXkkWwRiR0zV5Xwx3e+ZOU3u8jNbsItP+nCWT1b+3508Hs6ki3Vqax0XDp2FoVbdvPhXafSMq3hDgQVbd/L2E/X8Orc9SQE4hg1uCOjT+lEWnJCg8UgEq50JFtExHNK15YM7NycyQs38tTHhdzw8ny6Z6Zx42mdOTs3SzfMkLAUF2c89NNjOeevn3HfW0sZc8UJ9X4EeeXmXYz99CveXlyMAcNOzOGOM7rQqklyvb6uSLRSkS0iUS8+EMewvBwuOT6bqUuK+duMQm6fsIg/NP6SEf1yGNGvHW3SU/wOU+Qgx7RK45dndeXP767guc++ZtTgTiF/jbLyCqYXfMOrc9Yza00pKQkBRg7owKjBHfU3IXKUalRkm9lQ4K9AAPiHc+4vh8xPAl4CTgBKgcucc2u9efcA1wIVwG3OuemH26aZdaTqtr7NgfnAld6tfkVEjkp8II6Lj2/Lhb2zmbm6hJdnr+NvHxfy1MeFnNSpOef3bsPQXplkNNL1fiU8jD6lE4s27ODP764gp1kqQ3plHvU2yysqmbt2G+8t28zUxcVs33uAthkp/Oqsrlzev732f5EQOeKYbO8WuauAM6m68cA8YIRzbnnQMjcBxznnbjCz4cDFzrnLzKwnVTcu6Ae0AT6k6ja8/Ng2zWwiMNk5N8HMxgCLnXNPHy5Gje8TkbrasG0vr+dvYOqSTXy9dQ/xccYJ7TMY3KUFg7q05NjspgTqcfy2xmTLkewuK+fK5+awtGgnD//sOC7p27bW2yje8R2zvipl1ppSZqzYwrY9+0lOiOOM7q257MQcBh3TImzOUxAJZ7Xps2tSZA8Afu+cG+I9vwfAOffnoGWme8vMMrN4YDPQErg7eNnvl/NW+8E2gb8AJUCmc6780Nf+MeqwReRoOecoKP6Wd5ZuYuaqEgqKvwWqrkbUM6sJudlN6ZGVRk5GKm0zUslKTyYhcPTjuVVkS03s2neAUS/mM+frbZzfuw23n3EMx7Q6+DJ6lZWO0j372bTzOzZu/44Vm3exYvO3FBR/S9H27wBIT03glC4tOTs3k1O7tSQ1UaNGRWoj1Cc+ZgMbgp4XAf1/bBmvON5J1XCPbGD2Ietme9PVbbM5sMO7OcKhy4uI1BszIze7KbnZTfnN0O5s3V3G54VbWbh+B8s27mRi/gb27q84aJ2UhACNk+NpnBRPIM549bqTGvQqEBI70pITGD+qP0/OKGTszK+YuriYFo2TSEuOZ9+BCvbur2BPWTnllf89cGYGHVs0onfbdK4+uSMDOjWne2aajliLNJCI/QhrZqOB0QDt2rXzORoRiTYtGidxYZ9sLuxT9Tm/otJRvOM7NmzfS9G279i0cx+7yw6wu6yc3WUVVFRWkhBQ8SL1Jz4Qx51nduXyk9rx3rLNLNu4k737K0hJCJCaGCAlMZ7MJklkpaeQnZ5C55aNSUkMz5swicSCmhTZG4HgK+G39dqqW6bIGy7SlKoTIA+3bnXtpUC6mcV7R7Orey0AnHPPAM9A1VePNchDRKTOAnFGTrNUcpqlQme/o5FY1iotmV8M6OB3GCJyBDUZUDgP6GJmHc0sERgOTDlkmSnASG/6Z8AMVzXYewow3MySvKuGdAHm/tg2vXU+9raBt823656eiIiIiEjDO+KRbG+M9S3AdKout/e8c67AzB4A8p1zU4DngHFmVghso6poxltuIrAcKAduds5VAFS3Te8lfwNMMLMHgYXetkVEREREIoZuqy4i4iNdXUREJHLUps/W/YRFREREREJMRbaIiIiISIipyBYRERERCTEV2SIiIiIiIaYiW0REREQkxKLi6iJmVgKsq8OqLYCtIQ7HD8ojvCiP8BLuebR3zrX0O4iGpD5beYQZ5RFewj2PGvfZUVFk15WZ5UfDpbOUR3hRHuElWvKQ6HkvlUd4UR7hJVryAA0XEREREREJORXZIiIiIiIhFutF9jN+BxAiyiO8KI/wEi15SPS8l8ojvCiP8BItecT2mGwRERERkfoQ60eyRURERERCLiaLbDMbamYrzazQzO72Ox4AM3vezLaY2bKgtmZm9oGZrfZ+ZnjtZmZPePEvMbO+QeuM9JZfbWYjg9pPMLOl3jpPmJnVUx45ZvaxmS03swIzuz0SczGzZDOba2aLvTzu99o7mtkc77VfM7NErz3Je17oze8QtK17vPaVZjYkqL3B9kMzC5jZQjObFql5mNla731fZGb5XltE7VdSNw35t1JTpj47rHIx9dlhl4epzwbnXEw9gADwFdAJSAQWAz3DIK5TgL7AsqC2h4G7vem7gYe86XOAdwEDTgLmeO3NgDXezwxvOsObN9db1rx1z66nPLKAvt50GrAK6BlpuXjbbuxNJwBzvNecCAz32scAN3rTNwFjvOnhwGvedE9vH0sCOnr7XqCh90PgLuAVYJr3POLyANYCLQ5pi6j9So86ve/qs9Vn1yQP9dlhlgfqs2OyyB4ATA96fg9wj99xebF04OAOeyWQ5U1nASu96bHAiEOXA0YAY4Pax3ptWcCKoPaDlqvnnN4GzozkXIBUYAHQn6oL5Mcfui8B04EB3nS8t5wdun99v1xD7odAW+Aj4HRgmhdXJOaxlh922BG7X+lR4/ddfbb67NrmoD47PPJYS4z32bE4XCQb2BD0vMhrC0etnXObvOnNQGtv+sdyOFx7UTXt9cr72up4qo4oRFwu3td1i4AtwAdUffrf4Zwrr+a1/xOvN38n0PwIeTTUfvg48Gug0nvenMjMwwHvm9l8MxvttUXcfiW1pj5bfXaNqM8Ouzxivs+O9zsAqRnnnDMz53ccNWVmjYFJwB3OuW+Dh0pFSi7OuQqgj5mlA28C3X0OqdbM7Dxgi3Nuvpmd5nc8R2mQc26jmbUCPjCzFcEzI2W/ktgQafuj+uzwoD47usTikeyNQE7Q87ZeWzj6xsyyALyfW7z2H8vhcO1tq2mvF2aWQFVnPd45N9lrjshcAJxzO4CPqfqaLd3Mvv9wGvza/4nXm98UKKX2+YXaycAFZrYWmEDV149/jcA8cM5t9H5uoeofaD8ieL+SGlOfrT67VtRnh0Ue6rMhJsdkx1M1cL4j/x3038vvuLzYOnDw+L5HOPgEgYe96XM5+ASBuV57M+Brqk4OyPCmm3nzDj1B4Jx6ysGAl4DHD2mPqFyAlkC6N50C/Bs4D3idg08+ucmbvpmDTz6Z6E334uCTT9ZQdeJJg++HwGn89ySaiMoDaASkBU1/AQyNtP1Kjzq99+qz1WfXJA/12WGUB+qzq34PfgfgS9JVZ7Guomq81r1+x+PF9CqwCThA1diia6kaV/URsBr4MGjHMuApL/6lQF7Qdq4BCr3H1UHtecAyb52/4d2IqB7yGETVOKwlwCLvcU6k5QIcByz08lgG/NZr7+T9YRd6nV6S157sPS/05ncK2ta9XqwrCTr7uaH3Qw7usCMqDy/exd6j4PvXibT9So86v//qs9VnHykP9dlhlAfqs3HO6Y6PIiIiIiKhFotjskVERERE6pWKbBERERGREFORLSIiIiISYiqyRURERERCTEW2iIiIiEiIqcgWEREREQkxFdkiIiIiIiGmIltEREREJMT+H9xaWpAxy0/GAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot_losses()\n",
    "learn.recorder.plot_lr(show_moms=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('fine_tuned60kb_bwd')\n",
    "learn.save_encoder('fine_tuned_enc60kb_bwd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('fine_tuned60kb_bwd')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build/train classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labels</th>\n",
       "      <th>text</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Another trashy Grade Z quickie from the prolif...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>This show stinks. For parents, they usually wa...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>Just two comments....SEVEN years apart? Hardly...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>SOME NOT-SO-SPOILY SPOILERS AHEAD&lt;br /&gt;&lt;br /&gt;W...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>This is a very funny movie, easy to watch, tha...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    labels                                               text  rating\n",
       "0        0  Another trashy Grade Z quickie from the prolif...       1\n",
       "1        0  This show stinks. For parents, they usually wa...       2\n",
       "5        1  Just two comments....SEVEN years apart? Hardly...       7\n",
       "8        0  SOME NOT-SO-SPOILY SPOILERS AHEAD<br /><br />W...       4\n",
       "10       1  This is a very funny movie, easy to watch, tha...       7"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CLS_train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "imdb_data_clas = TextClasDataBunch.from_df(CLAS_PATH,CLS_train_df,CLS_valid_df, tokenizer=tokenizer,\n",
    "                                           vocab=imdb_data_lm.train_ds.vocab,label_cols='labels',text_cols='text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "del learn\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList\n",
       "y: CategoryList (25000 items)\n",
       "[Category 0, Category 0, Category 1, Category 0, Category 1]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas_bwd\n",
       "x: TextList (25000 items)\n",
       "[Text xxbos \n",
       "\n",
       " ) * 4 xxrep of ( 0 \n",
       "\n",
       " . possible that´s if , worse even reportedly is ´ toys demonic vs. dollman ´ sequel ? it xxunk , movies these watch we why that´s but , bad laughably . shot second every in reappears and disappears sunglasses xxunk how note : too , guy continuity the fired have should they . woman ft 50 the of attack since effects cheesiest the with , dull xxunk but , short mercifully ! bronx south in war gang a in involved gets and earth to ) ! ( head flying ugly an chases space outer from cop like - eastwood clint inch 13 xxunk tim . pyun albert prolific the from quickie z grade trashy another, Text xxbos . up grow to how know never 'll they . it see to kids your want n't wo you 2 of age the past but , entertaining 's it , kids for \n",
       "\n",
       " . overkill 's it . corny very and exaggeration of level highest the to said is syllable every . cheesy extremely also is show the \n",
       "\n",
       " . sad being to exposed been never have they because depressed become will trouble in kid a . happy always are they , therefore . 's xxunk the of world the in wrong ever is nothing xxup but . good 's that so , bright extremely is everything s , color as far as . painful 's it , bright so 's it ? bright it is \n",
       "\n",
       " . it at laugh and , joke a as intended not is which something say instead doodlebops the . no ? funny it is \n",
       "\n",
       " . anything teaching try n't do they , themselves about sing usually they . it 's that and sing doodlebops the . no ? educational it is \n",
       "\n",
       " . bright and , funny , educational usually is it . them for good something watch to kids their want usually they , parents for . stinks show this, Text xxbos ! fireworks expect n't do just . film the for price acceptable an be would xxunk $ . expenditure the begrudge not can i , store video regular my at week last xxunk $ mere a for up dvd xxup the picked having \n",
       "\n",
       " . moments key at particularly viewer the engage to fails just it . standard high a of overall production the and average than better is script the . suspect main the as \" house \" the with murders unconnected seemingly of couple a xxunk xxunk played had she if impression worthwhile more a made have might gibney ! intrigue of case a more . horror much without only - horror xxup amityville xxup the xxup of working - re a is basic most its at plot the , true \n",
       "\n",
       " . se per outing fright a as or entertainment as either - merit without not is thing the however that said having . appeal limited its in factor mitigating a is row xxup gantry xxup 13 of status telemovie budget - low the , mentioned been has as ! power - pulling relentless 's film the of evidence hardly ? apart years seven xxup . 4 xxrep comments two just, Text xxbos . wait ... wait n't ca you and movie interesting an like look it made trailer the thought you if . ticket matinée a worth 's it guess i , eh , movie a watch to want really you and theater the at better nothing 's there if . once least at seeing worth 's it , fan thriller hard - die a 're you if \n",
       "\n",
       " . it of 3 / 2 last the until lost almost was i so and on moved it as explanation enough n't was there except interesting it make to enough almost 's it and progresses movie the as out given slowly are plot the of details the . film the of most through \" ? huh \" of kind looks and drab really is gellar michelle sarah . clichés worn - well the from aside ... tolerable was one this thought i . movies \" scary \" the all screening of point a make and theater movie a at work i because free it saw i \" . return the \" about talking 'm i obviously , page \" return the \" the is this since but , years few last the in out come 's that thriller any almost about talking be could i \n",
       "\n",
       " ? structure decrepit other or barn old ratty a to , course of quietly , run ? implements metal among or platform wooden a under hide and off sneak quietly ? do you would what , murderer a was discovered just you someone by pursued being were you if \n",
       "\n",
       " ? stuff their through going start then and 's hello few a shout , in walk , door the open you , answer immediately xxup n't do they when and door 's stranger a on knock you do often how \n",
       "\n",
       " ? room their into breaks someone when bed a under hide would mind right their in who \n",
       "\n",
       " ? bathroom the to way their on floor dance crowded the of middle the through cut , club a at scared or sick or disoriented are they when , people do why \n",
       "\n",
       " ahead xxup spoilers xxup xxunk xxup so- - not xxup some xxup, Text xxbos . lately coming is good little the considering , filmmakers argentinian for \" bravo \" a is that add must i remark personal a on . end the at down little a gets it because 7 rated i but , 8 an deserved would it , overall good very is . anywhere laughable and understandable : worldwide hilarious are dialogs the of some . ) him on cheated wife his because troubled is diaz ( him assist also he while assignments his on ) luque by played ( diaz detective with go to has and peretti by played psychologist a is them of one that is twist the . sort argentinian of are comedy and jokes the but , weapon lethal like history partners police typical a is movie the . trademark his is humor of type the and recognizable is director the of work the . time the all almost you entertains that , watch to easy , movie funny very a is this]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas_bwd;\n",
       "\n",
       "Valid: LabelList\n",
       "y: CategoryList (25000 items)\n",
       "[Category 0, Category 0, Category 1, Category 0, Category 1]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas_bwd\n",
       "x: TextList (25000 items)\n",
       "[Text xxbos . film the out thru bigger more made have should they but good was that and the until with threesome a have to chance a got never he but . women other with him on cheating is who wife hot a got who sans horatio is for sorry felt i person only the . nasty very some and parts some at funny was it . it about commercials the seeing after movie ok a be will it thought i, Text xxbos . disappointment sad a . life of essence the for looking man young a of journey pretentious rather a into disintegrates it then but , ok were minutes 10 first the , well . standard similar a of be to film this expecting therefore was and \" neuf pont du amants les \" 's carax enjoyed i, Text xxbos . recommend xxup hearty a and 9 a , all in all . ) worthwhile pretty are gang the in guys the ( slant female slight a has it maybe so , girl a 'm i , yes and . ) bragging not 'm i but , surround in appreciated more much ( also design sound good . status cool super to it lifted , attention paying was director the means likely most which , cast great the but unremarkable was itself story the thought i . clearly , naysayers the with disagree i . scary , funny , out thought well , insulting never . get can you as find video a nifty as about it found and it about anything knowing not this rented i \n",
       "\n",
       " ? man gingerbread the liked somebody and ? type against playing fun having for rickman alan and thompson emma knocking ? quirky too xxup ? \" unforgettable \" and \" rounders \" from ? dahl john ? people you with matter the 's what, Text xxbos . it said he when \" even \" superfluous the use n't did xxunk down - watered 's it \" . it of rid get to glad was he even bet 'll i and . boy old - year - six a of brain the got 've you , sonny \" : line 's woolsey by illustrated best is comedy of level the but , laughs few a has \" nitwits the \" \n",
       "\n",
       " . bars the through other each to sing grable and wheeler . floor high a on cell jail a in is who , mary to talk and see can they that so stilts on are woolsey and wheeler , later . ) down back dance they which after ( staircase a up walking it sing they , time first the . mchugh jimmy and fields dorothy by written \" , heart my in music \" excellent the singing grable betty and wheeler feature both and , good are scenes two but . plot mystery murder tedious a with comedy dismal this directed , career distinguished a have to on went who , stevens george . one this not \n",
       "\n",
       " . funny were them of some and 30s the in together movies several made they but , pairing arbitrary an as me strike woolsey chomping - cigar the and wheeler voiced - baby the \n",
       "\n",
       " . killer real the reveal to machine truth his and newton to up 's it . him from stealing is everyone thinks who another and him from stealing is lake thinks who songwriter a , auditor 's lake , ) best willie ( janitor witted - slow seemingly a , detective private satisfied - self a , wife suspicious 's lake : are mystery murder wacky this in characters other the ? him threatened had who blackmailer the , widow black the is who ? lake shot really who but . jail to goes who , mary implicate to again conspire they . killer the is johnnie that think newton and mary make to conspire circumstances , dead up turns lake when . sweetheart 's johnnie is , ) grable betty ( mary , secretary 's lake . building the owns also who publisher song a , lake winfield to song a sell to wants johnnie . building office an of lobby the in stand cigar a at work both . inventor be - would a is ) woolsey robert ( newton ; songwriter be - would a is ) wheeler bert ( johnnie, Text xxbos . hanky a take and . film serious a . scenes quirky wonderfully some and , really else everyone and , hurt and streep by acting great . it respect and it love to movie a with agree to have n't do i : okay 's that but , face the in slapped felt i , forward moving and life 's one of charge taking of idea the by excited more is who one as . anyway smiling keep and , is it as life your accept simply to best be might it that suggest to guts the has film this]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas_bwd;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchRNNCore(\n",
       "    (encoder): Embedding(60004, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(60004, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=800, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (1): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (2): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (3): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=1200, bias=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "      (3): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=<fastai.layers.FlattenedLoss object at 0x7fa1985c13c8>, metrics=[<function accuracy at 0x7fa211f38620>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('/home/farzin/rnn_python_code/wiki103_from_download/clas_bwd'), model_dir='models', callback_fns=[<class 'fastai.basic_train.Recorder'>], callbacks=[RNNTrainer\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList\n",
       "y: CategoryList (25000 items)\n",
       "[Category 0, Category 0, Category 1, Category 0, Category 1]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas_bwd\n",
       "x: TextList (25000 items)\n",
       "[Text xxbos \n",
       "\n",
       " ) * 4 xxrep of ( 0 \n",
       "\n",
       " . possible that´s if , worse even reportedly is ´ toys demonic vs. dollman ´ sequel ? it xxunk , movies these watch we why that´s but , bad laughably . shot second every in reappears and disappears sunglasses xxunk how note : too , guy continuity the fired have should they . woman ft 50 the of attack since effects cheesiest the with , dull xxunk but , short mercifully ! bronx south in war gang a in involved gets and earth to ) ! ( head flying ugly an chases space outer from cop like - eastwood clint inch 13 xxunk tim . pyun albert prolific the from quickie z grade trashy another, Text xxbos . up grow to how know never 'll they . it see to kids your want n't wo you 2 of age the past but , entertaining 's it , kids for \n",
       "\n",
       " . overkill 's it . corny very and exaggeration of level highest the to said is syllable every . cheesy extremely also is show the \n",
       "\n",
       " . sad being to exposed been never have they because depressed become will trouble in kid a . happy always are they , therefore . 's xxunk the of world the in wrong ever is nothing xxup but . good 's that so , bright extremely is everything s , color as far as . painful 's it , bright so 's it ? bright it is \n",
       "\n",
       " . it at laugh and , joke a as intended not is which something say instead doodlebops the . no ? funny it is \n",
       "\n",
       " . anything teaching try n't do they , themselves about sing usually they . it 's that and sing doodlebops the . no ? educational it is \n",
       "\n",
       " . bright and , funny , educational usually is it . them for good something watch to kids their want usually they , parents for . stinks show this, Text xxbos ! fireworks expect n't do just . film the for price acceptable an be would xxunk $ . expenditure the begrudge not can i , store video regular my at week last xxunk $ mere a for up dvd xxup the picked having \n",
       "\n",
       " . moments key at particularly viewer the engage to fails just it . standard high a of overall production the and average than better is script the . suspect main the as \" house \" the with murders unconnected seemingly of couple a xxunk xxunk played had she if impression worthwhile more a made have might gibney ! intrigue of case a more . horror much without only - horror xxup amityville xxup the xxup of working - re a is basic most its at plot the , true \n",
       "\n",
       " . se per outing fright a as or entertainment as either - merit without not is thing the however that said having . appeal limited its in factor mitigating a is row xxup gantry xxup 13 of status telemovie budget - low the , mentioned been has as ! power - pulling relentless 's film the of evidence hardly ? apart years seven xxup . 4 xxrep comments two just, Text xxbos . wait ... wait n't ca you and movie interesting an like look it made trailer the thought you if . ticket matinée a worth 's it guess i , eh , movie a watch to want really you and theater the at better nothing 's there if . once least at seeing worth 's it , fan thriller hard - die a 're you if \n",
       "\n",
       " . it of 3 / 2 last the until lost almost was i so and on moved it as explanation enough n't was there except interesting it make to enough almost 's it and progresses movie the as out given slowly are plot the of details the . film the of most through \" ? huh \" of kind looks and drab really is gellar michelle sarah . clichés worn - well the from aside ... tolerable was one this thought i . movies \" scary \" the all screening of point a make and theater movie a at work i because free it saw i \" . return the \" about talking 'm i obviously , page \" return the \" the is this since but , years few last the in out come 's that thriller any almost about talking be could i \n",
       "\n",
       " ? structure decrepit other or barn old ratty a to , course of quietly , run ? implements metal among or platform wooden a under hide and off sneak quietly ? do you would what , murderer a was discovered just you someone by pursued being were you if \n",
       "\n",
       " ? stuff their through going start then and 's hello few a shout , in walk , door the open you , answer immediately xxup n't do they when and door 's stranger a on knock you do often how \n",
       "\n",
       " ? room their into breaks someone when bed a under hide would mind right their in who \n",
       "\n",
       " ? bathroom the to way their on floor dance crowded the of middle the through cut , club a at scared or sick or disoriented are they when , people do why \n",
       "\n",
       " ahead xxup spoilers xxup xxunk xxup so- - not xxup some xxup, Text xxbos . lately coming is good little the considering , filmmakers argentinian for \" bravo \" a is that add must i remark personal a on . end the at down little a gets it because 7 rated i but , 8 an deserved would it , overall good very is . anywhere laughable and understandable : worldwide hilarious are dialogs the of some . ) him on cheated wife his because troubled is diaz ( him assist also he while assignments his on ) luque by played ( diaz detective with go to has and peretti by played psychologist a is them of one that is twist the . sort argentinian of are comedy and jokes the but , weapon lethal like history partners police typical a is movie the . trademark his is humor of type the and recognizable is director the of work the . time the all almost you entertains that , watch to easy , movie funny very a is this]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas_bwd;\n",
       "\n",
       "Valid: LabelList\n",
       "y: CategoryList (25000 items)\n",
       "[Category 0, Category 0, Category 1, Category 0, Category 1]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas_bwd\n",
       "x: TextList (25000 items)\n",
       "[Text xxbos . film the out thru bigger more made have should they but good was that and the until with threesome a have to chance a got never he but . women other with him on cheating is who wife hot a got who sans horatio is for sorry felt i person only the . nasty very some and parts some at funny was it . it about commercials the seeing after movie ok a be will it thought i, Text xxbos . disappointment sad a . life of essence the for looking man young a of journey pretentious rather a into disintegrates it then but , ok were minutes 10 first the , well . standard similar a of be to film this expecting therefore was and \" neuf pont du amants les \" 's carax enjoyed i, Text xxbos . recommend xxup hearty a and 9 a , all in all . ) worthwhile pretty are gang the in guys the ( slant female slight a has it maybe so , girl a 'm i , yes and . ) bragging not 'm i but , surround in appreciated more much ( also design sound good . status cool super to it lifted , attention paying was director the means likely most which , cast great the but unremarkable was itself story the thought i . clearly , naysayers the with disagree i . scary , funny , out thought well , insulting never . get can you as find video a nifty as about it found and it about anything knowing not this rented i \n",
       "\n",
       " ? man gingerbread the liked somebody and ? type against playing fun having for rickman alan and thompson emma knocking ? quirky too xxup ? \" unforgettable \" and \" rounders \" from ? dahl john ? people you with matter the 's what, Text xxbos . it said he when \" even \" superfluous the use n't did xxunk down - watered 's it \" . it of rid get to glad was he even bet 'll i and . boy old - year - six a of brain the got 've you , sonny \" : line 's woolsey by illustrated best is comedy of level the but , laughs few a has \" nitwits the \" \n",
       "\n",
       " . bars the through other each to sing grable and wheeler . floor high a on cell jail a in is who , mary to talk and see can they that so stilts on are woolsey and wheeler , later . ) down back dance they which after ( staircase a up walking it sing they , time first the . mchugh jimmy and fields dorothy by written \" , heart my in music \" excellent the singing grable betty and wheeler feature both and , good are scenes two but . plot mystery murder tedious a with comedy dismal this directed , career distinguished a have to on went who , stevens george . one this not \n",
       "\n",
       " . funny were them of some and 30s the in together movies several made they but , pairing arbitrary an as me strike woolsey chomping - cigar the and wheeler voiced - baby the \n",
       "\n",
       " . killer real the reveal to machine truth his and newton to up 's it . him from stealing is everyone thinks who another and him from stealing is lake thinks who songwriter a , auditor 's lake , ) best willie ( janitor witted - slow seemingly a , detective private satisfied - self a , wife suspicious 's lake : are mystery murder wacky this in characters other the ? him threatened had who blackmailer the , widow black the is who ? lake shot really who but . jail to goes who , mary implicate to again conspire they . killer the is johnnie that think newton and mary make to conspire circumstances , dead up turns lake when . sweetheart 's johnnie is , ) grable betty ( mary , secretary 's lake . building the owns also who publisher song a , lake winfield to song a sell to wants johnnie . building office an of lobby the in stand cigar a at work both . inventor be - would a is ) woolsey robert ( newton ; songwriter be - would a is ) wheeler bert ( johnnie, Text xxbos . hanky a take and . film serious a . scenes quirky wonderfully some and , really else everyone and , hurt and streep by acting great . it respect and it love to movie a with agree to have n't do i : okay 's that but , face the in slapped felt i , forward moving and life 's one of charge taking of idea the by excited more is who one as . anyway smiling keep and , is it as life your accept simply to best be might it that suggest to guts the has film this]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas_bwd;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchRNNCore(\n",
       "    (encoder): Embedding(60004, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(60004, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=800, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (1): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (2): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (3): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=1200, bias=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "      (3): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=<fastai.layers.FlattenedLoss object at 0x7fa1985c13c8>, metrics=[<function accuracy at 0x7fa211f38620>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('/home/farzin/rnn_python_code/wiki103_from_download/clas_bwd'), model_dir='models', callback_fns=[<class 'fastai.basic_train.Recorder'>], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(60004, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(60004, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=800, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=1200, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       ")])\n",
       "bptt: 140\n",
       "alpha: 2.0\n",
       "beta: 1.0\n",
       "adjust: False], layer_groups=[Sequential(\n",
       "  (0): Embedding(60004, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(60004, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=800, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=1200, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       ")])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb_data_clas.batch_size = 16\n",
    "learn_cls = text_classifier_learner(imdb_data_clas,bptt,emb_sz,nh,nl,drop_mult=0.5,qrnn=True)\n",
    "learn_cls.load_encoder('fine_tuned_enc60kb_bwd')\n",
    "learn_cls.freeze()\n",
    "learn_cls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_find(learn_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VNXdx/HPL5ONLawJhoR9B9kjiLjhgqCP4F6oFhVb616rdWufx11rXSpubUXFvVorrQW1ota6gCCb7Mi+JexLgCRkm5znjxlwgJAJMJM7Sb7v1+u+mLvl/g6TmV/OPeeeY845REREKhLndQAiIhL7lCxERCQsJQsREQlLyUJERMJSshARkbCULEREJCwlCxERCUvJQkREwlKyEBGRsOK9DiBSmjVr5tq0aeN1GCIi1crs2bO3OedSwx1XY5JFmzZtmDVrltdhiIhUK2a2tjLH6TaUiIiEpWQhIiJhKVmIiEhYShYiIhKWkoWIiISlZCEiImEpWYiISFg15jkLibyiUj/ZO/eyaVchG3cVsmVPIQPbNaVPq8ZehyYiVUzJQg7hnGPS/I089OFitu4pOmT/pf0yuWtYF5rVT/IgOhHxgpKFHGDNtnz+718L+Wb5NnpmNuSeYV1o0agO6Q2TaZCcwLivV/HKlFVMXrSJ24d05ooTW+OLM6/DFpEoM+ec1zFERFZWltNwH0fm+3U7+XbldvYUlpJXVEJuQQmfLt5Moi+OO845fCJYsSWP+ycuYsqKbZzVNY3nf9qX5ASfByUQkWNlZrOdc1lhj1OyqH225RXx2L9/4P3Z2QAk+uKonxxP/aR4+rZqxD3ndqV5SnKFP8M5xxvT1nL/pEX0b9OEl6/MokFyQlWELyIRVNlkodtQtUhRqZ+/zVzPk5OXUlDs57rT2nPD4PakHMWXvJlx5UltaFQ3gdvfm8eol6bz+tX9aap2DJEaScmihnLOMXvtTj5dvJmVW/JYuTWP9Tv34i9zDOrQlAeGd6dDWoNjvs6I3hmkJCdw3VuzufTFabx1zQBaNKoTgRKISCzRbagaZm+xn3/NzeH1aWtZsnE3ifFxtGtWj/ap9WmfWo8+rRtzeqdUzCLbKD1zzQ7GvDqTlDoJvP3zAbRpVq9S5+3aW8K0ldv4atk29hSWMHpgG/q3bRLR2ETk8NRmUQvNWrODa16fxa69JXQ5rgGjB7bhgj4tqJtYNRXIhTm7GD1+Br44461rBtD5uANrLs45Nu0uZN76XL5fn8usNTuZuz4Xf5mjflI8CT5jZ0EJWa0bc8Pg9gzunBbxpCYiB1KyqGWKSv0MG/sNxf4ynrq0F/3bNvHki3b55j1c8cp3FJWW8frV/WlUN4FpK7fz7crtfLd6O5t3B57bSPAZ3Vs05JSOzTi1Uyq9Wzai1O94b9Z6xn29ipzcvTSqm0CdBB8JvjjifcZxKclktWnCCW0a07dVY+ol6S6qyLFSsqhlxn6+jLGfL+f1Mf05rVPYGRKjat32Ai5/ZTrZO/ey79crrUESJ7ZrSt9WjejVshHdWqSQFF9+d9sSfxkT525g9rqdlJSWUVrmKC4tY832fJZs3E2ZA1+ckdW6MZf0y+TcHulKHCJHScmiFlm5NY9hY79h6PHH8eyoPl6HA8CmXYW8MmUVrZrUZWD7ZrRPrReRms6ewhK+X5fLjNU7+GjBRlZvy6duoo+hxx/HL09tf8itLxGpmJJFDbEhdy/5RaUkJ/hITvBRL8l3QBuEc45RL01n8YbdfH77aaQ1qPj5iJrEOcecdTt5f3YOH87bQFFpGXcN68LVJ7UhTk+Vi1SKnrOoAbJ3FnDGk19R7C87YHtW68Zc1DeT83qk89mSzUxftYNHL+xRqxIFBJ716Ne6Cf1aN+E3Qzpx14QFPPThYr5cuoUnL+0V9sFCEak81Sxi2MMfLua1b9fw2MU9cc5RWFrGtj1FfLRgIyu25JEYH0d8nNE1PYW//3Jgrf9r2jnHX2es46EPF1MnwceLP8tSN1yRMHQbqprbU1jCwN9/wZld03hm5IHtEM45FubsZsKcbGat3cHTl/WmY3Pdq99nxZY8rn0j0IX4o1tO4biGqmGIHI5uQ1Vzf5u5nryiUq45ue0h+8yMHpkN6ZHZ0IPIYl+HtPqMG92PEc9P5Ya3Z/PutQNJjNc8XyLHQp8gDznneHDSYl6buvqA7aX+Ml6duob+bZvQM7ORR9FVbx3SGvCHS3oyZ10uj368xOtwRKo91SzCyMndyxvT1nDj4A5HNeBeRT5ZuInxwURRJ9HHT05oBcDkRZvJyd3Lved3i+j1apv/6dmCOWtzGT91NX1aNWJE7wyvQxKptpQsKpBbUMyV42ewYkseSb44bhvSOWI/u6C4lAc/XEzX9BRSGyTx238uJK1BMoO7pPHylFW0blqXs7o2j9j1aqt7zu3C/Oxc7p6wgLXbC+jTqhE9MxvRsI6GUxc5ElFNFmY2FHgG8AEvO+ceO2j/08Dg4GpdIM051yi4zw8sCO5b55wbHq04P/g+hzO7ph0wH0NhiZ9r35jNuu0F9MhoyKtT13DNKe0i9iXz3Bcr2LirkOdG9aFLegojx03jhrfncOfQzny/LpcHhnfXDHQRkOCL44XL+3LtG7P442fL9m/v1Lw+vz23K6d3TjvknHXbC5g0fwPOOeJ9gR5nLZvUZUi35hqrSmqtqPWGMjMfsAw4G8gGZgKjnHOLD3P8zUAf59yY4Hqec65+Za93tL2hVm/LZ/CTX9K4bgLXn96e0QPbkOiL46Z35vDxgk08N6oP7VLrcd6zU7jt7E7ccmbHI77GwVZsyWPYM18zvFcGT13WC4Atewq5+M/fsn7HXlKS45l2z5kawiLCdheWMH/9Lr5ft5OJ8zawfEse153WntuHdCLBF0dZmeO1b9fwxOSl7C3xH3L+xX0zeeTC4zUroNQonnedNbOBwP3OuXOC6/cAOOd+f5jjvwXuc859FlyvkmQBsCB7F09+upSvlm2leUoSPTIa8vmSLfzveV35+SntAPj567OYuWYHU+4afEwzwjnn+NkrM5iXncsXt59OaoMfJwtatTWPUS9NZ/TANtw4uMNRX0PCKyzx89CHi3n7u3X0bdWI3wzpzNOfL2Pmmp0M7pzKIxf2ILVBEqV+R0lZGeOnrGbs58vp3bIR437WjzQ98Cc1RCwki0uAoc65nwfXfwYMcM7dVM6xrYHpQKZzzh/cVgrMBUqBx5xzH5Rz3rXAtQCtWrXqt3bt2mOK+btV23l88lJmr93JmEFtD2hgnp+dy/Dnp3LHOZ2P6Yv84wUbueHtOTwwvDtXntTmkP3+MqfbT1Xow/kbuGfCAvYUlZKSHM9953fnor4Z5d5u+mThRm57bx4NkuN58WdZ9G6pnmpS/VW3ZHEXgURxc8i2DOdcjpm1A74AznTOrTzc9SL1UJ5zjlXb8mnX7NCB7656dQbzs3fxzZ2DD3uLaOXWPAqK/OU+A5FbUMyQp7+mWf0kJt40iHifei7HgrXb85kwJ4fLB7QKO0TID5t284s3ZrF5VxEPjOjOyBNaqh1DqrXKJotoflvlAC1D1jOD28ozEngndINzLif47yrgS6BKhlM1M9qn1i/3C+DmMzqyI7+Yt787sAaTW1DMm9PWMOKFqZz51Fdc+KepTF+1/ZDz75+4iB35xTx+SU8lihjSumk9bju7U6XGkupyXAoTbzyZE9s35Z5/LODO9+dTWE77hkhNE82aRTyBBu4zCSSJmcBPnXOLDjquC/AJ0NYFgzGzxkCBc67IzJoB04ARh2sch6ob7uOKl79j3vpcOjavT36Rn7yiUrbsKaTE7+hyXAMu6pvBuzPXk1tQwsSbBpHZuC4QeKbiurdmc+tZHbn1rE5Rj1Oiy1/meOY/y3n2P8vplp7CX67oR6umdb0OS+SIeX4bKhjEucBYAl1nxzvnHjGzB4FZzrmJwWPuB5Kdc3eHnHcS8CJQRqD2M9Y590pF16qqZLEwZxcPTFpEYnwc9RLjqZ8UT/OGyfxPz3S6twjcelq5NY8LXphKy8Z1mXD9SRQUlzLk6a9Jb5TMP28YRIJqFTXGf3/Ywq/e/Z4EXxxv/XwAXdNTvA5J5IjERLKoSrE2kOB/f9jCmNdncl6PdMqc4/PFW5h088manKcGWrU1j8tf/o69JX7eGNNfQ7RItRILbRa12uAuadx5Thc+nL+Rjxds4tazOypR1FDtUuvz3i8HUj8pnstf+o7Za3d4HZJIxClZRNF1p7Xj8gGtOKNLGtcGn9eQmqllk7q898uBNGuQxM9emcHUFdu8DkkkonQbSiSCtuwu5IpXvmPFljxuH9KZ609rX+snpZLYpttQIh5IS0lmwvUncW6PdJ6YvJSrX5vJjvxir8MSOWZKFiIR1iA5gedG9eGhC45n2srtnPfsNyzI3uV1WCLHRMlCJArMjJ+d2Jp/3HASAL9+by6l/jKPoxI5ekoWIlF0fEZDHhjenRVb8nhn5nqvwxE5akoWIlF2drfmDGjbhLGfLWNPYYnX4YgcFSULkSgzM/73vG5szy/mz18edixMkZimZCFSBXpkNuSiPhm8MmU1Obl7vQ5H5IgpWYhUkd+cE5jD/YlPfvA4EpEjp2QhUkVaNKrDz09pywdzNzBvfa7X4YgcESULkSp0/ekdaFovkccnq3Yh1YuShUgVqp8Uz42DOzB1xXaNHyXVipKFSBW7/MRWtGiYzOOTl1JTxmaTmk/JQqSKJcX7uPWsTsxbn8unizd7HY5IpShZiHjgor4ZtEutx5OTl+IvU+1CYp+ShYgH4n1x3H52Z5ZvyeNfc3O8DkckLCULEY8MO/44js9I4enPl1FcqkEGJbYpWYh4JC7O+M2QzqzfsZcJc7K9DkekQkoWIh46rVMq3Vuk8OrU1eoZJTFNyULEQ2bGlSe1YdnmPKat2u51OCKHpWQh4rHhvVrQpF4ir01d43UoIoelZCHiseQEHyNPaMnnSzazfkeB1+GIlEvJQiQGXHFia8yMt6av9ToUkXIpWYjEgBaN6nBO9+a8O3M9e4v9XocjcgglC5EYceXANuzaW8IHekhPYpCShUiM6N+2CV3TU3j92zXqRisxR8lCJEaYGVed1JofNu1h6gp1o5XYomQhEkNG9M4gvWEyT0z+QbULiSlKFiIxJDnBx6/P7sS87F18vGCT1+GI7KdkIRJjLu6bSafm9Xli8g+U+DXAoMQGJQuRGOOLM+4a2oU12wt4d+Z6r8MRAZQsRGLSGV3S6N+2Cc98vpz8olKvwxGJbrIws6FmttTMVpjZ3eXsf9rM5gaXZWaWG7LvSjNbHlyujGacIrHGzLh7WBe25RXx8jervQ5HJHrJwsx8wAvAMKAbMMrMuoUe45z7tXOut3OuN/Ac8I/guU2A+4ABQH/gPjNrHK1YRWJR31aNGXb8cYz7eiXb84q8DkdquWjWLPoDK5xzq5xzxcC7wIgKjh8FvBN8fQ7wmXNuh3NuJ/AZMDSKsYrEpNuHdCK/2M/b363zOhSp5aKZLDKA0Na57OC2Q5hZa6At8MWRnGtm15rZLDObtXXr1ogELRJLOqQ14LROqbw5fa2mXhVPxUoD90jgfefcEY2g5pwb55zLcs5lpaamRik0EW+NObktW/cU8dGCDV6HIrVYNJNFDtAyZD0zuK08I/nxFtSRnitSo53asRntU+vx6lSNGSXeiWaymAl0NLO2ZpZIICFMPPggM+sCNAamhWyeDAwxs8bBhu0hwW0itY6ZcfWgtszP3sXstTu9DkdqqaglC+dcKXATgS/5JcB7zrlFZvagmQ0POXQk8K4L+ZPJObcDeIhAwpkJPBjcJlIrXdQ3g5TkeF7V1Kvikfho/nDn3MfAxwdtu/eg9fsPc+54YHzUghOpRuomxjNqQCte/mY1Obl7yWhUx+uQpJaJlQZuEQlj9MA2ALwxbY2XYUgtpWQhUk1kBKdefee7deRpCBCpYkoWItXIdae1Z3dhKX/67wqvQ5FaRslCpBrpmdmIi/pk8PKU1azfUeB1OFKLKFmIVDN3DO2Mz4zH/v2D16FILaJkIVLNpDesw3WnteejBRuZsVo9yqVqKFmIVEPXntqO9IbJPPjhIsrK9FS3RJ+ShUg1VCfRx93DurAwZzcT5mR7HY7UAkoWItXU8F4t6NOqEY9PXsruwhKvw5EaTslCpJoyM+4/vzvb8op4avJSr8ORGk7JQqQa69WyEaNPbM0b09cyb31u+BNEjpKShUg1d/s5nUmtn8Rv/7mAUr8mSJLoULIQqeZSkhO47/zuLNqwm9e+XeN1OFJDKVmI1ADn9jiO0zun8sfPlrEhd6/X4UgNpGQhUgOYGQ+NOJ4y57h/4iKvw5EaSMlCpIZo2aQuN5/RkU8Xb2bayu1ehyM1jJKFSA1yzcltSW+YzO//vURPdktEKVmI1CDJCT5uH9KZ+dm7mDR/g9fhSA2iZCFSw1zYJ4Ou6Sk8MXkpRaV+r8ORKFu0YRfzs6P/jE2lkoWZtTezpODr083sFjNrFN3QRORo+OKM357bheyde3nj27VehyNRtGZbPleOn8Fv/j4Pf5RvO1a2ZjEB8JtZB2Ac0BL4a9SiEpFjckrHVE7tlMpzXywnt6DY63AkCjbvLuSKV77DX+b40+X98MVZVK9X2WRR5pwrBS4EnnPO3QGkRy8sETlW9wzrwp6iUp7/QlOw1jS7CkoY/coMduYX89rV/emQVj/q16xssigxs1HAlcCHwW0J0QlJRCKha3oKl/TN5I1pa8nRg3o1xt5iP2Nen8nqbfmMG51Fr5ZV0yJQ2WRxNTAQeMQ5t9rM2gJvRi8sEYmEW8/uBAZPf7bM61AkQp75z3LmrNvJ2JG9GdShWZVdt1LJwjm32Dl3i3PuHTNrDDRwzv0hyrGJyDHKaFSH0Se25h9zslm2eY/X4UgErNuRT7tm9Ti3R9W2BFS2N9SXZpZiZk2AOcBLZvbH6IYmIpFw4+AO1EuM50nNeVEj5Bf5qZcUX+XXrextqIbOud3ARcAbzrkBwFnRC0tEIqVxvUSuPbUdny7ezOy1O70OR45RflEp9RJjN1nEm1k6cBk/NnCLSDUx5uS2NKufxB8++QHnNAxIdZZXVBrTNYsHgcnASufcTDNrByyPXlgiEkn1kuK55cwOzFi9gy+XbfU6HDkGBcV+6iX5qvy6lW3g/rtzrqdz7vrg+irn3MXRDU1EImnkCa1o2aQOYz9frtpFNZYfyzULM8s0s3+a2ZbgMsHMMqMdnIhETmJ8HL84pR3z1ucyZ53m666u8opKqR+ryQJ4FZgItAguk4LbRKQaubhvJinJ8YyfutrrUOQolPrLKCoto25ijN6GAlKdc68650qDy2tAahTjEpEoqJcUz6j+rfhk4SY91V0N5RcHRhGO5ZrFdjO7wsx8weUKIOxUXGY21MyWmtkKM7v7MMdcZmaLzWyRmf01ZLvfzOYGl4mVjFNEwhh9UhsA3vh2jadxyJHLLyoFiN02C2AMgW6zm4CNwCXAVRWdYGY+4AVgGNANGGVm3Q46piNwDzDIOdcduDVk917nXO/gMryScYpIGBmN6jC0+3G8M2Pd/i8fqR5iPlk459Y654Y751Kdc2nOuQuAcL2h+gMrgj2nioF3gREHHfML4AXn3M7gdbYcYfwichTGnNyG3YWl/GNOttehyBHYdxuqXgy3WZTntjD7M4D1IevZwW2hOgGdzGyqmU03s6Eh+5LNbFZw+wXHEKeIHKRvq8b0ymzIq1PXaK7uaiTmaxaHEYmZNuKBjsDpwCgCY07tG2+3tXMuC/gpMNbM2h8SgNm1wYQya+tWPWgkUllmxpiT27JqWz5f6SG9aiMvmCxiuYG7POH+HMkhMKPePpnBbaGygYnOuRLn3GpgGYHkgXMuJ/jvKuBLoM8hATg3zjmX5ZzLSk1V5yyRI3Fuj3TSGybz8EeL1XZRTRQUB96nmOs6a2Z7zGx3OcseAs9bVGQm0NHM2ppZIjCSwLMaoT4gUKvAzJoRuC21yswah8z53QwYBCw+0sKJyOEl+OJ46rJerN6Wzz3/WBD2qe7i0jL+NnMdu/aWVFGEcrC8ohjtOuuca+CcSylnaeCcqzDa4DSsNxEYU2oJ8J5zbpGZPWhm+3o3TSbQLXcx8F/gDufcdqArMMvM5gW3P+acU7IQibCT2jfjtrM7MXHeBt76bl2Fx770zSrumrCA3/5jQRVFJwfzss0iqld0zn0MfHzQtntDXjsCDeW3HXTMt0CPaMYmIgE3nN6BWWt38tCkxfTMaFjuNJ3rthfw7H+W06x+Eh8t2MjQeRs4v1e4mwsSaQXBZFEnIcZuQ4lIzRcXZzx9WW9SGyRxw9tzyC0oPmC/c457Jy4kPs744MaT6NWyEf/3r4Vs2VPoUcS1V16Rn3qJPuLiItG/6MgoWYgIjesl8sLlfdmyp5DLXpzGqq15+/d9snATXy7dyq/P7kRm47o8dWlPCor9/O6fCzV6bRXzasRZULIQkaDeLRvx2tX92ZZXzPDnp/LJwk3kFZXywKTFdE1P4argMCEd0hrwmyGd+GzxZj6Ye3AHR4mm/GIlCxGJAYM6NOPDm0+mfVp9rntrNpf+ZRqb9xTy6IXHE+/78evimpPb0a91Y+771yKydxZ4GHHtEqhZVH17BShZiMhBWjSqw3u/PJHLB7RiycbdjOrfij6tGh9wjC/OePLSXjhgzGsz2V2o7rRVIb/I78n826BkISLlSIr38ciFPfjw5pO5//zu5R7Ttlk9/nJFP1ZtzeeGt+ZQ4i+r4ihrn/xibyY+AiULEanA8RkNSYw//NfEoA7N+P1FPZiyYhu/+2f4B/vk2OQXlVLXo2ThzVVFpMa4NKsl63cU8OwXK2jdtB43Du7gdUg1Vl6Rn/pqsxCR6urXZ3fiwj4ZPDF5KbPW7PA6nBqroLhUbRYiUn2ZGY9ceDzpDZO5f9IiDXseBWVljoJiv2e3oZQsRCQi6ibGc/ewLizM2c37szWpUqTlF+8bnly3oUSkmhveqwX9Wjfm8ck/sEfdaSOqYN8seapZiEh1Z2bcd343tuUV8/wXK7wOp0bZN/GR2ixEpEbomdmIS/tlMn7qalZvy/c6nBrDy+HJQclCRKLgjqGdSfTF8fCHi/XsRYTsr1mozUJEaoq0BsncelYn/vPDFt6cvtbrcGqEguAseboNJSI1yjUnt+Wsrmk8OGkx363a7nU41d6+3lC6DSUiNUpcnPHHn/SmVdO63PD2HDbk7vU6pGpt320ojQ0lIjVOSnICL43Oori0jF++OZvCEr/XIVVb+29Dqc1CRGqi9qn1efonvVmQs4u7JszX091HaV/Noq7aLESkpjqrW3PuOKcz/5q7gTsnzMevhHHE8otKqZPgw+fB/NugUWdFpIrccHp7ikvLeOY/yynxl/HUpb0OmH1PKpZf7PescRuULESkipgZvz67E4nxcTwxeSmlfsfYkb1JUMKoFC+nVAUlCxGpYjcO7kCiL45HPl5Cgs8YO7KP1yFVC/lF3g1PDmqzEBEP/OLUdtx8Rgc+mLtB819UkpdTqoKShYh45IbTO5DaIInHP1mqIUEqIb/IT10Pb0MpWYiIJ+ok+rjlzI7MWLODL5dt9TqcmBdos1DNQkRqoZ9ktaRVk7o8/slSPX8RRn5xKfXVZiEitVFifBy3D+nEko27mTR/g9fhxDTdhhKRWu38ni3oclwD/vjZMkr8ZV6HE5Occ2rgFpHaLS7OuHNoZ9ZuL+Ddmeu9Dicm7S3x45x3I86CkoWIxIDBndMY0LYJj360hDnrdnodTszJ83iWPFCyEJEYYGY8/9O+pKUkMea1mazYssfrkGJK/v6Jj9RmISK1XGqDJN4cM4D4uDh+9soMzX8Rwuv5tyHKycLMhprZUjNbYWZ3H+aYy8xssZktMrO/hmy/0syWB5croxmniMSGVk3r8vqYE8grLGX0+BnszC/2OqSYkO/xxEcQxWRhZj7gBWAY0A0YZWbdDjqmI3APMMg51x24Nbi9CXAfMADoD9xnZo2jFauIxI7uLRoybnQW63YUcONf52g4c36cUrVuDb0N1R9Y4Zxb5ZwrBt4FRhx0zC+AF5xzOwGcc1uC288BPnPO7Qju+wwYGsVYRSSGDGzflIdGdOfbldsZ9/Uqr8PxXF6wzaJG1iyADCC0H1x2cFuoTkAnM5tqZtPNbOgRnCsiNdhlWS05r0c6T326lLnrc70Ox1MFNb3NohLigY7A6cAo4CUza1TZk83sWjObZWaztm7V2DIiNYmZ8ehFPWieksyv3v1+f/fR2mh/19kaOtxHDtAyZD0zuC1UNjDROVfinFsNLCOQPCpzLs65cc65LOdcVmpqakSDFxHvNayTwDMje7N+RwH3frDQ63A8s7/rbA2d/Ggm0NHM2hL4oh8J/PSgYz4gUKN41cyaEbgttQpYCTwa0qg9hEBDuIjUMlltmnDLmR0Z+/lytuUXc1xKEk3qJZHWIImL+2bSsG6C1yFGXUFxKUnxcZ5OQxu1ZOGcKzWzm4DJgA8Y75xbZGYPArOccxOD+4aY2WLAD9zhnNsOYGYPEUg4AA865zRDikgtddPgDmzLK2L22lyWbtrNjvxiSvyOb5ZvZfxVJ2BmXocYVXkeD08OUZ5W1Tn3MfDxQdvuDXntgNuCy8HnjgfGRzM+Eake4n1xPHxBj/3rzjlembKahz9awsR5GxjRu2b3f/F6/m3wvoFbROSImRlXD2pLr5aNeGDSYnbU8If38ov9njZug5KFiFRTvjjj8Yt7sqewhAcnLfI6nKjyepY8ULIQkWqs83ENuP70DnwwdwP/Xbol/AnVVCwkC2+vLiJyjG4c3J6PF2zkf/+5kMtPbMWW3UVszSvC73fcPawLbZrV8zrEY5Zf7CejsdosRESOWlK8jz9c3JPNuwt5/JOlTJidzZINu5myYhtXvzazRgxGmF9U6nmbhWoWIlLt9WvdmFn/exZJ8T7qBAfbm7lmB5e/9B2/fGs2b17Tn6R4b/8yPxax0HVWNQsRqREa1U3cnygATmjThCcu7cmM1Tu4e8ICAj31qx/nHAXFfs+7zqpmISI11ojeGazbXsBTny2jddO63HpWJ69k6JMvAAAPaUlEQVRDOmJFpWX4y5xqFiIi0XTTGR24pF8mYz9fzrcrtnkdzhGLhUEEQclCRGo4M+PhC46nZZM63DtxEcWlZV6HdEQK9g8iqGQhIhJVyQk+7j+/Oyu25PHat6u9DueI5O2fUlVdZ0VEou7Mrs05s0saYz9fzsZdew/YV1bmKCzxexRZxX6cUlU1CxGRKnHf+d0pLXM88tGS/dvmrs/l/OenMOixLw5JIrEgPwZmyQMlCxGpRVo1rcsNp7fnw/kb+WThRv7vg4Vc+KepbMsrYm+Jn9v+Ng9/WWx1sc2Pgfm3QclCRGqZ605rT8smdbjurTm8/d1arjqpDZ/fdhr3D+/OtFXbefHrlV6HeIB9NYu6iXrOQkSkyiQn+Hjykl68MmU1N5/RkR6ZDQG4tF8mXy3dyh8/Xcag9s3o1bKRx5EG/NjArZqFiEiVGtCuKeNGZ+1PFBDoYvvohT1Ia5DEr979fv9f9F4rKFabhYhITGlYN4GxI/uwbkcBd06YT1Gp9z2ktucXk5wQR2K8t1/XShYiIiH6t23CHed04aP5G7nkz9NYuz3f03gWbdhN1/QUT2MAJQsRkUNcf3p7XhqdxbodBZz37BQ+nL/BkzjKyhyLcnbRI6Nh+IOjTMlCRKQcZ3drzke3nEyn5vW56a/fM/bzZVUew6pt+eQX+5UsRERiWWbjuvztlwM5r0c6f/pyJTuqeCKlBTm5AAc0xHtFyUJEpAIJvjh+dVZHikvLeG/W+iq99oLs3SQnxNEhtX6VXrc8ShYiImF0at6AAW2b8Nb0tVX6hPeCnFy6pacQ7/P+q9r7CEREqoErT2pD9s69fLl0S5Vcz1/mWLRhNz0zY+PhQCULEZFKOLtbc5qnJPH6tLWH7Ptk4Sbmrc+N6PVWbc2joNjP8THQuA1KFiIilZLgi+On/Vvz9bKtrN7247MXL361kuvems2IF6Zy3ZuzWbElb/++rXuKGD9lNVeOn8HCnF1HdL0FweN7xkDjNmhsKBGRShs1oCXP/3c5b01fy//9Tzdenbqa3//7B87rmU6ntAaM+3olny3ZzAW9M9iWV8SUFdvwlzkSfMZdE4qYeNPJ+OKsUtean72LOgk+2sdA4zYoWYiIVFpag2SGHp/O32etJ71hMg9/tIRzujdn7E96k+CL4/ITW/H8Fyt4+7u1pDVI5rrT2nFB7wyWbNrDLe98zzsz1nHFia0rda2FObvo3iKl0skl2pQsRESOwOiBrZk0bwMPf7SEM7qk8dyoviQEeys1q5/E/cO7c/ewLiT64ogLftF3SKvP29PX8uSnSzmvRzqN6yVWeI19jds/OaFl1MtTWWqzEBE5AlmtG3NS+6ac3a05f7q8b7kD/CUn+PYnCgiMaPvAiO7sKSzliU+Xhr3Gyq157C3xx0x7BahmISJyRMyMt38+ALMjuz3U5bgURg9szWvfrmHUCa0qfCp7fnagcTsWhvnYRzULEZEjdKSJYp9bz+pE03qJ3DtxIWUVPNy3MGcXdRN9tIuRxm1QshARqTIN6yRw19AufL8ul48XbjzscfOzc2OqcRuinCzMbKiZLTWzFWZ2dzn7rzKzrWY2N7j8PGSfP2T7xGjGKSJSVS7qm0nbZvX4y1crce7Q2kWpv4zFG3fTIyM2ntzeJ2rJwsx8wAvAMKAbMMrMupVz6N+cc72Dy8sh2/eGbB8erThFRKqSL8649tR2LMzZzdQV2w/Zv3JrPoUlZfTI9H7Co1DRrFn0B1Y451Y554qBd4ERUbyeiEi1cGGfDFIbJPGXr1Yesu+b5VuB2GrchugmiwwgdDzf7OC2g11sZvPN7H0zC+1UnGxms8xsupldUN4FzOza4DGztm7dGsHQRUSiJznBx5hBbZmyYhsLsn8cBuSHTbt58tOlnNiuScw8ub2P1w3ck4A2zrmewGfA6yH7WjvnsoCfAmPNrP3BJzvnxjnnspxzWampqVUTsYhIBFx+YisaJMXzl68DtYs9hSVc/9YcUpITeHZUn6PucRUt0UwWOUBoTSEzuG0/59x251xRcPVloF/Ivpzgv6uAL4E+UYxVRKRKpSQn8NMTW/HvBRtZsy2fuybMZ92OAp4b1Ye0Bsleh3eIaCaLmUBHM2trZonASOCAXk1mlh6yOhxYEtze2MySgq+bAYOAxVGMVUSkyl0zqC3xcXFc+eoMPl6wiTvP6cyAdk29DqtcUXuC2zlXamY3AZMBHzDeObfIzB4EZjnnJgK3mNlwoBTYAVwVPL0r8KKZlRFIaI8555QsRKRGSUtJ5uJ+GbwzYz1DujXn2lPbeR3SYVl5/Xyro6ysLDdr1iyvwxAROSJbdhfy6rdruO609jSsk1Dl1zez2cH24QppbCgREQ+lpSRz19AuXocRlte9oUREpBpQshARkbCULEREJCwlCxERCUvJQkREwlKyEBGRsJQsREQkLCULEREJq8Y8wW1mW4G15exqCOwKsy10vbzX+/5tBmw7yhDLi6Oyx4Qrw+HKU94x0SxDRfsr+j8/eD3cay/KEInfo9DXR1uGaP4eHbxe0WcBYrMMlSlPrH2eK7serc9Ca+dc+GG7nXM1egHGhdsWul7e65B/Z0UyjsoeE64MhyvPYcoStTJUtL+i//PKvAdelyESv0eRKEM0f48qGXfotpgrQ2XKE2uf58quV/Vn4eClNtyGmlSJbZPCvC7vZ0QijsoeE64MhytPRcccjXA/o6L9Ff2fH7xemddH62jLEInfo8pcP5xo/h4dvF6TPguhr2OtDJVdr+rPwgFqzG2oqmBms1wlBtyKZSpDbFAZvFfd44eqLUNtqFlE0jivA4gAlSE2qAzeq+7xQxWWQTULEREJSzULEREJq9YmCzMbb2ZbzGzhUZzbz8wWmNkKM3vWQmZWN7ObzewHM1tkZo9HNupD4oh4GczsfjPLMbO5weXcyEd+QBxReR+C+283MxecmjdqovQ+PGRm84Pvwadm1iLyke+PIRrxPxH8HMw3s3+aWaPIR35AHNEow6XBz3GZmUWtXeBYYj/Mz7vSzJYHlytDtlf4eQnraLtdVfcFOBXoCyw8inNnACcCBvwbGBbcPhj4HEgKrqdVwzLcD/ymOr8PwX0tCUzpuxZoVt3KAKSEHHML8JdqFv8QID74+g/AH6rhe9AV6Ax8CWTFWuzBuNoctK0JsCr4b+Pg68YVlbOyS62tWTjnviYw7/d+ZtbezD4xs9lm9o2ZHTJ9lZmlE/ggT3eBd+AN4ILg7usJzBdeFLzGlmpYhioVxTI8DdwJRL1RLhplcM7tDjm0HlEsR5Ti/9Q5Vxo8dDqQGa34o1iGJc65pdGM+1hiP4xzgM+cczucczuBz4ChkfjM19pkcRjjgJudc/2A3wB/KueYDCA7ZD07uA2gE3CKmX1nZl+Z2QlRjbZ8x1oGgJuCtw/Gm1nj6IV6WMdUBjMbAeQ45+ZFO9AKHPP7YGaPmNl64HLg3ijGWp5I/B7tM4bAX7JVLZJlqGqVib08GcD6kPV95TnmcmoO7iAzqw+cBPw95FZe0hH+mHgC1b8TgROA98ysXTCTR12EyvBn4CECf8k+BDxF4MNeJY61DGZWF/gtgdsgnojQ+4Bz7nfA78zsHuAm4L6IBVmBSMUf/Fm/A0qBtyMTXaWvG7EyVLWKYjezq4FfBbd1AD42s2JgtXPuwmjGpWTxozgg1znXO3SjmfmA2cHViQS+TEOr1JlATvB1NvCPYHKYYWZlBMZu2RrNwEMccxmcc5tDznsJ+DCaAZfjWMvQHmgLzAt+0DKBOWbW3zm3Kcqx7xOJ36VQbwMfU0XJggjFb2ZXAf8DnFlVfzCFiPR7UJXKjR3AOfcq8CqAmX0JXOWcWxNySA5wesh6JoG2jRyOtZzRarSpDgvQhpBGJeBb4NLgawN6Hea8gxuKzg1uvw54MPi6E4HqoFWzMqSHHPNr4N3q9j4cdMwaotzAHaX3oWPIMTcD71ez+IcCi4HUaP/fR/v3iCg3cB9t7By+gXs1gcbtxsHXTSpTzrAxVtUbGWsL8A6wESghUCO4hsBfpJ8A84K/6Pce5twsYCGwEnieHx9uTATeCu6bA5xRDcvwJrAAmE/gL6/06laGg45ZQ/R7Q0XjfZgQ3D6fwBg+GdUs/hUE/liaG1yi1psrimW4MPizioDNwORYip1ykkVw+5jg//8K4Ooj+bxUtOgJbhERCUu9oUREJCwlCxERCUvJQkREwlKyEBGRsJQsREQkLCULqdHMLK+Kr/eymXWL0M/yW2DU2YVmNincyK1m1sjMbojEtUUOpq6zUqOZWZ5zrn4Ef168+3GAvKgKjd3MXgeWOeceqeD4NsCHzrnjqyI+qV1Us5Bax8xSzWyCmc0MLoOC2/ub2TQz+97MvjWzzsHtV5nZRDP7AviPmZ1uZl+a2fsWmLPh7X1zAwS3ZwVf5wUHA5xnZtPNrHlwe/vg+gIze7iStZ9p/DhQYn0z+4+ZzQn+jBHBYx4D2gdrI08Ej70jWMb5ZvZABP8bpZZRspDa6BngaefcCcDFwMvB7T8Apzjn+hAY5fXRkHP6Apc4504LrvcBbgW6Ae2AQeVcpx4w3TnXC/ga+EXI9Z9xzvXgwJFAyxUcz+hMAk/UAxQCFzrn+hKYQ+WpYLK6G1jpnOvtnLvDzIYAHYH+QG+gn5mdGu56IuXRQIJSG50FdAsZ0TMlONJnQ+B1M+tIYNTdhJBzPnPOhc45MMM5lw1gZnMJjO0z5aDrFPPjQIyzgbODrwfy41wCfwWePEycdYI/OwNYQmBuAgiM7fNo8Iu/LLi/eTnnDwku3wfX6xNIHl8f5noih6VkIbVRHHCic64wdKOZPQ/81zl3YfD+/5chu/MP+hlFIa/9lP9ZKnE/Ngoe7piK7HXO9Q4Ouz4ZuBF4lsD8FqlAP+dciZmtAZLLOd+A3zvnXjzC64ocQrehpDb6lMBIrgCY2b6hoBvy47DNV0Xx+tMJ3P4CGBnuYOdcAYGpVW83s3gCcW4JJorBQOvgoXuABiGnTgbGBGtNmFmGmaVFqAxSyyhZSE1X18yyQ5bbCHzxZgUbfRcTGFoe4HHg92b2PdGtdd8K3GZm8wlMYLMr3AnOue8JjEA7isD8FllmtgAYTaCtBefcdmBqsKvtE865Twnc5poWPPZ9DkwmIpWmrrMiVSx4W2mvc86Z2UhglHNuRLjzRLykNguRqtcPeD7YgymXKpy2VuRoqWYhIiJhqc1CRETCUrIQEZGwlCxERCQsJQsREQlLyUJERMJSshARkbD+H1Y7u0yDVmEWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_cls.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 01:20 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.377595</th>\n",
       "    <th>0.262159</th>\n",
       "    <th>0.899200</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_cls.fit_one_cycle(1, 1e-1, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 02:36 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.448531</th>\n",
       "    <th>0.298936</th>\n",
       "    <th>0.877680</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>0.369926</th>\n",
       "    <th>0.259925</th>\n",
       "    <th>0.905760</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_cls.fit_one_cycle(2, 1e-1, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 02:52 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.357873</th>\n",
       "    <th>0.686789</th>\n",
       "    <th>0.885000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>0.290814</th>\n",
       "    <th>0.220855</th>\n",
       "    <th>0.921440</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_cls.freeze_to(-2)\n",
    "learn_cls.fit_one_cycle(2, slice(1e-1/2.6,1e-1), moms=(0.8,0.7), pct_start=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_cls.save('second_bwd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 05:10 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.322946</th>\n",
       "    <th>0.400557</th>\n",
       "    <th>0.908200</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>0.294446</th>\n",
       "    <th>0.337991</th>\n",
       "    <th>0.922640</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>0.236637</th>\n",
       "    <th>0.818094</th>\n",
       "    <th>0.924080</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_cls.freeze_to(-3)\n",
    "learn_cls.fit_one_cycle(3, slice(1e-2/(2.6**2),3e-2), moms=(0.9,0.7), pct_start=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 05:06 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.296635</th>\n",
       "    <th>0.235310</th>\n",
       "    <th>0.910840</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>0.289384</th>\n",
       "    <th>2.694395</th>\n",
       "    <th>0.912960</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>0.277817</th>\n",
       "    <th>0.264012</th>\n",
       "    <th>0.923920</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_cls.fit_one_cycle(3, slice(1e-2/(2.6**2),3e-2), moms=(0.9,0.7), pct_start=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_cls.save('third_bwd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList\n",
       "y: CategoryList (25000 items)\n",
       "[Category 0, Category 0, Category 1, Category 0, Category 1]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas\n",
       "x: TextList (25000 items)\n",
       "[Text xxbos another trashy grade z quickie from the prolific albert pyun . tim xxunk 13 inch clint eastwood - like cop from outer space chases an ugly flying head ( ! ) to earth and gets involved in a gang war in south bronx ! mercifully short , but xxunk dull , with the cheesiest effects since attack of the 50 ft woman . they should have fired the continuity guy , too : note how xxunk sunglasses disappears and reappears in every second shot . laughably bad , but that´s why we watch these movies , xxunk it ? sequel ´ dollman vs. demonic toys ´ is reportedly even worse , if that´s possible . \n",
       "\n",
       " 0 ( of xxrep 4 * ) \n",
       "\n",
       ", Text xxbos this show stinks . for parents , they usually want their kids to watch something good for them . it is usually educational , funny , and bright . \n",
       "\n",
       " is it educational ? no . the doodlebops sing and that 's it . they usually sing about themselves , they do n't try teaching anything . \n",
       "\n",
       " is it funny ? no . the doodlebops instead say something which is not intended as a joke , and laugh at it . \n",
       "\n",
       " is it bright ? it 's so bright , it 's painful . as far as color , s everything is extremely bright , so that 's good . but xxup nothing is ever wrong in the world of the xxunk 's . therefore , they are always happy . a kid in trouble will become depressed because they have never been exposed to being sad . \n",
       "\n",
       " the show is also extremely cheesy . every syllable is said to the highest level of exaggeration and very corny . it 's overkill . \n",
       "\n",
       " for kids , it 's entertaining , but past the age of 2 you wo n't want your kids to see it . they 'll never know how to grow up ., Text xxbos just two comments xxrep 4 . xxup seven years apart ? hardly evidence of the film 's relentless pulling - power ! as has been mentioned , the low - budget telemovie status of 13 xxup gantry xxup row is a mitigating factor in its limited appeal . having said that however the thing is not without merit - either as entertainment or as a fright outing per se . \n",
       "\n",
       " true , the plot at its most basic is a re - working of xxup the xxup amityville xxup horror - only without much horror . more a case of intrigue ! gibney might have made a more worthwhile impression if she had played xxunk xxunk a couple of seemingly unconnected murders with the \" house \" as the main suspect . the script is better than average and the production overall of a high standard . it just fails to engage the viewer particularly at key moments . \n",
       "\n",
       " having picked the xxup dvd up for a mere $ xxunk last week at my regular video store , i can not begrudge the expenditure . $ xxunk would be an acceptable price for the film . just do n't expect fireworks !, Text xxbos xxup some xxup not - so- xxup xxunk xxup spoilers xxup ahead \n",
       "\n",
       " why do people , when they are disoriented or sick or scared at a club , cut through the middle of the crowded dance floor on their way to the bathroom ? \n",
       "\n",
       " who in their right mind would hide under a bed when someone breaks into their room ? \n",
       "\n",
       " how often do you knock on a stranger 's door and when they do n't xxup immediately answer , you open the door , walk in , shout a few hello 's and then start going through their stuff ? \n",
       "\n",
       " if you were being pursued by someone you just discovered was a murderer , what would you do ? quietly sneak off and hide under a wooden platform or among metal implements ? run , quietly of course , to a ratty old barn or other decrepit structure ? \n",
       "\n",
       " i could be talking about almost any thriller that 's come out in the last few years , but since this is the \" the return \" page , obviously i 'm talking about \" the return . \" i saw it free because i work at a movie theater and make a point of screening all the \" scary \" movies . i thought this one was tolerable ... aside from the well - worn clichés . sarah michelle gellar is really drab and looks kind of \" huh ? \" through most of the film . the details of the plot are slowly given out as the movie progresses and it 's almost enough to make it interesting except there was n't enough explanation as it moved on and so i was almost lost until the last 2 / 3 of it . \n",
       "\n",
       " if you 're a die - hard thriller fan , it 's worth seeing at least once . if there 's nothing better at the theater and you really want to watch a movie , eh , i guess it 's worth a matinée ticket . if you thought the trailer made it look like an interesting movie and you ca n't wait ... wait ., Text xxbos this is a very funny movie , easy to watch , that entertains you almost all the time . the work of the director is recognizable and the type of humor is his trademark . the movie is a typical police partners history like lethal weapon , but the jokes and comedy are of argentinian sort . the twist is that one of them is a psychologist played by peretti and has to go with detective diaz ( played by luque ) on his assignments while he also assist him ( diaz is troubled because his wife cheated on him ) . some of the dialogs are hilarious worldwide : understandable and laughable anywhere . is very good overall , it would deserved an 8 , but i rated 7 because it gets a little down at the end . on a personal remark i must add that is a \" bravo \" for argentinian filmmakers , considering the little good is coming lately .]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas;\n",
       "\n",
       "Valid: LabelList\n",
       "y: CategoryList (25000 items)\n",
       "[Category 0, Category 0, Category 1, Category 0, Category 1]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas\n",
       "x: TextList (25000 items)\n",
       "[Text xxbos i thought it will be a ok movie after seeing the commercials about it . it was funny at some parts and some very nasty . the only person i felt sorry for is horatio sans who got a hot wife who is cheating on him with other women . but he never got a chance to have a threesome with until the and that was good but they should have made more bigger thru out the film ., Text xxbos i enjoyed carax 's \" les amants du pont neuf \" and was therefore expecting this film to be of a similar standard . well , the first 10 minutes were ok , but then it disintegrates into a rather pretentious journey of a young man looking for the essence of life . a sad disappointment ., Text xxbos what 's the matter with you people ? john dahl ? from \" rounders \" and \" unforgettable \" ? xxup too quirky ? knocking emma thompson and alan rickman for having fun playing against type ? and somebody liked the gingerbread man ? \n",
       "\n",
       " i rented this not knowing anything about it and found it about as nifty a video find as you can get . never insulting , well thought out , funny , scary . i disagree with the naysayers , clearly . i thought the story itself was unremarkable but the great cast , which most likely means the director was paying attention , lifted it to super cool status . good sound design also ( much more appreciated in surround , but i 'm not bragging ) . and yes , i 'm a girl , so maybe it has a slight female slant ( the guys in the gang are pretty worthwhile ) . all in all , a 9 and a hearty xxup recommend ., Text xxbos johnnie ( bert wheeler ) is a would - be songwriter ; newton ( robert woolsey ) is a would - be inventor . both work at a cigar stand in the lobby of an office building . johnnie wants to sell a song to winfield lake , a song publisher who also owns the building . lake 's secretary , mary ( betty grable ) , is johnnie 's sweetheart . when lake turns up dead , circumstances conspire to make mary and newton think that johnnie is the killer . they conspire again to implicate mary , who goes to jail . but who really shot lake ? who is the black widow , the blackmailer who had threatened him ? the other characters in this wacky murder mystery are : lake 's suspicious wife , a self - satisfied private detective , a seemingly slow - witted janitor ( willie best ) , lake 's auditor , a songwriter who thinks lake is stealing from him and another who thinks everyone is stealing from him . it 's up to newton and his truth machine to reveal the real killer . \n",
       "\n",
       " the baby - voiced wheeler and the cigar - chomping woolsey strike me as an arbitrary pairing , but they made several movies together in the 30s and some of them were funny . \n",
       "\n",
       " not this one . george stevens , who went on to have a distinguished career , directed this dismal comedy with a tedious murder mystery plot . but two scenes are good , and both feature wheeler and betty grable singing the excellent \" music in my heart , \" written by dorothy fields and jimmy mchugh . the first time , they sing it walking up a staircase ( after which they dance back down ) . later , wheeler and woolsey are on stilts so that they can see and talk to mary , who is in a jail cell on a high floor . wheeler and grable sing to each other through the bars . \n",
       "\n",
       " \" the nitwits \" has a few laughs , but the level of comedy is best illustrated by woolsey 's line : \" sonny , you 've got the brain of a six - year - old boy . and i 'll bet even he was glad to get rid of it . \" it 's watered - down xxunk did n't use the superfluous \" even \" when he said it ., Text xxbos this film has the guts to suggest that it might be best to simply accept your life as it is , and keep smiling anyway . as one who is more excited by the idea of taking charge of one 's life and moving forward , i felt slapped in the face , but that 's okay : i do n't have to agree with a movie to love it and respect it . great acting by streep and hurt , and everyone else really , and some wonderfully quirky scenes . a serious film . and take a hanky .]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchRNNCore(\n",
       "    (encoder): Embedding(60004, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(60004, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=800, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (1): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (2): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (3): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=1200, bias=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "      (3): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=<fastai.layers.FlattenedLoss object at 0x7fdb4702be80>, metrics=[<function accuracy at 0x7fdac5aab8c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('/home/farzin/rnn_python_code/wiki103_from_download/clas'), model_dir='models', callback_fns=[<class 'fastai.basic_train.Recorder'>], callbacks=[RNNTrainer\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList\n",
       "y: CategoryList (25000 items)\n",
       "[Category 0, Category 0, Category 1, Category 0, Category 1]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas\n",
       "x: TextList (25000 items)\n",
       "[Text xxbos another trashy grade z quickie from the prolific albert pyun . tim xxunk 13 inch clint eastwood - like cop from outer space chases an ugly flying head ( ! ) to earth and gets involved in a gang war in south bronx ! mercifully short , but xxunk dull , with the cheesiest effects since attack of the 50 ft woman . they should have fired the continuity guy , too : note how xxunk sunglasses disappears and reappears in every second shot . laughably bad , but that´s why we watch these movies , xxunk it ? sequel ´ dollman vs. demonic toys ´ is reportedly even worse , if that´s possible . \n",
       "\n",
       " 0 ( of xxrep 4 * ) \n",
       "\n",
       ", Text xxbos this show stinks . for parents , they usually want their kids to watch something good for them . it is usually educational , funny , and bright . \n",
       "\n",
       " is it educational ? no . the doodlebops sing and that 's it . they usually sing about themselves , they do n't try teaching anything . \n",
       "\n",
       " is it funny ? no . the doodlebops instead say something which is not intended as a joke , and laugh at it . \n",
       "\n",
       " is it bright ? it 's so bright , it 's painful . as far as color , s everything is extremely bright , so that 's good . but xxup nothing is ever wrong in the world of the xxunk 's . therefore , they are always happy . a kid in trouble will become depressed because they have never been exposed to being sad . \n",
       "\n",
       " the show is also extremely cheesy . every syllable is said to the highest level of exaggeration and very corny . it 's overkill . \n",
       "\n",
       " for kids , it 's entertaining , but past the age of 2 you wo n't want your kids to see it . they 'll never know how to grow up ., Text xxbos just two comments xxrep 4 . xxup seven years apart ? hardly evidence of the film 's relentless pulling - power ! as has been mentioned , the low - budget telemovie status of 13 xxup gantry xxup row is a mitigating factor in its limited appeal . having said that however the thing is not without merit - either as entertainment or as a fright outing per se . \n",
       "\n",
       " true , the plot at its most basic is a re - working of xxup the xxup amityville xxup horror - only without much horror . more a case of intrigue ! gibney might have made a more worthwhile impression if she had played xxunk xxunk a couple of seemingly unconnected murders with the \" house \" as the main suspect . the script is better than average and the production overall of a high standard . it just fails to engage the viewer particularly at key moments . \n",
       "\n",
       " having picked the xxup dvd up for a mere $ xxunk last week at my regular video store , i can not begrudge the expenditure . $ xxunk would be an acceptable price for the film . just do n't expect fireworks !, Text xxbos xxup some xxup not - so- xxup xxunk xxup spoilers xxup ahead \n",
       "\n",
       " why do people , when they are disoriented or sick or scared at a club , cut through the middle of the crowded dance floor on their way to the bathroom ? \n",
       "\n",
       " who in their right mind would hide under a bed when someone breaks into their room ? \n",
       "\n",
       " how often do you knock on a stranger 's door and when they do n't xxup immediately answer , you open the door , walk in , shout a few hello 's and then start going through their stuff ? \n",
       "\n",
       " if you were being pursued by someone you just discovered was a murderer , what would you do ? quietly sneak off and hide under a wooden platform or among metal implements ? run , quietly of course , to a ratty old barn or other decrepit structure ? \n",
       "\n",
       " i could be talking about almost any thriller that 's come out in the last few years , but since this is the \" the return \" page , obviously i 'm talking about \" the return . \" i saw it free because i work at a movie theater and make a point of screening all the \" scary \" movies . i thought this one was tolerable ... aside from the well - worn clichés . sarah michelle gellar is really drab and looks kind of \" huh ? \" through most of the film . the details of the plot are slowly given out as the movie progresses and it 's almost enough to make it interesting except there was n't enough explanation as it moved on and so i was almost lost until the last 2 / 3 of it . \n",
       "\n",
       " if you 're a die - hard thriller fan , it 's worth seeing at least once . if there 's nothing better at the theater and you really want to watch a movie , eh , i guess it 's worth a matinée ticket . if you thought the trailer made it look like an interesting movie and you ca n't wait ... wait ., Text xxbos this is a very funny movie , easy to watch , that entertains you almost all the time . the work of the director is recognizable and the type of humor is his trademark . the movie is a typical police partners history like lethal weapon , but the jokes and comedy are of argentinian sort . the twist is that one of them is a psychologist played by peretti and has to go with detective diaz ( played by luque ) on his assignments while he also assist him ( diaz is troubled because his wife cheated on him ) . some of the dialogs are hilarious worldwide : understandable and laughable anywhere . is very good overall , it would deserved an 8 , but i rated 7 because it gets a little down at the end . on a personal remark i must add that is a \" bravo \" for argentinian filmmakers , considering the little good is coming lately .]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas;\n",
       "\n",
       "Valid: LabelList\n",
       "y: CategoryList (25000 items)\n",
       "[Category 0, Category 0, Category 1, Category 0, Category 1]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas\n",
       "x: TextList (25000 items)\n",
       "[Text xxbos i thought it will be a ok movie after seeing the commercials about it . it was funny at some parts and some very nasty . the only person i felt sorry for is horatio sans who got a hot wife who is cheating on him with other women . but he never got a chance to have a threesome with until the and that was good but they should have made more bigger thru out the film ., Text xxbos i enjoyed carax 's \" les amants du pont neuf \" and was therefore expecting this film to be of a similar standard . well , the first 10 minutes were ok , but then it disintegrates into a rather pretentious journey of a young man looking for the essence of life . a sad disappointment ., Text xxbos what 's the matter with you people ? john dahl ? from \" rounders \" and \" unforgettable \" ? xxup too quirky ? knocking emma thompson and alan rickman for having fun playing against type ? and somebody liked the gingerbread man ? \n",
       "\n",
       " i rented this not knowing anything about it and found it about as nifty a video find as you can get . never insulting , well thought out , funny , scary . i disagree with the naysayers , clearly . i thought the story itself was unremarkable but the great cast , which most likely means the director was paying attention , lifted it to super cool status . good sound design also ( much more appreciated in surround , but i 'm not bragging ) . and yes , i 'm a girl , so maybe it has a slight female slant ( the guys in the gang are pretty worthwhile ) . all in all , a 9 and a hearty xxup recommend ., Text xxbos johnnie ( bert wheeler ) is a would - be songwriter ; newton ( robert woolsey ) is a would - be inventor . both work at a cigar stand in the lobby of an office building . johnnie wants to sell a song to winfield lake , a song publisher who also owns the building . lake 's secretary , mary ( betty grable ) , is johnnie 's sweetheart . when lake turns up dead , circumstances conspire to make mary and newton think that johnnie is the killer . they conspire again to implicate mary , who goes to jail . but who really shot lake ? who is the black widow , the blackmailer who had threatened him ? the other characters in this wacky murder mystery are : lake 's suspicious wife , a self - satisfied private detective , a seemingly slow - witted janitor ( willie best ) , lake 's auditor , a songwriter who thinks lake is stealing from him and another who thinks everyone is stealing from him . it 's up to newton and his truth machine to reveal the real killer . \n",
       "\n",
       " the baby - voiced wheeler and the cigar - chomping woolsey strike me as an arbitrary pairing , but they made several movies together in the 30s and some of them were funny . \n",
       "\n",
       " not this one . george stevens , who went on to have a distinguished career , directed this dismal comedy with a tedious murder mystery plot . but two scenes are good , and both feature wheeler and betty grable singing the excellent \" music in my heart , \" written by dorothy fields and jimmy mchugh . the first time , they sing it walking up a staircase ( after which they dance back down ) . later , wheeler and woolsey are on stilts so that they can see and talk to mary , who is in a jail cell on a high floor . wheeler and grable sing to each other through the bars . \n",
       "\n",
       " \" the nitwits \" has a few laughs , but the level of comedy is best illustrated by woolsey 's line : \" sonny , you 've got the brain of a six - year - old boy . and i 'll bet even he was glad to get rid of it . \" it 's watered - down xxunk did n't use the superfluous \" even \" when he said it ., Text xxbos this film has the guts to suggest that it might be best to simply accept your life as it is , and keep smiling anyway . as one who is more excited by the idea of taking charge of one 's life and moving forward , i felt slapped in the face , but that 's okay : i do n't have to agree with a movie to love it and respect it . great acting by streep and hurt , and everyone else really , and some wonderfully quirky scenes . a serious film . and take a hanky .]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchRNNCore(\n",
       "    (encoder): Embedding(60004, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(60004, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=800, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (1): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (2): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (3): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=1200, bias=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "      (3): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=<fastai.layers.FlattenedLoss object at 0x7fdb4702be80>, metrics=[<function accuracy at 0x7fdac5aab8c8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('/home/farzin/rnn_python_code/wiki103_from_download/clas'), model_dir='models', callback_fns=[<class 'fastai.basic_train.Recorder'>], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(60004, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(60004, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=800, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=1200, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       ")])\n",
       "bptt: 140\n",
       "alpha: 2.0\n",
       "beta: 1.0\n",
       "adjust: False], layer_groups=[Sequential(\n",
       "  (0): Embedding(60004, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(60004, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=800, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=1200, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       ")])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn_cls.load('third_bwd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 07:54 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.312428</th>\n",
       "    <th>0.247771</th>\n",
       "    <th>0.923800</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>0.237559</th>\n",
       "    <th>0.193471</th>\n",
       "    <th>0.929160</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>0.251314</th>\n",
       "    <th>0.191734</th>\n",
       "    <th>0.931760</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_cls.unfreeze()\n",
    "learn_cls.fit_one_cycle(3, slice(1e-2/(2.6**4),3e-2), moms=(0.8,0.7), pct_start=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_cls.save('final_bwd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del learn_cls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Investigate mis-labeled cases in train set. \n",
    "Are they marginal reviews?  Other details that could be better addressed?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList\n",
       "y: CategoryList (25000 items)\n",
       "[Category 0, Category 0, Category 1, Category 0, Category 1]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas\n",
       "x: TextList (25000 items)\n",
       "[Text xxbos another trashy grade z quickie from the prolific albert pyun . tim xxunk 13 inch clint eastwood - like cop from outer space chases an ugly flying head ( ! ) to earth and gets involved in a gang war in south bronx ! mercifully short , but xxunk dull , with the cheesiest effects since attack of the 50 ft woman . they should have fired the continuity guy , too : note how xxunk sunglasses disappears and reappears in every second shot . laughably bad , but that´s why we watch these movies , xxunk it ? sequel ´ dollman vs. demonic toys ´ is reportedly even worse , if that´s possible . \n",
       "\n",
       " 0 ( of xxrep 4 * ) \n",
       "\n",
       ", Text xxbos this show stinks . for parents , they usually want their kids to watch something good for them . it is usually educational , funny , and bright . \n",
       "\n",
       " is it educational ? no . the doodlebops sing and that 's it . they usually sing about themselves , they do n't try teaching anything . \n",
       "\n",
       " is it funny ? no . the doodlebops instead say something which is not intended as a joke , and laugh at it . \n",
       "\n",
       " is it bright ? it 's so bright , it 's painful . as far as color , s everything is extremely bright , so that 's good . but xxup nothing is ever wrong in the world of the xxunk 's . therefore , they are always happy . a kid in trouble will become depressed because they have never been exposed to being sad . \n",
       "\n",
       " the show is also extremely cheesy . every syllable is said to the highest level of exaggeration and very corny . it 's overkill . \n",
       "\n",
       " for kids , it 's entertaining , but past the age of 2 you wo n't want your kids to see it . they 'll never know how to grow up ., Text xxbos just two comments xxrep 4 . xxup seven years apart ? hardly evidence of the film 's relentless pulling - power ! as has been mentioned , the low - budget telemovie status of 13 xxup gantry xxup row is a mitigating factor in its limited appeal . having said that however the thing is not without merit - either as entertainment or as a fright outing per se . \n",
       "\n",
       " true , the plot at its most basic is a re - working of xxup the xxup amityville xxup horror - only without much horror . more a case of intrigue ! gibney might have made a more worthwhile impression if she had played xxunk xxunk a couple of seemingly unconnected murders with the \" house \" as the main suspect . the script is better than average and the production overall of a high standard . it just fails to engage the viewer particularly at key moments . \n",
       "\n",
       " having picked the xxup dvd up for a mere $ xxunk last week at my regular video store , i can not begrudge the expenditure . $ xxunk would be an acceptable price for the film . just do n't expect fireworks !, Text xxbos xxup some xxup not - so- xxup xxunk xxup spoilers xxup ahead \n",
       "\n",
       " why do people , when they are disoriented or sick or scared at a club , cut through the middle of the crowded dance floor on their way to the bathroom ? \n",
       "\n",
       " who in their right mind would hide under a bed when someone breaks into their room ? \n",
       "\n",
       " how often do you knock on a stranger 's door and when they do n't xxup immediately answer , you open the door , walk in , shout a few hello 's and then start going through their stuff ? \n",
       "\n",
       " if you were being pursued by someone you just discovered was a murderer , what would you do ? quietly sneak off and hide under a wooden platform or among metal implements ? run , quietly of course , to a ratty old barn or other decrepit structure ? \n",
       "\n",
       " i could be talking about almost any thriller that 's come out in the last few years , but since this is the \" the return \" page , obviously i 'm talking about \" the return . \" i saw it free because i work at a movie theater and make a point of screening all the \" scary \" movies . i thought this one was tolerable ... aside from the well - worn clichés . sarah michelle gellar is really drab and looks kind of \" huh ? \" through most of the film . the details of the plot are slowly given out as the movie progresses and it 's almost enough to make it interesting except there was n't enough explanation as it moved on and so i was almost lost until the last 2 / 3 of it . \n",
       "\n",
       " if you 're a die - hard thriller fan , it 's worth seeing at least once . if there 's nothing better at the theater and you really want to watch a movie , eh , i guess it 's worth a matinée ticket . if you thought the trailer made it look like an interesting movie and you ca n't wait ... wait ., Text xxbos this is a very funny movie , easy to watch , that entertains you almost all the time . the work of the director is recognizable and the type of humor is his trademark . the movie is a typical police partners history like lethal weapon , but the jokes and comedy are of argentinian sort . the twist is that one of them is a psychologist played by peretti and has to go with detective diaz ( played by luque ) on his assignments while he also assist him ( diaz is troubled because his wife cheated on him ) . some of the dialogs are hilarious worldwide : understandable and laughable anywhere . is very good overall , it would deserved an 8 , but i rated 7 because it gets a little down at the end . on a personal remark i must add that is a \" bravo \" for argentinian filmmakers , considering the little good is coming lately .]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas;\n",
       "\n",
       "Valid: LabelList\n",
       "y: CategoryList (25000 items)\n",
       "[Category 0, Category 0, Category 1, Category 0, Category 1]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas\n",
       "x: TextList (25000 items)\n",
       "[Text xxbos i thought it will be a ok movie after seeing the commercials about it . it was funny at some parts and some very nasty . the only person i felt sorry for is horatio sans who got a hot wife who is cheating on him with other women . but he never got a chance to have a threesome with until the and that was good but they should have made more bigger thru out the film ., Text xxbos i enjoyed carax 's \" les amants du pont neuf \" and was therefore expecting this film to be of a similar standard . well , the first 10 minutes were ok , but then it disintegrates into a rather pretentious journey of a young man looking for the essence of life . a sad disappointment ., Text xxbos what 's the matter with you people ? john dahl ? from \" rounders \" and \" unforgettable \" ? xxup too quirky ? knocking emma thompson and alan rickman for having fun playing against type ? and somebody liked the gingerbread man ? \n",
       "\n",
       " i rented this not knowing anything about it and found it about as nifty a video find as you can get . never insulting , well thought out , funny , scary . i disagree with the naysayers , clearly . i thought the story itself was unremarkable but the great cast , which most likely means the director was paying attention , lifted it to super cool status . good sound design also ( much more appreciated in surround , but i 'm not bragging ) . and yes , i 'm a girl , so maybe it has a slight female slant ( the guys in the gang are pretty worthwhile ) . all in all , a 9 and a hearty xxup recommend ., Text xxbos johnnie ( bert wheeler ) is a would - be songwriter ; newton ( robert woolsey ) is a would - be inventor . both work at a cigar stand in the lobby of an office building . johnnie wants to sell a song to winfield lake , a song publisher who also owns the building . lake 's secretary , mary ( betty grable ) , is johnnie 's sweetheart . when lake turns up dead , circumstances conspire to make mary and newton think that johnnie is the killer . they conspire again to implicate mary , who goes to jail . but who really shot lake ? who is the black widow , the blackmailer who had threatened him ? the other characters in this wacky murder mystery are : lake 's suspicious wife , a self - satisfied private detective , a seemingly slow - witted janitor ( willie best ) , lake 's auditor , a songwriter who thinks lake is stealing from him and another who thinks everyone is stealing from him . it 's up to newton and his truth machine to reveal the real killer . \n",
       "\n",
       " the baby - voiced wheeler and the cigar - chomping woolsey strike me as an arbitrary pairing , but they made several movies together in the 30s and some of them were funny . \n",
       "\n",
       " not this one . george stevens , who went on to have a distinguished career , directed this dismal comedy with a tedious murder mystery plot . but two scenes are good , and both feature wheeler and betty grable singing the excellent \" music in my heart , \" written by dorothy fields and jimmy mchugh . the first time , they sing it walking up a staircase ( after which they dance back down ) . later , wheeler and woolsey are on stilts so that they can see and talk to mary , who is in a jail cell on a high floor . wheeler and grable sing to each other through the bars . \n",
       "\n",
       " \" the nitwits \" has a few laughs , but the level of comedy is best illustrated by woolsey 's line : \" sonny , you 've got the brain of a six - year - old boy . and i 'll bet even he was glad to get rid of it . \" it 's watered - down xxunk did n't use the superfluous \" even \" when he said it ., Text xxbos this film has the guts to suggest that it might be best to simply accept your life as it is , and keep smiling anyway . as one who is more excited by the idea of taking charge of one 's life and moving forward , i felt slapped in the face , but that 's okay : i do n't have to agree with a movie to love it and respect it . great acting by streep and hurt , and everyone else really , and some wonderfully quirky scenes . a serious film . and take a hanky .]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchRNNCore(\n",
       "    (encoder): Embedding(60004, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(60004, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=800, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (1): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (2): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (3): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=1200, bias=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "      (3): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=<fastai.layers.FlattenedLoss object at 0x7f0ea3f00470>, metrics=[<function accuracy at 0x7f0e1d96f620>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('/home/farzin/rnn_python_code/wiki103_from_download/clas'), model_dir='models', callback_fns=[<class 'fastai.basic_train.Recorder'>], callbacks=[RNNTrainer\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList\n",
       "y: CategoryList (25000 items)\n",
       "[Category 0, Category 0, Category 1, Category 0, Category 1]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas\n",
       "x: TextList (25000 items)\n",
       "[Text xxbos another trashy grade z quickie from the prolific albert pyun . tim xxunk 13 inch clint eastwood - like cop from outer space chases an ugly flying head ( ! ) to earth and gets involved in a gang war in south bronx ! mercifully short , but xxunk dull , with the cheesiest effects since attack of the 50 ft woman . they should have fired the continuity guy , too : note how xxunk sunglasses disappears and reappears in every second shot . laughably bad , but that´s why we watch these movies , xxunk it ? sequel ´ dollman vs. demonic toys ´ is reportedly even worse , if that´s possible . \n",
       "\n",
       " 0 ( of xxrep 4 * ) \n",
       "\n",
       ", Text xxbos this show stinks . for parents , they usually want their kids to watch something good for them . it is usually educational , funny , and bright . \n",
       "\n",
       " is it educational ? no . the doodlebops sing and that 's it . they usually sing about themselves , they do n't try teaching anything . \n",
       "\n",
       " is it funny ? no . the doodlebops instead say something which is not intended as a joke , and laugh at it . \n",
       "\n",
       " is it bright ? it 's so bright , it 's painful . as far as color , s everything is extremely bright , so that 's good . but xxup nothing is ever wrong in the world of the xxunk 's . therefore , they are always happy . a kid in trouble will become depressed because they have never been exposed to being sad . \n",
       "\n",
       " the show is also extremely cheesy . every syllable is said to the highest level of exaggeration and very corny . it 's overkill . \n",
       "\n",
       " for kids , it 's entertaining , but past the age of 2 you wo n't want your kids to see it . they 'll never know how to grow up ., Text xxbos just two comments xxrep 4 . xxup seven years apart ? hardly evidence of the film 's relentless pulling - power ! as has been mentioned , the low - budget telemovie status of 13 xxup gantry xxup row is a mitigating factor in its limited appeal . having said that however the thing is not without merit - either as entertainment or as a fright outing per se . \n",
       "\n",
       " true , the plot at its most basic is a re - working of xxup the xxup amityville xxup horror - only without much horror . more a case of intrigue ! gibney might have made a more worthwhile impression if she had played xxunk xxunk a couple of seemingly unconnected murders with the \" house \" as the main suspect . the script is better than average and the production overall of a high standard . it just fails to engage the viewer particularly at key moments . \n",
       "\n",
       " having picked the xxup dvd up for a mere $ xxunk last week at my regular video store , i can not begrudge the expenditure . $ xxunk would be an acceptable price for the film . just do n't expect fireworks !, Text xxbos xxup some xxup not - so- xxup xxunk xxup spoilers xxup ahead \n",
       "\n",
       " why do people , when they are disoriented or sick or scared at a club , cut through the middle of the crowded dance floor on their way to the bathroom ? \n",
       "\n",
       " who in their right mind would hide under a bed when someone breaks into their room ? \n",
       "\n",
       " how often do you knock on a stranger 's door and when they do n't xxup immediately answer , you open the door , walk in , shout a few hello 's and then start going through their stuff ? \n",
       "\n",
       " if you were being pursued by someone you just discovered was a murderer , what would you do ? quietly sneak off and hide under a wooden platform or among metal implements ? run , quietly of course , to a ratty old barn or other decrepit structure ? \n",
       "\n",
       " i could be talking about almost any thriller that 's come out in the last few years , but since this is the \" the return \" page , obviously i 'm talking about \" the return . \" i saw it free because i work at a movie theater and make a point of screening all the \" scary \" movies . i thought this one was tolerable ... aside from the well - worn clichés . sarah michelle gellar is really drab and looks kind of \" huh ? \" through most of the film . the details of the plot are slowly given out as the movie progresses and it 's almost enough to make it interesting except there was n't enough explanation as it moved on and so i was almost lost until the last 2 / 3 of it . \n",
       "\n",
       " if you 're a die - hard thriller fan , it 's worth seeing at least once . if there 's nothing better at the theater and you really want to watch a movie , eh , i guess it 's worth a matinée ticket . if you thought the trailer made it look like an interesting movie and you ca n't wait ... wait ., Text xxbos this is a very funny movie , easy to watch , that entertains you almost all the time . the work of the director is recognizable and the type of humor is his trademark . the movie is a typical police partners history like lethal weapon , but the jokes and comedy are of argentinian sort . the twist is that one of them is a psychologist played by peretti and has to go with detective diaz ( played by luque ) on his assignments while he also assist him ( diaz is troubled because his wife cheated on him ) . some of the dialogs are hilarious worldwide : understandable and laughable anywhere . is very good overall , it would deserved an 8 , but i rated 7 because it gets a little down at the end . on a personal remark i must add that is a \" bravo \" for argentinian filmmakers , considering the little good is coming lately .]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas;\n",
       "\n",
       "Valid: LabelList\n",
       "y: CategoryList (25000 items)\n",
       "[Category 0, Category 0, Category 1, Category 0, Category 1]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas\n",
       "x: TextList (25000 items)\n",
       "[Text xxbos i thought it will be a ok movie after seeing the commercials about it . it was funny at some parts and some very nasty . the only person i felt sorry for is horatio sans who got a hot wife who is cheating on him with other women . but he never got a chance to have a threesome with until the and that was good but they should have made more bigger thru out the film ., Text xxbos i enjoyed carax 's \" les amants du pont neuf \" and was therefore expecting this film to be of a similar standard . well , the first 10 minutes were ok , but then it disintegrates into a rather pretentious journey of a young man looking for the essence of life . a sad disappointment ., Text xxbos what 's the matter with you people ? john dahl ? from \" rounders \" and \" unforgettable \" ? xxup too quirky ? knocking emma thompson and alan rickman for having fun playing against type ? and somebody liked the gingerbread man ? \n",
       "\n",
       " i rented this not knowing anything about it and found it about as nifty a video find as you can get . never insulting , well thought out , funny , scary . i disagree with the naysayers , clearly . i thought the story itself was unremarkable but the great cast , which most likely means the director was paying attention , lifted it to super cool status . good sound design also ( much more appreciated in surround , but i 'm not bragging ) . and yes , i 'm a girl , so maybe it has a slight female slant ( the guys in the gang are pretty worthwhile ) . all in all , a 9 and a hearty xxup recommend ., Text xxbos johnnie ( bert wheeler ) is a would - be songwriter ; newton ( robert woolsey ) is a would - be inventor . both work at a cigar stand in the lobby of an office building . johnnie wants to sell a song to winfield lake , a song publisher who also owns the building . lake 's secretary , mary ( betty grable ) , is johnnie 's sweetheart . when lake turns up dead , circumstances conspire to make mary and newton think that johnnie is the killer . they conspire again to implicate mary , who goes to jail . but who really shot lake ? who is the black widow , the blackmailer who had threatened him ? the other characters in this wacky murder mystery are : lake 's suspicious wife , a self - satisfied private detective , a seemingly slow - witted janitor ( willie best ) , lake 's auditor , a songwriter who thinks lake is stealing from him and another who thinks everyone is stealing from him . it 's up to newton and his truth machine to reveal the real killer . \n",
       "\n",
       " the baby - voiced wheeler and the cigar - chomping woolsey strike me as an arbitrary pairing , but they made several movies together in the 30s and some of them were funny . \n",
       "\n",
       " not this one . george stevens , who went on to have a distinguished career , directed this dismal comedy with a tedious murder mystery plot . but two scenes are good , and both feature wheeler and betty grable singing the excellent \" music in my heart , \" written by dorothy fields and jimmy mchugh . the first time , they sing it walking up a staircase ( after which they dance back down ) . later , wheeler and woolsey are on stilts so that they can see and talk to mary , who is in a jail cell on a high floor . wheeler and grable sing to each other through the bars . \n",
       "\n",
       " \" the nitwits \" has a few laughs , but the level of comedy is best illustrated by woolsey 's line : \" sonny , you 've got the brain of a six - year - old boy . and i 'll bet even he was glad to get rid of it . \" it 's watered - down xxunk did n't use the superfluous \" even \" when he said it ., Text xxbos this film has the guts to suggest that it might be best to simply accept your life as it is , and keep smiling anyway . as one who is more excited by the idea of taking charge of one 's life and moving forward , i felt slapped in the face , but that 's okay : i do n't have to agree with a movie to love it and respect it . great acting by streep and hurt , and everyone else really , and some wonderfully quirky scenes . a serious film . and take a hanky .]...\n",
       "Path: /home/farzin/rnn_python_code/wiki103_from_download/clas;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchRNNCore(\n",
       "    (encoder): Embedding(60004, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(60004, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=800, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (1): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (2): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (3): QRNNLayer(\n",
       "        (linear): WeightDropout(\n",
       "          (module): Linear(in_features=1111, out_features=1200, bias=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "      (3): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=<fastai.layers.FlattenedLoss object at 0x7f0ea3f00470>, metrics=[<function accuracy at 0x7f0e1d96f620>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('/home/farzin/rnn_python_code/wiki103_from_download/clas'), model_dir='models', callback_fns=[<class 'fastai.basic_train.Recorder'>], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(60004, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(60004, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=800, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=1200, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       ")])\n",
       "bptt: 140\n",
       "alpha: 2.0\n",
       "beta: 1.0\n",
       "adjust: False], layer_groups=[Sequential(\n",
       "  (0): Embedding(60004, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(60004, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=800, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=3333, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): QRNNLayer(\n",
       "    (linear): WeightDropout(\n",
       "      (module): Linear(in_features=1111, out_features=1200, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       ")])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn_cls.load('final')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_cls.data.batch_size = 8"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "How to check that this works properly with the train data and the test data?\n",
    "What is the smallest example?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<DatasetType.Fix: 5>"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DatasetType.Fix"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "y_hat, y_true = learn_cls.get_preds(DatasetType.Fix,ordered=True,)\n",
    "trn_df = CLS_train_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat, y_true = learn_cls.get_preds(DatasetType.Valid,ordered=True)\n",
    "trn_df = CLS_valid_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_df['pred0'] = to_np(y_hat)[:,0]\n",
    "trn_df['pred1'] = to_np(y_hat)[:,1]\n",
    "trn_df['pred_label']= to_np(y_hat).argmax(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_np(y_true)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn_df.labels.values[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labels</th>\n",
       "      <th>text</th>\n",
       "      <th>rating</th>\n",
       "      <th>pred0</th>\n",
       "      <th>pred1</th>\n",
       "      <th>pred_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>I thought it will be a Ok movie after seeing t...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.906678</td>\n",
       "      <td>0.093322</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>I enjoyed Carax's \"Les Amants du Pont Neuf\" an...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.994287</td>\n",
       "      <td>0.005713</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>What's the matter with you people? John Dahl? ...</td>\n",
       "      <td>10</td>\n",
       "      <td>0.016465</td>\n",
       "      <td>0.983535</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>Johnnie (Bert Wheeler) is a would-be songwrite...</td>\n",
       "      <td>3</td>\n",
       "      <td>0.898954</td>\n",
       "      <td>0.101046</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>This film has the guts to suggest that it migh...</td>\n",
       "      <td>8</td>\n",
       "      <td>0.002427</td>\n",
       "      <td>0.997573</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>I saw Ray when it first came out. Why? Partly,...</td>\n",
       "      <td>7</td>\n",
       "      <td>0.535921</td>\n",
       "      <td>0.464079</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>Who ARE the people that star in this thing? Ne...</td>\n",
       "      <td>8</td>\n",
       "      <td>0.007283</td>\n",
       "      <td>0.992717</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>One of Starewicz's longest and strangest short...</td>\n",
       "      <td>8</td>\n",
       "      <td>0.008810</td>\n",
       "      <td>0.991190</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>The three names that mean the most to this fil...</td>\n",
       "      <td>7</td>\n",
       "      <td>0.004281</td>\n",
       "      <td>0.995719</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>By far one of the best sci-fi films out there....</td>\n",
       "      <td>10</td>\n",
       "      <td>0.005068</td>\n",
       "      <td>0.994932</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>This was an excellent movie with good acting a...</td>\n",
       "      <td>10</td>\n",
       "      <td>0.000194</td>\n",
       "      <td>0.999806</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>What makes watching and reviewing films a plea...</td>\n",
       "      <td>9</td>\n",
       "      <td>0.000040</td>\n",
       "      <td>0.999960</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>Following is a little-known 1998 British film,...</td>\n",
       "      <td>7</td>\n",
       "      <td>0.163729</td>\n",
       "      <td>0.836271</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>This film is a sleeper because Rod Steiger's i...</td>\n",
       "      <td>8</td>\n",
       "      <td>0.000496</td>\n",
       "      <td>0.999504</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>I rented this movie expecting it to suck, and ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999786</td>\n",
       "      <td>0.000214</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1</td>\n",
       "      <td>Blademaster is definitely a memorable entry in...</td>\n",
       "      <td>10</td>\n",
       "      <td>0.001734</td>\n",
       "      <td>0.998266</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0</td>\n",
       "      <td>Sitting in a big wing chair with a huge book i...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.985897</td>\n",
       "      <td>0.014103</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>This probably ranks in my Top-5 list of the fu...</td>\n",
       "      <td>9</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.999981</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0</td>\n",
       "      <td>Everything about this movie is bad. everything...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999866</td>\n",
       "      <td>0.000134</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0</td>\n",
       "      <td>They must be. I'll list them so that you can c...</td>\n",
       "      <td>2</td>\n",
       "      <td>0.569170</td>\n",
       "      <td>0.430830</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    labels                                               text  rating  \\\n",
       "0        0  I thought it will be a Ok movie after seeing t...       1   \n",
       "1        0  I enjoyed Carax's \"Les Amants du Pont Neuf\" an...       4   \n",
       "2        1  What's the matter with you people? John Dahl? ...      10   \n",
       "3        0  Johnnie (Bert Wheeler) is a would-be songwrite...       3   \n",
       "4        1  This film has the guts to suggest that it migh...       8   \n",
       "5        1  I saw Ray when it first came out. Why? Partly,...       7   \n",
       "6        1  Who ARE the people that star in this thing? Ne...       8   \n",
       "7        1  One of Starewicz's longest and strangest short...       8   \n",
       "8        1  The three names that mean the most to this fil...       7   \n",
       "9        1  By far one of the best sci-fi films out there....      10   \n",
       "10       1  This was an excellent movie with good acting a...      10   \n",
       "11       1  What makes watching and reviewing films a plea...       9   \n",
       "12       1  Following is a little-known 1998 British film,...       7   \n",
       "13       1  This film is a sleeper because Rod Steiger's i...       8   \n",
       "14       0  I rented this movie expecting it to suck, and ...       1   \n",
       "15       1  Blademaster is definitely a memorable entry in...      10   \n",
       "16       0  Sitting in a big wing chair with a huge book i...       1   \n",
       "17       1  This probably ranks in my Top-5 list of the fu...       9   \n",
       "18       0  Everything about this movie is bad. everything...       1   \n",
       "19       0  They must be. I'll list them so that you can c...       2   \n",
       "\n",
       "       pred0     pred1  pred_label  \n",
       "0   0.906678  0.093322           0  \n",
       "1   0.994287  0.005713           0  \n",
       "2   0.016465  0.983535           1  \n",
       "3   0.898954  0.101046           0  \n",
       "4   0.002427  0.997573           1  \n",
       "5   0.535921  0.464079           0  \n",
       "6   0.007283  0.992717           1  \n",
       "7   0.008810  0.991190           1  \n",
       "8   0.004281  0.995719           1  \n",
       "9   0.005068  0.994932           1  \n",
       "10  0.000194  0.999806           1  \n",
       "11  0.000040  0.999960           1  \n",
       "12  0.163729  0.836271           1  \n",
       "13  0.000496  0.999504           1  \n",
       "14  0.999786  0.000214           0  \n",
       "15  0.001734  0.998266           1  \n",
       "16  0.985897  0.014103           0  \n",
       "17  0.000019  0.999981           1  \n",
       "18  0.999866  0.000134           0  \n",
       "19  0.569170  0.430830           0  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn_df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.93176"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1.-(trn_df.pred_label != trn_df.labels).sum()/len(trn_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_df.to_csv('bwd_training_classification.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Indeed, many of the marginal ratings (4 & 7) are a greater % of the errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1     141\n",
       "2      83\n",
       "3     118\n",
       "4     280\n",
       "7     345\n",
       "8     185\n",
       "9      74\n",
       "10    188\n",
       "Name: rating, dtype: int64"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn_df[trn_df.pred_label != trn_df.labels].rating.value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1     5100\n",
       "2     2284\n",
       "3     2420\n",
       "4     2696\n",
       "7     2496\n",
       "8     3009\n",
       "9     2263\n",
       "10    4732\n",
       "Name: rating, dtype: int64"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn_df.rating.value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1     0.027647\n",
       "2     0.036340\n",
       "3     0.048760\n",
       "4     0.103858\n",
       "7     0.138221\n",
       "8     0.061482\n",
       "9     0.032700\n",
       "10    0.039730\n",
       "Name: rating, dtype: float64"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn_df[trn_df.pred_label != trn_df.labels].rating.value_counts().sort_index() / trn_df.rating.value_counts().sort_index()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7 fasta.ai1 DEV",
   "language": "python",
   "name": "fastai1_dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}