Finetune LM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using seed: 558132\n"
     ]
    }
   ],
   "source": [
    "from prod.set_seed import random_ctl\n",
    "seed = random_ctl(558132)\n",
    "\n",
    "from fastai.text import *\n",
    "from fastai.callbacks import SaveModelCallback\n",
    "from fastai.layers import LabelSmoothingCrossEntropy\n",
    "\n",
    "import sentencepiece as spm #https://github.com/google/sentencepiece\n",
    "import fire\n",
    "\n",
    "from prod.sp_tok import *\n",
    "from prod.bin_metrics import Fbeta_binary\n",
    "\n",
    "def split_rebal_data(all_texts_df:DataFrame, clas_col:str='is_humor', split_seed:int=None):\n",
    "    ## rebalance cases\n",
    "    print('Number of positive samples:', (all_texts_df.loc[:,clas_col] == 1).sum())\n",
    "    print('Number of negative samples:',  (all_texts_df.loc[:,clas_col] == 0).sum())\n",
    "    print('Total samples:', len(all_texts_df))\n",
    "\n",
    "    if split_seed: np.random.seed(split_seed)\n",
    "    idx = np.random.permutation(len(all_texts_df))\n",
    "    test_cut = int(0.15 * len(idx))\n",
    "    valid_cut = int(0.15 * len(idx-test_cut))\n",
    "\n",
    "    df_train_all = all_texts_df.iloc[idx[:-(valid_cut+test_cut)],:]\n",
    "    df_valid     = all_texts_df.iloc[idx[ -(valid_cut+test_cut):-test_cut],:]\n",
    "    df_test      = all_texts_df.iloc[idx[-test_cut:],:]\n",
    "\n",
    "    print('Test prevalence(n = %d):'%len(df_test),df_test.loc[:,clas_col].sum()/ len(df_test))\n",
    "    print('Valid prevalence(n = %d):'%len(df_valid),df_valid.loc[:,clas_col].sum()/ len(df_valid))\n",
    "    print('Train all prevalence(n = %d):'%len(df_train_all), df_train_all.loc[:,clas_col].sum()/ len(df_train_all))\n",
    "    print('all samples (n = %d)'%len(all_texts_df))\n",
    "    assert len(all_texts_df) == (len(df_test)+len(df_valid)+len(df_train_all)),'math didnt work'\n",
    "\n",
    "    ## assumes that negative is less frequent case.  Generalize?\n",
    "    rows_pos = df_train_all.loc[:,clas_col] == 1\n",
    "    df_train_pos = df_train_all.loc[rows_pos]\n",
    "    df_train_neg = df_train_all.loc[~rows_pos]\n",
    "    print(f'Train Pos Cases:{df_train_pos.shape},Train Neg Cases:{df_train_neg.shape}')\n",
    "    df_resample_pos = df_train_pos.sample(n=len(df_train_neg),replace=True,\n",
    "                                          axis=0,random_state=np.random.get_state()[1][0]).reset_index(drop=True)\n",
    "    df_train = pd.concat([df_resample_pos,df_train_neg],axis=0) #randomized again in DataBunch?\n",
    "    print('Train prevalence (n = %d):'%len(df_train), df_train.loc[:,clas_col].sum()/ len(df_train))\n",
    "\n",
    "    return df_train, df_valid, df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = './data/rnn'\n",
    "sp_model = 'all_tweets_es_0609'\n",
    "batch_size = 64\n",
    "backward = 0\n",
    "dropmult = 1.0\n",
    "qrnn = True\n",
    "n_hid:int=2304\n",
    "flat_loss = True\n",
    "enc_name = 'twitter_es_enc_QRNN_0609_drop1_seed0_finetune'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Default Rules:\n",
      " ['fixup', 'replace_rep', 'replace_wrep', 'deal_caps', 'spec_add_spaces', 'rm_useless_spaces', 'sub_nl'] \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "PATH = Path(model_path)\n",
    "#torch.backends.cudnn.enabled=False\n",
    "\n",
    "defaults.text_spec_tok.append(NL) #add a New Line special char\n",
    "sp_vocab = Vocab( get_itos(sp_model) )\n",
    "mycust_tok = CustomTokenizer(SPTokenizer, sp_model, pre_rules=default_rules)\n",
    "\n",
    "train_text = pd.read_csv('./data/haha_2019_train.csv')#.loc[:,'text']\n",
    "test_text = pd.read_csv('./data/haha_2019_test.csv')#.loc[:,'text']\n",
    "\n",
    "raw_text = pd.concat([train_text.loc[:,'text'], test_text.loc[:,'text']],axis=0)\n",
    "\n",
    "print(\"Default Rules:\\n\",[x.__name__ for x in default_rules],\"\\n\\n\")\n",
    "for rule in default_rules: raw_text = raw_text.apply(lambda x: rule(str(x)))\n",
    "all_texts_df = pd.DataFrame(raw_text)\n",
    "all_texts_df.columns = ['new_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_texts = all_texts_df['new_text'].values.squeeze()\n",
    "idx = np.random.permutation(len(all_texts))\n",
    "cut = int(0.1 * len(idx))\n",
    "train_df = pd.DataFrame({'text':all_texts[idx[cut:]], 'labels':[0] * (len(all_texts)-cut)}, columns=['labels','text'])\n",
    "valid_df = pd.DataFrame({'text':all_texts[idx[:cut]], 'labels':[0] * cut}, columns=['labels','text'])\n",
    "print(f'Train:{train_df.shape}, valid:{valid_df.shape}')\n",
    "\n",
    "keyword_args = {'bs':batch_size}\n",
    "data = TextLMDataBunch.from_df(PATH, train_df, valid_df, \n",
    "                               tokenizer=mycust_tok, vocab=sp_vocab,\n",
    "                               text_cols='text', label_cols='labels', backwards=backward,\n",
    "                               **keyword_args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>idx</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>▁doy ▁cuenta ▁que ▁deb í ▁haber ▁sido ▁más ▁específico ... ▁ xxbos ▁el ▁hijo ▁le ▁dice ▁al ▁papá : ▁- pap á ▁, ▁soy ▁gay ▁! ▁, ▁y ▁el ▁papá ▁le ▁responde : ▁- gay ▁ricky ▁martin ▁que ▁tiene ▁plata ▁, ▁usted ▁lo ▁que ▁es ▁, ▁es ▁un ▁pobre ▁marica ▁! ▁! ▁ xxbos ▁soy ▁una ▁tremenda ▁paj era .. ▁😠 ▁ xxbos ▁# ▁elf ís ico a tra ep</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>xxbos ▁celebro ▁la ▁idea ▁de ▁gobierno ▁federal ▁de ▁ex trad itar ▁al ▁cha po . ▁sin ▁duda ▁será ▁útil ▁tener ▁a ▁alguien ▁del ▁otro ▁lado ▁que ▁nos ▁ayude ▁a ▁hacer ▁un ▁túnel . ▁ xxbos ▁una ▁mujer ▁enojada ▁es ▁más ▁peligrosa ▁que ▁cualquier ▁grupo ▁terrorista . ▁ xxbos ▁arriba ▁todos ▁! ▁que ▁hoy ▁gana ▁mexico oo ▁! ▁! ▁xxnl ▁# ▁mexico vs cor ea ▁ xxbos ▁mi ▁psiquiatra ▁me ▁dijo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>▁ xxbos ▁\" o ▁sea ▁, ▁sí ▁me ▁gustas ▁, ▁pero ▁en ▁vez ▁de ▁decír telo ▁, ▁te ▁voy ▁a ▁mandar ▁un ▁chingo ▁de ▁indirectas ▁y ▁mensajes ▁ocul tos \" ▁— mujeres . ▁ xxbos ▁cerra ▁y ▁vamos ▁, ▁muy ▁maduro ▁ xxbos ▁rico ▁con ▁lap top ▁- ▁va ▁a ▁estudiar xxnl ▁pobre ▁con ▁lap top ▁- ▁acaba ▁de ▁robar xxnl ▁rico ▁sin ▁camisa ▁- ▁modelo xxnl ▁pobre ▁sin ▁camisa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>▁no ▁es ▁la ▁ xxup ▁indic ada . ▁ xxbos ▁alex ia ▁esta ▁extremadamente ▁insoportable ▁😭 ▁ xxbos ▁— ▁¡ buenas ▁noches ▁! ▁me ▁da ▁una ▁cerveza ▁light ▁y ▁unos ▁cigar ros ▁men tol ados . ▁xxnl ▁—¿ también ▁unas ▁ko tex ▁joven azo ▁? ▁ xxbos ▁— hola ▁bebé ▁, ▁te ▁quiero ▁mucho ▁y ▁te ▁amo . ▁xxnl ▁—¿ otra ▁vez ▁le ▁pe gaste ▁al ▁carro ▁verdad ▁? ▁xxnl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>▁camino ▁de ▁cintura .. xxnl ▁de ▁mor on ▁, ▁para ▁la ▁plata ... ▁ xxbos ▁se ▁encuentran ▁dos ▁gl ánd ulas ▁y ▁le ▁dice ▁una ▁a ▁otra : xxnl ▁- ▁¿ qué ▁haces ▁? ▁xxnl ▁y ▁le ▁contesta : xxnl ▁- ▁s ▁ xxrep ▁6 ▁h ▁, ▁secreto ▁! ▁xxnl ▁# ▁chistes ▁# ▁humo ... ▁ xxbos ▁mmm ▁oye ▁ya ▁me ▁dio ▁sueño ▁, ▁ya ▁me ▁voy ▁a ▁dormir ▁hablamos</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'emb_sz': 400, 'n_hid': 2304, 'n_layers': 3, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.1, 'hidden_p': 0.15, 'input_p': 0.25, 'embed_p': 0.02, 'weight_p': 0.2, 'tie_weights': True, 'out_bias': True}\n",
      "SequentialRNN(\n",
      "  (0): AWD_LSTM(\n",
      "    (encoder): Embedding(30000, 400, padding_idx=1)\n",
      "    (encoder_dp): EmbeddingDropout(\n",
      "      (emb): Embedding(30000, 400, padding_idx=1)\n",
      "    )\n",
      "    (rnns): ModuleList(\n",
      "      (0): QRNN(\n",
      "        (layers): ModuleList(\n",
      "          (0): QRNNLayer(\n",
      "            (linear): WeightDropout(\n",
      "              (module): Linear(in_features=800, out_features=6912, bias=True)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (1): QRNN(\n",
      "        (layers): ModuleList(\n",
      "          (0): QRNNLayer(\n",
      "            (linear): WeightDropout(\n",
      "              (module): Linear(in_features=2304, out_features=6912, bias=True)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (2): QRNN(\n",
      "        (layers): ModuleList(\n",
      "          (0): QRNNLayer(\n",
      "            (linear): WeightDropout(\n",
      "              (module): Linear(in_features=2304, out_features=1200, bias=True)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (input_dp): RNNDropout()\n",
      "    (hidden_dps): ModuleList(\n",
      "      (0): RNNDropout()\n",
      "      (1): RNNDropout()\n",
      "      (2): RNNDropout()\n",
      "    )\n",
      "  )\n",
      "  (1): LinearDecoder(\n",
      "    (decoder): Linear(in_features=400, out_features=30000, bias=True)\n",
      "    (output_dp): RNNDropout()\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "config = awd_lstm_lm_config.copy()\n",
    "config['qrnn'] = qrnn\n",
    "config['n_hid'] = n_hid\n",
    "print(config)\n",
    "learn = language_model_learner(data, AWD_LSTM, drop_mult=dropmult, pretrained=False, config=config)\n",
    "if flat_loss: learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy)\n",
    "print(learn.model)\n",
    "\n",
    "'''\n",
    "-rw-rw-r-- 1 farzin farzin 416M Jun  9 14:35 ./data/rnn/models/twitter_es_504940.pth\n",
    "-rw-rw-r-- 1 farzin farzin 139M Jun  9 15:58 ./data/rnn/models/twitter_es_enc_QRNN_0609_drop1_seed0_finetune.pth\n",
    "'''\n",
    "_ = learn.load('twitter_es_370743')  #load the LM, finetune from here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'60,484,320'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_params = sum(p.numel() for p in learn.model.parameters() if p.requires_grad)\n",
    "f'{total_params:,}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "lr_find(learn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VdW5//HPk5mEJEAm5nlWGTQoKGipStVqHWqt1mq1eqm9tbXDtb1tf7/e3g732tpeh9qK6M+htg514HWtVcShCgpUgsqkIBCQIQlJgIQQMuf5/XFOMI0BDpAzJd/363VeOdl77bOfszjkOWvttdcyd0dERORIEqIdgIiIxAclDBERCYkShoiIhEQJQ0REQqKEISIiIVHCEBGRkCSF88XNbCtQA7QAze5e2GF/X+BBYBRQD3zV3deGcqyIiERWWBNG0Gx3rzzEvh8B77n7pWY2Hvg9cHaIx4qISARFu0tqIvAagLuvB4abWUF0QxIRkc6Eu4XhwCIzc+A+d5/fYf8q4DJgiZmdCgwDBgO7QjgWADObC8wFyMjIOGX8+PHheSciIt3QypUrK909L5SyFs6pQcxskLvvNLN84GXgm+6+uN3+LOAuYCqwBhgP/Iu7v3ekYztTWFjoRUVFYXs/IiLdjZmtDPUacVhbGO6+M/iz3MwWAKcCi9vt3wdcD2BmBmwBikM5VkREIits1zDMLMPMMtueA3OAtR3K9DGzlOCvNwKL3X1fKMeKiEhkhbOFUQAsCDQcSAIec/eFZnYTgLvPAyYAjwSvU6wDbjjcsWGMVUREjiBsCcPdi4HJnWyf1+75MmBsqMeKiEj0RHtYrYiIxAklDBERCYkShoiIhEQJQ0Qkjr38/i7mvbE5IudSwhARiWMvrinlj0u3RuRcShgiInGspLqOAX16ReRcShgiInGstLqeAdlpETmXEoaISJxyd0qr6xmkFoaIiBzO7tpGGptb1cIQEZHDK6mqA9A1DBERObySqnoABmYrYYiIyGGUVre1MNQlJSIih1FaXU9KUgI5GSlHLtwFlDBEROJUSVUdA7PTCC4FEXZKGCIicaqkqo4BEbp+AUoYIiJxq7S6PmLXL0AJQ0QkLjW3tLJrX33ERkiBEoaISFwqr2mg1WFghO7BACUMEZG4FOkhtaCEISISl3ZG+KY9UMIQEYlLpVVqYYiISAhKq+vpnZpEVlpyxM6phCEiEodKquoYGMHWBShhiIjEpZLqyN60B2FOGGa21czWmNl7ZlbUyf6+ZrbAzFab2dtmdmK7feeZ2QYz22Rm/x7OOEVE4k1pVX3EWxhJETjHbHevPMS+HwHvufulZjYe+D1wtpklBp+fC+wAVpjZc+7+fgTiFRGJafVNLeyubexeLYwQTAReA3D39cBwMysATgU2uXuxuzcCTwAXRy9MEZHYUVYdHFIbwZv2IPwJw4FFZrbSzOZ2sn8VcBmAmZ0KDAMGA4OA7e3K7Qhu+wQzm2tmRWZWVFFR0aXBi4jEopLgTXsDI7Q0a5twJ4yZ7n4ycD7wDTM7s8P+24A+ZvYe8E3gXaDlaE7g7vPdvdDdC/Py8rokaBGRWNa20l6klmZtE9ZrGO6+M/iz3MwWEOhqWtxu/z7gegALTOi+BSgGegFD2r3UYGBnOGMVEYkXB2/a6y4tDDPLMLPMtufAHGBthzJ9zKxtqagbgcXBJLICGGNmI4L7rwSeC1esIiLxpKS6nn4ZKaQlJ0b0vOFsYRQAC4IrQSUBj7n7QjO7CcDd5wETgEfMzIF1wA3Bfc1mdjPwEpAIPOju68IYq4hI3CitjvxNexDGhOHuxcDkTrbPa/d8GTD2EMe/ALwQrvhEROJVaVU9Q3PSI37eaA+rFRGRo9S2lnekKWGIiMSRmvomahqaIz5CCpQwRETiSmnwpr1Ij5ACJQwRkbhSEhxSO0gtDBEROZyDLQwlDBEROZySqjoSDAoyUyN+biUMEZE4UlJVT35mGkmJkf/zrYQhIhJHSqvrIrqOd3tKGCIicaS0uj7i05q3UcIQEYkT7h61m/ZACUNEJG7sqW2kobk14ivttVHCEBGJE6UHV9pTC0NERA6j5OA6GGphiIjIYZRGaS3vNkoYIiJxoqS6jpTEBHIyUo5cOAyUMERE4kRJVT39s9NISLConF8JQ0QkTpRW1UVllto2ShgiInGitLo+KrPUtlHCEBGJAy2tTtm++qhNCwJKGCIicaGipoGWVo/akFpQwhARiQs7qw4A0btpD5QwRETiwpbKQMIYnpMRtRiUMERE4sCWyv0kJRhD+qVHLYakcL64mW0FaoAWoNndCzvszwb+BAwNxvIbd38ouK8FWBMsus3dPxfOWEVEYllxRS1D+6WTHIWFk9qENWEEzXb3ykPs+wbwvrtfZGZ5wAYz+7O7NwJ17j4lAvGJiMS8LZW1jMiNXncURL9LyoFMMzOgN7AHaI5uSCIisaW11dlSWcvIvO6dMBxYZGYrzWxuJ/vvASYAJQS6n25x99bgvjQzKzKz5WZ2yaFOYGZzg+WKKioquvwNiIhEW0l1HQ3NrYzI7R3VOMLdJTXT3XeaWT7wspmtd/fF7fZ/BngP+DQwKlhmibvvA4YFjx0JvGZma9x9c8cTuPt8YD5AYWGhh/n9iIhEXHFFLUD3bmG4+87gz3JgAXBqhyLXA896wCZgCzC+w7HFwOvA1HDGKiISq7ZUBhNGd72GYWYZZpbZ9hyYA6ztUGwbcHawTAEwDig2s75mlhrcngucAbwfrlhFRGJZccV+eqcmkZeZGtU4wtklVQAsCFzPJgl4zN0XmtlNAO4+D/g58LCZrQEM+IG7V5rZ6cB9ZtZKIKnd5u5KGCLSIxUHR0gF/55GTdgSRrAraXIn2+e1e15CoOXRscxS4KRwxSYiEk+KK2opHN432mFEfVitiIgcRn1TCyXVdVG/BwOUMEREYtpHuw/gDiPzojukFpQwRERiWnHFfiD6I6RACUNEJKYVB4fUqktKREQOq7iiloKsVDJSIzH13+EpYYiIxLAtlfsZGeUpQdooYYiIxLDiylpGRHlKkDZKGCIiMWpvbSNVB5pi4oI3KGGIiMSs4srgCCm1MERE5HDaZqmN9rTmbZQwRERiVHFlbWAd7769oh0KoIQhIhKztlTUMjQnnaQoruPdXmxEISIin1AcQ0NqQQlDRCQmtbQ6W3cfiJkL3qCEISISk0qq6mhsbo2ZIbWghCEiEpNiaQ6pNkoYIiIx6OAstTEwrXkbJQwRkRi0pbKWzNQkcnunRDuUg5QwRERiUHFFLSPzor+Od3tKGCIiMWhLZW1MXb8AJQwRkZhT19jCzqq6mLp+AUoYIiIxZ+vu2BshBUoYIiIxp23SwVi6aQ/CnDDMbKuZrTGz98ysqJP92Wb2VzNbZWbrzOz6dvu+YmYbg4+vhDNOEZFYsiU4rXmstTAisUjsbHevPMS+bwDvu/tFZpYHbDCzPwO9gf8ACgEHVprZc+6+NwLxiohEVXFFLQOy00hPif463u1Fu0vKgUwLjBvrDewBmoHPAC+7+55gkngZOC96YYqIRE5xDI6QgvAnDAcWmdlKM5vbyf57gAlACbAGuMXdW4FBwPZ25XYEt32Cmc01syIzK6qoqOja6EVEomD7ngMMy0mPdhifEO6EMdPdTwbOB75hZmd22P8Z4D1gIDAFuMfMso7mBO4+390L3b0wLy+vS4IWEYmWhuYWdtc2MiA7NhZNai+sCcPddwZ/lgMLgFM7FLkeeNYDNgFbgPHATmBIu3KDg9tERLq18n0NAPTPTotyJJ8UtoRhZhlmltn2HJgDrO1QbBtwdrBMATAOKAZeAuaYWV8z6xs89qVwxSoiEitKq+sB6J8VewkjpEvwZjYK2OHuDWb2KWAS8Ed3rzrMYQXAguA8KEnAY+6+0MxuAnD3ecDPgYfNbA1gwA/aRlSZ2c+BFcHX+pm77znqdyciEmdKq+sAGBCDLYxQx2w9AxSa2WhgPvC/wGPABYc6wN2LgcmdbJ/X7nkJgdZDZ8c/CDwYYnwiIt3Crn3BFkYMJoxQu6Ra3b0ZuBT4nbvfCgwIX1giIj1TaXU9vVOTyExLjnYonxBqwmgys6uArwDPB7fF3rsREYlzZdX1FGSlRjuMToWaMK4HZgC/dPctZjYCeDR8YYmI9Eyl1fUxOaQWQryG4e7vA98CCI5aynT3X4UzMBGRnmjXvnpGj86NdhidCqmFYWavm1mWmfUD3gHuN7P/CW9oIiI9S3NLK+U1DTE5QgpC75LKdvd9wGUEhtOeBpwTvrBERHqeyv2NtLQ6BTF4DwaEnjCSzGwAcAUfX/QWEZEuFMv3YEDoCeNnBO603uzuK8xsJLAxfGGJiPQ8ZdWxew8GhH7R+yngqXa/FwOfD1dQIiI9UVnwpr1YHSUV6kXvwWa2wMzKg49nzGxwuIMTEelJyqrrSUlKoG96bN7mFmqX1EPAcwSmIR8I/DW4TUREukhpdT39s9IIzsEXc0JNGHnu/pC7NwcfDwNafEJEpAuVVdfH7PULCD1h7DazL5tZYvDxZWB3OAMTEelpyvbVx+wIKQg9YXyVwJDaMqAUuBy4LkwxiYj0OO4eaGHE6D0YEGLCcPeP3P1z7p7n7vnufgkaJSUi0mX21DbS2NLaLbqkOvPdLotCRKSHa1tprzt0SXUmNi/ji4jEoY8XTorNezDg+BKGd1kUIiI9XCyv5d3msHd6m1kNnScGA2I3DYqIxJmy6noSE4y8zNhcPAmOkDDcPTNSgYiI9GSl1fXkZ6aSmBC7vf0hzSUlPUt9Uwtl1fWUVNdRWlVPaXUdJdX15GSkcPVpw2J6FIdIvNq1L7Zv2gMlDAnaW9vIwnVlPL+6hGWbd9PaoSOyX0YKVQcauff1zVw0eSA3zBzBiYOyoxOsSDdUWl3H2ILY7tRRwjgK7s6rH5SzqWI/kwZnM2lwH3qnHrkK65ta2Fyxn03l+9m4az8by2uormvCMMwIPDASEoyB2WmMyM1geG4GI3MzGJqTTmpSYljez776Jhat28Xzq0t4c2Mlza3OiNwM5p45ilF5GQzs04sB2WkMyO5Fr5REtu85wENvbeXJFdtY8O5OZozM4YaZI5g0OJv01CTSkxNJ6NCcbm5pZV99M9V1TVTXNTEiJ4PsGJ1YTSRa3J3S6nrOHBvbMy6FNWGY2VagBmgBmt29sMP+W4Gr28UygcC8VXuOdGyklVTV8ZP/XcsrH5Qf3JZgMLYgkylD+jBpcB8AKvc3sHt/A5X7G6nc30DZvnq27zlw8Bt7YoIxPCednIxUnFbcA6MK3J3mVmfdzmp21zYePIcZ5GSk0lm3Zl5mKuP7ZzG+fybjB2Qyrn8meb1TDztx2bbdB3jlg128un4Xb2/ZQ1OLM6hPL26YNYKLJg3khIFZhzx+SL90fnLRRG45ZwxPrtjGQ29t5cY/Fv1TmfSURNJTkkhJNPbVN7O/ofkTMT903TS1TkTaqWlo5kBjS0zfgwFg7uEbHRv8o1/o7pUhlL0I+I67f/poj21TWFjoRUVFRy54FFpanUeWbuW3izbQ6vDdc8dy6cmDWLOzmve2VfHu9ipWba+iuq7p4DFZaUnkZqaSm5FKXlYqo/N6M7YgkzEFvRmek0FK0uFHM1fXNbG1spatu2sprqilvKb+E2XcYWdVHRvKaiivaTi4vW96MgVZaeRlppLbO5Xc3ink9k5l74EmXv1gFxvL9wMwOr83Z4/PZ84J/Tl5aJ9jmh2zqaWVNzZUsKumntqGZmobWjjQ2Mz+hhYam1vJ6pVEdq/kg4+UpAT+628fUF3XxLxrTmHWmNj+NiUSKR/uqmHOHYu5+6qpfG7ywIie28xWhvqFPJa6pK4CHo92EO2tK6nmR8+uYdWOas4am8cvLjmRIf3SAZg9Lp/Z4/IBaG11duytIznJ6JeRctxdSNm9kpk8pA+Th/QJqfye2kbWl+1jfWkNG8v3U1FTT8X+Rooraqnc30BDcytJCcZpI/tx5alDOWdCPsNyMo4rRoDkxATOmVhwVMcUDuvHdQ+9zfUPreDXl0/ispO1rIpIWRzcgwHhTxgOLDIzB+5z9/mdFTKzdOA84OajPTZcHl3+ET99bh1905O5+6qpXDRpwCG/hSckGENz0iMZ3j/pl5HC6aNyOX1U7if2uTu1jS0YkBHC9ZZw65+dxl9umsFNj67ku39ZRWl1Pf/6qVGHrFt3p6KmgeLKWrYEW10jczO4eMog0pLDc21HJNLK4mBaEAh/wpjp7jvNLB942czWu/viTspdBLzl7nuO9lgzmwvMBRg6dOhxB+zu/GbRBn7/982cPT6f314xmT7pKcf9utFiZiFdmI+krLRkHr7+VG59ehW3v7SBkqo6Zo3Jo2J/AxU1Hz9Kq+vYWllLbWPLwWOTEozmVue2F9fzpdOGcs304TE/FFHkSNru8s7Pit2b9iDMCcPddwZ/lpvZAuBUoLOEcSUduqNCPTbY8pgPgWsYxxNvU0srP3p2DU+t3MGV04bwi0tOJCnxeGZPkUNJSUrgjium0D8rjfsWF/Pnf2wD2i7yB6679M9OY9rwfozMy2BEbuAxILsXK7bu4cE3t/CH1zdz3xvFXHDSAL46cwRTQuzCE4k1ZfvqyO19/N3Z4Ra2hGFmGUCCu9cEn88BftZJuWzgLODLR3tsVzrQ2My//vkdXt9QwS1nj+Hb54yJ2WUSu4uEBOOHF0zgc1MG4g75man0y0g5YpKePjKH6SNz2Lb7AI8s28qTK7bz3KoSJg/pw7XTh/HZSQPUXSVxpay6noIYv34B4W1hFAALgn90k4DH3H2hmd0E4O7zguUuBRa5e+2Rjg1XoLv3N/DVR4pYs6OK/7r0JL502vF3bUnoThh4bENsh+ak838vnMh3zh3L00XbeXT5R3zvqVX88oUPuKJwCFefNvTgIIUjqalvYtnm3YzK783I3Ax9WZCIKq2uZ3Df2J+eL6zDaiPtWIbV7q1t5LJ7l1JSVcc9XzqZc49y1I/EDndn6ebd/HHZVl5+fxcAnx5fwPVnDOf0UTmdJoH6phb+tPwjfv/3Tew9EBgand0rmSlD+jB1aB+mDu3LpEHZ9M2I3+tYEvum/GwRF04awC8uOSni547XYbVR0Sc9mXMnFvCZEwo4ZVi/aIcjx8HMOGN0LmeMzmVnVR2P/eMjHn97O698sIsx+b259vThXDZ1EBmpSbS0Os++s4M7X9nIzqo6Zo3J5cZZI9lVXc+72/fy7rYq7np1I23fp/pnpTF+QCbj+2cxYUAmEwZkMSa/t1oictzqm1qoOtDEgBheB6NNj29hSPdW39TC86tLeWTpVtbsrCYzLYlLpgxiefFuNpYHpnj5wXnjOWP0J4ck19Q3sXpHNWt3VrO+rIYPSvexuWI/TS2B/zPj+2fytbNGcuGkgSRrcIQco62VtXzqN6/zmy9M5vJTIn9fkloYIkFpyYlcfspgPn/yIN7ZVsXDS7fy+NvbGNovnT9cfTLnn9j/kK2EzLTkgy2WNo3NrWyu2M872/byyNKtfOfJVfzmpQ+5cdYIvjhtCOkp+i8lRycelmZto0+39AhmxinD+nLKsL4c+PxJpCYlHtO6AylJCUwYkMWEAVlcNW0or60vZ94bm/nPv77P3a9u5LrTRzD3zJH0StEoLQlN2b46gLi4n0jtaOlx0lOSumSRmoQE45yJBTz99dN5+qYZnDKsL3e88iFz7nyDNz6s6IJIpSeIh6VZ2yhhiHSBwuH9eOAr03j8X6aTnJDAVx58m289/i4V7SaGFOnMrup6MtOSYmLqniNRwhDpQjNG5fDit2dxy9ljWLi2jLN/+zqPv72N1o4rUokElVbXx8X1C1DCEOlyqUmJfOfcsbxwyywmDMjih8+u4bJ7l7Js8+5ohyYxqGxfPf3jYEgtKGGIhM3o/N48MXc6t18+ibLqeq66fznXPvg2a3dWRzs0iSGl1fUMiIPrF6CEIRJWZsYXCofw+q2f4scXTGD1jiou/N2bfOOxdyiu2B/t8CTKmlpaqdzfQEGcdEnF/lUWkW4gLTmRfzlzJF88dQgPLC7mgTe3sHBtGWeOyeXiKYM4d2JBXFz0lK5VXtOAe3zcgwFKGCIRlZWWzHfnjOOaGcN56K0t/O97JXz7yfdIS07gnAkFXDxlEGeOzY35aa6la5RVx889GKCEIRIVeZmpfP+88fzbnHGs3LaX594r4W9rSnl+dSlpyQmMKwjMW9U2f9X4/pmaALEbiqe7vEEJQySqEhKMacP7MW14P35y0UTe3FTJ4g8r2FBWw8sf7OLJou0Hy356fD6/vnwSub1je1U2CV28rOXdRglDJEYkJyYwe1w+s8flA8H1zPc3sL60hqKP9jLvjc2cf9cS7rhiCjPHfHKyRIk/W3fXkpmaRHav5GiHEhKNkhKJUWZGfmYaZ47N47vnjuW5m88gu1cy1zz4D361cD1NLa3RDlGO0/rSGsb1z4ybafKVMETixPj+Wfz15plcOW0I976+mSvuW8b2PQeiHZYcI3dnQ1kN4wdkRjuUkClhiMSRXimJ/Pdlk7jnS1PZtGs/F9y1hFufWsULa0rZV98U7fDkKOysqqOmoZlx/bOiHUrIdA1DJA5dOGkgkwf34faXNvDSujKeWrmDpITAFO6zx+dz/on9GZaTEe0w5TA2lNUAMKF//LQwlDBE4tSQfuncfdVUmltaeXd7FX9fX87fN1Rw24vruePlD/nD1Sdz9gStUR+r1gcTxtg4ShjqkhKJc0mJCUwb3o/vnzeeF2+ZxZs/mM3YgkzmPrqSZ1buiHZ4cgjry2oY1KcXWWnxMUIKlDBEup3BfdN5fO50po/sx/eeWsUDS4qjHZJ0YkPZPsbHUesClDBEuqXeqUk8eN00LjipP7/42wfc9uJ63LUmR6xoaG5hc0VtXI2QAl3DEOm2UpMS+d1VJ9MnfS3z3tjM3tpGfnnpiSQl6ntitG0ur6Wl1eNqhBSEOWGY2VagBmgBmt29sMP+W4Gr28UyAchz9z1mdh5wF5AIPODut4UzVpHuKDHB+OUlJ5KTkcLvXtvEtj0H+O0VkxnYJz4W7Omu1pftA+JrhBREpktqtrtP6ZgsANz99uC+KcAPgTeCySIR+D1wPjARuMrMJkYgVpFux8z43pxx/PrySazaUcV5dy7muVUl0Q6rR9tQVkNKYgLDc+Nr6HMstU2vAh4PPj8V2OTuxe7eCDwBXBy1yES6gSsKh/DiLbMYld+bbz3+Lt9+4l2q63SzXzR8UFbD6PzeJMdZ92C4o3VgkZmtNLO5hypkZunAecAzwU2DgO3tiuwIbuvs2LlmVmRmRRUVFV0Utkj3NCwng6e+NoPvnDOWv64u5fw7F2ut8SiIxxFSEP6EMdPdTybQtfQNMzvzEOUuAt5y9z1HewJ3n+/uhe5emJeXdzyxivQISYkJ3HLOGJ6+aQYpSQlcdf9yzr9rCXe9spEPd9VoNFWY7a1tZNe+BsbFYcII60Vvd98Z/FluZgsIdDUt7qTolXzcHQWwExjS7vfBwW0i0kWmDu3L3741i8ff3sbCtWXc+eqH3PHKh4zMy+C8E/pz8ZRBcflHLda13eE9fkB8jZCCMCYMM8sAEty9Jvh8DvCzTsplA2cBX263eQUwxsxGEEgUVwJfClesIj1VRmoSN84ayY2zRlK+r56X1pWxcF0Z9y0u5g+vb2b6yH589YwRnD2hgMSE+JiCO9a1jZCKxy6pcLYwCoAFwXnek4DH3H2hmd0E4O7zguUuBRa5e23bge7ebGY3Ay8RGFb7oLuvC2OsIj1eflYa18wYzjUzhrOntpG/FG3nj0u3MvfRlQzp14uvzBjOFdOGxNVUFrFoQ1kNfdOTyc+Mv5UTrTv1VxYWFnpRUVG0wxDpNppbWln0/i4eemsLK7buJT0lkU+Pz+essXmcNS6P/Mz4WFo0llz8+7folZzAE3NnRDsUAMxsZWe3PXRGd3qLyCElJSZwwUkDuOCkAazdWc2fln/Eq+vLeX51KQAnDMzirLF5nDuxgKlD+0Y52tjX2ups3FXDFYVDjlw4BilhiEhIThyUzW2fn0Rrq/N+6T7e+LCCNzZUHLzecdHkgfzkwonkxWFXS6Rs33uAA40tcXn9ApQwROQoJSQYJw7K5sRB2Xxj9miq65p46K0t/OHvm1n8YQU//uwEvnDK4LhZpzqSPiiN3xFSEFt3eotIHMrulcy3zxnLC7fMZEx+b77/9GqufuAfbK2sPfLBPcyGshrMYGxB72iHckyUMESkS4zOz+QvX5vBLy45kTU7qvnMnYt5ZOnWaIcVU9aX7WNYv3TSU+Kzc0cJQ0S6TEKC8eXpw3j5u2dxxuhc/uO5dfxqodbiaLOhrCaub4ZUwhCRLtc/O437ry3kS6cN5d7XN/PDZ9fQ3NIa7bCiqq6xhS27axkfZ2tgtBef7SIRiXkd1+LYe6CRu66cSlpyYrRDi4qN5TW4x+cd3m3UwhCRsGlbi+MnF07kpXW7uP6hFdTU98wp1dfH+QgpUMIQkQj46swR3PHFyazYuocv3f8Pyqrrox1SxK0vqyEtOYGh/dKjHcoxU8IQkYi4dOpg7r+2kE3l+5lzxxs8vXJHj7oYvr5sH+MKMuN6EkclDBGJmNnj83nxllmM65/Jvz21ihseKWLXvu7f2nB31sf5CClQwhCRCBuem8GTc2fwfy+cyNLNlZz7P92/tVFR08Ce2kbGxfEIKVDCEJEoSEgwbpg5ghdvOZOxBYHWxvUPrzi4VkR3s6w4sAzuKcPie4JGJQwRiZoRuRk8+bVAa2Pl1r2cf9cSbn7sHTaV7492aF3qzY2VZPdK5qRB2dEO5bjoPgwRiarEYGvj8ycP4v4lxTz01lZeWFPKJVMH8e2zxzI0J35HFUHg+sWSjZWcMTonri94g1oYIhIj+qSncOtnxrPk+7O5YeYI/ra6lE//9nV+9tf3OdDYHO3wjtnmiv2U7atn5ui8aIdy3JQwRCSm5PRO5cefnciS78/mi9OG8OBbWzj/riX8I3gdIN4s2VgJwKwxuVGO5PgpYYhITMrPSuOXl57EE3On4w5fnL+cnz63Lu5aG0s2VjI8J50hcXzDXhslDBGkJp12AAAMt0lEQVSJadNH5rDw27O47vThPLx0K+fduYRlm+OjtdHY3Mry4t3M7AatC1DCEJE4kJ6SxE8/dwJPzp2OGVx1/3L+5+UPaW2N7Xs33tm2lwONLcwaE//XL0AJQ0TiyGkjc3jxlll8/uTB3P3qRuY+ujKmJzN8c2MliQnGjFE50Q6lSyhhiEhcSU9J4jdfmMRPL5rI3zeUc8nv36K4Ijbv21iyqZIpQ/qQlZYc7VC6RFgThpltNbM1ZvaemRUdosyngvvXmdkbR3OsiPRMZsZ1Z4zgTzecxt4DTVx8z1u8tn5XtMP6J1UHGlm9o4qZo7vH9QuITAtjtrtPcffCjjvMrA/wB+Bz7n4C8IVQjxURmTEqh+duPoOhOenc8EgRd77yIU0xsrLf0s27ce8ew2nbRLtL6kvAs+6+DcDdy6Mcj4jEmcF903n6ptO5ZMog7nxlI5+9ewlvb9kT7bBYsrGCzNQkJg/pE+1Quky4E4YDi8xspZnN7WT/WKCvmb0eLHPtURwLgJnNNbMiMyuqqKjo4vBFJB70Sknkji9O4YFrC6ltaOGK+5Zx61Or2FPbGJV42qYDmT4qh+TEaH8v7zrhnktqprvvNLN84GUzW+/uizuc/xTgbKAXsMzMlrv7hyEcC4C7zwfmAxQWFsb2GDsRCatzJhZw+ugcfvfaJu5fXMzLH+zi388bzxWFQ0iI4DxOH+0+wI69dXztzJERO2ckhDX1ufvO4M9yYAFwaociO4CX3L3W3SuBxcDkEI8VEfmE9JQkfnDeeF64ZRZj8zP592fXcNOfVlLf1BKxGJZsDPR2zOwm91+0CVvCMLMMM8tsew7MAdZ2KPa/wEwzSzKzdOA04IMQjxUROaSxBZk8+bXp/J/PTuDlD3Zx7f97m+q6yNyzsWRjJYP79mJ4nM+021E4WxgFwJtmtgp4G/ibuy80s5vM7CYAd/8AWAisDpZ5wN3XHurYMMYqIt2QmXHjrJHcfeVU3t2+lyvmLQv7krDNLa0s27ybWWNyMYvv6cw7Cts1DHcvJti91GH7vA6/3w7cHsqxIiLH4qLJA+mbnsLXHi3isj8s5Y83nMqovN5hOdeqHVXUNDR3i+nMO+o+l+9FRA5j5phcnvzaDBqaW7j83qW8t70qLOdZsrESMzhjdPeYDqQ9JQwR6TFOHJTNM18/ncy0ZK6cv4xfLVzfpUNv99Y28uw7O5k0uA990lO67HVjhRKGiPQow3IyeObrp3POhALmvbGZmb96jdteXM/u/Q3H9bp1jS3c8MgKyvbV8+MLJnRRtLHF3LvPrQuFhYVeVKRpp0QkNBt31XD3a5t4fnUJvZITuWbGMObOGklO79Sjep3mllZu+tNKXl1fzr1Xn8x5Jw4IU8Rdz8xWhjr9khKGiPR4m8pr+N1rm3huVQnpyYncMHMEN545MqRZZt2dHz67hidWbOfnl5zINdOHRSDirnM0CUNdUiLS443Oz+SuK6fy8nfO4lPj8rn7tU2c+eu/M3/x5iPe8HfHKxt5YsV2bp49Ou6SxdFSC0NEpIM1O6q5fdEGFn9YQf+sNL519hhmjMohKy2JrF7JB+eH+vM/PuLHC9ZyReFgfvX5SXF534W6pEREusDy4t38euF63tn2z0Nw01MSyUpLprymnk+Ny2f+NaeQFKeTDB5Nwgj35IMiInFr+sgcnvn66RR9tJede+uormtiX10T++qbqK5rondqMv/2mbFxmyyOlhKGiMhhmBnThvdj2vBoRxJ9PSMtiojIcVPCEBGRkChhiIhISJQwREQkJEoYIiISEiUMEREJiRKGiIiERAlDRERC0q2mBjGzCqAKqD5EkexO9oWy7XC/5wKVxxLvYXQW0/GWP1yZQ+07Uj103BbuejlUDMdb/mjrRp+ZY//MxHq9hHpMqO//cNsPVxcd94Wzboa5e2jrybp7t3oA849mXyjbDvc7UBTJ93Cs5Y+2XkKph07qIqz1Eit1o8/MsX9mYr1ejrdujmb7Eeqi476YqJvu2CX116PcF8q2I/3e1Y729UMpf7T10tn2I9VVuOvlWM4RjrrRZ+bYPzOxXi+hHhPq+z/c9sPVRSz+X+peXVLRYGZFHuJMjz2J6uXQVDedU70cWqzUTXdsYUTa/GgHEKNUL4emuumc6uXQYqJu1MIQEZGQqIUhIiIhUcIQEZGQKGG0Y2YPmlm5ma09hmNPMbM1ZrbJzO62dov7mtk3zWy9ma0zs193bdThF456MbOfmtlOM3sv+Lig6yMPv3B9ZoL7v2dmbma5XRdxZITpM/NzM1sd/LwsMrOBXR95eIWpXm4P/n1ZbWYLzKxP10ceoITxzx4GzjvGY+8F/gUYE3ycB2Bms4GLgcnufgLwm+MPM+IepovrJegOd58SfLxwfCFGzcOEoW7MbAgwB9h2nPFFy8N0fb3c7u6T3H0K8Dzwk+MNMgoepuvr5WXgRHefBHwI/PA4YzwkJYx23H0xsKf9NjMbZWYLzWylmS0xs/EdjzOzAUCWuy/3wCiCPwKXBHd/HbjN3RuC5ygP77voemGql24hjHVzB/B9IC5HpYSjXtx9X7uiGcRh3YSpXha5e3Ow6HJgcLjiV8I4svnAN939FODfgD90UmYQsKPd7zuC2wDGArPM7B9m9oaZTQtrtJFzvPUCcHOwGf2gmfUNX6gRd1x1Y2YXAzvdfVW4A42w4/7MmNkvzWw7cDXx2cLoTFf8X2rzVeDFLo8wKClcL9wdmFlv4HTgqXbdy6lH+TJJQD9gOjAN+IuZjfQ4Hs/cRfVyL/BzAt8Sfw78lsCHPa4db92YWTrwIwLdUd1GF31mcPcfAz82sx8CNwP/0WVBRkFX1UvwtX4MNAN/7proPkkJ4/ASgKpgn+lBZpYIrAz++hyBP37tm4GDgZ3B5zuAZ4MJ4m0zayUwkVhFOAMPs+OuF3ff1e64+wn0SXcHx1s3o4ARwKrgH5DBwDtmdqq7l4U59nDqiv9L7f0ZeIE4Txh0Ub2Y2XXAhcDZYf0y2tUTWsX7AxgOrG33+1LgC8HnRuDidWfHvU2gFWEEmoQXBLffBPws+HwssJ3gDZPx9AhDvQxoV+Y7wBPRfo+xUjcdymwFcqP9HmOhXoAx7cp8E3g62u8xRurlPOB9IC/ssUe78mLpATwOlAJNBFoGNxD4trcQWBX8R/nJIY4tBNYCm4F72pICkAL8KbjvHeDT0X6fMVIvjwJrgNUEvkENiNT7ifW66VAmLhNGmD4zzwS3ryYwcd6gaL/PGKmXTQS+iL4XfMwLV/yaGkREREKiUVIiIhISJQwREQmJEoaIiIRECUNEREKihCEiIiFRwpBuzcz2R/h8D5jZxC56rZbgzKxrzeyvR5qF1Mz6mNm/dsW5RTqjYbXSrZnZfnfv3YWvl+QfT/QWVu1jN7NHgA/d/ZeHKT8ceN7dT4xEfNLzqIUhPY6Z5ZnZM2a2Ivg4I7j9VDNbZmbvmtlSMxsX3H6dmT1nZq8Br5rZp8zsdTN7OrgOwZ/brU3wupkVBp/vD06Wt8rMlptZQXD7qODva8zsFyG2gpbx8eSEvc3sVTN7J/gaFwfL3AaMCrZKbg+WvTX4Hleb2X92YTVKD6SEIT3RXQTW4pgGfB54ILh9PTDL3acSmAn1v9odczJwubufFfx9KvBtYCIwEjijk/NkAMvdfTKwmMBaBm3nv8vdT+KfZyDtVHBeobMJ3BEPUA9c6u4nA7OB3wYT1r8Dmz2wvsitZjaHwLoJpwJTgFPM7MwjnU/kUDT5oPRE5wAT280OmhWcNTQbeMTMxhCYRTe53TEvu3v7dQzedvcdAGb2HoH5gd7scJ5GPp5UcSVwbvD5DD5e++IxDr2oVq/gaw8CPiCwUA4E5hL6r+Af/9bg/oJOjp8TfLwb/L03gQSy+BDnEzksJQzpiRKA6e5e336jmd0D/N3dLw1eD3i93e7aDq/R0O55C53/X2ryjy8SHqrM4dS5+5TglOcvAd8A7iawFkQecIq7N5nZViCtk+MN+G93v+8ozyvSKXVJSU+0iMBspwCYWdvU0tl8PGX0dWE8/3ICXWEAVx6psLsfAL4FfM/MkgjEWR5MFrOBYcGiNUBmu0NfAr4abD1hZoPMLL+L3oP0QEoY0t2lm9mOdo/vEvjjWxi8EPw+gSnoAX4N/LeZvUt4W9/fBr5rZquB0UD1kQ5w93cJzNJ6FYG1IArNbA1wLYFrL7j7buCt4DDc2919EYEur2XBsk/zzwlF5KhoWK1IhAW7mOrc3c3sSuAqd7/4SMeJRJuuYYhE3inAPcGRTVV0g6VppWdQC0NEREKiaxgiIhISJQwREQmJEoaIiIRECUNEREKihCEiIiH5/9K8hx4S/g9YAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot(skip_end=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 01:22 <p><table style='width:375px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "    <th>time</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>0</th>\n",
       "    <th>5.344512</th>\n",
       "    <th>5.130066</th>\n",
       "    <th>0.291400</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>5.205573</th>\n",
       "    <th>5.073628</th>\n",
       "    <th>0.298674</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>4.987444</th>\n",
       "    <th>5.022674</th>\n",
       "    <th>0.306565</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>4.761755</th>\n",
       "    <th>4.993356</th>\n",
       "    <th>0.312356</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>4.588805</th>\n",
       "    <th>4.995852</th>\n",
       "    <th>0.313393</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(5, 3e-3, moms=(0.8,0.7), wd=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 02:47 <p><table style='width:375px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "    <th>time</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>0</th>\n",
       "    <th>4.541148</th>\n",
       "    <th>4.997848</th>\n",
       "    <th>0.313419</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>4.547158</th>\n",
       "    <th>4.997664</th>\n",
       "    <th>0.313445</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>4.538640</th>\n",
       "    <th>4.998408</th>\n",
       "    <th>0.313550</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>4.529515</th>\n",
       "    <th>5.000189</th>\n",
       "    <th>0.314194</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>4.512976</th>\n",
       "    <th>5.001172</th>\n",
       "    <th>0.314154</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>5</th>\n",
       "    <th>4.508440</th>\n",
       "    <th>5.003024</th>\n",
       "    <th>0.313826</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>6</th>\n",
       "    <th>4.488965</th>\n",
       "    <th>5.003732</th>\n",
       "    <th>0.313839</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>7</th>\n",
       "    <th>4.482460</th>\n",
       "    <th>5.004355</th>\n",
       "    <th>0.313787</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>8</th>\n",
       "    <th>4.478616</th>\n",
       "    <th>5.004660</th>\n",
       "    <th>0.313708</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>9</th>\n",
       "    <th>4.483404</th>\n",
       "    <th>5.004200</th>\n",
       "    <th>0.313774</th>\n",
       "    <th>00:16</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(10, 1e-4, moms=(0.7,0.6), wd=0.1, pct_start=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder(enc_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(f\"twitter_es_finetune_{seed}{'_bwd' if backward else ''}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}