Ensemble 5 fold First Seed Regr models Big Drop LM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "\n",
    "from fastai.text import *\n",
    "# from fastai.callbacks import SaveModelCallback\n",
    "import sentencepiece as spm #https://github.com/google/sentencepiece\n",
    "\n",
    "from prod.sp_tok import *\n",
    "#torch.backends.cudnn.enabled=False #for quant.313v machine.  Not sure why..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path('./data/rnn')\n",
    "sp_model = './all_tweets_es_0609'\n",
    "\n",
    "defaults.text_spec_tok.append(NL) #add a New Line special char\n",
    "sp_vocab = Vocab( get_itos(sp_model) )\n",
    "mycust_tok = CustomTokenizer(SPTokenizer,sp_model,pre_rules=default_rules)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('./data/haha_2019_test.csv')\n",
    "test_df['funniness_average'] = 0\n",
    "test_df['funniness_average'].iloc[0] = 0.1\n",
    "test_df['funniness_average'].iloc[1] = 1.2\n",
    "test_df['funniness_average'].iloc[2] = 3.4\n",
    "## apply the rules\n",
    "raw_text = test_df.loc[:,'text']\n",
    "\n",
    "print(\"Default Rules:\\n\",[x.__name__ for x in default_rules],\"\\n\\n\")\n",
    "for rule in default_rules:\n",
    "    raw_text = raw_text.apply(lambda x: rule(str(x)))    \n",
    "test_df['new_text'] =  raw_text #tokenizer adds xxbos\n",
    "\n",
    "data_tst = TextClasDataBunch.from_df(PATH, test_df, test_df,\n",
    "                               tokenizer=mycust_tok, vocab=sp_vocab,\n",
    "                               text_cols='new_text', label_cols='funniness_average')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_wt_fnames = [x.split('/')[-1] for x in glob.glob( str(PATH/'models')+'/haha_regr_0610_mix_fld*.pth')]\n",
    "len(model_wt_fnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "!grep Reg ./prod/out_regr_tmp200.txt | head -n5 > ./by_seed_reg_perf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reg Fold: 0 RndSeed: 460304,0.5666607022285461\r\n",
      "Reg Fold: 1 RndSeed: 460304,0.5495392680168152\r\n",
      "Reg Fold: 2 RndSeed: 460304,0.5185738801956177\r\n",
      "Reg Fold: 3 RndSeed: 460304,0.5452352166175842\r\n",
      "Reg Fold: 4 RndSeed: 460304,0.5569671988487244\r\n"
     ]
    }
   ],
   "source": [
    "!head by_seed_reg_perf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>mse</th>\n",
       "      <th>rnd_seed</th>\n",
       "      <th>nfold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Reg Fold: 0 RndSeed: 460304</td>\n",
       "      <td>0.566661</td>\n",
       "      <td>460304</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Reg Fold: 1 RndSeed: 460304</td>\n",
       "      <td>0.549539</td>\n",
       "      <td>460304</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Reg Fold: 2 RndSeed: 460304</td>\n",
       "      <td>0.518574</td>\n",
       "      <td>460304</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Reg Fold: 3 RndSeed: 460304</td>\n",
       "      <td>0.545235</td>\n",
       "      <td>460304</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Reg Fold: 4 RndSeed: 460304</td>\n",
       "      <td>0.556967</td>\n",
       "      <td>460304</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            id       mse rnd_seed nfold\n",
       "0  Reg Fold: 0 RndSeed: 460304  0.566661   460304     0\n",
       "1  Reg Fold: 1 RndSeed: 460304  0.549539   460304     1\n",
       "2  Reg Fold: 2 RndSeed: 460304  0.518574   460304     2\n",
       "3  Reg Fold: 3 RndSeed: 460304  0.545235   460304     3\n",
       "4  Reg Fold: 4 RndSeed: 460304  0.556967   460304     4"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## be more picky about the models.  Take the top quarter in validation. Does better val -> better test?\n",
    "valid_perf = pd.read_csv('by_seed_reg_perf', names = ['id','mse'])\n",
    "valid_perf['rnd_seed'] = valid_perf.id.str.split(':',expand=True).iloc[:,-1].str.strip()\n",
    "valid_perf['nfold'] = valid_perf.id.str.split(' ',expand=True).iloc[:,2]\n",
    "valid_perf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Freq of mse over all Random Seeds (100) \\n Vert line = 25th p-tile')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEXCAYAAABWNASkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHmZJREFUeJzt3Xu4HFWd7vHvm4RAuAkCboEQAgIPREHFCDg6uo9XcJTo8QaDaBSNOuLljDoHjx5BvII6oyOMEo8KooKIl4kaBkTZoggOqEBMAhoiJgHkjrAhMYb8zh9rbam0u+/du7NX3s/z9LO7aq2uWquq9tvV1d2rFRGYmVlZpgy6AWZm1nsOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjncJylJT5f0O0mjkl4y6PZsziTNl/SzynRI2m+QbcrtmJ3bMm3QbemV2m3d5mPfJOnTvW7TOOt5m6TT+r2eQXO4NyHpZklrc4iO3fYYdLuAU4EzImL7iPjuoBtTCkkjktbl/XyXpG9L2n3Q7eqWpGdI+rmkP0m6R9IVkp466HaNkTQdeD/wicq8hZJulLRR0vxxHvO/JP1R0v2SviRp60rZbEmXSXpI0g2Snlt56BeA4yQ9po9dGjiHe2tenEN07HZrbYUBnH3tDSyd4HUOnJJ+H7cnRsT2wH7A9sAn+7y+vpK0I/B94LPAo4E9gQ8Cfx5ku2rMA26IiFsq864D/gn4VW1lSS8ATgKeQ/pf2JfUpzHnAb8GdgHeB1woaTeAiFgHXAS8pvfd2Hw43DtUeUl9gqRVwI/z/CPyGdJ9kq6TNFx5zD6SfiLpAUk/lHSGpK82WMcbJa3IZ1qLxl4xSLqJdDB/L59hbj3OY2+W9B5J10t6UNIXJQ1Juiiv/1JJO+e620j6qqS7c7uvljSUyx6VH3ubpFskfVjS1Drt3VrSpyXdmm+fHmubpOWSXlSpO03SnZIObWG7jUj6iKQrgIdy32vXfZKkm3Lflkl6ad2d16KIuA/4LvCkynoOk3RlbudteR9Or5SHpDcrXTK7T9KZkpTLpkr6ZH5FsBL4h5o+7JH38z15v7+xUnaKpG/m/fSApCWSDpD0Xkl3SFot6fl1unJA7s95EfFwRKyNiEsi4vrK8l+f99G9ki6WtHel7MB8vN6jdCb9ykrZLrnN90v6b+BxlTJJ+rfcvvtzm59Qp41HAT+p2f5nRsSPgHXj1H8t8MWIWBoR9wIfAubn9R4AHAqcnPv6LWAJ8LLK40eo2f7FiQjfGtyAm4HnjjN/NhDAV4DtgBmkM6K7gReSnjifl6d3y4+5EvhXYGvgmcADwFfrrPfZwF2kg3Rr0lnX5c3aVVN+FTCU23UH6QzoycA2pCejk3PdNwHfA7YFpgJPAXbMZd8Bzsp9fAzw38Cb6qzz1LzOxwC7AT8HPpTLPgB8rVL3H4Dl+X6z7TYCrAIeD0wDthpn3a8A9siPfxXwILB7LpsP/KxSN4D96vRhBHhDvr8LcCnwn5XypwBH5HbMBpYD76xZ9veBnYBZwJ3AkbnszcANwF6kM+jLcv1pufxy4D/y/nlSfuyzc9kppJB7QV73V4Dfk85KtwLeCPy+Tp92zNvzHFKI7lxTPg9YARyUl/1+4Oe5bDtgNfC6XPZk0nE5J5efD1yQ6z0BuGVsW+e2/jJvC+Xl716njVcDr6hT9jNgfs2864BXVaZ3zdtyF+Cl5GOrUn4G8NnK9KHAPYPOl37eBt6Azf1GCslR4L58+26ePzsfTPtW6v5v4Nyax19MOsuYBWwAtquUfZ364f5F4PTK9PbAX4DZlXY1C/fjKtPfAj5XmX5bpS+vJwXxITXLGCK9dJ9RmXcscFmddd4EvLAy/QLg5nx/P9KT2bZ5+mvAB5ptt3x/BDi1zf12LTAv359Pe+H+EPCnXO9aYFaD9bwT+E7Nsp9Rmb4AOCnf/zHw5krZ83P9aaTAfxjYoVL+MeDsfP8U4IeVshfn43Jqnt4hL2unOu08CDgbWJOPw0XAUC67CDihUndK3gZ7k54of1qzrLOAk0knAn8BDqyUfZRHwv3ZwG9JT4ZTmuyv35GfBMcpGy/cb6rWJz3BBen/8njgqpr6Hxnblnl6f+Dhdo6pyXbzZZnWvCQidsq32k+mrK7c3xt4RX45fp+k+4BnALuTzirvjYgHK/X/0GCde1TLI2KUdPa1Zxvtvr1yf+0409vn++eSwvT8fDnldElb5f5sBdxW6c9ZpDPzpm3O9/fI7V9BOst9saRtgaNJT27QeLuNqW7nvyHpNZKurTz+CaSzuU68PSIeBRwC7AzMrKznAEnfV34jjxRmtev5Y+X+Qzyynfeo6Ud1W+1BOpN8oKa8ur9r999dEfFwZZrKujYREcsjYn5EzCRtmz2AsU+m7A18prLt7iGdae+Zyw6v2TfHAY8lvTqbVq9PEfFj0hnzmcAdSm+Q7jhe+4B7SU9QrRolvSIZM3b/gXHKxsqr23YH0hN4sRzu3asOq7madAa6U+W2XUR8HLgN2FnSdpX6sxos91bSPxYA+XG7kF729lRE/CUiPhgRc4C/A15EerNpNenMfddKf3aMiMe30mZS/6pvPp9HOvOfByzLgQ+Nt9tfm1mv/fn68BeAE4FdImIn4DekgOpYRCwBPgz89bo58DnSpZX9I2JH4P+0sZ7bSGfoY6r7/1bg0ZJ2qCnvx/6+gXQWP3b9ezXpUlt1+8+IiJ/nsp/UlG0fEW8hXTba0KBPRMS/R8RTgDmka//vqdOs63N5q5YCT6xMPxG4PSLuzmX71mzLJ7LpBxAOIl3aKZbDvbe+SjozfUF+82wbScOSZkbEH4BrgA9Kmi7pGaSX1vWcB7xO0pPym5IfBX4RETf3utGS/oekg5XeKL2f9FJ7Y0TcBlwCfErSjpKmSHqcpGc1aPP7Je0maVfSdfbqG8bnky5FvIVHztqhwXZrsQvbkcL/ztyf1/FIcHXrHNLlqaPz9A6kbTQq6UBSX1p1AfB2STOV3sw+aawgIlaTLo19LPf/EOAENt1+HclviL5rbHtK2ov0JHtVrvJ54L2SHp/LHyXpFbns+8ABko6XtFW+PVXSQflVw7eBUyRtK2kO6RLk2HqfKunw/CrwQdJ7BhvrNHMxsMlxlf9PtiE9eW6Vt8tYZn0FOEHSHEk7kd4nOBsgIn5Lupx2cn7MS0mvwr5VWfyzSJejiuVw76H8DzqPdDZ3J+ms5z08sp3/ETic9LL3ZNIBWm9ZlwL/l3RA3kb6FMIxfWr6Y4ELSaG1nPSphXNz2WuA6cAy0kvnC9n0cknVh0lPYNeTPp3wqzwPgPxkcSXp1cE3KvObbbeGImIZ8Km87NuBg4ErWnlsC8teD3yGtC8A3k3ajw+QXi18o85Dx/MF0uWv60jb5ts15ceSrhnfSnoj++R8HHTrAdJx9wtJD5JC/TfAuwAi4jvAaaTLcvfnsqNy2QOkJ+Rjcrv+mOuOfULrRNKloD+SwvXLlfXumPt8L+lyzd1UPsde43vAgdr0OySXkC43/R2wMN9/Zm7XfwGnk96UXpWXf3LlsccAc/O6Pw68PCLGnvy3Ib15f06DbTbpKb+5YAMg6RTSG3uvHnRbzAZN0gLSp3De2ef1vA3YKyL+pZ/rGbRivvZsZpNbRCycoPV8diLWM2i+LGNmViBfljEzK5DP3M3MCuRwt82K0jgyb8j3j5N0yaDb1C/qYnjcDtc3S2ksoql5+q/b2srjcLdxKQ2T+vpx5r9D0jUdLrOtMIuIr0VEvcGwJpzS4GZjA2jdqTSQ1+6V8lMk/UWbDg+9by6b8LHblQaP++tQtxGxKn8B6eFGj7MyONytnnMYf0jU4+ng88ETGWp9tDPp89azSd/EfYBNP9cN8I3YdHjolRPcRjPA4W71nQs8Q5sO/TqH9E2/8/J03eGA81n6FUpDvt5N+rLP54Gn5TPa+5o1oPZMXw2G083ldYet7YWIuCgivhkR90fEQ6RxU57e4sMvz3/vy/1/WqXdn8xt/r2ko+otIJ+Jv1dpSON7JX05fyFnvLrnkoYCGBsW+l+avXro9/azieVwt3FFxBrSt/+Or8w+HlgcEXfl6bNJY4vsRxoK9vlA9Rru4cBK0tf3X00a8vbKfEa7U4dNexHwVNKTzCtJI08iaewbrv+TNKDVT8lPQuNRZSCscW4n1XtcjWfytz+Y8uJ82WappLfU1IU0auP2EXFlnj4cuJE0+NjpwBerT1jjOI7U58eRxmJ5/3iVIuJ40jc3x35o5vRGHWl3+9kkMOhhKX3bfG+kQL4x359CCouX5umGwwGThtldVbO8+VSG3q2zzhEeGU99k/o0Hk637rC1fdo2h5CGkfj7yrw5pNEWp5K+Mn8bcGwum01l7PZK/1ZUprfNdR5bZ503s+mQwS8EbmrQxpupDAtd24aabT2h28+3/t985m6NfBvYXdIRwDApfH6Qy1oZDrjhML0dqjecbqNha3tK6ce1LwLeERE/HZsfEcsi4tZIv3b0c9KYNC9vsri/9ifSpR6oM2xvVju87tivc11UeRP3uDa6M2bCtp9NjBLe5LI+iYiHJF1IemN1BnB+pIG0YNPhgDfUW0ST6V5aDXwkIr7WSmVJow2KPxoRH63zuL1Jv870oYg4d7w6FcEjwwH3qu+1w+veChAR412rb2edbW0/2/z5zN2aOYf0azwvo/IpmWh/OGBIIzbOVOU3R3uo0bC1fyM2/URL7a1esO9J+jWlMyLi8+OUz5O0s5LDgLcD/5mL7yQNd/s3v//aprcqDRn8aNJP7DUalfL2NtbX1vazzZ/D3Zq5nPSLNWsi4uqasnaGA4YUjEuBP0q6q0G9tkWDYWt76A2ksDyl+ln2SvkxpN8ifYA0nPNpEXFObt9DpJ96uyJf+jiiwzZ8nfSkupL0U3MfblD3Y6Tx9e+T9O5GC52g7WcTyGPLmE0Skm4mvQHaizHerXA+czczK5DD3cysQL4sY2ZWIJ+5m5kVaGCfc991111j9uzZfVv+gw8+yHbbbde35W/utuT+D7rvK+98EIB9d5v4Ngy674O0pfT9l7/85V0RsVuzegML99mzZ3PNNR2NHNuSkZERhoeH+7b8zd2W3P9B9/1VZ6VhY77xpqc1qdl7g+77IG0pfZf0h1bq+bKMmVmBHO5mZgVyuJuZFcjhbmZWIIe7mVmBHO5mZgVqGu6SviTpDkm/qVMuSf8uaYWk6yUd2vtmmplZO1o5cz8bOLJB+VHA/vm2APhc980yM7NuNA33iLic9JNb9cwDvhLJVcBOkhqN6W1mZn3Wi2+o7smmv+u4Js+7rbaipAWks3uGhoYYGRnpaIUPrKv3q26P2Lh+Ld/7r94Pe73DNpPjlwlHR0c73r6TXbd9b+X4auTue9IvEfbj+GumX8d9P/Xqf6rd/d7tfu7GROTIhCZVRCwEFgLMnTs3Ov2q8KXLbm9aZ+2qJcyYdXBHy29keM5Qz5fZD1vKV7HH023fWzm+Gpmy/AYAZsw6sKvldKJfx30/9ep/qt393u1+7sZE5EgvPi1zC5v+aO/MPM/MzAakF+G+CHhN/tTMEcCf8o8nm5nZgDS9LCPpPGAY2FXSGuBkYCuA/Avwi4EXkn4Y+CHgdf1qrJmZtaZpuEfEsU3KA3hrz1pkZmZd8zdUzcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzArUU7pKOlHSjpBWSThqnfJakyyT9WtL1kl7Y+6aamVmrmoa7pKnAmcBRwBzgWElzaqq9H7ggIp4MHAP8R68bamZmrWvlzP0wYEVErIyI9cD5wLyaOgHsmO8/Cri1d000M7N2TWuhzp7A6sr0GuDwmjqnAJdIehuwHfDc8RYkaQGwAGBoaIiRkZE2m5usXbehaZ2N69eydtWSjpbfyMgdy3u+zH4YHR3tePtOdt32vZXjq5GN69an5fTh+Gu67j4d9/3Uq/+pdvd7t/u5GxORI62EeyuOBc6OiE9JehpwrqQnRMTGaqWIWAgsBJg7d24MDw93tLJLl93etM7aVUuYMevgjpbfyPCcoZ4vsx9GRkbodPtOdt32vZXjq5Epy28AYMasA7taTif6ddz3U6/+p9rd793u525MRI60clnmFmCvyvTMPK/qBOACgIi4EtgG2LUXDTQzs/a1Eu5XA/tL2kfSdNIbpotq6qwCngMg6SBSuN/Zy4aamVnrmoZ7RGwATgQuBpaTPhWzVNKpko7O1d4FvFHSdcB5wPyIiH412szMGmvpmntELAYW18z7QOX+MuDpvW2amZl1yt9QNTMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCtRSuEs6UtKNklZIOqlOnVdKWiZpqaSv97aZZmbWjmnNKkiaCpwJPA9YA1wtaVFELKvU2R94L/D0iLhX0mP61WAzM2uulTP3w4AVEbEyItYD5wPzauq8ETgzIu4FiIg7ettMMzNrRyvhviewujK9Js+rOgA4QNIVkq6SdGSvGmhmZu1relmmjeXsDwwDM4HLJR0cEfdVK0laACwAGBoaYmRkpKOVrV23oWmdjevXsnbVko6W38jIHct7vsx+GB0d7Xj7Tnbd9r2V46uRjevWp+X04fhruu4+Hff91Kv/qXb3e7f7uRsTkSOthPstwF6V6Zl5XtUa4BcR8Rfg95J+Swr7q6uVImIhsBBg7ty5MTw83FGjL112e9M6a1ctYcasgztafiPDc4Z6vsx+GBkZodPtO9l12/dWjq9Gpiy/AYAZsw7sajmd6Ndx30+9+p9qd793u5+7MRE50splmauB/SXtI2k6cAywqKbOd0ln7UjalXSZZmUP22lmZm1oGu4RsQE4EbgYWA5cEBFLJZ0q6ehc7WLgbknLgMuA90TE3f1qtJmZNdbSNfeIWAwsrpn3gcr9AP4538zMbMD8DVUzswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswK1FO6SjpR0o6QVkk5qUO9lkkLS3N410czM2tU03CVNBc4EjgLmAMdKmjNOvR2AdwC/6HUjzcysPa2cuR8GrIiIlRGxHjgfmDdOvQ8BpwHretg+MzPrwLQW6uwJrK5MrwEOr1aQdCiwV0T8QNJ76i1I0gJgAcDQ0BAjIyNtNxhg7boNTetsXL+WtauWdLT8RkbuWN7zZfbD6Ohox9t3suu2760cX41sXLc+LacPx1/TdffpuO+nXv1Ptbvfu93P3ZiIHGkl3BuSNAX4V2B+s7oRsRBYCDB37twYHh7uaJ2XLru9aZ21q5YwY9bBHS2/keE5Qz1fZj+MjIzQ6fad7LrteyvHVyNTlt8AwIxZB3a1nE7067jvp179T7W737vdz92YiBxp5bLMLcBelemZed6YHYAnACOSbgaOABb5TVUzs8FpJdyvBvaXtI+k6cAxwKKxwoj4U0TsGhGzI2I2cBVwdERc05cWm5lZU03DPSI2ACcCFwPLgQsiYqmkUyUd3e8GmplZ+1q65h4Ri4HFNfM+UKfucPfNMjOzbvgbqmZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYFaCndJR0q6UdIKSSeNU/7PkpZJul7SjyTt3fummplZq5qGu6SpwJnAUcAc4FhJc2qq/RqYGxGHABcCp/e6oWZm1rpWztwPA1ZExMqIWA+cD8yrVoiIyyLioTx5FTCzt800M7N2TGuhzp7A6sr0GuDwBvVPAC4ar0DSAmABwNDQECMjI621ssbadRua1tm4fi1rVy3paPmNjNyxvOfL7IfR0dGOt+9k123fWzm+Gtm4bn1aTh+Ov6br7tNx30+9+p9qd793u5+7MRE50kq4t0zSq4G5wLPGK4+IhcBCgLlz58bw8HBH67l02e1N66xdtYQZsw7uaPmNDM8Z6vky+2FkZIROt+9k123fWzm+Gpmy/AYAZsw6sKvldKJfx30/9ep/qt393u1+7sZE5Egr4X4LsFdlemaetwlJzwXeBzwrIv7cm+aZmVknWrnmfjWwv6R9JE0HjgEWVStIejJwFnB0RNzR+2aamVk7moZ7RGwATgQuBpYDF0TEUkmnSjo6V/sEsD3wTUnXSlpUZ3FmZjYBWrrmHhGLgcU18z5Quf/cHrfLzMy64G+ompkVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVqKVwl3SkpBslrZB00jjlW0v6Ri7/haTZvW6omZm1rmm4S5oKnAkcBcwBjpU0p6baCcC9EbEf8G/Aab1uqJmZta6VM/fDgBURsTIi1gPnA/Nq6swDzsn3LwSeI0m9a6aZmbVjWgt19gRWV6bXAIfXqxMRGyT9CdgFuKtaSdICYEGeHJV0YyeNbtGutevfwmzJ/d8s+v7Dwax2s+j7gGwpfd+7lUqthHvPRMRCYOFErEvSNRExdyLWtTnakvvvvrvv1tplmVuAvSrTM/O8cetImgY8Cri7Fw00M7P2tRLuVwP7S9pH0nTgGGBRTZ1FwGvz/ZcDP46I6F0zzcysHU0vy+Rr6CcCFwNTgS9FxFJJpwLXRMQi4IvAuZJWAPeQngAGbUIu/2zGtuT+u+9bpi25739DPsE2MyuPv6FqZlYgh7uZWYEmZbi3MBzCfEl3Sro2396Q5z9J0pWSlkq6XtKrJr713emi73tL+lWet1TSmye+9d3ptO+V8h0lrZF0xsS1une66b+khyvzaz8Qsdnrsu+zJF0iabmkZVvM8CgRMalupDd1bwL2BaYD1wFzaurMB84Y57EHAPvn+3sAtwE7DbpPE9T36cDW+f72wM3AHoPu00T0vVL+GeDrjepsrrdu+w+MDroPA+z7CPC8fH97YNtB92kibpPxzL2V4RDGFRG/jYjf5fu3AncAu/Wtpb3XTd/XR8Sf8+TWTL5XbR33HUDSU4Ah4JI+ta/fuur/JNdx3/M4WNMi4ocAETEaEQ/1r6mbj8n2Dw7jD4ew5zj1XpYvvVwoaa/aQkmHkc4CbupPM/uiq75L2kvS9XkZp+UnuMmi475LmgJ8Cnh3/5vZN90e99tIukbSVZJe0teW9l43fT8AuE/StyX9WtIn8mCIxZuM4d6K7wGzI+IQ0hAf51QLJe0OnAu8LiI2DqB9/VS37xGxOs/fD3itpKEBtbFf6vX9n4DFEbFmYC2bGI2O+70jfTX/H4FPS3rcIBrYR/X6Pg34e9IT+1NJl3bmD6KBE20yhnvT4RAi4u7KJYj/BzxlrEzSjsAPgPdFxFV9bmuvddX3Sp1bgd+QDvrJopu+Pw04UdLNwCeB10j6eH+b23Nd7fuIuCX/XUm6Bv3kfja2x7rp+xrg2nxJZwPwXeDQPrd38zDoi/7t3kjPxCuBfXjkzZXH19TZvXL/pcBV+f504EfAOwfdjwH0fSYwI9/fGfgtcPCg+zQRfa+pM5/J+YZqN/t+Zx55M31X4HfUvCG5Od+67PvUXH+3PP1l4K2D7tNE3CZ0VMheiNaGQ3i7pKOBDaThEObnh78SeCawi6SxefMj4tqJ7EOnuuz7QcCnJAUg4JMRsWTCO9GhLvs+6fVg358laSPp1frHI2LZhHeiQ930PSIelvRu4EeSBPwS+MIg+jHRPPyAmVmBJuM1dzMza8LhbmZWIIe7mVmBHO5mZgVyuJuZFcjhbmZWIIe7mVmB/j/kcHflLEkSXAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "valid_perf.mse.hist(alpha=0.3)\n",
    "#valid_perf.groupby('rnd_seed')['mse'].mean().hist(alpha=0.3)\n",
    "plt.axvline(x=valid_perf.mse.quantile(0.25))\n",
    "#valid_perf[valid_perf.F1 >= valid_perf.F1.quantile(0.75)].F1.hist(alpha=0.3)\n",
    "plt.title('Freq of mse over all Random Seeds (100) \\n Vert line = 25th p-tile')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['haha_regr_0610_mix_fld0_460304.pth',\n",
       " 'haha_regr_0610_mix_fld1_460304.pth',\n",
       " 'haha_regr_0610_mix_fld2_460304.pth',\n",
       " 'haha_regr_0610_mix_fld3_460304.pth',\n",
       " 'haha_regr_0610_mix_fld4_460304.pth']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seed_grouped_F1 = valid_perf.groupby('rnd_seed')['mse'].mean()\n",
    "selected_seeds = seed_grouped_F1[seed_grouped_F1<=seed_grouped_F1.quantile(1.)].index.values.tolist()\n",
    "seed_fold_pair = valid_perf[valid_perf.rnd_seed.isin(selected_seeds)].loc[:,['rnd_seed','nfold']].values\n",
    "model_wt_fnames = []\n",
    "for (seed,fold) in seed_fold_pair:\n",
    "    model_wt_fnames.append(f'haha_regr_0610_mix_fld{fold}_{seed}.pth')\n",
    "model_wt_fnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "ensemble_preds = []\n",
    "for wt_fname in model_wt_fnames:\n",
    "    config = awd_lstm_clas_config.copy()\n",
    "    config['qrnn'] = True\n",
    "    config['n_hid'] = 2304\n",
    "    \n",
    "    learn = text_classifier_learner(data_tst, AWD_LSTM, drop_mult=0.7,pretrained=False,\n",
    "                               config=config)\n",
    "    learn = learn.load(wt_fname.split('.')[0]) #load weights\n",
    "    preds_t,y_t,losses_t = learn.get_preds(DatasetType.Valid,with_loss=True, ordered=True)\n",
    "    ensemble_preds.append(to_np(preds_t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 6000, 1)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(ensemble_preds).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 2.052521],\n",
       "        [-0.00616 ],\n",
       "        [ 2.543163],\n",
       "        [ 0.354687],\n",
       "        ...,\n",
       "        [ 0.424061],\n",
       "        [ 0.215067],\n",
       "        [ 0.92576 ],\n",
       "        [ 2.105393]],\n",
       "\n",
       "       [[ 1.478124],\n",
       "        [ 0.112399],\n",
       "        [ 2.65174 ],\n",
       "        [ 0.302537],\n",
       "        ...,\n",
       "        [ 0.495565],\n",
       "        [ 0.262895],\n",
       "        [ 1.166142],\n",
       "        [ 1.414182]],\n",
       "\n",
       "       [[ 1.009712],\n",
       "        [ 0.188823],\n",
       "        [ 2.353325],\n",
       "        [ 0.201784],\n",
       "        ...,\n",
       "        [ 0.72875 ],\n",
       "        [ 0.160609],\n",
       "        [ 1.113645],\n",
       "        [ 1.923212]],\n",
       "\n",
       "       [[ 0.82088 ],\n",
       "        [ 0.105517],\n",
       "        [ 2.345455],\n",
       "        [ 0.19966 ],\n",
       "        ...,\n",
       "        [ 0.595372],\n",
       "        [ 0.239313],\n",
       "        [ 1.301585],\n",
       "        [ 1.396813]],\n",
       "\n",
       "       [[ 1.071244],\n",
       "        [ 0.315523],\n",
       "        [ 2.371071],\n",
       "        [ 0.191901],\n",
       "        ...,\n",
       "        [ 0.69947 ],\n",
       "        [ 0.335064],\n",
       "        [ 1.011055],\n",
       "        [ 1.270143]]], dtype=float32)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(ensemble_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.286496],\n",
       "       [0.14322 ],\n",
       "       [2.452951],\n",
       "       [0.250114],\n",
       "       ...,\n",
       "       [0.588644],\n",
       "       [0.24259 ],\n",
       "       [1.103637],\n",
       "       [1.621949]], dtype=float32)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ens_mean = np.array(ensemble_preds).mean(axis=0)\n",
    "ens_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>is_humor</th>\n",
       "      <th>funniness_average</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tweet1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tweet2</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tweet3</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tweet4</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tweet5</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  is_humor  funniness_average\n",
       "0  tweet1         1                3.0\n",
       "1  tweet2         0                3.0\n",
       "2  tweet3         1                3.0\n",
       "3  tweet4         0                3.0\n",
       "4  tweet5         0                3.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_df = pd.read_csv('sub0610_mixup.csv')\n",
    "out_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((6000, 3), (6000, 1))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_df.shape, ens_mean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_df.funniness_average = ens_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>is_humor</th>\n",
       "      <th>funniness_average</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tweet1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.286496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tweet2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.143220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tweet3</td>\n",
       "      <td>1</td>\n",
       "      <td>2.452951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tweet4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.250114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tweet5</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.267732</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  is_humor  funniness_average\n",
       "0  tweet1         1           1.286496\n",
       "1  tweet2         0           0.143220\n",
       "2  tweet3         1           2.452951\n",
       "3  tweet4         0           0.250114\n",
       "4  tweet5         0          -0.267732"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_df.to_csv('sub0610_mixup.csv',header=True,index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  adding: sub0610_mixup.csv (deflated 66%)\r\n"
     ]
    }
   ],
   "source": [
    "!zip  sub0610_mixup.zip sub0610_mixup.csv "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp sub0610_mixup.zip /mnt/awcnas4_AWX/tmp/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}