Ensemble 200 models more LM train top Quartile F1 ens Unigram BigDrop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'numpy.core._multiarray_umath'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'numpy.core._multiarray_umath'"
     ]
    },
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'numpy.core._multiarray_umath'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'numpy.core._multiarray_umath'"
     ]
    },
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'numpy.core._multiarray_umath'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'numpy.core._multiarray_umath'"
     ]
    },
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'numpy.core._multiarray_umath'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'numpy.core._multiarray_umath'"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "\n",
    "from fastai.text import *\n",
    "# from fastai.callbacks import SaveModelCallback\n",
    "import sentencepiece as spm #https://github.com/google/sentencepiece\n",
    "\n",
    "from prod.sp_tok import *\n",
    "\n",
    "torch.backends.cudnn.enabled=False #for quant.313v machine.  Not sure why..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path('./data/rnn')\n",
    "sp_model = './all_tweets_es_0607_unigram'\n",
    "\n",
    "defaults.text_spec_tok.append(NL) #add a New Line special char\n",
    "sp_vocab = Vocab( get_itos(sp_model) )\n",
    "mycust_tok = CustomTokenizer(SPTokenizer,sp_model,pre_rules=default_rules)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('./data/haha_2019_test.csv')\n",
    "test_df['is_humor'] = 0\n",
    "test_df['is_humor'].iloc[0] = 1 #create two classes in data.\n",
    "## apply the rules\n",
    "raw_text = test_df.loc[:,'text']\n",
    "\n",
    "print(\"Default Rules:\\n\",[x.__name__ for x in default_rules],\"\\n\\n\")\n",
    "for rule in default_rules:\n",
    "    raw_text = raw_text.apply(lambda x: rule(str(x)))    \n",
    "test_df['new_text'] =  raw_text #tokenizer adds xxbos\n",
    "\n",
    "data_tst = TextClasDataBunch.from_df(PATH, test_df, test_df,\n",
    "                               tokenizer=mycust_tok, vocab=sp_vocab,\n",
    "                               text_cols='new_text', label_cols='is_humor')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_wt_fnames = [x.split('/')[-1] for x in glob.glob( str(PATH/'models')+'/haha_clas_bd_*.pth')]\n",
    "len(model_wt_fnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "!grep Clas ./prod/out_fwd_tmp30.txt > ./by_seed_fwd_perf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>acc</th>\n",
       "      <th>F1</th>\n",
       "      <th>rnd_seed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Clas RndSeed: 974110</td>\n",
       "      <td>0.855833</td>\n",
       "      <td>0.808465</td>\n",
       "      <td>974110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Clas RndSeed: 625561</td>\n",
       "      <td>0.853333</td>\n",
       "      <td>0.806497</td>\n",
       "      <td>625561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Clas RndSeed: 903002</td>\n",
       "      <td>0.855556</td>\n",
       "      <td>0.811046</td>\n",
       "      <td>903002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Clas RndSeed: 813876</td>\n",
       "      <td>0.859167</td>\n",
       "      <td>0.811741</td>\n",
       "      <td>813876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Clas RndSeed: 64525</td>\n",
       "      <td>0.856667</td>\n",
       "      <td>0.813043</td>\n",
       "      <td>64525</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     id       acc        F1 rnd_seed\n",
       "0  Clas RndSeed: 974110  0.855833  0.808465   974110\n",
       "1  Clas RndSeed: 625561  0.853333  0.806497   625561\n",
       "2  Clas RndSeed: 903002  0.855556  0.811046   903002\n",
       "3  Clas RndSeed: 813876  0.859167  0.811741   813876\n",
       "4   Clas RndSeed: 64525  0.856667  0.813043    64525"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## be more picky about the models.  Take the top quarter in validation. Does better val -> better test?\n",
    "valid_perf = pd.read_csv('by_seed_fwd_perf', names = ['id','acc','F1'])\n",
    "valid_perf['rnd_seed'] = valid_perf.id.str.split(':',expand=True).iloc[:,-1].str.strip()\n",
    "valid_perf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Freq of F1 over all Random Seeds (200) \\n Vert line = 75th p-tile')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEXCAYAAACNj66GAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHWdJREFUeJzt3XmcXGWd7/HPN4SQJokEDfZlCwF1xChIpK/giNpx31DvLA5cFXGZiPcOg1e97jODI4L6khEcHDWOCrK1gOK+orSI4kJAtiQ4LCEJO0IgHYIx5Dd/PE/LSVHVXd1dp6uf5Pt+verVVXWeOuf3nKr69qlz6jyliMDMzMoxrdsFmJnZ2Di4zcwK4+A2MyuMg9vMrDAObjOzwji4zcwK4+AuhKRnS/ovSUOSXtPteiabpOMlnZWvL5AUkqZPgbr6Ja3tdh2dVF3X43jsSZLe0emamiznZElvr3s5U5WDu4GkVZI25oAcvuzR7bqAfwVOi4jZEfGNxokj1S1pqaTrJW2RdPRkFz7ZGtbFHZJOlzS723VNlKRXS/qdpAck3SPpp5L27XZdwyTtBhwFfD7fPlTSjyXdK+luSedL2r3SXpI+LukP+fJxSapMP0jSMkkP5r8HVRb3SeADkmZMVv+mEgd3c4fngBy+3NbYoAtbe/sA143SplXdVwH/B7ii1grbNEnr7vCImA0cBCwC3j8Jy6yNpCcCXwHeBewC7At8Bni4m3U1OBr4XkRszLd3BZYCC0iv3/XAlyvtlwCvAZ4OHAgcDrwNIAfyN4Gz8nzOAL45HNQRcTuwEnhVnR2aqhzcbap8PH+LpNXAT/P9h0r6paR1kq6S1F95zL6SfiZpfd7yOG2kj6CS/l7SDXkL5VuVLeYbgf2Ab+etyJ3GUntEfCYifgI81EY/d5H0lbyFdIukD0maJmmn3MenVdrulrdsH59vvzJvEa7L6+TASttVkt4r6WpgQ7PwlnSqpDV5i3KZpOeMpZ8t+n4H8ENSgA8v5xWSrszLWSPp+Mq04ef5jZJW5y3bD1am9+Qt+PskLQf+Z0MfniJpMK+D6yS9qjLtdEn/Ien7+Xn8haT/IemUPL+Vkha16MpBwM0R8ZNI1kfE1yJidZ73NEnvk3Rj3no9T9JjK8tu+3UKzKtMmynprDzPdZJ+K6m3RY0vA35WWfffj4jzI+KBiHgQOA14dqX9G4GTI2JtRNwKnEwKf4B+YDpwSkT8MSI+DQh4fuXxg8ArWtSyTXNwj93zgKcAL5G0J/Bd4ATgscC7ga8pfWQEOAdYRnojfIT0Qm1K0vOBk4DXArsDtwADABHxBGA1j2xR/7GGfg37d9IW3X6kvh4FvCkv8+vAkZW2rwV+FhF35cD5EmmL6XGkj8vfavgncyTpjTY3IjY3WfZvSQH1WNK6O1/SzIl0RtJepEC5oXL3htyvubmet+vRxw0OA54MvAD4Z0lPyff/C/CEfHkJledU0o7At4EfAY8HjgXOlvTkynxfC3yI9Jr4I3AZ6ZPQPOAC4N9adOUKYH9Jn5K0WI/e9XMsaev1ecAewH2kLXIm+Dp9I+n1sDfpeT0G2EhzBwDXt5gG8Fy2/tT4VNKnwWFX5fuGp10dW4/JcXVlOsAK0tb69icifKlcgFXAELAuX76R718ABLBfpe17gTMbHv9D0ot9PrAZmFWZdg5wVovlfhH4ROX2bOBPwIJKXS8ca90NbS4Fjh5hHjsAm4CFlfveBgzm6y8EbqxM+wVwVL7+WeAjDfO7Hnhepb43j/G5uA94er5+/PC6qzwX00dZF+tzu5+Q/lm0Ws4pwKca5r1XZfpvgCPy9ZuAl1amLQHW5uvPAe4AplWmnwscn6+fDnyhMu1YYEXl9gHAuhHqPBQ4D7ib9OnpdGB2nrYCeEGl7e759TN9Iq9T4M3AL4ED23i+/gTs32LagcC9wHMq9z1cbQ88Ka97Af8EDDTM4+zhdZlvvwi4aSyvqW3l4i3u5l4TEXPzpXFLbE3l+j7A3+aPkOskrSNtqe1O3uqJiA2V9reMsMw9qtMjYgj4A7Bnh+puxzxgx4Y6b6nUcDGws6RDJC0gbR1fmKftA7yrYV3sTerXsOq6exRJ75a0QtL9+fG7UPnYPkaviYg5pI/c+7P1x/9DJF2cdwfdT9qKbFzOHZXrD5L+kULqT7Uf1XW1B7AmIrY0TK8+h3dWrm9scrvlQdSI+FVEvDYidiP9k3guMLwbZx/gwsq6X0EKxl4m9jo9kxTyA5Juk/SJ/MmimfuAOY13Ku2f/z5wXET8vDJpCHhM5fZjgKFIqdw4bXj6+srtOaSNlO2Og3vsqh/d1pC2ZOZWLrMi4mPA7cCukmZV2s8fYb63kd5gAOTHPQ64tYO1j+Ye0lbTPpX75g/XEBEPk7b4jsyX70TE8BtpDfDRhnWxc0ScW5lXy6Eo8/7s95B2JewaEXOB+0lbX+MWET8jbZl+snL3OcC3gL0jYhfgc2NYzu2kf0jDqs/pbcDekqY1TO/4cxgRvyXtuho+5rAGeFnD+p8Zad/xuF+nEfGniPhwRCwE/hJ4JWk3UzNXA39RvUPSPsBFpE9jZza0v46td3U8nUd2pVwHHCip+rwcyNa7Wp7C1rtathsO7ok5Czhc0ksk7ZAP5PRL2isibgEuBz4saYakw0hHzVs5F3iT0legdgJOBH4dEasmWmRe/kxSOO2Y63zUc18J5o9KmpPfdO/M/Rx2DvB3wOvy9WFfAI7JW7OSNCsfBHzUFlgLc0gf2e8Gpkv6Zx69xTVepwAvkjQcEnOAeyPiIUnPBP73GOZ1HvB+Sbvm/efHVqb9mrR1/h5JO+YDgIeTj1VMhKTDlA5eDx8I3p/0jYpf5SafIz1v++Tpu0l6dZ427tdp3p9+gKQdgAdI/9irnyiqvkfaxz782D1JB/FPi4jPNWn/FeCdkvZUOhD/LtI/WUgHHh8G/lHpwPg/5Pt/Wnn880hb8tufbu+rmWoXWuxLpsV+VeAQ0pH0e0mh811gfp62H/Bz0se+H5OOqjfdx53bHwPcmOf1Hbbez9q0rnamk94E0XDpb9F2V9Ib/W7Slto/U9lnm9vckGuc0XD/S0kHGNeRtuTOB+a0Wf8OpIObD+THvqf6GMa+j/uFDfd9Fvhavv43pN0B6/N6Pm2keef199Z8fWdS4KwDlgP/n7yPO09/an493J+n/6/KtNOBEyq330o+fpBvPxHY3KJPTyMd+Lwzv55WAR8HdszTp5H+yV6f+3UjcOJEX6ekT1bXkw7o3gl8eoT1Pg9YC/Tk2/+S1+VQ9VJpL+ATuaZ783VVpi8iHTTdSDo4u6gybfe8rBnNatnWL8orwSaB0tfOnhgRr+92LWZ1kHQicFdEnFLzck4mHSj/jzqXM1V1/ZRhM9t2RMQHJmk575qM5UxV3sdtZlYY7yoxMyuMt7jNzArj4LZJozSGx1vz9ddJ+lG3a6qLpKMlXTqJy5uvNP7JDvn2n9e1bXsc3NshpcGM3tzk/uMkXT7OeY4pqCLi7Ih48XiWVYf8j6Q6JO6DSoNNHZynHy/pTw1t9svTJn18cKVBu144fDsiVkcax2YqjRZoNXFwb5/OoPnZb2/I08ZkMgOrLvkfyZ+HxCUNg3sTWw+F+9XYetjcm7pTrW3vHNzbpzOBw4bPsgOQtJB0SvG5+fYukr4o6XZJt0o6ofIx/GilIUk/JekPwFdJZ+49K2+Jjjp+ROMWet5iPUbpV37WSfpM9XRnSW9WGsfkPkk/rNZekzcCX4n2jt5fkv+uy/1/1vAESZ/MNd8s6WWtZpC3oN8vaXlu/2W1GBlR0pmk09KHh/l9z2hb/V1Yf1YjB/d2KCLWkgaMekPl7jeQBsG/J98+nXQK+hNJZ7C9mHSm37BDSFukvcDrSWd9Xpa3ROeOs7RXksa3PpA0ZslLIP3yC/AB4K+A3Uhn+Z3bYh6oMphSk8v7Risih9pzSWdIVh2uNFb6ddr6Z7Oem//Ozf2/LN8+hHTW4TzSWYFfrP4zauJ1uc9PII358aFmjSLiDWw9zO8nRunPmNafTX0O7u3XGeTgVhq35HX5PpQGyn858I6I2BARdwGfAo6oPP62iPj3iNgcj/ziyUR9LCLWRfpxgIt55McPjgFOiogVkcbxPhE4qNVWY2w9mFLj5WNt1HEU8POIuLly33mkQY12A/6eNEb3kc0eXHFLRHwh73c+g3SadqsfIYA0pseaiLgX+Chbj30+EWNafzb1Obi3X18Hdpd0KGno051J41dAGh1wR+B2PTIM6OdJPw4wbMQhWsep1VCq+wCnVmq5lzTOxViGvB2Lo2jY1x8RyyPitoh4OCJ+CZxKGvNkJH/uT6RfgIERhm3l0cPFDv8C0vAv5gxJel27naiY7PVnNSv+oJKNT0Q8KOkCUkj1kAat35QnryH9Osu8aP5LNfDoIVrrPJNreMjYs9tpLGlohMknRsSJIzz22aTAvGCUxQwP+D98vRMah4u9DSAimu0bH8syx7T+bOrzFvf27QzSEK1/TWULM9IPsf4IOFnSY5R+z/AJkp7XYj6QRo7bS/X86vbnSEOpPhX+fOD0b1s1bvjmR+OlZWhnbySNIlgdsH/4F9Z3VfJM4B9JP2YLabS9LaRR9ibi/0raS+m3Ij9IOujbyp1jWN6Y1p9NfQ7u7dslpOFH10YamL/qKGAGaWjS+0hboLuPMK+fkga5v0PSPSO0G7OIuJA0hOmApAeAa0m/I9lR+Vscr6X5VyKPIA1nu5500PLjEXFGru9B0j7pX+TdEYeOs4RzSP8wbyINy3rCCG1PAj6Ul/fukWY6WevPJo/HKjGbAiStIo35fVG3a7Gpz1vcZmaFcXCbmRXGu0rMzArjLW4zs8LU8j3uefPmxYIFC+qYde02bNjArFmzul3GuJVeP5Tfh27Uf9PdGwDYb7fOLNfPweRbtmzZPRGxWzttawnuBQsWcPnl4xodtOsGBwfp7+/vdhnjVnr9UH4fulH/330+DY/y1bc9a5SW7fFzMPkk3dJuW+8qMTMrjIPbzKwwDm4zs8I4uM3MCuPgNjMrjIPbzKwwbQW3pLmSLlD6dfAV1d/UMzOzydXu97hPBX4QEX+Tx1veucaazMxsBKMGt6RdSD+GejRA/pWUTSM9xszM6jPqIFOSDgKWkgbUfzqwDDguIjY0tFsCLAHo7e09eGBgoJaC6zY0NMTs2SP9LODUVnr9UH4fmtW//qFWvwDXGadembaljlvUmR8g2rJpI9Nm9ExoHnNmdu+XEUt8DS1evHhZRPS107adNTsdeAZwbET8WtKpwPuAf6o2ioilpICnr68vSjvddFiJp8pWlV4/lN+HZvVftPzOWpc5bcVKAHrm79+R+W1cfQ098w+Y0Dz6F470g/b1Kv01NJp2Dk6uJf201a/z7QtIQW5mZl0wanBHxB3AGklPzne9gLTbxMzMuqDdnVDHAmfnb5TcBLypvpLMzGwkbQV3RPwOaGunuZmZ1ctnTpqZFcbBbWZWGAe3mVlhHNxmZoVxcJuZFcbBbWZWGAe3mVlhHNxmZoVxcJuZFcbBbWZWGAe3mVlhHNxmZoVxcJuZFcbBbWZWGAe3mVlhHNxmZoVxcJuZFcbBbWZWGAe3mVlhHNxmZoVxcJuZFcbBbWZWGAe3mVlhHNxmZoVxcJuZFWZ6O40krQLWAw8DmyOir86izMystbaCO1scEffUVomZmbXFu0rMzAqjiBi9kXQzcB8QwOcjYmmTNkuAJQC9vb0HDwwMdLjUyTE0NMTs2bO7Xca4lV4/lN+HZvWvf2hzrcs89cpNABy3aEZH5rdl00amzeiZ0DzmzBzLB/rOKvE1tHjx4mXt7oZud80eFhG3Sno88GNJKyPikmqDHOZLAfr6+qK/v38sNU8Zg4ODlFo7lF8/lN+HZvVftPzOWpc5bcVKAHrm79+R+W1cfQ098w+Y0Dz6F/Z2pJbxKP01NJq2dpVExK35713AhcAz6yzKzMxaGzW4Jc2SNGf4OvBi4Nq6CzMzs+ba2VXSC1woabj9ORHxg1qrMjOzlkYN7oi4CXj6JNRiZmZt8NcBzcwK4+A2MyuMg9vMrDAObjOzwji4zcwK4+A2MyuMg9vMrDAObjOzwji4zcwK4+A2MyuMg9vMrDAObjOzwji4zcwK4+A2MyuMg9vMrDAObjOzwji4zcwK4+A2MyuMg9vMrDAObjOzwji4zcwK4+A2MyuMg9vMrDAObjOzwji4zcwK4+A2MytM28EtaQdJV0r6Tp0FmZnZyMayxX0csKKuQszMrD1tBbekvYBXAP9ZbzlmZjYaRcTojaQLgJOAOcC7I+KVTdosAZYA9Pb2HjwwMNDhUifH0NAQs2fP7nYZ41ZK/esf2txy2pZNG5k2o2cSq+msbtR/6pWbADhu0YyOzK8TfZgzc3pHahmPUt4HVYsXL14WEX3ttB11zUp6JXBXRCyT1N+qXUQsBZYC9PX1RX9/y6ZT2uDgIKXWDuXUf9HyO1tO27j6GnrmHzCJ1XRWN+qftmIlAD3z9+/I/DrRh/6FvR2pZTxKeR+MVzu7Sp4NvErSKmAAeL6ks2qtyszMWho1uCPi/RGxV0QsAI4AfhoRr6+9MjMza8rf4zYzK8yYjh5ExCAwWEslZmbWFm9xm5kVxsFtZlYYB7eZWWEc3GZmhXFwm5kVxsFtZlYYB7eZWWEc3GZmhXFwm5kVxsFtZlYYB7eZWWEc3GZmhXFwm5kVxsFtZlYYB7eZWWEc3GZmhXFwm5kVxsFtZlYYB7eZWWEc3GZmhXFwm5kVxsFtZlYYB7eZWWEc3GZmhXFwm5kVZtTgljRT0m8kXSXpOkkfnozCzMysuelttPkj8PyIGJK0I3CppO9HxK9qrs3MzJoYNbgjIoChfHPHfIk6izIzs9aUcnmURtIOwDLgicBnIuK9TdosAZYA9Pb2HjwwMNDhUifH0NAQs2fP7nYZ4zaW+tc/tLnmasZny6aNTJvR0+0yxq0b9Z965SYAjls0oyPz60Qf5sxs5wN9PUp8Hy9evHhZRPS107atNRsRDwMHSZoLXCjpaRFxbUObpcBSgL6+vujv7x9b1VPE4OAgpdYOY6v/ouV31lvMOG1cfQ098w/odhnj1o36p61YCUDP/P07Mr9O9KF/YW9HahmP0t/HoxnTt0oiYh1wMfDSesoxM7PRtPOtkt3yljaSeoAXASvrLszMzJprZ1fJ7sAZeT/3NOC8iPhOvWWZmVkr7Xyr5Gpg0STUYmZmbfCZk2ZmhXFwm5kVxsFtZlYYB7eZWWEc3GZmhXFwm5kVxsFtZlYYB7eZWWEc3GZmhXFwm5kVxsFtZlYYB7eZWWEc3GZmhXFwm5kVxsFtZlYYB7eZWWEc3GZmhXFwm5kVxsFtZlYYB7eZWWEc3GZmhXFwm5kVxsFtZlYYB7eZWWEc3GZmhXFwm5kVZtTglrS3pIslLZd0naTjJqMwMzNrbnobbTYD74qIKyTNAZZJ+nFELK+5NjMza2LULe6IuD0irsjX1wMrgD3rLszMzJpTRLTfWFoAXAI8LSIeaJi2BFgC0Nvbe/DAwEDnqpwE6x/aDMCWTRuZNqOny9WMX+n1Q/l96Eb9p165CYDjFs3oyPw60Yc5M9v5QF+PoaEhZs+e3bXlj8fixYuXRURfO23bXrOSZgNfA97RGNoAEbEUWArQ19cX/f397c56Srho+Z0AbFx9DT3zD+hyNeNXev1Qfh+6Uf+0FSsB6Jm/f0fm14k+9C/s7Ugt4zE4OEhpGTQWbX2rRNKOpNA+OyK+Xm9JZmY2kna+VSLgi8CKiPi3+ksyM7ORtLPF/WzgDcDzJf0uX15ec11mZtbCqPu4I+JSQJNQi5mZtcFnTpqZFcbBbWZWGAe3mVlhHNxmZoVxcJuZFcbBbWZWGAe3mVlhHNxmZoVxcJuZFcbBbWZWGAe3mVlhHNxmZoVxcJuZFcbBbWZWGAe3mVlhHNxmZoVxcJuZFcbBbWZWGAe3mVlhHNxmZoVxcJuZFcbBbWZWGAe3mVlhHNxmZoVxcJuZFWbU4Jb0JUl3Sbp2MgoyM7ORtbPFfTrw0prrMDOzNo0a3BFxCXDvJNRiZmZtmN6pGUlaAiwB6O3tZXBwcFzzWf/Q5k6VNC5bNm1k4+prulrDRJReP5Tfh27Uv+WhTQAdW24n+jB414qO1DIe9z+wnm//4KJJX+6cmR2L1BF1bCkRsRRYCtDX1xf9/f3jms9Fy+/sVEnjsnH1NfTMP6CrNUxE6fVD+X3oRv3TVqwEoGf+/h2ZXyf60L+wtyO1jMe3f3BRV15Dk9Vnf6vEzKwwDm4zs8K083XAc4HLgCdLWivpLfWXZWZmrYy6jzsijpyMQszMrD3eVWJmVhgHt5lZYRzcZmaFcXCbmRXGwW1mVhgHt5lZYRzcZmaFcXCbmRXGwW1mVhgHt5lZYRzcZmaFcXCbmRXGwW1mVhgHt5lZYRzcZmaFcXCbmRXGwW1mVhgHt5lZYRzcZmaFcXCbmRXGwW1mVhgHt5lZYRzcZmaFcXCbmRXGwW1mVhgHt5lZYdoKbkkvlXS9pBskva/uoszMrLVRg1vSDsBngJcBC4EjJS2suzAzM2uunS3uZwI3RMRNEbEJGABeXW9ZZmbWyvQ22uwJrKncXgsc0thI0hJgSb45JOn6iZfXFfOAe7pdxASUXj+U34eu1f/jzs3Kz8Hk26fdhu0Ed1siYimwtFPz6xZJl0dEX7frGK/S64fy+1B6/VB+H0qvfzTt7Cq5Fdi7cnuvfJ+ZmXVBO8H9W+BJkvaVNAM4AvhWvWWZmVkro+4qiYjNkv4B+CGwA/CliLiu9sq6p/TdPaXXD+X3ofT6ofw+lF7/iBQR3a7BzMzGwGdOmpkVxsFtZlaYbTq4RztVX9J8SRdLulLS1ZJeXpn2/vy46yW9pHL/XEkXSFopaYWkZxXYh/8n6TpJ10o6V9LMqVa/pMfl+4ckndbwmIMlXZPn+WlJqqv+OvogaWdJ382voeskfayk+hse+y1J19ZZf119kDRD0lJJv8/PxV/X3Y+OiYht8kI6kHojsB8wA7gKWNjQZinw9nx9IbCqcv0qYCdg3zyfHfK0M4C35uszgLkl9YF0QtXNQE9udx5w9BSsfxZwGHAMcFrDY34DHAoI+D7wsin6HDTtA7AzsLjyGvp5XX2o6znI0/8KOAe4tq71X/Pr6MPACfn6NGBenf3o5GVb3uJu51T9AB6Tr+8C3JavvxoYiIg/RsTNwA3AMyXtAjwX+CJARGyKiHUl9SFPmw70SJpOCpHbqMe464+IDRFxKfBQtbGk3YHHRMSvIr3jvgK8pqb6a+lDRDwYERfn65uAK0jnRxRRP4Ck2cA7gRNqqruqlj4AbwZOyu22REQxZ1puy8Hd7FT9PRvaHA+8XtJa4HvAsaM8dl/gbuDL+SPZf0qaVUPtwzreh4i4FfgksBq4Hbg/In7U+dJb19DQ5nia1z/SPNeOMs9OqqMPfyZpLnA48JOJldlSXfV/BDgZeLADNY6m433I6x3gI5KukHS+pN4O1Vu7bTm423EkcHpE7AW8HDhT0kjrZDrwDOCzEbEI2AB0e5jbMfVB0q6krZV9gT2AWZJePymVNjfW52AqGlcf8ieec4FPR8RNNdc4krG+hg4CnhARF05WgW0Yz3t5L+CXEfEM4DLSBk0RSnuDjEU7p+q/hbSPl4i4DJhJGpym1WPXAmsj4tf5/gtIQV6XOvrwQuDmiLg7Iv4EfB34y1qqn1j9I82zuluh7iEY6ujDsKXAf0XEKR2os5U66n8W0CdpFXAp8BeSBjtUbzN19OEPpE8LX8+3z6fe93JHbcvB3c6p+quBFwBIegrpyb47tztC0k6S9gWeBPwmIu4A1kh6cn78C4DlJfUhtz80f7NB+bErpmD9TUXE7cADkg7N9R8FfLOO4rOO9yG3O4G0L/YdHa94a3U8B5+NiD0iYgHpwN/vI6K/htqH1dGHAL4N9Oe76n4vd1a3j47WeSF9ZPo96Yj0B/N9/wq8Kh45+vwL0lHq3wEvrjz2g/lx11M54g8cBFwOXA18A9i1wD58GFgJXAucCew0RetfBdwLDJE+7SzM9/fl2m8ETiOfAVxKH0hbjEH6h/m7fHlrKfU3zHsBNX+rpMbX0T7AJaT38k+A+XX3o1MXn/JuZlaYbXlXiZnZNsnBbWZWGAe3mVlhHNxmZoVxcJuZFcbBbWZWGAe3mVlh/htUBok84ZPaUwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "valid_perf.F1.hist(alpha=0.3)\n",
    "plt.axvline(x=valid_perf.F1.quantile(0.75))\n",
    "#valid_perf[valid_perf.F1 >= valid_perf.F1.quantile(0.75)].F1.hist(alpha=0.3)\n",
    "plt.title('Freq of F1 over all Random Seeds (200) \\n Vert line = 75th p-tile')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_rnd_seeds = valid_perf[valid_perf.F1 >= valid_perf.F1.quantile(0.75)].rnd_seed.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_wt_fnames = [f'haha_clas_bd_{x}.pth' for x in selected_rnd_seeds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "ensemble_preds = []\n",
    "for wt_fname in model_wt_fnames:\n",
    "    config = awd_lstm_clas_config.copy()\n",
    "    config['qrnn'] = True\n",
    "    config['n_hid'] = 2304\n",
    "    \n",
    "    learn = text_classifier_learner(data_tst, AWD_LSTM, drop_mult=0.7,pretrained=False,\n",
    "                               config=config)\n",
    "    learn = learn.load(wt_fname.split('.')[0]) #load weights\n",
    "    preds_t,y_t,losses_t = learn.get_preds(DatasetType.Valid,with_loss=True, ordered=True)\n",
    "    ensemble_preds.append(to_np(preds_t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8, 6000, 2)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(ensemble_preds).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[0.188219, 0.811781],\n",
       "        [0.997719, 0.002281],\n",
       "        [0.076236, 0.923764],\n",
       "        [0.935197, 0.064803],\n",
       "        ...,\n",
       "        [0.991071, 0.008929],\n",
       "        [0.935688, 0.064312],\n",
       "        [0.763841, 0.236159],\n",
       "        [0.173104, 0.826896]],\n",
       "\n",
       "       [[0.27887 , 0.72113 ],\n",
       "        [0.887463, 0.112537],\n",
       "        [0.049133, 0.950867],\n",
       "        [0.914154, 0.085846],\n",
       "        ...,\n",
       "        [0.695903, 0.304097],\n",
       "        [0.96248 , 0.03752 ],\n",
       "        [0.254974, 0.745026],\n",
       "        [0.229806, 0.770194]],\n",
       "\n",
       "       [[0.27974 , 0.72026 ],\n",
       "        [0.909635, 0.090365],\n",
       "        [0.076918, 0.923082],\n",
       "        [0.930934, 0.069065],\n",
       "        ...,\n",
       "        [0.805898, 0.194102],\n",
       "        [0.942865, 0.057135],\n",
       "        [0.471402, 0.528598],\n",
       "        [0.281978, 0.718022]],\n",
       "\n",
       "       [[0.136267, 0.863733],\n",
       "        [0.954988, 0.045012],\n",
       "        [0.035078, 0.964922],\n",
       "        [0.934271, 0.065729],\n",
       "        ...,\n",
       "        [0.954321, 0.045679],\n",
       "        [0.930976, 0.069024],\n",
       "        [0.30551 , 0.69449 ],\n",
       "        [0.31738 , 0.68262 ]],\n",
       "\n",
       "       [[0.137899, 0.862101],\n",
       "        [0.980893, 0.019107],\n",
       "        [0.044148, 0.955852],\n",
       "        [0.92079 , 0.07921 ],\n",
       "        ...,\n",
       "        [0.958689, 0.041311],\n",
       "        [0.921452, 0.078548],\n",
       "        [0.381537, 0.618463],\n",
       "        [0.160419, 0.839581]],\n",
       "\n",
       "       [[0.209379, 0.790621],\n",
       "        [0.990771, 0.009229],\n",
       "        [0.053013, 0.946987],\n",
       "        [0.920997, 0.079003],\n",
       "        ...,\n",
       "        [0.921859, 0.078141],\n",
       "        [0.902531, 0.097469],\n",
       "        [0.256077, 0.743923],\n",
       "        [0.245554, 0.754446]],\n",
       "\n",
       "       [[0.294131, 0.705869],\n",
       "        [0.980485, 0.019515],\n",
       "        [0.070283, 0.929717],\n",
       "        [0.92732 , 0.07268 ],\n",
       "        ...,\n",
       "        [0.989052, 0.010948],\n",
       "        [0.927522, 0.072478],\n",
       "        [0.392563, 0.607437],\n",
       "        [0.331471, 0.668529]],\n",
       "\n",
       "       [[0.202826, 0.797174],\n",
       "        [0.990941, 0.009059],\n",
       "        [0.045737, 0.954263],\n",
       "        [0.930755, 0.069245],\n",
       "        ...,\n",
       "        [0.980196, 0.019804],\n",
       "        [0.930874, 0.069126],\n",
       "        [0.414254, 0.585746],\n",
       "        [0.278889, 0.721111]]], dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(ensemble_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.215916, 0.784084],\n",
       "       [0.961612, 0.038388],\n",
       "       [0.056318, 0.943682],\n",
       "       [0.926802, 0.073198],\n",
       "       ...,\n",
       "       [0.912124, 0.087876],\n",
       "       [0.931799, 0.068201],\n",
       "       [0.40502 , 0.59498 ],\n",
       "       [0.252325, 0.747675]], dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ens_mean = np.array(ensemble_preds).mean(axis=0)\n",
    "ens_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_df = test_df['id'].to_frame().copy()\n",
    "out_df['is_humor'] = ens_mean.argmax(axis=1)\n",
    "out_df['funniness_average'] = 3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>is_humor</th>\n",
       "      <th>funniness_average</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tweet1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tweet2</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tweet3</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tweet4</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tweet5</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  is_humor  funniness_average\n",
       "0  tweet1         1                3.0\n",
       "1  tweet2         0                3.0\n",
       "2  tweet3         1                3.0\n",
       "3  tweet4         0                3.0\n",
       "4  tweet5         0                3.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_df.to_csv('sub0608_Unigram_clas.csv',header=True,index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}