Ensemble 20 Seeds select best F1 0610.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "\n",
    "from fastai.text import *\n",
    "import sentencepiece as spm #https://github.com/google/sentencepiece\n",
    "\n",
    "from prod.sp_tok import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path('./data/rnn')\n",
    "sp_model = './all_tweets_es_0609'\n",
    "\n",
    "defaults.text_spec_tok.append(NL) #add a New Line special char\n",
    "sp_vocab = Vocab( get_itos(sp_model) )\n",
    "mycust_tok = CustomTokenizer(SPTokenizer,sp_model,pre_rules=default_rules)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('./data/haha_2019_test.csv')\n",
    "test_df['is_humor'] = 0\n",
    "test_df['is_humor'].iloc[0] = 1 #create two classes in data.\n",
    "\n",
    "raw_text = test_df.loc[:,'text']\n",
    "print(\"Default Rules:\\n\",[x.__name__ for x in default_rules],\"\\n\\n\")\n",
    "for rule in default_rules:\n",
    "    raw_text = raw_text.apply(lambda x: rule(str(x)))    \n",
    "test_df['new_text'] =  raw_text #tokenizer adds xxbos\n",
    "\n",
    "data_tst = TextClasDataBunch.from_df(PATH, test_df, test_df,\n",
    "                               tokenizer=mycust_tok, vocab=sp_vocab,\n",
    "                               text_cols='new_text', label_cols='is_humor')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15,\n",
       " ['haha_clas_0609_fld0_105422.pth',\n",
       "  'haha_clas_0609_fld0_389450.pth',\n",
       "  'haha_clas_0609_fld1_248367.pth',\n",
       "  'haha_clas_0609_fld4_389450.pth',\n",
       "  'haha_clas_0609_fld2_248367.pth'])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_wt_fnames = [x.split('/')[-1] for x in glob.glob( str(PATH/'models')+'/haha_clas_0609_fld*.pth')]\n",
    "len(model_wt_fnames), model_wt_fnames[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "!grep Clas ./prod/out_fwd_1.txt > ./by_seed_fwd_perf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>acc</th>\n",
       "      <th>F1</th>\n",
       "      <th>rnd_seed</th>\n",
       "      <th>nfold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Clas Fold: 0 RndSeed: 389450</td>\n",
       "      <td>0.858958</td>\n",
       "      <td>0.813797</td>\n",
       "      <td>389450</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Clas Fold: 1 RndSeed: 389450</td>\n",
       "      <td>0.868125</td>\n",
       "      <td>0.821648</td>\n",
       "      <td>389450</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Clas Fold: 2 RndSeed: 389450</td>\n",
       "      <td>0.871042</td>\n",
       "      <td>0.828729</td>\n",
       "      <td>389450</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Clas Fold: 3 RndSeed: 389450</td>\n",
       "      <td>0.863958</td>\n",
       "      <td>0.815693</td>\n",
       "      <td>389450</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Clas Fold: 4 RndSeed: 389450</td>\n",
       "      <td>0.857292</td>\n",
       "      <td>0.816662</td>\n",
       "      <td>389450</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             id       acc        F1 rnd_seed nfold\n",
       "0  Clas Fold: 0 RndSeed: 389450  0.858958  0.813797   389450     0\n",
       "1  Clas Fold: 1 RndSeed: 389450  0.868125  0.821648   389450     1\n",
       "2  Clas Fold: 2 RndSeed: 389450  0.871042  0.828729   389450     2\n",
       "3  Clas Fold: 3 RndSeed: 389450  0.863958  0.815693   389450     3\n",
       "4  Clas Fold: 4 RndSeed: 389450  0.857292  0.816662   389450     4"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## be more picky about the models.  Take the top quarter in validation. Does better val -> better test?\n",
    "valid_perf = pd.read_csv('by_seed_fwd_perf', names = ['id','acc','F1'])\n",
    "valid_perf['rnd_seed'] = valid_perf.id.str.split(':',expand=True).iloc[:,-1].str.strip()\n",
    "valid_perf['nfold'] = valid_perf.id.str.split(' ',expand=True).iloc[:,2]\n",
    "valid_perf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Freq of F1 over all Random Seeds (20) \\n Vert line = MaxValue')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEXCAYAAABWNASkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XuYHGWd9vHvnYSQQAJBwBEJEBA8REGRWQLraeIReOWwq6+CCrKK0dV4ua+6K6jrIqIr7LKKC4rxVRFcGYOKG90gCiYiahCQk0lAIyIknBEwE4IR+e0fz9NSNNPTPd09M52H+3NdfU1X1dNVv66qubu6qvtpRQRmZlaWSRNdgJmZdZ/D3cysQA53M7MCOdzNzArkcDczK5DD3cysQA73gkl6gaRfSxqSdMRE1zPeJJ0o6av5/hxJIWlKD9Q1IGntRNfRTdV13cZj/1XSP3Shhj5JqyVt2em8SuBw7wJJN0vamEO0dnvqRNcFnAScEREzIuLb9RNHqlvSIkk3SnpE0rHjXfh4q1sXd0g6W9KMia6rU5IOl3SNpD9IukfSDyXtPtF11UjaETgG+HwePkDSDyT9XtLdks6XtFOlvSSdIunefDtFkgAi4k5gGbBgIp5Lr3G4d8+hOURrt9vqG0zAUeNuwMombRrVfS3wTuAXY1phi8Zp3R0aETOA5wH7AieMwzLHjKQ9gXOA9wHbArsDZwJ/nsi66hwLLI2IjXl4O2ARMIe0/64HvlxpvwA4AngusA9wKPD2yvT/qht+wnK4j6HKqYC3SroF+GEef4Ckn0q6X9K1kgYqj9ld0o8krc9HMGeM9HZX0tskrclHOksqR96/AfYAvpOPRkf1VjUizoyIS4CHWnie20o6Jx9p/U7ShyVNkrRlfo7PqbTdMR8hPzkPvzofWd6f18k+lbY3S/qApOuADcMFvKTTJd2aj0yvkvSi0TzPBs/9DuAiUsjXlvN/JF2dl3OrpBMr02rb+c2SbslHyB+qTJ+e3wncJ2kV8Fd1z+FZkpbndbBS0mGVaWdL+qykC/N2/Imkp0j6dJ7fDZL2bfBUngf8NiIuiWR9RHwzIm7J854k6XhJv8lHwYslPamy7Jb3U2CHyrRpkr6a53m/pCsk9TWo8WDgR5V1f2FEnB8Rf4iIB4EzgBdU2r8ZOC0i1kbEOuA00gtEzeXAHpJ2a7C8J46I8K3DG3Az8PJhxs8BgnT0tDUwHdgZuBc4hPTi+oo8vGN+zM+A/wC2BF5MOnL5aoPlvhS4B3h+bv+fwKXN6mp1em5zGXBskzbnAP8NzMzP+VfAW/O0LwEfr7R9F/C9fH9f4C5gHjCZ9I97M7Blpb5rgF2A6Q2W/SZge2AK6Qj1DmBannZibd1VtsWUZusCmA1cD5xemT4A7J232T7AncARdfP+Qt7GzwX+CDwrT/8k8GPgSfm5/BJYm6dtAawBPghMzdt0PfCMPP3svI33A6aRDhB+SzqVMRk4GVjW4DntQXpx/hQwH5hRN/09wIr8fLcknRo5L09rez8lHTl/B9gq17gfsE2DGu8G/mqEfesfgBWV4QeAeZXhfmB93WOuAw6b6FyY6NuEF1DCLQfDEHB/vn07j6/90+9RafsB4Ny6x19ECrZdgYeBrSvTvkbjcP8icGpleAbwJ2BOpa5m4f64uuvajBju+Z93EzC3Mu7twPJ8/+XAbyrTfgIck+9/DvhY3fxuBF5Sqe8to9wW9wHPzfdPZHThPpRDKoBLgFkjLOfTwKfq5j27Mv3nwJH5/k3AQZVpC3g03F9EekGaVJl+HnBivn828IXKtHcDqyvDewP3j1DnAcBiUog+lOc3I09bDbys0nanvP9M6WQ/Bd4C/BTYp4Xt9SfgmQ2m7QP8HnhRZdyfq+2BvfK613D72BP55tMy3XNERMzKt/pPptxaub8b8H/z29X7Jd0PvJD0j/VU4L6I2FBp/7sRlvnU6vSIGCIdXe3cpbpbsQPp6LNa5+8qNSwDtpI0T9Ic0qmCC/K03YD31a2LXUjPq6a67h5H0vuVPiHxQH78tlROEYzSERExk3SU/kwee6phnqRl+dTTA8A7hlnOHZX7D5JebCE9n+rzqK6rpwK3RsQjddOr2/DOyv2Nwww3vPAbESsi4nURsSPpheTFQO2U0W7ABZV1v5oUnn10tp+eS3ohGJR0m6RTJW3RoMT7SO/4HkPpesGFwHsi4seVSUPANpXhbYChyKmezSQdrDyhOdzHR3XHu5V0RDSrcts6Ij4J3A5sJ2nrSvtdR5jvbaR/QgDy47YH1nWx9mbuIR19Vc9x7lqrISL+TDpyPCrfvhsR63O7W0mnbKrrYquIOK8yr4bdlubz6/8EvA7YLiJmkd62q5MnFBE/Ih3h/ntl9NeAJcAuEbEtcNYolnM76UWrprpNbwN2kTSpbnrXt2FEXAF8C6hdA7kVOLhu/U+LdC677f00Iv4UER+NiLnAXwOvJp1GGs51wNOrI/L58otJ7+rOrWu/knTaq+a5VD40kK/L7En6QMATmsN9/H0VOFTSqyRNzhefBiTNjojfAVcCH5U0VdILSZ8GaOQ84O8kPS9fMP0EcHlE3NxpkXn500gBtkWu83H7SyW8Py5pZv7HfG9+njVfA14PvDHfr/kC8I58VCxJW+cLl487kmtgJun0wN3AFEkf4bFHdZ34NPAKSbUgmQn8PiIekrQ/8IZRzGsxcIKk7STNJp1aqbmcdJT/T5K2yBctDwUGO30Ckl6odMG9dvH6mcBhpPPskF6gPl67+Kh0sfvwPK3t/VTSfEl7S5oM/IH04l99Z1K1FHhJ5bE7k64rnBERZw3T/hzgvZJ2VvrwwPtIL8Q1+wM35xqf0Bzu4ywibgUOJ11Au5t0hPSPPLot3kC6wPh74F9IO3OjeV0M/DPwTdLR1NOAI7tU6vdJb/n/mvTRtI2kt/TDeTewgXRu+TJSgH+pUuflefpTSW+1a+OvBN5G+kTEfaQLi8eOosaLgO+RLuD+jnROecTTOK2KiLtJ6/4jedQ7gZMkrc/jFo9idh/N9f2WtF7/cjQaEZtIwXgw6V3QZ0nni2/o9DmQTk0cBlwvaYi0ri4ATs3TTye9G/l+fl4rSPtep/vpU4BvkIJ9NenTMPVH4DXnAIdImp6HjyNdCD5Rle9fVNp/nnSx9nrShen/yeNq3kh60XrC02NPVVmvUfrI3Z4R8aaJrsVsLEj6BHBXRHy6w/k8mfRCsm9ENP0Ib+km/KvYZvbEFhEf7NJ87gKe1Y15lcCnZczMCuTTMmZmBfKRu5lZgRzu1pOU+lo5Lt9/o6TvT3RNmxOlPmlOnug6bOI43G1EuWOqtwwz/j2SrmxznsdKuqzV9hHxXxHxynaWNRby571D0gV145+bxy/vcP4HSNqgYbocVuq8bGEn87cnBoe7NfMVhv924dF52qioB34so0vuBg6UtH1l3JtJn7nvSESsANYCr62OV+pdcy7py2tmI3K4WzPnAi+sdqEqaS6pU6fz8vC2kr4o6XZJ6ySdnL+dWDtK/4mkT0m6F/g66UsmB+YvqDTtA6T+SD8fHb9D6Vem7pd0piRVpr9Fqb+Z+yRdpLHp/nUT8G3yl8by8309qT/xau0NuySWtFTSaZXhQUm1L38N96J6DKnv83tz+/OVfljkAUmXSnr2cIUO904pr8M98/0tJf27UnfFd0o6q/KlIttMOdxtRBGxltT519GV0UeTQuaePHw2qRuAPUnd+L6S9E3Dmnmkb6/2kbrofQfws0g/DjKrzdJeTeoXfR9S3zKvgvTLQ6RvVf4tsCOpq92GR7qqdIw1zO34JjWcw6MB/CrSNybrf6TlClJnaU8ifXP3fKVuHSD1nni0pJdKeiPpq/PvydPOBV4saZdc5yTSt0Kr75YuJPWK+GTSj6o85oVlFD5J6t/leaRtuDOPfjPXNlcT3S2lb71/IwXyjfn+JOAW4G/ycB+p7/LplfZHkfsYJ3UncEvd/I4FLmuyzOXAccO1J3Um9sLK8GLg+Hz/QnJf8pV6HwR26+L6GODRLnt/DTyD1BfMG0kvastHeOxfuiTOw68hfbX/nupzytMuBj6Y77+CdCpoiwbznZXXy7Z5+Gzg5EbrO7fdk9R30AbgaZVpB5J+5GPC9z3f2r/5yN1a8S1gJ0kHkIJtK1KfHpB6g9wCuF2Pdg37edLRZE1X+nup06h73d2A0yu1/J4UYKPpBnk0zgUWkn4M44L6iWreJfF3SH3i3xgR9ReZv8Kj75iOBgYj4k95vpMlfVLpV5T+QOqPHkbf3fGOpO15VWWdfS+Pt81YKRe3bAxFxIOSvkE6BTGdFDKb8uRbSUfuO0TEw41m0WS4m2rdCLd0iqKuU6p6n4iITzSZxbmkDs/OyeupOu9al8QvA1ZGxCOS7uOxXQV/nNS51u6SjorHdnf8LeCzkuaTTjMNVKa9gdSx18tJwb4t6V3BcN0QbyAFeK2up1Sm3UPqFO7Zkbr6tUI43K1VXyGFzRaksAIgIm7Pn0E/TdI/k35MYXfSrxL9aNg5pR+bmC1pauVFolvOAj4m6ZqIWClpW+CVEXH+cI0j/SB22yLit5JeQrqmUK++S+LjqXRJLOnFwN+R+iTfg/TDGZfWQjYiNuQX1S8Dv4vUi2Z13n8k/TjLVqTunhu5Fni2pOcBN5B+oapW/yOSvgB8StLCiLhLqdvd50TERaNZF9ZbfFrGWnUp6Ycw1kb60YeqY0i//7mKdPT4DdIv9jTyQ9IPLNwh6Z4R2o1aRFwAnEL6FaA/kC5yHtzNZQyzzMsiov5CKozQJbGkbUgXZBdGxLpIvzb0ReDL1U/+kF5Ud+PxXT+fk+e5jrTeV9BARPwKOIl0Dv/XpG6Zqz5AevexIq+zi0nXEWwz5r5lzMwK5CN3M7MCOdzNzArkcDczK5DD3cysQBP2Ucgddtgh5syZMyHL3rBhA1tvvfWELLsVvV4f9H6Nrq9zvV7j5lrfTXdvAGCPHdur/aqrrronIpp/yWyivhq73377xURZtmzZhC27Fb1eX0Tv1+j6OtfrNW6u9b3urJ/G6876advzBa4Mdz9gZvbE5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MytQ03CX9CVJd0n6ZYPpkvQZSWskXSfp+d0v08zMRqOVI/ezgYNGmH4w6Xcc9wIWAJ/rvCwzM+tE03CPiEtJP1XWyOGkX6GJiFgBzJI0Ul/eZmY2xlrqz13SHOC7EfGcYaZ9F/hk5N9/lHQJ8IF47K/G1NouIB3d09fXt9/g4GBbRa9/qNGvubXmkU0bmTR1eluPnTlt7HtsGBoaYsaMjn4gaMz1eo2d1Nfp/tWKRvvgeOxfrSp5G8PYb+dG2/j0qzcxeZI4YV57GTR//vyrIqK/Wbtx3ZMiYhGwCKC/vz8GBgbams/Fq+7sqI6Nt1zP9F33buuxA3P7Olp2K5YvX06762a89HqNndTX6f7Vikb74HjsX60qeRvD2G/nRtt40uobmLXVVAYGDhzT5Xfj0zLrgF0qw7PzODMzmyDdCPclwDH5UzMHAA9ExO1dmK+ZmbWp6WkZSecBA8AOktYC/wJsARARZwFLgUNIP7D7IOnX3M3MbAI1DfeIOKrJ9ADe1bWKzMysY/6GqplZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mVqCWwl3SQZJulLRG0vHDTN9V0jJJV0u6TtIh3S/VzMxa1TTcJU0GzgQOBuYCR0maW9fsw8DiiNgXOBL4bLcLNTOz1rVy5L4/sCYiboqITcAgcHhdmwC2yfe3BW7rXolmZjZaioiRG0ivBQ6KiOPy8NHAvIhYWGmzE/B9YDtga+DlEXHVMPNaACwA6Ovr229wcLCtotc/9HBbj6t5ZNNGJk2d3tZjZ06b0tGyWzE0NMSMGTPGfDmd6PUaO6mv0/2rFY32wfHYv1pV8jaGsd/Ojbbx6VdvYvIkccK89jJo/vz5V0VEf7N23dqTjgLOjojTJB0InCvpORHxSLVRRCwCFgH09/fHwMBAWwu7eNWdHRW78Zbrmb7r3m09dmBuX0fLbsXy5ctpd92Ml16vsZP6Ot2/WtFoHxyP/atVJW9jGPvt3GgbT1p9A7O2msrAwIFjuvxWTsusA3apDM/O46reCiwGiIifAdOAHbpRoJmZjV4r4X4FsJek3SVNJV0wXVLX5hbgZQCSnkUK97u7WaiZmbWuabhHxMPAQuAiYDXpUzErJZ0k6bDc7H3A2yRdC5wHHBvNTuabmdmYaemce0QsBZbWjftI5f4q4AXdLc3MzNrlb6iamRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFaincJR0k6UZJayQd36DN6yStkrRS0te6W6aZmY3GlGYNJE0GzgReAawFrpC0JCJWVdrsBZwAvCAi7pP05LEq2MzMmmvlyH1/YE1E3BQRm4BB4PC6Nm8DzoyI+wAi4q7ulmlmZqPRSrjvDNxaGV6bx1U9HXi6pJ9IWiHpoG4VaGZmo6eIGLmB9FrgoIg4Lg8fDcyLiIWVNt8F/gS8DpgNXArsHRH3181rAbAAoK+vb7/BwcG2il7/0MNtPa7mkU0bmTR1eluPnTmt6Zmsjg0NDTFjxowxX04ner3GTurrdP9qRaN9cDz2r1aVvI1h7Ldzo218+tWbmDxJnDCvvQyaP3/+VRHR36xdK3vSOmCXyvDsPK5qLXB5RPwJ+K2kXwF7AVdUG0XEImARQH9/fwwMDLSw+Me7eNWdbT2uZuMt1zN9173beuzA3L6Olt2K5cuX0+66GS+9XmMn9XW6f7Wi0T44HvtXq0rexjD227nRNp60+gZmbTWVgYEDx3T5rZyWuQLYS9LukqYCRwJL6tp8GxgAkLQD6TTNTV2s08zMRqFpuEfEw8BC4CJgNbA4IlZKOknSYbnZRcC9klYBy4B/jIh7x6poMzMbWUsn+CJiKbC0btxHKvcDeG++mZnZBPM3VM3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCtRSuEs6SNKNktZIOn6Edq+RFJL6u1eimZmNVtNwlzQZOBM4GJgLHCVp7jDtZgLvAS7vdpFmZjY6rRy57w+siYibImITMAgcPky7jwGnAA91sT4zM2uDImLkBtJrgYMi4rg8fDQwLyIWVto8H/hQRLxG0nLg/RFx5TDzWgAsAOjr69tvcHCwraLXP/RwW4+reWTTRiZNnd7WY2dOm9LRslsxNDTEjBkzxnw5nej1Gjupr9P9qxWN9sHx2L9aVfI2hrHfzo228elXb2LyJHHCvPYyaP78+VdFRNNT3x3vSZImAf8BHNusbUQsAhYB9Pf3x8DAQFvLvHjVnW09rmbjLdczfde923rswNy+jpbdiuXLl9PuuhkvvV5jJ/V1un+1otE+OB77V6tK3sYw9tu50TaetPoGZm01lYGBA8d0+a2cllkH7FIZnp3H1cwEngMsl3QzcACwxBdVzcwmTivhfgWwl6TdJU0FjgSW1CZGxAMRsUNEzImIOcAK4LDhTsuYmdn4aBruEfEwsBC4CFgNLI6IlZJOknTYWBdoZmaj19I594hYCiytG/eRBm0HOi/LzMw64W+ompkVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBWop3CUdJOlGSWskHT/M9PdKWiXpOkmXSNqt+6WamVmrmoa7pMnAmcDBwFzgKElz65pdDfRHxD7AN4BTu12omZm1rpUj9/2BNRFxU0RsAgaBw6sNImJZRDyYB1cAs7tbppmZjYYiYuQG0muBgyLiuDx8NDAvIhY2aH8GcEdEnDzMtAXAAoC+vr79BgcH2yp6/UMPt/W4mkc2bWTS1OltPXbmtCkdLbsVQ0NDzJgxY8yX04ler7GT+jrdv1rRaB8cj/2rVSVvYxj77dxoG59+9SYmTxInzGsvg+bPn39VRPQ3a9fVPUnSm4B+4CXDTY+IRcAigP7+/hgYGGhrORevurPNCpONt1zP9F33buuxA3P7Olp2K5YvX06762a89HqNndTX6f7Vikb74HjsX60qeRvD2G/nRtt40uobmLXVVAYGDhzT5bcS7uuAXSrDs/O4x5D0cuBDwEsi4o/dKc/MzNrRyjn3K4C9JO0uaSpwJLCk2kDSvsDngcMi4q7ul2lmZqPRNNwj4mFgIXARsBpYHBErJZ0k6bDc7N+AGcD5kq6RtKTB7MzMbBy0dM49IpYCS+vGfaRy/+VdrsvMzDrgb6iamRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRWopXCXdJCkGyWtkXT8MNO3lPT1PP1ySXO6XaiZmbWuabhLmgycCRwMzAWOkjS3rtlbgfsiYk/gU8Ap3S7UzMxa18qR+/7Amoi4KSI2AYPA4XVtDge+ku9/A3iZJHWvTDMzG40pLbTZGbi1MrwWmNeoTUQ8LOkBYHvgnmojSQuABXlwSNKN7RTdBTtQV1uP6fX6oPdrdH2d6/UaN+v6Fr+j7fnu1kqjVsK9ayJiEbBoPJc5HElXRkT/RNfRSK/XB71fo+vrXK/X6PpG1sppmXXALpXh2XncsG0kTQG2Be7tRoFmZjZ6rYT7FcBeknaXNBU4ElhS12YJ8OZ8/7XADyMiulemmZmNRtPTMvkc+kLgImAy8KWIWCnpJODKiFgCfBE4V9Ia4PekF4BeNuGnhpro9fqg92t0fZ3r9Rpd3wjkA2wzs/L4G6pmZgVyuJuZFWizDPcWukPYVdIySVdLuk7SIXn89nn8kKQz6h7z+tx2paRTKuOPlXS3pGvy7bgxrvEVkq6SdH3++9LKY/bL49dI+kzti2KSniTpB5J+nf9u12P1nShpXWUdHjJB9X1c0q2ShurmNeruM8a5vp7YByVtJel/JN2Q/08+2UvrsEl9PbEO87TvSbo213iWUi8Abf0fjygiNqsb6aLub4A9gKnAtcDcujaLgL/P9+cCN+f7WwMvBN4BnFFpvz1wC7BjHv4K8LJ8/9hq23GocV/gqfn+c4B1lcf8HDgAEHAhcHAefypwfL5/PHBKj9V3IvD+Hlh/BwA7AUN183oncFa+fyTw9R6rryf2QWArYH6+PxX4cWUbT/g6bFJfT6zDPLxN/ivgm8CR7fwfN7ttjkfurXSHEMA2+f62wG0AEbEhIi4DHqprvwfw64i4Ow9fDLxmgmq8OiJuy+NXAtPzUdFOpJ1iRaStfw5wRG5X7f7hK5XxvVLfaHW9vjxtRUTcPszyRtt9xnjX146u1xgRD0bEstxmE/AL0vdeoAfWYZP62jFW2/kPefwU0otG7VMto/0/HtHmGO7DdYewc12bE4E3SVoLLAXe3WSea4BnSJqj9CWsI3jsF7dek99yfUPSLsPPYkxqfA3wi4j4Y3782gbz7KuEwh1AX4/VB7Awr8MvtfB2cyzqa2l5EfEwUOs+o1fqg97YB/9C0izgUOCS+uX1wjocpj7ooXUo6SLgLmA96cUQRv9/PKLNMdxbcRRwdkTMBg4hfQa/4XONiPuAvwe+TnordzPw5zz5O8CciNgH+AGPvrKOaY2Snk3qXfPto5lpPmruxudbu1nf54CnAc8DbgdO67H6xkI36+upfTAfAJ0HfCYibupSLWNdX0+tw4h4FekU3JbAS6nTjf/jzTHcW+kO4a3AYoCI+BkwjdSJT0MR8Z2ImBcRBwI3Ar/K4++tvOL+f2C/sa5R0mzgAuCYiPhNZZ7Vt5jVed6ZT4uQ/97VS/VFxJ0R8eeIeAT4Aunt7njX19Ly1Fr3GeNaXw/tgzWLSKcxPz3c8npgHT6uvh5ch0TEQ8B/8+ipntH+H4+skxP2E3Ejnae6CdidRy9yPLuuzYXAsfn+s0jnwVSZfix1F1eAJ+e/2wHXAE/PwztV2vwNsGIsawRm5fZ/O8x86y9YHpLH/xuPvRBzao/VV12H/w8YnIj6Ko+tv2D5Lh57MXBxj9XXS/vgyaSLgJN6cR2OUF9PrENgRq2WPP+vAwvb+T9uWn8nD56oG+ntz69IV7I/lMedBByW788FfpJX7jXAKyuPvZnURcIQ6Rza3Dz+PGBVvh1Zaf+vpAsi1wLLgGeOZY3Ah4ENeVztVnvh6Qd+med5Bo9+w3h70rnFX5MuBj+px+o7F7geuI7UD9FOE1TfqXmbP5L/npjHTwPOJ117+TmwR4/V1xP7IOnINYDVlfHH9co6bFJfr6z5dMpfAAAARUlEQVTDPlJ/XdeR/lf+E5jS7v/xSDd3P2BmVqDN8Zy7mZk14XA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrED/CxB7Gh4Aa2Y9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "valid_perf.groupby('rnd_seed')['F1'].mean().hist(alpha=0.3)\n",
    "plt.axvline(x=valid_perf.groupby('rnd_seed')['F1'].mean().max())\n",
    "plt.title('Freq of F1 over all Random Seeds (20) \\n Vert line = MaxValue')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    3.000000\n",
       "mean     0.821271\n",
       "std      0.001892\n",
       "min      0.819306\n",
       "25%      0.820367\n",
       "50%      0.821429\n",
       "75%      0.822254\n",
       "max      0.823080\n",
       "Name: F1, dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_perf.groupby('rnd_seed')['F1'].mean().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['haha_clas_0609_fld0_248367.pth',\n",
       " 'haha_clas_0609_fld1_248367.pth',\n",
       " 'haha_clas_0609_fld2_248367.pth',\n",
       " 'haha_clas_0609_fld3_248367.pth',\n",
       " 'haha_clas_0609_fld4_248367.pth']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seed_grouped_F1 = valid_perf.groupby('rnd_seed')['F1'].mean()\n",
    "\n",
    "## best of all sampled seeds\n",
    "selected_seeds = seed_grouped_F1[seed_grouped_F1>=seed_grouped_F1.max()].index.values.tolist()\n",
    "seed_fold_pair = valid_perf[valid_perf.rnd_seed.isin(selected_seeds)].loc[:,['rnd_seed','nfold']].values\n",
    "model_wt_fnames = []\n",
    "for (seed,fold) in seed_fold_pair:\n",
    "    model_wt_fnames.append(f'haha_clas_0609_fld{fold}_{seed}.pth')\n",
    "model_wt_fnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ensemble_preds = []\n",
    "for wt_fname in model_wt_fnames:\n",
    "    config = awd_lstm_clas_config.copy()\n",
    "    config['qrnn'] = True\n",
    "    config['n_hid'] = 2304\n",
    "    \n",
    "    learn = text_classifier_learner(data_tst, AWD_LSTM, drop_mult=0.7,pretrained=False,\n",
    "                               config=config)\n",
    "    learn = learn.load(wt_fname.split('.')[0]) #load weights\n",
    "    preds_t,y_t,losses_t = learn.get_preds(DatasetType.Valid,with_loss=True, ordered=True)\n",
    "    ensemble_preds.append(to_np(preds_t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 6000, 2)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(ensemble_preds).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.238477, 0.761523],\n",
       "       [0.921819, 0.078181],\n",
       "       [0.079422, 0.920578],\n",
       "       [0.976196, 0.023804],\n",
       "       ...,\n",
       "       [0.635782, 0.364218],\n",
       "       [0.950017, 0.049983],\n",
       "       [0.34033 , 0.65967 ],\n",
       "       [0.235313, 0.764687]], dtype=float32)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ens_mean = np.array(ensemble_preds).mean(axis=0)\n",
    "ens_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_df = test_df['id'].to_frame().copy()\n",
    "out_df['is_humor'] = ens_mean.argmax(axis=1)\n",
    "out_df['funniness_average'] = 3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>is_humor</th>\n",
       "      <th>funniness_average</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tweet1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tweet2</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tweet3</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tweet4</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tweet5</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  is_humor  funniness_average\n",
       "0  tweet1         1                3.0\n",
       "1  tweet2         0                3.0\n",
       "2  tweet3         1                3.0\n",
       "3  tweet4         0                3.0\n",
       "4  tweet5         0                3.0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_df.to_csv('sub0610_finetune_bestSeed.csv',header=True,index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}