movie-review-sentiment-analysis-kernels-only-update1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "from fastai.learner import *\n",
    "\n",
    "import torchtext\n",
    "from torchtext import vocab, data\n",
    "from torchtext.datasets import language_modeling\n",
    "\n",
    "from fastai.rnn_reg import *\n",
    "from fastai.rnn_train import *\n",
    "from fastai.nlp import *\n",
    "from fastai.lm_rnn import *\n",
    "\n",
    "import dill as pickle\n",
    "import spacy\n",
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "\n",
    "PATH = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fastai\r\n",
      "labeled_text\r\n",
      "models\r\n",
      "movie-review-sentiment-analysis-kernels-only.ipynb\r\n",
      "movie-review-sentiment-analysis-kernels-only-transferred-model.ipynb\r\n",
      "movie-review-sentiment-analysis-kernels-only-update1.ipynb\r\n",
      "sampleSubmission.csv\r\n",
      "test.tsv\r\n",
      "TEXT_full.pkl\r\n",
      "TEXT_imdb.pkl\r\n",
      "TEXT.pkl\r\n",
      "tmp\r\n",
      "train\r\n",
      "train.tsv\r\n",
      "valid\r\n"
     ]
    }
   ],
   "source": [
    "PATH = \"\"\n",
    "!ls {PATH}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(f'{PATH}train.tsv', sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.read_csv(f'{PATH}test.tsv',sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PhraseId</th>\n",
       "      <th>SentenceId</th>\n",
       "      <th>Phrase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>156061</td>\n",
       "      <td>8545</td>\n",
       "      <td>An intermittently pleasing but mostly routine ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>156062</td>\n",
       "      <td>8545</td>\n",
       "      <td>An intermittently pleasing but mostly routine ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>156063</td>\n",
       "      <td>8545</td>\n",
       "      <td>An</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>156064</td>\n",
       "      <td>8545</td>\n",
       "      <td>intermittently pleasing but mostly routine effort</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>156065</td>\n",
       "      <td>8545</td>\n",
       "      <td>intermittently pleasing but mostly routine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>156066</td>\n",
       "      <td>8545</td>\n",
       "      <td>intermittently pleasing but</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>156067</td>\n",
       "      <td>8545</td>\n",
       "      <td>intermittently pleasing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>156068</td>\n",
       "      <td>8545</td>\n",
       "      <td>intermittently</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>156069</td>\n",
       "      <td>8545</td>\n",
       "      <td>pleasing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>156070</td>\n",
       "      <td>8545</td>\n",
       "      <td>but</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PhraseId  SentenceId                                             Phrase\n",
       "0    156061        8545  An intermittently pleasing but mostly routine ...\n",
       "1    156062        8545  An intermittently pleasing but mostly routine ...\n",
       "2    156063        8545                                                 An\n",
       "3    156064        8545  intermittently pleasing but mostly routine effort\n",
       "4    156065        8545         intermittently pleasing but mostly routine\n",
       "5    156066        8545                        intermittently pleasing but\n",
       "6    156067        8545                            intermittently pleasing\n",
       "7    156068        8545                                     intermittently\n",
       "8    156069        8545                                           pleasing\n",
       "9    156070        8545                                                but"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PhraseId</th>\n",
       "      <th>SentenceId</th>\n",
       "      <th>Phrase</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>A series of escapades demonstrating the adage ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>A series of escapades demonstrating the adage ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>A series</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>series</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PhraseId  SentenceId                                             Phrase  \\\n",
       "0         1           1  A series of escapades demonstrating the adage ...   \n",
       "1         2           1  A series of escapades demonstrating the adage ...   \n",
       "2         3           1                                           A series   \n",
       "3         4           1                                                  A   \n",
       "4         5           1                                             series   \n",
       "\n",
       "   Sentiment  \n",
       "0          1  \n",
       "1          2  \n",
       "2          2  \n",
       "3          2  \n",
       "4          2  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop incomplete sentences in training set\n",
    "def dropIncomplete (df, label=\"SentenceId\", keepLocation=0):\n",
    "    sentences = []\n",
    "    for i in df[label].unique():  \n",
    "        sentences.append(df[df[label] == i].reset_index(drop=True).loc[keepLocation])\n",
    "    df_result = pd.DataFrame(sentences)\n",
    "    return df_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = dropIncomplete(df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = dropIncomplete(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.DataFrame(df_train[\"Phrase\"]).reset_index(drop=True)\n",
    "df_test = pd.DataFrame(df_test[\"Phrase\"]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Phrase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A series of escapades demonstrating the adage ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>This quiet , introspective and entertaining in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Even fans of Ismail Merchant 's work , I suspe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A positively thrilling combination of ethnogra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Aggressive self-glorification and a manipulati...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Phrase\n",
       "0  A series of escapades demonstrating the adage ...\n",
       "1  This quiet , introspective and entertaining in...\n",
       "2  Even fans of Ismail Merchant 's work , I suspe...\n",
       "3  A positively thrilling combination of ethnogra...\n",
       "4  Aggressive self-glorification and a manipulati..."
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Phrase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>An intermittently pleasing but mostly routine ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Kidman is really the only thing that 's worth ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Once you get into its rhythm ... the movie bec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I kept wishing I was watching a documentary ab...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Kinnear does n't aim for our sympathy , but ra...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Phrase\n",
       "0  An intermittently pleasing but mostly routine ...\n",
       "1  Kidman is really the only thing that 's worth ...\n",
       "2  Once you get into its rhythm ... the movie bec...\n",
       "3  I kept wishing I was watching a documentary ab...\n",
       "4  Kinnear does n't aim for our sympathy , but ra..."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_full = df_train.append(df_test).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Phrase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11830</th>\n",
       "      <td>At its worst , it implodes in a series of very...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11831</th>\n",
       "      <td>Maybe I found the proceedings a little bit too...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11832</th>\n",
       "      <td>Moretti 's compelling anatomy of grief and the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11833</th>\n",
       "      <td>Montias ... pumps a lot of energy into his nic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11834</th>\n",
       "      <td>Not sweet enough to liven up its predictable s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11835</th>\n",
       "      <td>Nasty , ugly , pointless and depressing , even...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11836</th>\n",
       "      <td>With tightly organized efficiency , numerous f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11837</th>\n",
       "      <td>They should have called it Gutterball .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11838</th>\n",
       "      <td>A long-winded , predictable scenario .</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Phrase\n",
       "11830  At its worst , it implodes in a series of very...\n",
       "11831  Maybe I found the proceedings a little bit too...\n",
       "11832  Moretti 's compelling anatomy of grief and the...\n",
       "11833  Montias ... pumps a lot of energy into his nic...\n",
       "11834  Not sweet enough to liven up its predictable s...\n",
       "11835  Nasty , ugly , pointless and depressing , even...\n",
       "11836  With tightly organized efficiency , numerous f...\n",
       "11837            They should have called it Gutterball .\n",
       "11838             A long-winded , predictable scenario ."
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_full[df_full.shape[1]-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11839, 1)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_full.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19.047977025086578"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#average number of words in each sentence\n",
    "np.mean(df_full[\"Phrase\"].apply(lambda x: len(x.split(\" \"))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "spacy_tok = spacy.load('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT = data.Field(lower=True, tokenize=\"spacy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bbtt = 70\n",
    "bs=64; bptt=19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8529, 1), (3310, 1))"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "md = LanguageModelData.from_dataframes(path=PATH,field=TEXT,col=\"Phrase\",train_df=df_full, val_df=df_test,\n",
    "                                       test_df=df_test,bs=bs, bptt=bptt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(TEXT, open(f'{PATH}TEXT_full.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0m\u001b[01;36mfastai\u001b[0m@\r\n",
      "\u001b[01;34mlabeled_text\u001b[0m/\r\n",
      "\u001b[01;34mmodels\u001b[0m/\r\n",
      "movie-review-sentiment-analysis-kernels-only.ipynb\r\n",
      "movie-review-sentiment-analysis-kernels-only-transferred-model.ipynb\r\n",
      "movie-review-sentiment-analysis-kernels-only-update1.ipynb\r\n",
      "sampleSubmission.csv\r\n",
      "test.tsv\r\n",
      "TEXT_full.pkl\r\n",
      "TEXT_imdb.pkl\r\n",
      "TEXT.pkl\r\n",
      "\u001b[01;34mtmp\u001b[0m/\r\n",
      "\u001b[01;34mtrain\u001b[0m/\r\n",
      "train.tsv\r\n",
      "\u001b[01;34mvalid\u001b[0m/\r\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(219, 17902, 1, 268557)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(md.trn_dl), md.nt, len(md.trn_ds), len(md.trn_ds[0].text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<unk>', '<pad>', '<', '>', 'eos', '.', 'the', ',', 'a', 'and', 'of', 'to']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT.vocab.itos[:12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT.vocab.stoi['to']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_sz = 200\n",
    "nh = 500\n",
    "nl = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.Adam, betas = (0.7, 0.99))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner = md.get_model(opt_fn, em_sz, nh, nl, dropouti=0.05, dropout=0.05,\n",
    "                       wdrop=0.1, dropoute=0.02, dropouth=0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.reg_fn = partial(seq2seq_reg, alpha=2, beta=1)\n",
    "learner.clip=0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b13b67bc243246a59834f3b3156b8b1b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=15), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                              \n",
      "    0      5.86874    5.733352  \n",
      "    1      5.510582   5.323128                              \n",
      "    2      5.295627   5.201433                              \n",
      "    3      5.168229   4.97615                               \n",
      "    4      4.932089   4.783189                              \n",
      "    5      4.769538   4.66728                               \n",
      "    6      4.731221   4.637032                              \n",
      "    7      4.808199   4.667988                              \n",
      "    8      4.614614   4.441589                              \n",
      "    9      4.488423   4.356996                              \n",
      "    10     4.338571   4.237494                              \n",
      "    11     4.175133   4.046654                              \n",
      "    12     4.087235   3.968333                              \n",
      "    13     4.02391    3.914336                              \n",
      "    14     4.037097   3.898286                              \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([3.89829])]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit(3e-3, 4, wds=1e-6, cycle_len=1, cycle_mult=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.save_encoder('adam1_movieReview_bptt19_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a039f3fc9a884b3ca4b1607b2f75c48f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                              \n",
      "    0      4.241308   4.082727  \n",
      "    1      4.171602   4.013448                              \n",
      "    2      4.09656    4.053461                              \n",
      "    3      3.848913   3.698589                              \n",
      "    4      3.795498   3.746261                              \n",
      "    5      3.596768   3.458892                              \n",
      "    6      3.451824   3.334224                              \n",
      "    7      3.361029   3.272225                              \n",
      "    8      3.435255   3.234743                              \n",
      "    9      3.345696   3.212821                              \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([3.21282])]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit(3e-3, 1, wds=1e-6, cycle_len=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.save_encoder('adam3_movieReview_bptt19_10_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load_encoder('adam3_movieReview_bptt19_10_enc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'i want to say it wil be'"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#use our torchtext field to numericalize it so we can feed it to our language model.\n",
    "m = learner.model\n",
    "ss = \"\"\"I want to say it wil be\"\"\"\n",
    "s = [TEXT.preprocess(ss)]\n",
    "t=TEXT.numericalize(s)\n",
    "' '.join(s[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "#steps to test a language model\n",
    "# Set batch size to 1\n",
    "m[0].bs=1\n",
    "# Turn off dropout\n",
    "m.eval()\n",
    "# Reset hidden state\n",
    "m.reset()\n",
    "# Get predictions from model\n",
    "res,*_ = m(t)\n",
    "# Put the batch size back to what it was\n",
    "m[0].bs=bs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.', 'about', '--', 'a', 'so', ':', 'long', ',', 'in', 'made']"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Top 10 predictions\n",
    "nexts = torch.topk(res[-1], 10)[1]\n",
    "[TEXT.vocab.itos[o] for o in to_np(nexts)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I want to say it wil be \n",
      "\n",
      ". < eos > a movie that 's too busy written and too funny to be funny , but it ...\n"
     ]
    }
   ],
   "source": [
    "#generate a bit more text\n",
    "print(ss,\"\\n\")\n",
    "for i in range(20):\n",
    "    n=res[-1].topk(2)[1]\n",
    "    n = n[1] if n.data[0]==0 else n[0]\n",
    "    print(TEXT.vocab.itos[n.data[0]], end=' ')\n",
    "    res, *_ =m(n[0].unsqueeze(0))\n",
    "print(\"...\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Phrase</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story .</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A series of escapades demonstrating the adage that what is good for the goose</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A series</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>series</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                         Phrase  \\\n",
       "0  A series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story .   \n",
       "1                                                                                                                 A series of escapades demonstrating the adage that what is good for the goose   \n",
       "2                                                                                                                                                                                      A series   \n",
       "3                                                                                                                                                                                             A   \n",
       "4                                                                                                                                                                                        series   \n",
       "\n",
       "   Sentiment  \n",
       "0          1  \n",
       "1          2  \n",
       "2          2  \n",
       "3          2  \n",
       "4          2  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Process training set data with sentiment label\n",
    "df_trainWLabels = pd.read_csv(\"train.tsv\", sep=\"\\t\")\n",
    "#df_trainWLabels = dropIncomplete(df_trainWLabels)\n",
    "df_trainWLabels.drop([\"PhraseId\", \"SentenceId\"], axis = 1,inplace=True)\n",
    "df_trainWLabels.reset_index(inplace=True, drop=True)\n",
    "df_trainWLabels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "156060"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_trainWLabels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_suffled = df_trainWLabels.sample(frac=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Phrase</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>119366</th>\n",
       "      <td>too hard to be emotional</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127268</th>\n",
       "      <td>Once folks started hanging out at the barbershop</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21882</th>\n",
       "      <td>... if it had been only half-an-hour long or a TV special , the humor would have been fast and furious -- at ninety minutes , it drags .</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83765</th>\n",
       "      <td>decides to fight her bully of a husband</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83789</th>\n",
       "      <td>should pay reparations to viewers .</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                          Phrase  \\\n",
       "119366                                                                                                                  too hard to be emotional   \n",
       "127268                                                                                          Once folks started hanging out at the barbershop   \n",
       "21882   ... if it had been only half-an-hour long or a TV special , the humor would have been fast and furious -- at ninety minutes , it drags .   \n",
       "83765                                                                                                    decides to fight her bully of a husband   \n",
       "83789                                                                                                        should pay reparations to viewers .   \n",
       "\n",
       "        Sentiment  \n",
       "119366          2  \n",
       "127268          2  \n",
       "21882           1  \n",
       "83765           2  \n",
       "83789           1  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_suffled.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def IntoLabeledFolders(df, folderPath, validRatio=0.3, contentIndex=0, labelIndex=1):\n",
    "    count = 1\n",
    "    L = len(df)\n",
    "    cutPoint = int(round((1-validRatio)*L))\n",
    "\n",
    "    for index, row in df.iterrows():\n",
    "        phrase=row[contentIndex]\n",
    "        label = row[labelIndex]\n",
    "        if (count<cutPoint):\n",
    "            filename = os.path.join(folderPath,\"train\", str(label), str(count))+\".txt\"\n",
    "        else:\n",
    "            filename = os.path.join(folderPath,\"valid\", str(label), str(count))+\".txt\"\n",
    "        os.makedirs(os.path.dirname(filename), exist_ok=True)\n",
    "        with open(filename,\"w\") as f:\n",
    "            f.write(phrase)\n",
    "        count = count + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "IntoLabeledFolders(df=df_train_suffled, folderPath = PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['0', '1', '2', '3', '4']"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show labels ###Start here if already created labeled folders\n",
    "labelList = !ls {PATH}train\n",
    "labelList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[4910, 19140, 55597, 23204, 6390]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Count number of samples under each label training\n",
    "labelCount = []\n",
    "for i in labelList:\n",
    "    trn_files = !ls {PATH}train/{i}\n",
    "    labelCount.append(len(trn_files))\n",
    "labelCount"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2162, 8133, 23985, 9723, 2816]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Count number of samples under each label in validation\n",
    "labelCount = []\n",
    "for i in labelList:\n",
    "    trn_files = !ls {PATH}valid/{i}\n",
    "    labelCount.append(len(trn_files))\n",
    "labelCount"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['quite fun in places']"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Review sentences\n",
    "label = 3\n",
    "fileIndex = 40\n",
    "FileList = !ls {PATH}train/{label}\n",
    "sample = !cat {PATH}train/{label}/{FileList[fileIndex]}\n",
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiLabeledMovieReview(torchtext.data.Dataset):\n",
    "    \"\"\"Create a MovieReview dataset instance given a path and fields.\n",
    "    Arguments:\n",
    "        path: Path to the dataset's highest level directory\n",
    "        text_field: The field that will be used for text data.\n",
    "        label_field: The field that will be used for label data.\n",
    "        Remaining keyword arguments: Passed to the constructor of\n",
    "            data.Dataset.\n",
    "    \"\"\"\n",
    "    def __init__(self, path, text_field, label_field, **kwargs):\n",
    "        fields = [('text', text_field), ('label', label_field)]\n",
    "        examples = []\n",
    "        for label in ['0','1','2','3','4']:\n",
    "            fnames = glob.glob(os.path.join(path, label, \"*.txt\"))\n",
    "            assert fnames, f\"can't find file under {path}/{label}\"\n",
    "            for fname in fnames:\n",
    "                with open(fname, \"r\") as f: text = f.readline()\n",
    "                examples.append(data.Example.fromlist([text,label], fields))\n",
    "        super().__init__(examples, fields, **kwargs)\n",
    "    \n",
    "    @staticmethod\n",
    "    def sort_key(ex): return len(ex.text)\n",
    "    \n",
    "    @classmethod\n",
    "    def splits(cls, text_field, label_field, root='.data', \n",
    "               train='train', test='test', **kwargs):\n",
    "        return super().splits(\n",
    "            root, text_field=text_field, label_field=label_field,\n",
    "            train=train, validation=None, test=test, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "MovieReviewLabel = data.Field(sequential=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT = pickle.load(open(f\"{PATH}TEXT_full.pkl\",\"rb\")) #open in read and bineary mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = MultiLabeledMovieReview.splits(TEXT, MovieReviewLabel,PATH,train=\"train\", test=\"valid\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = splits[0].examples[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('0', 'worst -- and only -- killer website movie')"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.label, \" \".join(t.text[:30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 64; bptt = 19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "md2 = TextData.from_splits(PATH, splits, bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0m\u001b[01;36mfastai\u001b[0m@\r\n",
      "\u001b[01;34mlabeled_text\u001b[0m/\r\n",
      "\u001b[01;34mmodels\u001b[0m/\r\n",
      "movie-review-sentiment-analysis-kernels-only.ipynb\r\n",
      "movie-review-sentiment-analysis-kernels-only-transferred-model.ipynb\r\n",
      "movie-review-sentiment-analysis-kernels-only-update1.ipynb\r\n",
      "sampleSubmission.csv\r\n",
      "test.tsv\r\n",
      "TEXT_full.pkl\r\n",
      "TEXT_imdb.pkl\r\n",
      "TEXT.pkl\r\n",
      "\u001b[01;34mtmp\u001b[0m/\r\n",
      "\u001b[01;34mtrain\u001b[0m/\r\n",
      "train.tsv\r\n",
      "\u001b[01;34mvalid\u001b[0m/\r\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.Adam, betas=(0.7, 0.99))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_sz = 200  # size of each embedding vector\n",
    "nh = 500     # number of hidden activations per layer\n",
    "nl = 3       # number of layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "m3 = md2.get_model(opt_fn, 1500, bptt, emb_sz=em_sz, n_hid=nh, n_layers=nl, \n",
    "           dropout=0.1, dropouti=0.4, wdrop=0.5, dropoute=0.05, dropouth=0.3)\n",
    "m3.reg_fn = partial(seq2seq_reg, alpha=2, beta=1)\n",
    "m3.load_encoder('adam3_movieReview_bptt19_10_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "m3.clip=25.\n",
    "lrs=np.array([1e-4,1e-4,1e-4,1e-3,1e-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe3181888b6047fa950bfb03b46fec19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                     \n",
      "    0      1.225756   1.454782   0.379485  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13b7a4717cbf47df8c1b977709fc8a73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                     \n",
      "    0      1.19055    1.374208   0.410391  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.37421]), 0.4103907096704536]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# freeze/unfreeze and differential learning rate\n",
    "m3.freeze_to(-1)\n",
    "m3.fit(lrs/2, 1, metrics=[accuracy])\n",
    "m3.unfreeze()\n",
    "m3.fit(lrs, 1, metrics=[accuracy], cycle_len=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "261578211ec547f2a05495be7d35e9fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=14), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                     \n",
      "    0      1.194742   1.355661   0.421916  \n",
      "    1      1.190895   1.323928   0.436148                     \n",
      "    2      1.176648   1.313545   0.438876                     \n",
      "    3      1.133999   1.307074   0.444976                     \n",
      "    4      1.153755   1.281625   0.46397                      \n",
      "    5      1.12497    1.286287   0.457966                     \n",
      "    6      1.102612   1.266341   0.472022                     \n",
      "    7      1.090571   1.258767   0.473763                     \n",
      "    8      1.098813   1.238849   0.483113                     \n",
      "    9      1.061897   1.241319   0.486762                     \n",
      "    10     1.046057   1.250746   0.484095                     \n",
      "    11     1.014976   1.212742   0.49873                       \n",
      "    12     1.016663   1.272033   0.485856                     \n",
      "    13     0.998064   1.207591   0.503764                      \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.20759]), 0.5037638492083111]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m3.fit(lrs, 7, metrics=[accuracy], cycle_len=2, cycle_save_name='MovieReview')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d057ba37f37479c884e13cf6fa83898",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                      \n",
      "    0      1.032701   1.236182   0.515319  \n",
      "    1      1.026307   1.221975   0.507638                      \n",
      "    2      0.989347   1.210637   0.504696                      \n",
      "    3      0.981331   1.179499   0.521223                      \n",
      "    4      0.974019   1.179538   0.516481                      \n",
      "    5      0.934968   1.176682   0.528881                      \n",
      "    6      0.944647   1.153137   0.540309                      \n",
      "    7      0.939814   1.122244   0.540852                      \n",
      "    8      0.929467   1.135205   0.543101                      \n",
      "    9      0.915657   1.126405   0.540739                      \n",
      "    10     0.954687   1.145194   0.525871                      \n",
      "    11     0.949187   1.183899   0.516627                      \n",
      "    12     0.926991   1.158074   0.546305                      \n",
      "    13     0.917253   1.129904   0.539476                      \n",
      "    14     0.899947   1.138538   0.548058                      \n",
      "    15     0.893655   1.089156   0.554742                      \n",
      "    16     0.858723   1.137278   0.556254                      \n",
      "    17     0.870513   1.109819   0.55512                       \n",
      "    18     0.863162   1.115408   0.560294                      \n",
      "    19     0.881463   1.076111   0.560382                      \n",
      "    20     0.900346   1.195322   0.533693                      \n",
      "    21     0.885085   1.115115   0.563246                      \n",
      "    22     0.878241   1.138096   0.548402                      \n",
      "    23     0.879573   1.08946    0.572766                      \n",
      "    24     0.85648    1.071554   0.565661                      \n",
      "    25     0.848028   1.06256    0.571185                      \n",
      "    26     0.854573   1.059278   0.57788                       \n",
      "    27     0.843253   1.07072    0.573092                      \n",
      "    28     0.840027   1.053285   0.576614                      \n",
      "    29     0.833189   1.07951    0.573417                      \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.07951]), 0.5734173470388986]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m3.fit(lrs, 3, metrics=[accuracy], cycle_len=10, cycle_save_name='MovieReview')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc44997755c64a9b99a84fc225499cad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                      \n",
      "    0      0.900049   1.31804    0.529656  \n",
      "    1      0.906962   1.133928   0.563387                      \n",
      "    2      0.871039   1.124645   0.560782                      \n",
      "    3      0.872574   1.070637   0.580854                      \n",
      "    4      0.826138   1.091729   0.57487                       \n",
      "    5      0.83213    1.092103   0.560083                      \n",
      "    6      0.799855   1.025597   0.592756                      \n",
      "    7      0.805425   1.024155   0.591203                      \n",
      "    8      0.820428   1.016166   0.592013                      \n",
      "    9      0.779574   1.028961   0.593652                      \n",
      "    10     0.884673   1.26474    0.5508                        \n",
      "    11     0.850128   1.083796   0.575192                      \n",
      "    12     0.82217    1.137643   0.565252                      \n",
      "    13     0.824535   1.052093   0.591333                      \n",
      "    14     0.779113   1.044278   0.591102                      \n",
      "    15     0.777209   1.023938   0.596508                      \n",
      "    16     0.766308   1.061431   0.581653                      \n",
      "    17     0.758851   1.06384    0.594211                      \n",
      "    18     0.75263    1.06078    0.600139                      \n",
      "    19     0.747051   1.041808   0.598059                      \n",
      "    20     0.799364   1.08172    0.594071                      \n",
      "    21     0.811608   1.081755   0.588924                      \n",
      "    22     0.783963   1.037228   0.589141                      \n",
      "    23     0.797419   1.069875   0.592563                      \n",
      "    24     0.757051   1.08244    0.598788                      \n",
      "    25     0.736384   1.068745   0.601245                      \n",
      "    26     0.749729   1.089612   0.606563                      \n",
      "    27     0.730677   1.041227   0.605802                      \n",
      "    28     0.732193   1.034491   0.606174                      \n",
      "    29     0.713091   1.058857   0.603928                      \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.05886]), 0.6039281361324342]"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m3.fit(lrs*2, 3, metrics=[accuracy], cycle_len=10, cycle_save_name='MovieReview')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using a transfered model trained on IMDB data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT = pickle.load(open(\"TEXT_imdb.pkl\",\"rb\")) #TEXT field for IMDB NLP model\n",
    "splits = MultiLabeledMovieReview.splits(TEXT, MovieReviewLabel, PATH,train=\"train\", test=\"valid\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 64; bptt = 70\n",
    "md2 = TextData.from_splits(PATH, splits, bs)\n",
    "opt_fn = partial(optim.Adam, betas=(0.7, 0.99))\n",
    "em_sz = 200  # size of each embedding vector\n",
    "nh = 500     # number of hidden activations per layer\n",
    "nl = 3       # number of layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "m3 = md2.get_model(opt_fn, 1500, bptt, emb_sz=em_sz, n_hid=nh, n_layers=nl, \n",
    "           dropout=0.1, dropouti=0.4, wdrop=0.5, dropoute=0.05, dropouth=0.3)\n",
    "m3.reg_fn = partial(seq2seq_reg, alpha=2, beta=1)\n",
    "m3.load_encoder('adam3_imdb_10_enc') #NLP model trained on IMDB dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "m3.clip=25\n",
    "lrs=np.array([1e-4,1e-4,1e-4,1e-3,1e-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31f237e113c84a8caf5e0404d608ea28",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                     \n",
      "    0      1.139295   1.263814   0.432896  \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "627106aa00db414f8c6851a41d130148",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                     \n",
      "    0      1.070088   1.167292   0.488185  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.16729]), 0.4881846352137183]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# freeze/unfreeze and differential learning rate\n",
    "m3.freeze_to(-1)\n",
    "m3.fit(lrs/2, 1, metrics=[accuracy])\n",
    "m3.unfreeze()\n",
    "m3.fit(lrs, 1, metrics=[accuracy], cycle_len=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b33a7e5edd5c4c09a9f645ec47b48337",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=14), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                     \n",
      "    0      1.052536   1.141739   0.509442  \n",
      "    1      1.03386    1.147596   0.511788                      \n",
      "    2      1.005644   1.210041   0.488842                      \n",
      "    3      0.961546   1.168524   0.5204                        \n",
      "    4      0.96673    1.154298   0.534447                      \n",
      "    5      0.949491   1.172889   0.531779                      \n",
      "    6      0.956332   1.20941    0.512285                      \n",
      "    7      0.932446   1.18231    0.534402                      \n",
      "    8      0.924049   1.2296     0.52465                       \n",
      "    9      0.901105   1.169018   0.539088                      \n",
      "    10     0.905614   1.207651   0.517092                      \n",
      "    11     0.88481    1.211402   0.545079                      \n",
      "    12     0.885735   1.226406   0.537763                      \n",
      "    13     0.887049   1.188989   0.545634                      \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.18899]), 0.5456343977969432]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m3.fit(lrs, 7, metrics=[accuracy], cycle_len=2, cycle_save_name='MovieReview')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e719b36a343c469b9bbbae8d6a1057d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                      \n",
      "    0      0.919168   1.194043   0.536609  \n",
      "    1      0.892308   1.167078   0.558539                      \n",
      "    2      0.886449   1.177978   0.542236                      \n",
      "    3      0.889609   1.19618    0.541768                      \n",
      "    4      0.852477   1.223079   0.535612                      \n",
      "    5      0.862832   1.18232    0.541867                      \n",
      "    6      0.84305    1.166159   0.552757                      \n",
      "    7      0.832242   1.183578   0.551902                      \n",
      "    8      0.816114   1.201127   0.555013                      \n",
      "    9      0.833731   1.172674   0.55678                       \n",
      "    10     0.854878   1.226053   0.541263                      \n",
      "    11     0.864414   1.158709   0.535323                      \n",
      "    12     0.832049   1.222211   0.538389                      \n",
      "    13     0.820791   1.193392   0.545381                      \n",
      "    14     0.813213   1.188233   0.530849                      \n",
      "    15     0.812891   1.208803   0.546514                      \n",
      "    16     0.804022   1.174116   0.555962                      \n",
      "    17     0.799992   1.183702   0.548964                      \n",
      "    18     0.77872    1.187479   0.553884                      \n",
      "    19     0.7697     1.179102   0.55526                       \n",
      "    20     0.850102   1.178752   0.549675                      \n",
      "    21     0.834315   1.14718    0.55191                       \n",
      "    22     0.798101   1.245953   0.556615                      \n",
      "    23     0.812178   1.198965   0.554173                      \n",
      "    24     0.795626   1.224418   0.554593                      \n",
      "    25     0.77021    1.23547    0.540587                      \n",
      "    26     0.76067    1.229097   0.551618                      \n",
      "    27     0.753727   1.217464   0.555558                      \n",
      "    28     0.750583   1.195503   0.558326                      \n",
      "    29     0.765192   1.187818   0.556804                      \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.18782]), 0.5568040695153538]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Not overfitting yet, continue training\n",
    "m3.fit(lrs, 3, metrics=[accuracy], cycle_len=10, cycle_save_name='MovieReview')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}