Train_GloVe_BiLSTM_kFold_CV.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GloVe-BiLSTM wit Cross Validation\n",
    "\n",
    "This Notebook presents the training of the GloVe-BiLSTM model with the use of k-fold cross validation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import random\n",
    "import time\n",
    "from torchtext.legacy import data\n",
    "from sklearn.model_selection import KFold\n",
    "import numpy as np\n",
    "import torch.optim as optim\n",
    "\n",
    "# This class loads the dataset\n",
    "class load_data(object):\n",
    "\n",
    "    # Initialize the parameters\n",
    "    def __init__(self, SEED=1234):\n",
    "        torch.manual_seed(SEED)\n",
    "        torch.cuda.manual_seed(SEED)\n",
    "        torch.backends.cudnn.enabled = False \n",
    "        torch.backends.cudnn.benchmark = False\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "\n",
    "        # Create Field objects\n",
    "        # use Spacy tokenizer\n",
    "        # specify English language\n",
    "        # need lenghts for padded sequences\n",
    "        PHRASE = data.Field(tokenize = 'spacy',\n",
    "                        tokenizer_language = 'en_core_web_sm',\n",
    "                        include_lengths = True)\n",
    "        SENTIMENT = data.LabelField()\n",
    "\n",
    "        # Create tuples representing the columns\n",
    "        fields = [\n",
    "                (None, None), # ignore PhraseId column\n",
    "                (None, None), # ignore SentenceId column\n",
    "                ('Phrase', PHRASE),\n",
    "                ('Sentiment', SENTIMENT)\n",
    "        ]\n",
    "\n",
    "        # Load the dataset in tsv format\n",
    "        self.train_ds, self.test_ds = data.TabularDataset.splits(\n",
    "                        path = './data',\n",
    "                        train = 'train.tsv',\n",
    "                        test = 'test_mapped.tsv',\n",
    "                        format = 'tsv',\n",
    "                        fields = fields,\n",
    "                        skip_header = True\n",
    "        )\n",
    "        \n",
    "        self.SEED = SEED\n",
    "\n",
    "    # Shhuffle the data using k-fold Cross Validation\n",
    "    def get_fold_data(self, num_folds):\n",
    "        \n",
    "        # Create Field objects\n",
    "        PHRASE = data.Field(tokenize = 'spacy',\n",
    "                        tokenizer_language = 'en_core_web_sm',\n",
    "                        include_lengths = True)\n",
    "        SENTIMENT = data.LabelField()\n",
    "\n",
    "        # Create tuples representing the columns\n",
    "        fields = [\n",
    "                (None, None), # ignore PhraseId column\n",
    "                (None, None), # ignore SentenceId column\n",
    "                ('Phrase', PHRASE),\n",
    "                ('Sentiment', SENTIMENT)\n",
    "        ]\n",
    "        \n",
    "        # k-Fold Cross Validation\n",
    "        kf = KFold(n_splits=num_folds, shuffle=True, random_state = random.seed(self.SEED))\n",
    "        train_data_arr = np.array(self.train_ds.examples)\n",
    "\n",
    "        for train_index, val_index in kf.split(train_data_arr):\n",
    "            yield(\n",
    "                PHRASE,\n",
    "                SENTIMENT,\n",
    "                data.Dataset(train_data_arr[train_index], fields=fields),\n",
    "                data.Dataset(train_data_arr[val_index], fields=fields),\n",
    "            )\n",
    "    \n",
    "    def get_test_data(self):\n",
    "        return self.test_ds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "class LSTM(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers, \n",
    "                 bidirectional, dropout, pad_idx):\n",
    "        \n",
    "        super().__init__()\n",
    "        \n",
    "        #Embedding layer\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx = pad_idx)\n",
    "        \n",
    "        #LSTM layer\n",
    "        self.rnn = nn.LSTM(embedding_dim, \n",
    "                           hidden_dim, \n",
    "                           num_layers=n_layers, \n",
    "                           bidirectional=bidirectional, \n",
    "                           dropout=dropout)\n",
    "        #Fully-Connected Linear Layer\n",
    "        self.fc = nn.Linear(hidden_dim * 2, output_dim)\n",
    "        \n",
    "        #Dropout functionality for reqularizotion\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, text, text_lengths):\n",
    "        \n",
    "        #text = [sent len, batch size]\n",
    "        \n",
    "        embedded = self.dropout(self.embedding(text))\n",
    "        \n",
    "        #embedded = [sent len, batch size, emb dim]\n",
    "        \n",
    "        #pack embedding sequence to process only the non-padded elements of our sequence\n",
    "        # lengths need to be on CPU!\n",
    "        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths.to('cpu'))\n",
    "        \n",
    "        packed_output, (hidden, cell) = self.rnn(packed_embedded)\n",
    "        \n",
    "        #unpack sequence; no used in our case\n",
    "        output, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_output)\n",
    "\n",
    "        #output = [sent len, batch size, hid dim * num directions]\n",
    "        #output over padding tokens are zero tensors\n",
    "        \n",
    "        #hidden = [num layers * num directions, batch size, hid dim]\n",
    "        #cell = [num layers * num directions, batch size, hid dim]\n",
    "        \n",
    "        #concat the final forward (hidden[-2,:,:]) and backward (hidden[-1,:,:]) hidden layers\n",
    "        #and apply dropout\n",
    "        \n",
    "        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1))\n",
    "                \n",
    "        #hidden = [batch size, hid dim * num directions]\n",
    "            \n",
    "        return self.fc(hidden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def categorical_accuracy(preds, y):\n",
    "    #Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n",
    "    \n",
    "    top_pred = preds.argmax(1, keepdim = True)\n",
    "    correct = top_pred.eq(y.view_as(top_pred)).sum()\n",
    "    acc = correct.float() / y.shape[0]\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train and Evaluation methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    for batch in iterator:\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        text, text_lengths = batch.Phrase\n",
    "        \n",
    "        predictions = model(text, text_lengths)\n",
    "        \n",
    "        loss = criterion(predictions, batch.Sentiment)\n",
    "        \n",
    "        acc = categorical_accuracy(predictions, batch.Sentiment)\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for batch in iterator:\n",
    "\n",
    "            text, text_lengths = batch.Phrase\n",
    "            \n",
    "            predictions = model(text, text_lengths)\n",
    "            \n",
    "            loss = criterion(predictions, batch.Sentiment)\n",
    "            \n",
    "            acc = categorical_accuracy(predictions, batch.Sentiment)\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def epoch_time(start_time, end_time):\n",
    "    # calculate the duration of the training time for one epoch\n",
    "    \n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** Running Training *****\n",
      "Now fold: 1 / 5\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_36708/3545866814.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     84\u001b[0m         \u001b[0mstart_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     85\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 86\u001b[1;33m         \u001b[0mtrain_loss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_acc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_iterator\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     87\u001b[0m         \u001b[0mvalid_loss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalid_acc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalid_iterator\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     88\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_36708/1413853967.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(model, iterator, optimizer, criterion)\u001b[0m\n\u001b[0;32m     18\u001b[0m         \u001b[0macc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcategorical_accuracy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSentiment\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 20\u001b[1;33m         \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     21\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\EnvP3\\lib\\site-packages\\torch\\_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[0;32m    305\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    306\u001b[0m                 inputs=inputs)\n\u001b[1;32m--> 307\u001b[1;33m         \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    308\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    309\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\EnvP3\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[0;32m    154\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[0;32m    155\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 156\u001b[1;33m         allow_unreachable=True, accumulate_grad=True)  # allow_unreachable flag\n\u001b[0m\u001b[0;32m    157\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    158\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "data_generator = load_data()\n",
    "_history = []\n",
    "device = None\n",
    "model = None\n",
    "criterion = None\n",
    "fold_index = 0\n",
    "\n",
    "N_FOLDS = 5\n",
    "\n",
    "for PHRASE, SENTIMENT, train_ds, val_ds in data_generator.get_fold_data(num_folds=N_FOLDS):\n",
    "    \n",
    "    print(\"***** Running Training *****\")\n",
    "    print(f\"Now fold: {fold_index + 1} / {N_FOLDS}\")\n",
    "\n",
    "    # Build vocabulary (using Glove pre-trained word embeddings, dimensinality = 100)\n",
    "    PHRASE.build_vocab(train_ds,\n",
    "                        vectors = \"glove.6B.100d\", \n",
    "                        unk_init = torch.Tensor.normal_)\n",
    "\n",
    "    sentiment_labels = np.array(['0', '1', '2', '3', '4'])\n",
    "\n",
    "    #SENTIMENT.build_vocab(train_ds) # old version\n",
    "    SENTIMENT.build_vocab(sentiment_labels) # Custom indexing is necessary for correct mapping; {'0': 0, '1': 1, '2': 2, '3': 3, '4': 4}\n",
    "\n",
    "    # Initialize model\n",
    "    INPUT_DIM = len(PHRASE.vocab)\n",
    "    EMBEDDING_DIM = 100\n",
    "    HIDDEN_DIM = 256\n",
    "    OUTPUT_DIM = len(SENTIMENT.vocab)\n",
    "    N_LAYERS = 2\n",
    "    BIDIRECTIONAL = True\n",
    "    DROPOUT = 0.5\n",
    "    PAD_IDX = PHRASE.vocab.stoi[PHRASE.pad_token]\n",
    "\n",
    "    model = LSTM(INPUT_DIM, \n",
    "                EMBEDDING_DIM, \n",
    "                HIDDEN_DIM, \n",
    "                OUTPUT_DIM, \n",
    "                N_LAYERS, \n",
    "                BIDIRECTIONAL, \n",
    "                DROPOUT, \n",
    "                PAD_IDX)\n",
    "\n",
    "    # replace the initial weights of the embedding layer with the pre-trained embeddings\n",
    "    pretrained_embeddings = PHRASE.vocab.vectors\n",
    "    model.embedding.weight.data.copy_(pretrained_embeddings)\n",
    "\n",
    "    UNK_IDX = PHRASE.vocab.stoi[PHRASE.unk_token]\n",
    "    #  initialize <unk> and <pad> tokens to all zeros\n",
    "    model.embedding.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "    model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "\n",
    "    # Init optimizer and loss function\n",
    "    optimizer = optim.Adam(model.parameters())\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "    # Only necessary for GPU\n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "    model = model.to(device)\n",
    "    criterion = criterion.to(device)\n",
    "\n",
    "    BATCH_SIZE = 32\n",
    "\n",
    "    # Creating the iterators\n",
    "    train_iterator = data.Iterator(train_ds, \n",
    "                                batch_size=BATCH_SIZE, \n",
    "                                sort = False,\n",
    "                                sort_key = lambda x: len(x.Phrase),\n",
    "                                sort_within_batch = True,\n",
    "                                device = device)\n",
    "    valid_iterator = data.Iterator(val_ds, \n",
    "                                batch_size=BATCH_SIZE, \n",
    "                                sort = False,\n",
    "                                sort_key = lambda x: len(x.Phrase),\n",
    "                                sort_within_batch = True,\n",
    "                                device = device) \n",
    "\n",
    "    N_EPOCHS = 10\n",
    "\n",
    "    best_valid_loss = float('inf')\n",
    "\n",
    "    for epoch in range(N_EPOCHS):\n",
    "        start_time = time.time()\n",
    "\n",
    "        train_loss, train_acc = train(model, train_iterator, optimizer, criterion)\n",
    "        valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)\n",
    "    \n",
    "        end_time = time.time()\n",
    "\n",
    "        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "\n",
    "        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "        print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
    "        print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')\n",
    "\n",
    "        if valid_loss < best_valid_loss:\n",
    "            best_valid_loss = valid_loss\n",
    "            torch.save(model.state_dict(), 'model_kFold.pt')\n",
    "\n",
    "        _history.append([valid_loss, valid_acc])\n",
    "    \n",
    "    fold_index += 1\n",
    "    \n",
    "_history = np.asarray(_history)\n",
    "loss = np.mean(_history[:, 0])\n",
    "acc = np.mean(_history[:, 1])\n",
    "    \n",
    "print('***** Cross Validation Result *****')\n",
    "print(f'LOSS: {loss}, ACC: {acc}') "
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "09afb56c1b8549a6bdcc3ef42a2483a881d9725ada16af5f09354855406c79b5"
  },
  "kernelspec": {
   "display_name": "Python 3.7.9 64-bit ('EnvP3': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}