Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://www.google.com/search?q=python+transformer&rlz=1C1RXQR_enUS1000US1000&source=lnms&tbm=vid&sa=X&ved=2ahUKEwiUxKfx2MT7AhUGCjQIHdamC8QQ_AUoA3oECAEQBQ&biw=1536&bih=722&dpr=1.25#fpstate=ive&vld=cid:4d96f023,vid:ZIRmXkHp0-c \n",
    "\n",
    "https://github.com/angelosps/Question-Answering/blob/main/Question_Answering.ipynb\n",
    "\n",
    "\n",
    "Papers \n",
    "https://arxiv.org/pdf/1606.05250.pdf SQUADv2 dataset\n",
    "https://arxiv.org/pdf/1810.04805.pdf BERT paper -- Given how large the model is we'll be looking at fine tuning only the portion related to question answering "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: Could not find a version that satisfies the requirement torch==0.4.0 (from versions: 1.7.0, 1.7.1, 1.8.0, 1.8.1, 1.9.0, 1.9.1, 1.10.0, 1.10.1, 1.10.2, 1.11.0, 1.12.0, 1.12.1, 1.13.0, 1.13.1)\n",
      "ERROR: No matching distribution found for torch==0.4.0\n"
     ]
    }
   ],
   "source": [
    "!pip install torch==0.4.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#os.mkdir('squad')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://rajpurkar.github.io/SQuAD-explorer/dataset/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\vikto\\anaconda3\\lib\\site-packages\\requests\\__init__.py:91: RequestsDependencyWarning: urllib3 (1.26.14) or chardet (3.0.4) doesn't match a supported version!\n",
      "  RequestsDependencyWarning)\n"
     ]
    }
   ],
   "source": [
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train-v2.0.json:\tb'}]}'\n",
      "dev-v2.0.json:\tb'f.\"}]}]}'\n"
     ]
    }
   ],
   "source": [
    "for file in ['train-v2.0.json', 'dev-v2.0.json']:\n",
    "    res = requests.get(f'{url}{file}')\n",
    "    with open(f'squad/{file}', 'wb') as f: \n",
    "        for chunk in res.iter_content(chunk_size=10):\n",
    "            f.write(chunk)\n",
    "    print(f'{file}:\\t{chunk}')\n",
    "              \n",
    "              "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_squad(path):\n",
    "    with open(path, 'rb') as f:\n",
    "        squad_dict = json.load(f)\n",
    "        \n",
    "    contexts = []\n",
    "    questions = []\n",
    "    answers = []\n",
    "\n",
    "    for group in squad_dict['data']:\n",
    "        for passage in group['paragraphs']:\n",
    "            context = passage['context']\n",
    "            for qa in passage['qas']:\n",
    "                question = qa['question']\n",
    "                if 'plausible_answers' in qa.keys():\n",
    "                    access = 'plausible_answers'\n",
    "                else:\n",
    "                    access = 'answers'\n",
    "                for answer in qa[access]:\n",
    "                    contexts.append(context)\n",
    "                    questions.append(question)\n",
    "                    answers.append(answer)\n",
    "    return contexts, questions, answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_contexts, train_questions, train_answers = read_squad('squad/train-v2.0.json')\n",
    "dev_contexts, dev_questions, dev_answers = read_squad('squad/dev-v2.0.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_end_idx(answers, contexts):\n",
    "    for answer, context in zip(answers, contexts):\n",
    "        gold_text = answer['text'] #Answer we're looking for \n",
    "        start_idx = answer['answer_start']\n",
    "        end_idx = start_idx + len(gold_text) #Ideal end index \n",
    "        \n",
    "        #Check to make sure we're not off by a few characters \n",
    "        if context[start_idx:end_idx] == gold_text:\n",
    "            answer['answer_end'] = end_idx\n",
    "        else:\n",
    "            for n in [1,2]:\n",
    "                if context[start_idx-n:end_idx-n] == gold_text:\n",
    "                    answer['answer_start'] = start_idx-n\n",
    "                    answer['answer_end'] = end_idx-n\n",
    "                    \n",
    "add_end_idx(train_answers, train_contexts)\n",
    "add_end_idx(dev_answers, dev_contexts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenize/Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from transformers import BertTokenizerFast\n",
    "\n",
    "tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_encoding = tokenizer(train_contexts, train_questions, truncation=True, padding=True)\n",
    "val_encoding = tokenizer(dev_contexts, dev_questions, truncation=True, padding=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_toke_positions(encodings, answers):\n",
    "    start_positions = []\n",
    "    end_positions = []\n",
    "    for i in range(len(answers)):\n",
    "        start_positions.append(encodings.char_to_token(i, answers[i]['answer_start']))\n",
    "        end_positions.append(encodings.char_to_token(i, answers[i]['answer_end']))\n",
    "        \n",
    "        if start_positions[-1] is None:\n",
    "            start_positions[-1] = tokenizer.model_max_length\n",
    "        \n",
    "        go_back = 1\n",
    "        while end_positions[-1] is None:\n",
    "            end_positions[-1] = encodings.char_to_token(i, answers[i]['answer_end']-go_back)\n",
    "            go_back += 1\n",
    "            \n",
    "            \n",
    "        encodings.update({\n",
    "            'start_positions': start_positions,\n",
    "            'end_positions' : end_positions\n",
    "        })\n",
    "        \n",
    "add_toke_positions(train_encoding, train_answers)\n",
    "add_toke_positions(val_encoding, dev_answers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "\n",
    "class SquadDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, encodings):\n",
    "        self.encodings = encodings \n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        return {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.encodings.input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = SquadDataset(train_encoding)\n",
    "val_dataset = SquadDataset(val_encoding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "# Fine-tune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using a model of type bert to instantiate a model of type distilbert. This is not supported for all configurations of models and can yield errors.\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing DistilBertForQuestionAnswering: ['bert.embeddings.LayerNorm.bias', 'bert.encoder.layer.0.attention.output.dense.bias', 'bert.encoder.layer.8.attention.output.LayerNorm.bias', 'bert.encoder.layer.6.attention.self.query.weight', 'bert.encoder.layer.6.attention.self.value.bias', 'bert.encoder.layer.6.intermediate.dense.weight', 'bert.encoder.layer.5.attention.self.key.bias', 'bert.embeddings.token_type_embeddings.weight', 'bert.encoder.layer.9.attention.self.value.weight', 'bert.encoder.layer.3.output.LayerNorm.weight', 'bert.encoder.layer.4.attention.output.LayerNorm.weight', 'bert.encoder.layer.9.attention.self.value.bias', 'bert.encoder.layer.6.intermediate.dense.bias', 'bert.encoder.layer.2.intermediate.dense.weight', 'bert.pooler.dense.bias', 'bert.encoder.layer.1.output.dense.weight', 'bert.encoder.layer.11.attention.self.query.bias', 'bert.encoder.layer.4.output.LayerNorm.weight', 'bert.encoder.layer.4.attention.output.dense.bias', 'bert.encoder.layer.2.output.LayerNorm.weight', 'bert.encoder.layer.10.intermediate.dense.weight', 'cls.predictions.transform.dense.bias', 'bert.encoder.layer.1.output.LayerNorm.weight', 'bert.encoder.layer.9.attention.output.LayerNorm.bias', 'bert.encoder.layer.8.attention.self.value.weight', 'bert.encoder.layer.1.attention.self.query.bias', 'bert.encoder.layer.3.attention.output.LayerNorm.weight', 'bert.encoder.layer.2.attention.self.value.weight', 'bert.encoder.layer.3.attention.self.key.weight', 'bert.encoder.layer.11.attention.output.LayerNorm.weight', 'bert.encoder.layer.11.attention.self.key.weight', 'bert.encoder.layer.8.output.dense.bias', 'bert.encoder.layer.10.attention.self.query.weight', 'bert.encoder.layer.10.attention.output.LayerNorm.weight', 'bert.encoder.layer.5.attention.self.value.bias', 'cls.predictions.bias', 'bert.encoder.layer.3.attention.self.key.bias', 'bert.encoder.layer.9.attention.output.LayerNorm.weight', 'bert.encoder.layer.8.attention.output.dense.weight', 'bert.encoder.layer.7.attention.self.query.bias', 'bert.encoder.layer.5.attention.output.dense.weight', 'bert.encoder.layer.3.output.dense.bias', 'bert.encoder.layer.11.intermediate.dense.weight', 'bert.encoder.layer.10.output.LayerNorm.bias', 'bert.encoder.layer.7.attention.output.dense.bias', 'bert.encoder.layer.9.output.LayerNorm.weight', 'bert.encoder.layer.0.attention.output.LayerNorm.bias', 'bert.encoder.layer.11.attention.self.value.bias', 'bert.encoder.layer.1.attention.self.value.weight', 'bert.encoder.layer.5.attention.output.LayerNorm.bias', 'bert.encoder.layer.11.intermediate.dense.bias', 'bert.encoder.layer.11.output.LayerNorm.bias', 'bert.encoder.layer.6.output.LayerNorm.weight', 'bert.embeddings.word_embeddings.weight', 'bert.encoder.layer.0.output.LayerNorm.bias', 'bert.encoder.layer.9.output.dense.weight', 'bert.encoder.layer.8.intermediate.dense.weight', 'bert.encoder.layer.4.intermediate.dense.bias', 'bert.encoder.layer.2.attention.output.dense.weight', 'bert.encoder.layer.7.attention.self.value.weight', 'bert.encoder.layer.1.output.LayerNorm.bias', 'bert.encoder.layer.8.output.LayerNorm.bias', 'bert.encoder.layer.8.attention.self.value.bias', 'bert.encoder.layer.7.attention.self.value.bias', 'bert.encoder.layer.8.intermediate.dense.bias', 'bert.encoder.layer.5.output.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'bert.encoder.layer.9.attention.self.query.weight', 'bert.encoder.layer.10.output.LayerNorm.weight', 'bert.embeddings.position_embeddings.weight', 'bert.encoder.layer.1.attention.self.value.bias', 'bert.encoder.layer.6.output.dense.bias', 'bert.encoder.layer.11.attention.output.dense.bias', 'bert.encoder.layer.3.intermediate.dense.weight', 'bert.encoder.layer.0.intermediate.dense.weight', 'bert.encoder.layer.4.output.dense.bias', 'bert.encoder.layer.0.output.dense.bias', 'bert.encoder.layer.10.attention.self.value.bias', 'bert.encoder.layer.10.attention.output.dense.weight', 'bert.encoder.layer.2.output.dense.bias', 'bert.encoder.layer.6.attention.self.key.bias', 'bert.encoder.layer.8.attention.self.key.weight', 'bert.encoder.layer.3.attention.self.value.bias', 'bert.encoder.layer.5.intermediate.dense.bias', 'bert.encoder.layer.1.attention.output.dense.bias', 'bert.encoder.layer.8.attention.output.LayerNorm.weight', 'bert.encoder.layer.5.output.dense.bias', 'bert.encoder.layer.3.intermediate.dense.bias', 'cls.seq_relationship.weight', 'bert.encoder.layer.9.attention.output.dense.bias', 'bert.encoder.layer.2.output.LayerNorm.bias', 'bert.encoder.layer.0.intermediate.dense.bias', 'bert.encoder.layer.3.attention.output.dense.bias', 'bert.encoder.layer.4.output.LayerNorm.bias', 'bert.encoder.layer.1.intermediate.dense.weight', 'bert.encoder.layer.6.output.LayerNorm.bias', 'bert.encoder.layer.11.attention.output.LayerNorm.bias', 'bert.encoder.layer.0.attention.output.LayerNorm.weight', 'bert.encoder.layer.7.attention.output.LayerNorm.bias', 'bert.encoder.layer.5.attention.output.LayerNorm.weight', 'bert.encoder.layer.3.attention.output.LayerNorm.bias', 'bert.encoder.layer.9.output.dense.bias', 'bert.encoder.layer.10.output.dense.weight', 'bert.encoder.layer.1.attention.self.key.weight', 'bert.encoder.layer.2.attention.self.key.bias', 'bert.encoder.layer.11.attention.output.dense.weight', 'bert.encoder.layer.7.attention.output.dense.weight', 'bert.encoder.layer.6.attention.self.query.bias', 'bert.encoder.layer.4.attention.output.LayerNorm.bias', 'bert.encoder.layer.7.output.LayerNorm.weight', 'bert.encoder.layer.11.attention.self.value.weight', 'bert.encoder.layer.1.intermediate.dense.bias', 'bert.encoder.layer.4.intermediate.dense.weight', 'bert.encoder.layer.3.attention.self.value.weight', 'bert.encoder.layer.3.attention.output.dense.weight', 'bert.encoder.layer.2.attention.output.dense.bias', 'bert.encoder.layer.7.intermediate.dense.weight', 'bert.encoder.layer.9.attention.self.key.bias', 'bert.encoder.layer.9.output.LayerNorm.bias', 'bert.encoder.layer.8.output.LayerNorm.weight', 'bert.encoder.layer.4.attention.self.key.weight', 'bert.encoder.layer.3.output.LayerNorm.bias', 'bert.encoder.layer.7.attention.self.query.weight', 'bert.encoder.layer.2.attention.self.query.weight', 'bert.encoder.layer.5.intermediate.dense.weight', 'bert.encoder.layer.10.attention.self.query.bias', 'bert.encoder.layer.10.intermediate.dense.bias', 'bert.encoder.layer.7.output.dense.weight', 'bert.encoder.layer.0.output.LayerNorm.weight', 'bert.encoder.layer.3.attention.self.query.weight', 'bert.encoder.layer.8.attention.output.dense.bias', 'bert.encoder.layer.9.attention.output.dense.weight', 'bert.encoder.layer.11.output.LayerNorm.weight', 'bert.encoder.layer.5.output.LayerNorm.bias', 'bert.encoder.layer.10.attention.self.value.weight', 'bert.encoder.layer.11.attention.self.key.bias', 'bert.encoder.layer.8.attention.self.key.bias', 'bert.encoder.layer.7.output.dense.bias', 'bert.encoder.layer.4.attention.self.key.bias', 'bert.encoder.layer.0.output.dense.weight', 'bert.encoder.layer.6.attention.output.dense.bias', 'bert.encoder.layer.4.attention.self.query.bias', 'bert.encoder.layer.0.attention.self.query.weight', 'bert.encoder.layer.9.attention.self.key.weight', 'bert.encoder.layer.1.attention.output.LayerNorm.weight', 'bert.encoder.layer.8.attention.self.query.weight', 'bert.encoder.layer.5.attention.self.key.weight', 'bert.encoder.layer.6.attention.output.LayerNorm.bias', 'bert.encoder.layer.11.attention.self.query.weight', 'bert.encoder.layer.4.attention.self.value.weight', 'bert.embeddings.LayerNorm.weight', 'bert.encoder.layer.8.attention.self.query.bias', 'bert.encoder.layer.3.output.dense.weight', 'bert.encoder.layer.4.attention.output.dense.weight', 'bert.encoder.layer.5.attention.self.value.weight', 'bert.encoder.layer.9.attention.self.query.bias', 'bert.encoder.layer.2.attention.output.LayerNorm.weight', 'bert.encoder.layer.0.attention.self.value.weight', 'bert.encoder.layer.4.attention.self.value.bias', 'bert.encoder.layer.2.attention.output.LayerNorm.bias', 'bert.encoder.layer.6.output.dense.weight', 'bert.encoder.layer.5.attention.output.dense.bias', 'bert.encoder.layer.10.attention.output.dense.bias', 'bert.encoder.layer.4.output.dense.weight', 'bert.encoder.layer.7.attention.self.key.weight', 'bert.encoder.layer.7.attention.output.LayerNorm.weight', 'bert.encoder.layer.9.intermediate.dense.bias', 'bert.encoder.layer.7.intermediate.dense.bias', 'bert.encoder.layer.6.attention.self.key.weight', 'bert.encoder.layer.3.attention.self.query.bias', 'bert.encoder.layer.1.attention.output.dense.weight', 'bert.encoder.layer.0.attention.output.dense.weight', 'bert.encoder.layer.7.attention.self.key.bias', 'bert.encoder.layer.4.attention.self.query.weight', 'bert.encoder.layer.9.intermediate.dense.weight', 'bert.encoder.layer.5.attention.self.query.bias', 'cls.seq_relationship.bias', 'bert.encoder.layer.11.output.dense.weight', 'bert.encoder.layer.2.attention.self.key.weight', 'bert.encoder.layer.10.attention.self.key.bias', 'bert.encoder.layer.6.attention.output.dense.weight', 'bert.encoder.layer.5.output.dense.weight', 'bert.encoder.layer.2.intermediate.dense.bias', 'bert.encoder.layer.5.attention.self.query.weight', 'bert.encoder.layer.6.attention.self.value.weight', 'bert.encoder.layer.2.attention.self.query.bias', 'bert.encoder.layer.6.attention.output.LayerNorm.weight', 'bert.encoder.layer.10.attention.output.LayerNorm.bias', 'bert.encoder.layer.0.attention.self.query.bias', 'bert.encoder.layer.0.attention.self.value.bias', 'bert.pooler.dense.weight', 'bert.encoder.layer.0.attention.self.key.weight', 'bert.encoder.layer.8.output.dense.weight', 'bert.encoder.layer.10.output.dense.bias', 'bert.encoder.layer.1.attention.self.query.weight', 'bert.encoder.layer.7.output.LayerNorm.bias', 'bert.encoder.layer.1.attention.output.LayerNorm.bias', 'bert.encoder.layer.2.attention.self.value.bias', 'bert.encoder.layer.1.output.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'bert.encoder.layer.1.attention.self.key.bias', 'bert.encoder.layer.2.output.dense.weight', 'bert.encoder.layer.0.attention.self.key.bias', 'bert.encoder.layer.10.attention.self.key.weight', 'cls.predictions.decoder.weight', 'bert.encoder.layer.11.output.dense.bias']\n",
      "- This IS expected if you are initializing DistilBertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of DistilBertForQuestionAnswering were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['transformer.layer.2.ffn.lin2.weight', 'transformer.layer.10.sa_layer_norm.bias', 'transformer.layer.7.attention.k_lin.weight', 'transformer.layer.6.attention.v_lin.weight', 'transformer.layer.9.attention.k_lin.weight', 'transformer.layer.6.output_layer_norm.bias', 'transformer.layer.3.attention.q_lin.weight', 'transformer.layer.11.sa_layer_norm.weight', 'transformer.layer.8.attention.out_lin.weight', 'transformer.layer.4.attention.v_lin.bias', 'transformer.layer.5.sa_layer_norm.weight', 'transformer.layer.6.attention.q_lin.bias', 'transformer.layer.11.output_layer_norm.bias', 'transformer.layer.3.ffn.lin1.bias', 'transformer.layer.4.attention.out_lin.weight', 'transformer.layer.7.attention.out_lin.weight', 'transformer.layer.9.ffn.lin1.bias', 'transformer.layer.2.attention.q_lin.bias', 'transformer.layer.8.ffn.lin2.bias', 'transformer.layer.7.output_layer_norm.bias', 'transformer.layer.8.attention.out_lin.bias', 'transformer.layer.5.attention.q_lin.weight', 'transformer.layer.1.attention.out_lin.bias', 'transformer.layer.8.attention.k_lin.weight', 'transformer.layer.11.ffn.lin2.bias', 'transformer.layer.2.attention.v_lin.weight', 'transformer.layer.10.output_layer_norm.bias', 'transformer.layer.3.ffn.lin1.weight', 'transformer.layer.0.ffn.lin1.weight', 'transformer.layer.8.ffn.lin1.bias', 'transformer.layer.6.attention.k_lin.weight', 'transformer.layer.6.attention.out_lin.weight', 'transformer.layer.11.attention.v_lin.weight', 'transformer.layer.4.attention.v_lin.weight', 'transformer.layer.0.sa_layer_norm.bias', 'transformer.layer.8.attention.q_lin.bias', 'transformer.layer.9.output_layer_norm.bias', 'transformer.layer.10.attention.v_lin.bias', 'transformer.layer.7.attention.out_lin.bias', 'transformer.layer.7.attention.v_lin.bias', 'transformer.layer.5.sa_layer_norm.bias', 'transformer.layer.9.attention.q_lin.weight', 'transformer.layer.5.attention.v_lin.weight', 'transformer.layer.6.attention.v_lin.bias', 'transformer.layer.6.attention.out_lin.bias', 'transformer.layer.1.ffn.lin1.bias', 'transformer.layer.2.attention.out_lin.bias', 'transformer.layer.0.attention.q_lin.bias', 'transformer.layer.3.attention.k_lin.bias', 'transformer.layer.4.output_layer_norm.weight', 'transformer.layer.5.output_layer_norm.bias', 'transformer.layer.9.sa_layer_norm.weight', 'transformer.layer.5.attention.out_lin.bias', 'transformer.layer.9.output_layer_norm.weight', 'transformer.layer.0.ffn.lin2.bias', 'transformer.layer.5.attention.out_lin.weight', 'transformer.layer.5.ffn.lin1.weight', 'transformer.layer.11.attention.k_lin.weight', 'transformer.layer.1.ffn.lin2.weight', 'transformer.layer.0.ffn.lin1.bias', 'transformer.layer.4.attention.q_lin.bias', 'transformer.layer.10.ffn.lin1.weight', 'transformer.layer.0.attention.v_lin.weight', 'transformer.layer.10.attention.out_lin.weight', 'transformer.layer.0.attention.k_lin.bias', 'transformer.layer.1.output_layer_norm.bias', 'transformer.layer.1.output_layer_norm.weight', 'transformer.layer.5.attention.v_lin.bias', 'transformer.layer.6.attention.q_lin.weight', 'transformer.layer.9.ffn.lin2.weight', 'transformer.layer.9.ffn.lin2.bias', 'transformer.layer.1.sa_layer_norm.bias', 'transformer.layer.7.ffn.lin1.weight', 'transformer.layer.3.attention.out_lin.bias', 'transformer.layer.1.ffn.lin2.bias', 'transformer.layer.4.attention.out_lin.bias', 'transformer.layer.2.sa_layer_norm.bias', 'transformer.layer.11.ffn.lin1.weight', 'transformer.layer.1.attention.q_lin.weight', 'transformer.layer.6.ffn.lin1.bias', 'transformer.layer.1.attention.out_lin.weight', 'transformer.layer.6.ffn.lin2.weight', 'transformer.layer.7.output_layer_norm.weight', 'transformer.layer.8.output_layer_norm.weight', 'transformer.layer.10.ffn.lin1.bias', 'transformer.layer.6.ffn.lin2.bias', 'transformer.layer.4.ffn.lin1.weight', 'transformer.layer.8.ffn.lin2.weight', 'transformer.layer.8.attention.k_lin.bias', 'transformer.layer.3.sa_layer_norm.weight', 'transformer.layer.11.attention.out_lin.weight', 'transformer.layer.4.ffn.lin2.bias', 'transformer.layer.1.attention.k_lin.weight', 'transformer.layer.4.attention.k_lin.weight', 'transformer.layer.11.attention.k_lin.bias', 'transformer.layer.11.output_layer_norm.weight', 'transformer.layer.3.output_layer_norm.weight', 'transformer.layer.5.ffn.lin1.bias', 'transformer.layer.7.attention.v_lin.weight', 'transformer.layer.7.attention.q_lin.weight', 'transformer.layer.1.attention.q_lin.bias', 'transformer.layer.11.attention.out_lin.bias', 'transformer.layer.11.attention.q_lin.weight', 'transformer.layer.10.ffn.lin2.bias', 'transformer.layer.11.ffn.lin1.bias', 'transformer.layer.8.ffn.lin1.weight', 'transformer.layer.4.ffn.lin2.weight', 'transformer.layer.11.sa_layer_norm.bias', 'transformer.layer.8.sa_layer_norm.weight', 'transformer.layer.9.attention.q_lin.bias', 'transformer.layer.11.attention.v_lin.bias', 'transformer.layer.2.ffn.lin1.bias', 'transformer.layer.10.ffn.lin2.weight', 'transformer.layer.11.ffn.lin2.weight', 'transformer.layer.0.sa_layer_norm.weight', 'transformer.layer.9.attention.out_lin.weight', 'transformer.layer.3.attention.k_lin.weight', 'transformer.layer.2.output_layer_norm.bias', 'transformer.layer.10.sa_layer_norm.weight', 'transformer.layer.6.sa_layer_norm.weight', 'transformer.layer.8.attention.q_lin.weight', 'transformer.layer.6.ffn.lin1.weight', 'transformer.layer.0.attention.out_lin.weight', 'transformer.layer.3.ffn.lin2.bias', 'qa_outputs.weight', 'transformer.layer.11.attention.q_lin.bias', 'transformer.layer.10.attention.v_lin.weight', 'transformer.layer.1.attention.k_lin.bias', 'embeddings.position_embeddings.weight', 'embeddings.word_embeddings.weight', 'transformer.layer.0.attention.v_lin.bias', 'transformer.layer.4.sa_layer_norm.weight', 'transformer.layer.0.attention.out_lin.bias', 'transformer.layer.2.attention.q_lin.weight', 'transformer.layer.2.ffn.lin1.weight', 'transformer.layer.7.attention.k_lin.bias', 'qa_outputs.bias', 'transformer.layer.5.attention.k_lin.bias', 'transformer.layer.6.attention.k_lin.bias', 'transformer.layer.7.ffn.lin2.bias', 'transformer.layer.5.ffn.lin2.bias', 'transformer.layer.0.attention.k_lin.weight', 'transformer.layer.2.attention.out_lin.weight', 'transformer.layer.4.sa_layer_norm.bias', 'transformer.layer.5.attention.k_lin.weight', 'embeddings.LayerNorm.bias', 'transformer.layer.8.attention.v_lin.weight', 'transformer.layer.8.attention.v_lin.bias', 'transformer.layer.9.sa_layer_norm.bias', 'transformer.layer.3.attention.q_lin.bias', 'transformer.layer.2.attention.k_lin.bias', 'transformer.layer.0.ffn.lin2.weight', 'transformer.layer.4.output_layer_norm.bias', 'transformer.layer.5.attention.q_lin.bias', 'transformer.layer.10.attention.q_lin.weight', 'transformer.layer.3.attention.v_lin.bias', 'transformer.layer.4.ffn.lin1.bias', 'transformer.layer.10.attention.q_lin.bias', 'transformer.layer.3.output_layer_norm.bias', 'transformer.layer.0.output_layer_norm.weight', 'transformer.layer.6.sa_layer_norm.bias', 'transformer.layer.7.ffn.lin2.weight', 'transformer.layer.5.output_layer_norm.weight', 'transformer.layer.2.attention.v_lin.bias', 'transformer.layer.1.attention.v_lin.weight', 'transformer.layer.7.sa_layer_norm.weight', 'transformer.layer.1.attention.v_lin.bias', 'transformer.layer.7.sa_layer_norm.bias', 'transformer.layer.9.attention.v_lin.bias', 'transformer.layer.4.attention.k_lin.bias', 'transformer.layer.1.sa_layer_norm.weight', 'transformer.layer.3.attention.v_lin.weight', 'transformer.layer.3.ffn.lin2.weight', 'transformer.layer.8.output_layer_norm.bias', 'transformer.layer.5.ffn.lin2.weight', 'transformer.layer.10.attention.k_lin.weight', 'transformer.layer.2.sa_layer_norm.weight', 'transformer.layer.8.sa_layer_norm.bias', 'transformer.layer.9.attention.v_lin.weight', 'transformer.layer.6.output_layer_norm.weight', 'transformer.layer.1.ffn.lin1.weight', 'transformer.layer.2.output_layer_norm.weight', 'transformer.layer.0.attention.q_lin.weight', 'transformer.layer.0.output_layer_norm.bias', 'transformer.layer.2.attention.k_lin.weight', 'transformer.layer.7.attention.q_lin.bias', 'transformer.layer.10.output_layer_norm.weight', 'transformer.layer.9.attention.out_lin.bias', 'transformer.layer.4.attention.q_lin.weight', 'transformer.layer.10.attention.out_lin.bias', 'embeddings.LayerNorm.weight', 'transformer.layer.3.sa_layer_norm.bias', 'transformer.layer.10.attention.k_lin.bias', 'transformer.layer.2.ffn.lin2.bias', 'transformer.layer.7.ffn.lin1.bias', 'transformer.layer.9.ffn.lin1.weight', 'transformer.layer.3.attention.out_lin.weight', 'transformer.layer.9.attention.k_lin.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "import torch \n",
    "from transformers import DistilBertForQuestionAnswering\n",
    "model = DistilBertForQuestionAnswering.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader \n",
    "from transformers import AdamW #Adam optimizer with weighted decay \n",
    "from tqdm import tqdm #Visualize training progress "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU Available:\tTrue\n",
      "GPU Count:\t1\n",
      "GPU Name:\tNVIDIA GeForce RTX 2080\n",
      "torch.cuda.memory_allocated: 0.000000GB\n",
      "torch.cuda.memory_reserved: 0.000000GB\n",
      "torch.cuda.max_memory_reserved: 0.000000GB\n"
     ]
    }
   ],
   "source": [
    "import torch \n",
    "print(\"GPU Available:\\t\"+str(torch.cuda.is_available()))\n",
    "print(\"GPU Count:\\t\"+str(torch.cuda.device_count()))\n",
    "print(\"GPU Name:\\t\"+str(torch.cuda.get_device_name()))\n",
    " \n",
    "print(\"torch.cuda.memory_allocated: %fGB\"%(torch.cuda.memory_allocated(0)/1024/1024/1024))\n",
    "print(\"torch.cuda.memory_reserved: %fGB\"%(torch.cuda.memory_reserved(0)/1024/1024/1024))\n",
    "print(\"torch.cuda.max_memory_reserved: %fGB\"%(torch.cuda.max_memory_reserved(0)/1024/1024/1024))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "327"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "torch.cuda.empty_cache()\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\vikto\\anaconda3\\lib\\site-packages\\transformers\\optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  FutureWarning,\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "model.to(device)\n",
    "model.train()\n",
    "optim = AdamW(model.parameters(), lr=1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU Memory Reset\n",
      "|===========================================================================|\n",
      "|                  PyTorch CUDA memory summary, device ID 0                 |\n",
      "|---------------------------------------------------------------------------|\n",
      "|            CUDA OOMs: 0            |        cudaMalloc retries: 0         |\n",
      "|===========================================================================|\n",
      "|        Metric         | Cur Usage  | Peak Usage | Tot Alloc  | Tot Freed  |\n",
      "|---------------------------------------------------------------------------|\n",
      "| Allocated memory      |  426468 KB |  426468 KB |  426468 KB |       0 B  |\n",
      "|       from large pool |  425984 KB |  425984 KB |  425984 KB |       0 B  |\n",
      "|       from small pool |     484 KB |     484 KB |     484 KB |       0 B  |\n",
      "|---------------------------------------------------------------------------|\n",
      "| Active memory         |  426468 KB |  426468 KB |  426468 KB |       0 B  |\n",
      "|       from large pool |  425984 KB |  425984 KB |  425984 KB |       0 B  |\n",
      "|       from small pool |     484 KB |     484 KB |     484 KB |       0 B  |\n",
      "|---------------------------------------------------------------------------|\n",
      "| GPU reserved memory   |  462848 KB |  462848 KB |  462848 KB |       0 B  |\n",
      "|       from large pool |  460800 KB |  460800 KB |  460800 KB |       0 B  |\n",
      "|       from small pool |    2048 KB |    2048 KB |    2048 KB |       0 B  |\n",
      "|---------------------------------------------------------------------------|\n",
      "| Non-releasable memory |   36379 KB |   36379 KB |  247037 KB |  210657 KB |\n",
      "|       from large pool |   34816 KB |   34816 KB |  244992 KB |  210176 KB |\n",
      "|       from small pool |    1563 KB |    1563 KB |    2045 KB |     481 KB |\n",
      "|---------------------------------------------------------------------------|\n",
      "| Allocations           |     199    |     199    |     199    |       0    |\n",
      "|       from large pool |      74    |      74    |      74    |       0    |\n",
      "|       from small pool |     125    |     125    |     125    |       0    |\n",
      "|---------------------------------------------------------------------------|\n",
      "| Active allocs         |     199    |     199    |     199    |       0    |\n",
      "|       from large pool |      74    |      74    |      74    |       0    |\n",
      "|       from small pool |     125    |     125    |     125    |       0    |\n",
      "|---------------------------------------------------------------------------|\n",
      "| GPU reserved segments |      20    |      20    |      20    |       0    |\n",
      "|       from large pool |      19    |      19    |      19    |       0    |\n",
      "|       from small pool |       1    |       1    |       1    |       0    |\n",
      "|---------------------------------------------------------------------------|\n",
      "| Non-releasable allocs |      18    |      18    |      19    |       1    |\n",
      "|       from large pool |      17    |      17    |      18    |       1    |\n",
      "|       from small pool |       1    |       1    |       1    |       0    |\n",
      "|===========================================================================|\n",
      "\n"
     ]
    }
   ],
   "source": [
    "torch.cuda.reset_peak_memory_stats() #Reset GPU Memory usage\n",
    "print(\"GPU Memory Reset\")\n",
    "print(torch.cuda.memory_summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                         | 0/8145 [00:06<?, ?it/s]\n"
     ]
    }
   ],
   "source": [
    "# Define the lists to hold the loss and accuracy values\n",
    "train_losses = []\n",
    "train_accs = []\n",
    "\n",
    "# Set up the training loop\n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "    loop = tqdm(train_loader, leave=True)\n",
    "    running_loss = 0.0\n",
    "    running_corrects = 0\n",
    "    \n",
    "    for batch in loop:\n",
    "        optim.zero_grad()\n",
    "        \n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        start_positions = batch['start_positions'].to(device)\n",
    "        end_positions = batch['end_positions'].to(device)\n",
    "        \n",
    "        outputs = model(input_ids, attention_mask=attention_mask,\n",
    "                        start_positions=start_positions,\n",
    "                        end_positions=end_positions)\n",
    "        \n",
    "        loss = outputs[0]\n",
    "        \n",
    "        #Calculate loss for every parameter \n",
    "        loss.backward()\n",
    "        optim.step() #Update gradients \n",
    "        \n",
    "        # Compute the batch accuracy\n",
    "        logits = outputs[1]\n",
    "        _, preds = torch.max(logits, 1)\n",
    "        corrects = torch.sum(preds == end_positions)\n",
    "        acc = corrects.double() / end_positions.size(0)\n",
    "        \n",
    "        # Record the batch loss and accuracy\n",
    "        running_loss += loss.item() * input_ids.size(0)\n",
    "        running_corrects += corrects\n",
    "        train_losses.append(loss.item())\n",
    "        train_accs.append(acc.item())\n",
    "        \n",
    "        #Visualize progress bar \n",
    "        loop.set_description(f'Epoch {epoch}')\n",
    "        loop.set_postfix(loss=loss.item(), acc=acc.item())\n",
    "        \n",
    "    # Compute the epoch loss and accuracy\n",
    "    epoch_loss = running_loss / len(train_dataset)\n",
    "    epoch_acc = running_corrects.double() / len(train_dataset)\n",
    "    \n",
    "    # Print the epoch loss and accuracy\n",
    "    print(f\"Epoch {epoch + 1} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "# Plot the loss and accuracy curves\n",
    "plt.plot(train_losses)\n",
    "plt.title(\"Training Loss\")\n",
    "plt.xlabel(\"Batch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.show()\n",
    "\n",
    "plt.plot(train_accs)\n",
    "plt.title(\"Training Accuracy\")\n",
    "plt.xlabel(\"Batch\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = 'model/bert-base-uncased_custom_epochs-'+str(epochs)\n",
    "os.mkdir(model_path)\n",
    "model.save_pretrained(model_path)\n",
    "tokenizer.save_pretrained(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_loader = DataLoader(val_dataset, batch_size=16)\n",
    "\n",
    "acc = []\n",
    "loop = tqdm(val_loader) #leave=True\n",
    "for batch in loop:\n",
    "    with torch.no_grad():\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        start_true = batch['start_positions'].to(device)\n",
    "        end_true = batch['end_positions'].to(device)\n",
    "\n",
    "        outputs = model(input_ids, attention_mask=attention_mask)\n",
    "\n",
    "        loss = outputs[0]\n",
    "\n",
    "        start_pred = torch.argmax(outputs['start_logits'], dim=1)\n",
    "        end_pred = torch.argmax(outputs['end_logits'], dim=1)\n",
    "        \n",
    "        acc.append(((start_pred == start_true).sum().true_divide(len(start_pred))).item())\n",
    "\n",
    "        acc.append(((end_pred == end_true).sum().true_divide(len(end_pred))).item())\n",
    "\n",
    "        \n",
    "aveAcc = sum(acc)/len(acc)\n",
    "print(\"Average Validation Accuracy:\\t%.2f%%\"%(100*aveAcc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Set the model to evaluation mode\n",
    "model.eval()\n",
    "\n",
    "# Create empty lists for true and predicted start and end positions\n",
    "start_true, end_true, start_pred, end_pred = [], [], [], []\n",
    "\n",
    "# Loop through the validation set and predict start and end positions\n",
    "with torch.no_grad():\n",
    "    for batch in val_loader:\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        start_true_batch = batch['start_positions'].to(device)\n",
    "        end_true_batch = batch['end_positions'].to(device)\n",
    "\n",
    "        outputs = model(input_ids, attention_mask=attention_mask)\n",
    "\n",
    "        start_pred_batch = torch.argmax(outputs['start_logits'], dim=1)\n",
    "        end_pred_batch = torch.argmax(outputs['end_logits'], dim=1)\n",
    "\n",
    "        start_true += start_true_batch.cpu().tolist()\n",
    "        end_true += end_true_batch.cpu().tolist()\n",
    "        start_pred += start_pred_batch.cpu().tolist()\n",
    "        end_pred += end_pred_batch.cpu().tolist()\n",
    "\n",
    "# Compute the confusion matrix\n",
    "cm_start = confusion_matrix(start_true, start_pred)\n",
    "cm_end = confusion_matrix(end_true, end_pred)\n",
    "\n",
    "# Plot the confusion matrix\n",
    "fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 5))\n",
    "sns.heatmap(cm_start, annot=True, cmap='Blues', fmt='g', ax=axes[0])\n",
    "sns.heatmap(cm_end, annot=True, cmap='Blues', fmt='g', ax=axes[1])\n",
    "axes[0].set_title('Confusion Matrix for Start Positions')\n",
    "axes[1].set_title('Confusion Matrix for End Positions')\n",
    "axes[0].set_xlabel('Predicted Label')\n",
    "axes[1].set_xlabel('Predicted Label')\n",
    "axes[0].set_ylabel('True Label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "# Initialize empty lists to store predicted and actual answers\n",
    "predicted_answers = []\n",
    "actual_answers = []\n",
    "\n",
    "# Iterate over the batches\n",
    "for batch in val_loader:\n",
    "    # Move the batch to the GPU\n",
    "    batch = {k:v.to(device) for k,v in batch.items()}\n",
    "\n",
    "    # Get the model's predictions for the current batch\n",
    "    with torch.no_grad():\n",
    "        outputs = model(**batch)\n",
    "        start_logits, end_logits = outputs.start_logits, outputs.end_logits\n",
    "\n",
    "    # Convert logits to probabilities\n",
    "    start_probs = torch.softmax(start_logits, dim=1).cpu().numpy()\n",
    "    end_probs = torch.softmax(end_logits, dim=1).cpu().numpy()\n",
    "\n",
    "    # Get the predicted start and end positions\n",
    "    start_pred, end_pred = np.argmax(start_probs, axis=1), np.argmax(end_probs, axis=1)\n",
    "\n",
    "    # Get the true start and end positions\n",
    "    start_true, end_true = batch['start_positions'].cpu().numpy(), batch['end_positions'].cpu().numpy()\n",
    "\n",
    "    # Convert the predicted and actual answers to text\n",
    "    for i in range(len(start_pred)):\n",
    "        predicted_answer = tokenizer.decode(batch['input_ids'][i][start_pred[i]:end_pred[i]+1], skip_special_tokens=True)\n",
    "        actual_answer = tokenizer.decode(batch['input_ids'][i][start_true[i]:end_true[i]+1], skip_special_tokens=True)\n",
    "        predicted_answers.append(predicted_answer)\n",
    "        actual_answers.append(actual_answer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot the histogram\n",
    "fig, ax = plt.subplots()\n",
    "ax.hist([predicted_answers, actual_answers], bins=20, label=['Predicted Answers', 'Actual Answers'])\n",
    "ax.legend(loc='upper right')\n",
    "ax.set_xlabel('Answer Text')\n",
    "ax.set_ylabel('Frequency')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Testing Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prediction(context, question):\n",
    "    inputs = tokenizer.encode_plus(question, context, return_tensors='pt').to(device)\n",
    "    outputs = model(**inputs)\n",
    "\n",
    "    answer_start = torch.argmax(outputs[0])  \n",
    "    answer_end = torch.argmax(outputs[1]) + 1 \n",
    "\n",
    "    answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(inputs['input_ids'][0][answer_start:answer_end]))\n",
    "\n",
    "    return answer\n",
    "\n",
    "def normalize_text(s):\n",
    "    \"\"\"Removing articles and punctuation, and standardizing whitespace are all typical text processing steps.\"\"\"\n",
    "    import string, re\n",
    "    def remove_articles(text):\n",
    "        regex = re.compile(r\"\\b(a|an|the)\\b\", re.UNICODE)\n",
    "        return re.sub(regex, \" \", text)\n",
    "    def white_space_fix(text):\n",
    "        return \" \".join(text.split())\n",
    "    def remove_punc(text):\n",
    "        exclude = set(string.punctuation)\n",
    "        return \"\".join(ch for ch in text if ch not in exclude)\n",
    "    def lower(text):\n",
    "        return text.lower()\n",
    "\n",
    "    return white_space_fix(remove_articles(remove_punc(lower(s))))\n",
    "\n",
    "def exact_match(prediction, truth):\n",
    "    return bool(normalize_text(prediction) == normalize_text(truth))\n",
    "\n",
    "def compute_f1(prediction, truth):\n",
    "    pred_tokens = normalize_text(prediction).split()\n",
    "    truth_tokens = normalize_text(truth).split()\n",
    "\n",
    "    # if either the prediction or the truth is no-answer then f1 = 1 if they agree, 0 otherwise\n",
    "    if len(pred_tokens) == 0 or len(truth_tokens) == 0:\n",
    "        return int(pred_tokens == truth_tokens)\n",
    "\n",
    "    common_tokens = set(pred_tokens) & set(truth_tokens)\n",
    "\n",
    "    # if there are no common tokens then f1 = 0\n",
    "    if len(common_tokens) == 0:\n",
    "        return 0\n",
    "\n",
    "    prec = len(common_tokens) / len(pred_tokens)\n",
    "    rec = len(common_tokens) / len(truth_tokens)\n",
    "\n",
    "    return round(2 * (prec * rec) / (prec + rec), 2)\n",
    "  \n",
    "def question_answer(context, question,answer):\n",
    "    prediction = get_prediction(context,question)\n",
    "    em_score = exact_match(prediction, answer)\n",
    "    f1_score = compute_f1(prediction, answer)\n",
    "\n",
    "    print(f'Question: {question}')\n",
    "    print(f'Prediction: {prediction}')\n",
    "    print(f'True Answer: {answer}')\n",
    "    print(f'Exact match: {em_score}')\n",
    "    print(f'F1 score: {f1_score}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Beyonce\n",
    "context = \"\"\"Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ bee-YON-say) (born September 4, 1981) is an American singer, \n",
    "          songwriter, record producer and actress. Born and raised in Houston, Texas, she performed in various singing \n",
    "          and dancing competitions as a child, and rose to fame in the late 1990s as lead singer of R&B girl-group Destiny\\'s Child. \n",
    "          Managed by her father, Mathew Knowles, the group became one of the world\\'s best-selling girl groups of all time. \n",
    "          Their hiatus saw the release of Beyoncé\\'s debut album, Dangerously in Love (2003), which established her as a solo artist worldwide, \n",
    "          earned five Grammy Awards and featured the Billboard Hot 100 number-one singles \"Crazy in Love\" and \"Baby Boy\".\"\"\"\n",
    "\n",
    "\n",
    "questions = [\"For whom the passage is talking about?\",\n",
    "             \"When did Beyonce born?\",\n",
    "             \"Where did Beyonce born?\",\n",
    "             \"What is Beyonce's nationality?\",\n",
    "             \"Who was the Destiny's group manager?\",\n",
    "             \"What name has the Beyoncé's debut album?\",\n",
    "             \"How many Grammy Awards did Beyonce earn?\",\n",
    "             \"When did the Beyoncé's debut album release?\",\n",
    "             \"Who was the lead singer of R&B girl-group Destiny's Child?\"]\n",
    "\n",
    "answers = [\"Beyonce Giselle Knowles - Carter\", \"September 4, 1981\", \"Houston, Texas\", \n",
    "           \"American\", \"Mathew Knowles\", \"Dangerously in Love\", \"five\", \"2003\", \n",
    "           \"Beyonce Giselle Knowles - Carter\"]\n",
    "\n",
    "for question, answer in zip(questions, answers):\n",
    "    question_answer(context, question, answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}