Code/Apply_Pretrained_Model/Pretrained_NoRBERT_Task1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Pretrained_NoRBERT_Task1.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_4NzfRZGk-Y-",
        "colab_type": "text"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tobhey/NoRBERT/blob/master/Code/Apply_Pretrained_Model/Pretrained_NoRBERT_Task1.ipynb)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IiDlHo-_CRac",
        "colab_type": "text"
      },
      "source": [
        "# Binary classification of functional and non-functional requirements"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "18lQiv4UBSdt",
        "colab_type": "text"
      },
      "source": [
        "This notebook can be used to load a pretrained model, trained on the task of binary classification of functional and non-functional requirements on the original Promise NFR dataset. It can be used to classify a given input."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fSq4Qamxdkz2",
        "colab_type": "text"
      },
      "source": [
        "Note: some cells are hidden and only the title is shown. To display the code, double-click the cell to switch the display mode."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d-1y5KaxYVmP",
        "colab_type": "text"
      },
      "source": [
        "## Init"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GQN5E-FoUqsv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Install dependencies {display-mode: \"form\"}\n",
        "!pip install fastai==1.0.61 fastcore==1.3.29 fastprogress==1.0.3 pytorch-transformers==1.2.0 sklearn==0.0 spacy==3.6.1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oeArf6jVVMzl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Import dependencies {display-mode: \"form\"}\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "from fastai import *\n",
        "from fastai.text import *\n",
        "from fastai.callbacks import *\n",
        "from sklearn.model_selection import train_test_split, StratifiedKFold\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.utils.multiclass import unique_labels\n",
        "\n",
        "from pytorch_transformers import BertTokenizer, BertPreTrainedModel, BertModel, BertConfig\n",
        "from pytorch_transformers import AdamW"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FgN8QQ8YYkpU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Define Config classes {display-mode: \"form\"}\n",
        "class Config(dict):\n",
        "    def __init__(self, **kwargs):\n",
        "        super().__init__(**kwargs)\n",
        "        for k, v in kwargs.items():\n",
        "            setattr(self, k, v)\n",
        "    \n",
        "    def set(self, key, val):\n",
        "        self[key] = val\n",
        "        setattr(self, key, val)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tIyeBgjnYp7M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# adapt this to your needs!\n",
        "config_data = Config(\n",
        "    root_folder = '.', # where is the root folder? Keep it that way if you want to load from Google Drive\n",
        "    model_path = '/models/', # where is the folder for the model(s); relative to the root\n",
        "    model_name = 'NoRBERT_Task1_NFR_e10_NoSampling.pkl', # what is the model name? \n",
        ")\n",
        "\n",
        "load_from_gdrive = True # True, if you want to use Google Drive; else, False\n",
        "gdrive_root_folder = '/content/drive/My Drive/Code/Apply_Pretrained_Model/' # Set this to the Google Drive path. Starts with '/content/drive/' and then usually 'My Drive/*' for the files in your Drive"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Wtzha3q7QjjU",
        "colab": {}
      },
      "source": [
        "#@title Check, if and what kind of GPU is used {display-mode: \"form\"}\n",
        "cuda_available = torch.cuda.is_available()\n",
        "if cuda_available:\n",
        "    curr_device = torch.cuda.current_device()\n",
        "    print(torch.cuda.get_device_name(curr_device))\n",
        "device = torch.device(\"cuda\" if cuda_available else \"cpu\")\n",
        "device"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LysOxgPvW0am",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Init loading from Google Drive, if set in config above {display-mode: \"form\"}\n",
        "if load_from_gdrive:\n",
        "    from google.colab import drive\n",
        "    # Connect to drive to load the corpus from there\n",
        "    drive.mount('/content/drive', force_remount=True)\n",
        "    config_data.root_folder = gdrive_root_folder"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "TNRRj6jIJrp2",
        "colab": {}
      },
      "source": [
        "#@title Load/Define NoRBERT classes {display-mode: \"form\"}\n",
        "class FastAiBertTokenizer(BaseTokenizer):\n",
        "    \"\"\"Wrapper around BertTokenizer to be compatible with fast.ai\"\"\"\n",
        "    def __init__(self, tokenizer: BertTokenizer, max_seq_len: int=512, **kwargs):\n",
        "        self._pretrained_tokenizer = tokenizer\n",
        "        self.max_seq_len = max_seq_len\n",
        "\n",
        "    def __call__(self, *args, **kwargs):\n",
        "        return self\n",
        "\n",
        "    def tokenizer(self, t:str):\n",
        "        \"\"\"Limits the maximum sequence length. Prepend with [CLS] and append [SEP]\"\"\"\n",
        "        return [\"[CLS]\"] + self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2] + [\"[SEP]\"]\n",
        "\n",
        "## \n",
        "\n",
        "class BertTokenizeProcessor(TokenizeProcessor):\n",
        "    \"\"\"Special Tokenizer, where we remove sos/eos tokens since we add that ourselves in the tokenizer.\"\"\"\n",
        "    def __init__(self, tokenizer):\n",
        "        super().__init__(tokenizer=tokenizer, include_bos=False, include_eos=False)\n",
        "\n",
        "class BertNumericalizeProcessor(NumericalizeProcessor):\n",
        "    \"\"\"Use a custom vocabulary to match the original BERT model.\"\"\"\n",
        "    def __init__(self, *args, **kwargs):\n",
        "        super().__init__(*args, vocab=Vocab(list(bert_tok.vocab.keys())), **kwargs)\n",
        "\n",
        "def get_bert_processor(tokenizer:Tokenizer=None, vocab:Vocab=None):\n",
        "    return [BertTokenizeProcessor(tokenizer=tokenizer),\n",
        "            NumericalizeProcessor(vocab=vocab)]\n",
        "\n",
        "class BertDataBunch(TextDataBunch):\n",
        "    @classmethod\n",
        "    def from_df(cls, path:PathOrStr, train_df:DataFrame, valid_df:DataFrame, test_df:Optional[DataFrame]=None,\n",
        "              tokenizer:Tokenizer=None, vocab:Vocab=None, classes:Collection[str]=None, text_cols:IntsOrStrs=1,\n",
        "              label_cols:IntsOrStrs=0, **kwargs) -> DataBunch:\n",
        "        \"Create a `TextDataBunch` from DataFrames.\"\n",
        "        p_kwargs, kwargs = split_kwargs_by_func(kwargs, get_bert_processor)\n",
        "        # use our custom processors while taking tokenizer and vocab as kwargs\n",
        "        processor = get_bert_processor(tokenizer=tokenizer, vocab=vocab, **p_kwargs)\n",
        "        if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols\n",
        "        src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),\n",
        "                      TextList.from_df(valid_df, path, cols=text_cols, processor=processor))\n",
        "        src = src.label_for_lm() if cls==TextLMDataBunch else src.label_from_df(cols=label_cols, classes=classes)\n",
        "        if test_df is not None: src.add_test(TextList.from_df(test_df, path, cols=text_cols))\n",
        "        return src.databunch(**kwargs)\n",
        "\n",
        "##\n",
        "\n",
        "class BertTextClassifier(BertPreTrainedModel):\n",
        "    def __init__(self, model_name, num_labels):\n",
        "        config = BertConfig.from_pretrained(model_name)\n",
        "        super(BertTextClassifier, self).__init__(config)\n",
        "        self.num_labels = num_labels\n",
        "        \n",
        "        self.bert = BertModel.from_pretrained(model_name, config=config)\n",
        "        \n",
        "        self.dropout = nn.Dropout(self.config.hidden_dropout_prob)\n",
        "        self.classifier = nn.Linear(self.config.hidden_size, num_labels)\n",
        "\n",
        "        #self.apply(self.init_weights)\n",
        "    \n",
        "    def forward(self, tokens, labels=None, position_ids=None, token_type_ids=None, attention_mask=None, head_mask=None):\n",
        "        outputs = self.bert(tokens, position_ids=position_ids, token_type_ids=token_type_ids, attention_mask=attention_mask, head_mask=head_mask)\n",
        "        \n",
        "        pooled_output = outputs[1]\n",
        "        # According to documentation of pytorch-transformers, pooled output might not be the best \n",
        "        # and you’re often better with averaging or pooling the sequence of hidden-states for the whole input sequence \n",
        "        #hidden_states = outputs[0]\n",
        "        #pooled_output = torch.mean(hidden_states, 1)\n",
        "\n",
        "        dropout_output = self.dropout(pooled_output)\n",
        "        logits = self.classifier(dropout_output)\n",
        "\n",
        "        activation = nn.Softmax(dim=1)\n",
        "        probs = activation(logits)   \n",
        "\n",
        "        return logits"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OK17yQAGV_bM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Load classifier {display-mode: \"form\"}\n",
        "classifier = load_learner(config_data.root_folder + config_data.model_path, config_data.model_name)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9IxRhtyyYbht",
        "colab_type": "text"
      },
      "source": [
        "## Predict"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CxsXG_HaXg1s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title {display-mode: \"form\"}\n",
        "def predict(classifier, text):\n",
        "    prediction = classifier.predict(text)\n",
        "    prediction_class = prediction[1]\n",
        "    label = classifier.data.classes[prediction_class]\n",
        "    return label"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UMMYic_pa5Sy",
        "colab_type": "code",
        "cellView": "both",
        "colab": {}
      },
      "source": [
        "#Set the labels, the classifier will output (in correct order!)\n",
        "labels = ['F', 'NFR']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HISh0sGGeVbj",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Set input requirement\n",
        "input_requirement =  'The system shall display Events in a vertical table by time.'  #@param {type: \"string\"}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ztKNDBCVaGfK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Predict output label {display-mode: \"form\"}\n",
        "pred_class = predict(classifier, input_requirement)\n",
        "print(labels[pred_class])"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}