BERT_training_with_cos_v1_CUDA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "colab": {
      "name": "BERT training with cos v1 CUDA.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hepbc/NLP/blob/master/BERT_training_with_cos_v1_CUDA.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CVC6AzBLDyBx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Mid4mGDElgI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%pip install transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YW1aMr7UDyB9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "# Load pre-trained model tokenizer (vocabulary)\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "#tokenizer = AutoTokenizer.from_pretrained(\"ipuneetrathore/bert-base-cased-finetuned-finBERT\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mlq29lJNFLsG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YL7FXWYbDyCC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lines = pd.read_csv(r\"Bert example tags expanded with cos.csv\", header=0)\n",
        "lines = lines.sample(frac=1)\n",
        "lines\n",
        "txt = lines[\"Text\"].tolist()\n",
        "text = []\n",
        "for line in txt:\n",
        "    text.append(line.lower())\n",
        "text"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hp9uRMY4DyCF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tags = []\n",
        "slot_map = {}\n",
        "slot_names = [\"[PAD]\"]\n",
        "slot_map[\"[PAD]\"] = 0 \n",
        "for i in range(len(lines)):\n",
        "    k = \"\"\n",
        "    for j in range(11):\n",
        "#        print(i,j)\n",
        "        slot = lines.iloc[i,j+2]\n",
        "#        print(slot)\n",
        "        if pd.isna(slot):\n",
        "            break\n",
        "        else:\n",
        "            if slot not in slot_names:\n",
        "                slot_names.append(slot)\n",
        "                slot_map[slot] = len(slot_map)\n",
        "            k = k + \" \" + str(slot)\n",
        "    tags.append(k)\n",
        "print(slot_names)\n",
        "print(slot_map)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "apPertl8DyCP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pickle"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4uFTHcntDyCU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(\"slot_and_intent_data\", \"wb\") as file:\n",
        "    pickle.dump(slot_names, file)\n",
        "    pickle.dump(slot_map, file)\n",
        "    #pickle.dump(intent_names, file)\n",
        "    #pickle.dump(intent_map, file)\n",
        "file.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nzzg85PmDyCW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def encode_token_labels(text_sequences, slot_names, tokenizer, slot_map,\n",
        "                        max_length):\n",
        "    encoded = np.zeros(shape=(len(text_sequences), max_length), dtype=np.int32)\n",
        "    for i, (text_sequence, word_labels) in enumerate(\n",
        "            zip(text_sequences, slot_names)):\n",
        "        encoded_labels = []\n",
        "        print(text_sequence, word_labels)\n",
        "        for word, word_label in zip(text_sequence.split(), word_labels.split()):\n",
        "            #print(word, word_label)\n",
        "            tokens = tokenizer.tokenize(word)\n",
        "            #print(tokens)\n",
        "            encoded_labels.append(slot_map[word_label])\n",
        "            #print(encoded_labels)\n",
        "            expand_label = word_label.replace(\"B-\", \"I-\")\n",
        "            if not expand_label in slot_map:\n",
        "                expand_label = word_label\n",
        "            encoded_labels.extend([slot_map[expand_label]] * (len(tokens) - 1))\n",
        "        encoded[i, 1:len(encoded_labels) + 1] = encoded_labels\n",
        "        print(tokenizer.tokenize(text_sequence))\n",
        "        print(encoded_labels)\n",
        "    return encoded\n",
        "\n",
        "\n",
        "#slot_train = encode_token_labels(text, tags, tokenizer, slot_map, 25)\n",
        "#slot_train = encode_token_labels(text[:75], tags, tokenizer, slot_map, 25)\n",
        "#slot_test = encode_token_labels(text[75:], tags, tokenizer, slot_map, 25)\n",
        "slot_ids = encode_token_labels(text, tags, tokenizer, slot_map, 25)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O76ohRGYDyCZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def encode_dataset(tokenizer, text_sequences, max_length):\n",
        "    token_ids = np.zeros(shape=(len(text_sequences), max_length),\n",
        "                         dtype=np.int32)\n",
        "    for i, text_sequence in enumerate(text_sequences):\n",
        "        print(text_sequence)\n",
        "        encoded = tokenizer.encode(text_sequence)\n",
        "        #print(len(encoded))\n",
        "        token_ids[i, 0:len(encoded)] = encoded\n",
        "    attention_masks = (token_ids != 0).astype(np.int32)\n",
        "    #return {\"input_ids\": token_ids, \"attention_masks\": attention_masks}\n",
        "    return(token_ids, attention_masks)\n",
        "\n",
        "# encoded_train = encode_dataset(tokenizer, text[:75], 25)\n",
        "# encoded_test = encode_dataset(tokenizer, text[75:], 25)\n",
        "inputs_ids, attention_masks = encode_dataset(tokenizer, text, 25)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0s68as7ADyCe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tqdm import tqdm, trange\n",
        "\n",
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "#from transformers import BertTokenizer, BertConfig\n",
        "\n",
        "#from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "torch.__version__\n",
        "\n",
        "MAX_LEN = 25\n",
        "bs = 32\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nTWRMXZ1DyCh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tr_inputs, val_inputs, tr_tags, val_tags = train_test_split(inputs_ids, slot_ids,\n",
        "                                                            random_state=2020, test_size=0.05)\n",
        "tr_masks, val_masks, _, _ = train_test_split(attention_masks, inputs_ids,\n",
        "                                             random_state=2020, test_size=0.05)\n",
        "\n",
        "tr_inputs = torch.tensor(tr_inputs)\n",
        "val_inputs = torch.tensor(val_inputs)\n",
        "tr_tags = torch.tensor(tr_tags)\n",
        "val_tags = torch.tensor(val_tags)\n",
        "tr_masks = torch.tensor(tr_masks)\n",
        "val_masks = torch.tensor(val_masks)\n",
        "\n",
        "train_data = TensorDataset(tr_inputs, tr_masks, tr_tags)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=bs)\n",
        "\n",
        "valid_data = TensorDataset(val_inputs, val_masks, val_tags)\n",
        "valid_sampler = SequentialSampler(valid_data)\n",
        "valid_dataloader = DataLoader(valid_data, sampler=valid_sampler, batch_size=bs)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_QI72Py5DyCj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(text[93])\n",
        "print(tokenizer.tokenize(text[93]))\n",
        "print(tags[93])\n",
        "print(slot_ids[93])\n",
        "print(inputs_ids[93])\n",
        "print(tokenizer.encode(tokenizer.tokenize(text[93])))\n",
        "# print(tr_tags[93])\n",
        "print(slot_map)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HvfuwmLQDyCm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(len(tr_inputs), len(tr_tags), len(tr_masks))\n",
        "print(len(train_data), len(train_sampler), len(train_dataloader))\n",
        "\n",
        "print(len(val_inputs), len(val_tags), len(val_masks))\n",
        "print(len(valid_data), len(valid_sampler), len(valid_dataloader))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a9s3SbPXF-yb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%pip install seqeval"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cNcqZ3lEDyCq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from seqeval.metrics import f1_score, accuracy_score"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mwf_CqO5DyCu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import transformers\n",
        "from transformers import BertForTokenClassification, AdamW\n",
        "\n",
        "print(transformers.__version__)\n",
        "\n",
        "model = BertForTokenClassification.from_pretrained(\n",
        "    \"bert-base-uncased\",\n",
        "    num_labels=len(slot_map),\n",
        "    output_attentions = False,\n",
        "    output_hidden_states = False\n",
        ")\n",
        "\n",
        "FULL_FINETUNING = True\n",
        "if FULL_FINETUNING:\n",
        "    param_optimizer = list(model.named_parameters())\n",
        "    no_decay = ['bias', 'gamma', 'beta']\n",
        "    optimizer_grouped_parameters = [\n",
        "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
        "         'weight_decay_rate': 0.01},\n",
        "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
        "         'weight_decay_rate': 0.0}\n",
        "    ]\n",
        "else:\n",
        "    param_optimizer = list(model.classifier.named_parameters())\n",
        "    optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n",
        "\n",
        "optimizer = AdamW(\n",
        "    optimizer_grouped_parameters,\n",
        "    lr=9e-5, #3e-5\n",
        "    eps=1e-8\n",
        ")\n",
        "\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "epochs = 25\n",
        "max_grad_norm = 1.0\n",
        "\n",
        "# Total number of training steps is number of batches * number of epochs.\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(\n",
        "    optimizer,\n",
        "    num_warmup_steps=0,\n",
        "    num_training_steps=total_steps\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-SgREau2DyCx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tag_values = slot_names\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "n_gpu = torch.cuda.device_count()\n",
        "print(device, n_gpu)\n",
        "torch.cuda.get_device_name(0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ld_qLg-bDyC0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## Store the average loss after each epoch so we can plot them.\n",
        "loss_values, validation_loss_values = [], []\n",
        "\n",
        "for _ in trange(epochs, desc=\"Epoch\"):\n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "    # Perform one full pass over the training set.\n",
        "\n",
        "    # Put the model into training mode.\n",
        "    model.train()\n",
        "    # Reset the total loss for this epoch.\n",
        "    total_loss = 0\n",
        "\n",
        "    # Training loop\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "        print(step)\n",
        "        # add batch to gpu\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "        #batch = tuple(t for t in batch)\n",
        "        #print(batch)\n",
        "        b_input_ids, b_input_mask, b_labels = batch\n",
        "        #print(type(b_input_ids))\n",
        "        ###############Bug fix code####################\n",
        "        b_input_ids = b_input_ids.type(torch.LongTensor)\n",
        "        b_input_mask = b_input_mask.type(torch.LongTensor)\n",
        "        b_labels = b_labels.type(torch.LongTensor)\n",
        "        #print(type(b_input_ids))\n",
        "        b_input_ids = b_input_ids.to(device)\n",
        "        b_input_mask = b_input_mask.to(device)\n",
        "        b_labels = b_labels.to(device)\n",
        "#          ############################################\n",
        "\n",
        "        # Always clear any previously calculated gradients before performing a backward pass.\n",
        "        model.zero_grad()\n",
        "        optimizer.zero_grad()\n",
        "        # forward pass\n",
        "        # This will return the loss (rather than the model output)\n",
        "        # because we have provided the `labels`.\n",
        "        outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "        # get the loss\n",
        "        loss = outputs[0]\n",
        "        # Perform a backward pass to calculate the gradients.\n",
        "        loss.backward()\n",
        "        # track train loss\n",
        "        total_loss += loss.item()\n",
        "        # Clip the norm of the gradient\n",
        "        # This is to help prevent the \"exploding gradients\" problem.\n",
        "        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n",
        "        # update parameters\n",
        "        optimizer.step()\n",
        "        # Update the learning rate.\n",
        "        scheduler.step()\n",
        "        print(\"done\")\n",
        "\n",
        "    # Calculate the average loss over the training data.\n",
        "    avg_train_loss = total_loss / len(train_dataloader)\n",
        "    print(\"Average train loss: {}\".format(avg_train_loss))\n",
        "\n",
        "    # Store the loss value for plotting the learning curve.\n",
        "    loss_values.append(avg_train_loss)\n",
        "\n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on\n",
        "    # our validation set.\n",
        "\n",
        "    # Put the model into evaluation mode\n",
        "    model.eval()\n",
        "    # Reset the validation loss for this epoch.\n",
        "    eval_loss, eval_accuracy = 0, 0\n",
        "    nb_eval_steps, nb_eval_examples = 0, 0\n",
        "    predictions , true_labels = [], []\n",
        "    for batch in valid_dataloader:\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "        b_input_ids, b_input_mask, b_labels = batch\n",
        "        \n",
        "        \n",
        "        ###############Bug fix code####################\n",
        "        b_input_ids = b_input_ids.type(torch.LongTensor)\n",
        "        b_input_mask = b_input_mask.type(torch.LongTensor)\n",
        "        b_labels = b_labels.type(torch.LongTensor)\n",
        "        print(type(b_input_ids))#, b_input_mask, b_labels))\n",
        "        b_input_ids = b_input_ids.to(device)\n",
        "        b_input_mask = b_input_mask.to(device)\n",
        "        b_labels = b_labels.to(device)\n",
        "        # Telling the model not to compute or store gradients,\n",
        "        # saving memory and speeding up validation\n",
        "        with torch.no_grad():\n",
        "            # Forward pass, calculate logit predictions.\n",
        "            # This will return the logits rather than the loss because we have not provided labels.\n",
        "            print(type(outputs))\n",
        "            outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "        # Move logits and labels to CPU\n",
        "        logits = outputs[1].detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "        # Calculate the accuracy for this batch of test sentences.\n",
        "        eval_loss += outputs[0].mean().item()\n",
        "        predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n",
        "        true_labels.extend(label_ids)\n",
        "\n",
        "    eval_loss = eval_loss / len(valid_dataloader)\n",
        "    validation_loss_values.append(eval_loss)\n",
        "    print(\"Validation loss: {}\".format(eval_loss))\n",
        "    pred_tags = [tag_values[p_i] for p, l in zip(predictions, true_labels)\n",
        "                                 for p_i, l_i in zip(p, l) if tag_values[l_i] != \"PAD\"]\n",
        "    valid_tags = [tag_values[l_i] for l in true_labels\n",
        "                                  for l_i in l if tag_values[l_i] != \"PAD\"]\n",
        "    print(\"Validation Accuracy: {}\".format(accuracy_score(pred_tags, valid_tags)))\n",
        "    print(\"Validation F1-Score: {}\".format(f1_score(pred_tags, valid_tags)))\n",
        "    print()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UyZk64NiQ2z2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"\\n * input_ids_tensor \\n \")\n",
        "print(b_input_ids)\n",
        "print(b_input_ids.device)\n",
        "\n",
        "print(\"\\n * segment_ids_tensor \\n \")\n",
        "print(b_input_mask)\n",
        "print(b_input_mask.device)\n",
        "\n",
        "print(\"\\n * input_mask_tensor \\n \")\n",
        "print(b_labels)\n",
        "print(b_labels.device)\n",
        "\n",
        "print(\"\\n * self.device \\n \")\n",
        "print(self.device)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MwlotIakDyC4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Saving model\n",
        "torch.save(model.state_dict(), \"NERwithTCv4.pt\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KiSgQPTFDyC8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Loading model\n",
        "model1 = BertForTokenClassification.from_pretrained(\n",
        "    \"bert-base-uncased\",\n",
        "    num_labels=len(slot_map),\n",
        "    output_attentions = False,\n",
        "    output_hidden_states = False\n",
        ")\n",
        "\n",
        "model1.load_state_dict(torch.load(\"NERwithTC.pt\"))\n",
        "model1.eval()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RlHrNjEkDyC_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def show_predictions(text, tokenizer, model, slot_names):\n",
        "    inputs = torch.tensor(tokenizer.encode(text))[None, :]  # batch_size = 1\n",
        "    inputs = inputs.type(torch.LongTensor)\n",
        "    #    b_input_mask = b_input_mask.type(torch.LongTensor)\n",
        "    #    b_labels = b_labels.type(torch.LongTensor)\n",
        "        #print(type(b_input_ids))\n",
        "    inputs = inputs.to(device)\n",
        "\n",
        "    print(inputs)\n",
        "    with torch.no_grad():\n",
        "        outputs = model(inputs)\n",
        "    #print(type(outputs))\n",
        "    #print(outputs)\n",
        "    slot_logits = outputs[0].detach().cpu().numpy()\n",
        "    #label_ids = b_labels.to('cpu').numpy()\n",
        "    #slot_logits = outputs[0]\n",
        "    #print(np.numpy(slot_logits\n",
        "    slot_ids = slot_logits.argmax(axis=-1)[0, 1:-1]\n",
        "    #print(slot_ids)\n",
        "    #intent_id = intent_logits.numpy().argmax(axis=-1)[0]\n",
        "    #print(\"## Intent:\", intent_id)#intent_names[intent_id])\n",
        "    print(\"## Slots:\")\n",
        "    for token, slot_id in zip(tokenizer.tokenize(text), slot_ids):\n",
        "        print(f\"{token:>10} : {slot_names[slot_id]}\")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mWYRyj4mDyDB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "usertxt = \"\"\n",
        "while usertxt != \"Exit\":\n",
        "    usertxt = input()\n",
        "    show_predictions(usertxt, tokenizer, model, slot_names)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yn7YE5uwDyDE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def decode_predictions1(text, tokenizer, slot_names, slot_ids):\n",
        "#info = {\"intent\": intent_names[intent_id]}\n",
        "    info = {}\n",
        "    #info[\"intent\"] = list(intent_names)[intent_id]\n",
        "    collected_slots = {}\n",
        "    active_slot_words = []\n",
        "    active_slot_name = None\n",
        "    prev_word_slot_name = \"\"\n",
        "    prev_slot_pos = \"\"\n",
        "    prev_slot_type = \"\"\n",
        "    words = text.split()\n",
        "    for i, word in enumerate(words):\n",
        "        print(word)\n",
        "        tokens = tokenizer.tokenize(word)\n",
        "        current_word_slot_ids = slot_ids[:len(tokens)]\n",
        "        slot_ids = slot_ids[len(tokens):]\n",
        "        print(current_word_slot_ids, slot_ids)\n",
        "        current_word_slot_name = slot_names[current_word_slot_ids[0]]\n",
        "        curr_slot_pos = current_word_slot_name[0]\n",
        "        curr_slot_type = current_word_slot_name[2:]\n",
        "        print(current_word_slot_name, curr_slot_type, prev_word_slot_name, prev_slot_type)\n",
        "        if len(prev_word_slot_name) != 0:\n",
        "            prev_slot_pos = prev_word_slot_name[0]\n",
        "            prev_slot_type = prev_word_slot_name[2:]            \n",
        "        # if I-series continuing add to slot words\n",
        "        if current_word_slot_name == prev_word_slot_name:\n",
        "            active_slot_words.append(word)\n",
        "        # if I-series after B-series of same slot type, add to slot words\n",
        "        elif curr_slot_pos == \"I\" and prev_slot_pos ==\"B\" and curr_slot_type == prev_slot_type:\n",
        "            active_slot_words.append(word)\n",
        "        # for all else, add previously collected slot to list\n",
        "        else:\n",
        "            if prev_slot_type in collected_slots:\n",
        "                #print(prev_slot_type)\n",
        "                slot_name = \" \".join(active_slot_words)\n",
        "                collected_slots[prev_slot_type].append(slot_name)\n",
        "                #all fundamentals to have an attribute or NA\n",
        "                if prev_slot_type == \"funda\":\n",
        "                    #check if attr came before funda\n",
        "#                     if curr_slot_type != \"attr\":\n",
        "#                         if \"attr\" in collected_slots:\n",
        "#                             collected_slots[\"attr\"].append(\"NA\")\n",
        "#                         else:\n",
        "#                             collected_slots[\"attr\"] = []\n",
        "#                             collected_slots[\"attr\"].append(\"NA\")\n",
        "                    if \"attr\" in collected_slots:\n",
        "                        if len(collected_slots[\"funda\"]) != len(collected_slots[\"attr\"]):\n",
        "                            collected_slots[\"attr\"].append(\"NA\")\n",
        "                    else:\n",
        "                        collected_slots[\"attr\"] = []\n",
        "                        if len(collected_slots[\"funda\"]) != len(collected_slots[\"attr\"]):\n",
        "                            collected_slots[\"attr\"].append(\"NA\")\n",
        "                        else:\n",
        "                            print(\"Each fundamental should have only one attribute.\")\n",
        "            else: \n",
        "                collected_slots[prev_slot_type] = []\n",
        "                slot_name = \" \".join(active_slot_words)\n",
        "                collected_slots[prev_slot_type].append(slot_name)\n",
        "                #all fundamentals to have an attribute or NA\n",
        "                if prev_slot_type == \"funda\":\n",
        "#                     if curr_slot_type != \"attr\":\n",
        "#                         if \"attr\" in collected_slots:\n",
        "#                             collected_slots[\"attr\"].append(\"NA\")\n",
        "#                         else:\n",
        "#                             collected_slots[\"attr\"] = []\n",
        "#                             collected_slots[\"attr\"].append(\"NA\")\n",
        "                    if \"attr\" in collected_slots:\n",
        "                        if len(collected_slots[\"funda\"]) != len(collected_slots[\"attr\"]):\n",
        "                            collected_slots[\"attr\"].append(\"NA\")\n",
        "                    else:\n",
        "                        collected_slots[\"attr\"] = []\n",
        "                        if len(collected_slots[\"funda\"]) != len(collected_slots[\"attr\"]):\n",
        "                            collected_slots[\"attr\"].append(\"NA\")\n",
        "                        else:\n",
        "                            print(\"Each fundamental should have only one attribute.\")\n",
        "                                \n",
        "            active_slot_words = []\n",
        "            active_slot_words.append(word)\n",
        "        print(\"Active: \", active_slot_words)    \n",
        "        prev_word_slot_name = current_word_slot_name\n",
        "    # handling last word\n",
        "    if curr_slot_type in collected_slots:\n",
        "        slot_name = \" \".join(active_slot_words)\n",
        "        collected_slots[curr_slot_type].append(slot_name)\n",
        "    else:\n",
        "        collected_slots[curr_slot_type] = []\n",
        "        slot_name = \" \".join(active_slot_words)\n",
        "        collected_slots[curr_slot_type].append(slot_name)\n",
        "    #adding NA in attributes if last word in fundamental\n",
        "    if curr_slot_type == \"funda\":\n",
        "        if \"attr\" in collected_slots:\n",
        "#             collected_slots[\"attr\"].append(\"NA\")\n",
        "#         else:\n",
        "#             collected_slots[\"attr\"] = []\n",
        "#             collected_slots[\"attr\"].append(\"NA\")\n",
        "            if len(collected_slots[\"funda\"]) != len(collected_slots[\"attr\"]):\n",
        "                collected_slots[\"attr\"].append(\"NA\")\n",
        "            else:\n",
        "                collected_slots[\"attr\"] = []\n",
        "                if len(collected_slots[\"funda\"]) != len(collected_slots[\"attr\"]):\n",
        "            #collected_slots[\"attr\"] = []\n",
        "                    collected_slots[\"attr\"].append(\"NA\")\n",
        "                else:\n",
        "                    print(\"Each fundamental should have only one attribute.\")                            \n",
        "    info[\"slots\"] = collected_slots\n",
        "    return info\n",
        "\n",
        "\n",
        "#def nlu(text, tokenizer, model, intent_names, slot_names):\n",
        "def nlu(text, tokenizer, model, slot_names):\n",
        "    #inputs = tf.constant(tokenizer.encode(text))[None, :]  # batch_size = 1\n",
        "    inputs = torch.tensor(tokenizer.encode(text))[None, :]  # batch_size = 1\n",
        "    print(inputs)\n",
        "    with torch.no_grad():\n",
        "        outputs = model(inputs)\n",
        "    #print(outputs)\n",
        "    slot_logits = outputs[0]\n",
        "    #print(slot_logits)\n",
        "    slot_ids = slot_logits.numpy().argmax(axis=-1)[0, 1:-1]\n",
        "    print(slot_ids)\n",
        "    #intent_id = intent_logits.numpy().argmax(axis=-1)[0]\n",
        "\n",
        "    #return decode_predictions1(text, tokenizer, intent_names, slot_names, intent_id, slot_ids)\n",
        "    return decode_predictions1(text, tokenizer, slot_names, slot_ids)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "db59zmkuDyDI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "usertxt = \"\"\n",
        "while usertxt != \"exit\":\n",
        "    usertxt = input().lower()\n",
        "    #print(nlu(usertxt, tokenizer, model1, intent_names, slot_names))\n",
        "    print(nlu(usertxt, tokenizer, model, slot_names))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ng0p9G0XDyDO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "words"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}