Dialogue Act Tagger (for dialogue analysis).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XShs4pFfPVaW"
   },
   "source": [
    "## Connecting to Drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2322,
     "status": "ok",
     "timestamp": 1645278527975,
     "user": {
      "displayName": "Thomas Bellucci",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "13668310691905895775"
     },
     "user_tz": -60
    },
    "id": "8fCZngyUPHEk",
    "outputId": "5a4dd98f-2696-4555-b485-d63b06ec6612"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "\n",
    "# Pilot annotations were stored in Google Drive in Data/pilot_annotations\n",
    "drive.mount('/content/gdrive')\n",
    "\n",
    "model_dir = '/content/gdrive/MyDrive/Thesis MSc AI VU - Thomas Bellucci/tests/models'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AU2eYT61lAzU"
   },
   "source": [
    "## Training Dialogue Act Tagger on MIDAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 340,
     "status": "ok",
     "timestamp": 1645277598745,
     "user": {
      "displayName": "Thomas Bellucci",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "13668310691905895775"
     },
     "user_tz": -60
    },
    "id": "O9XC29DDf2Yk",
    "outputId": "120dafa8-882f-40ac-d40a-0bb7a1235f15"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['how about another short piece of football news : EMPTY > how can you pick us knows now ## open_question_factual;',\n",
       " 'do you want to hear some fun facts about cats instead : EMPTY > yes ## pos_answer;command',\n",
       " 'did you know that : yes > i did ## pos_answer;']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "url = 'https://raw.githubusercontent.com/DianDYu/MIDAS_dialog_act/throw_exception_on_example_format_error/da_data/train.txt'\n",
    "raw_data = pd.read_csv(url, sep='\\n', header=None).values.flatten().tolist()\n",
    "raw_data[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1645277670638,
     "user": {
      "displayName": "Thomas Bellucci",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "13668310691905895775"
     },
     "user_tz": -60
    },
    "id": "wl0NhGNrjrSY",
    "outputId": "a5d0178e-7e28-4237-f2b1-b77b106490e9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('how about another short piece of football news',\n",
       "  'how can you pick us knows now',\n",
       "  'open_question_factual'),\n",
       " ('do you want to hear some fun facts about cats instead',\n",
       "  'yes',\n",
       "  'pos_answer'),\n",
       " ('do you want to hear some fun facts about cats instead', 'yes', 'command')]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "def split_line(line):\n",
    "    items = re.split(' : | > | ## ', line.strip())\n",
    "    if len(items) != 4:\n",
    "        return []\n",
    "    turn0, _, turn1, acts = items\n",
    "    acts = acts.strip().split(';')\n",
    "    return [(turn0, turn1, act) for act in acts if act.strip() != '']\n",
    "\n",
    "data = []\n",
    "for line in raw_data:\n",
    "    for item in split_line(line):\n",
    "        data.append(item)\n",
    "data[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 359,
     "status": "ok",
     "timestamp": 1645277672784,
     "user": {
      "displayName": "Thomas Bellucci",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "13668310691905895775"
     },
     "user_tz": -60
    },
    "id": "xi0zcYrPmCPf",
    "outputId": "af4684eb-4ae5-4170-f9f6-ec52f0671b74"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10927"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1645277675895,
     "user": {
      "displayName": "Thomas Bellucci",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "13668310691905895775"
     },
     "user_tz": -60
    },
    "id": "C0aPV-7dk_n_",
    "outputId": "9e12059b-3d62-4d1c-f743-175d89769fb8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'abandon',\n",
       " 'apology',\n",
       " 'appreciation',\n",
       " 'back-channeling',\n",
       " 'closing',\n",
       " 'command',\n",
       " 'comment',\n",
       " 'complaint',\n",
       " 'dev_command',\n",
       " 'hold',\n",
       " 'neg_answer',\n",
       " 'nonsense',\n",
       " 'open_question_factual',\n",
       " 'open_question_opinion',\n",
       " 'opening',\n",
       " 'opinion',\n",
       " 'other',\n",
       " 'other_answers',\n",
       " 'pos_answer',\n",
       " 'respond_to_apology',\n",
       " 'statement',\n",
       " 'thanking',\n",
       " 'yes_no_question'}"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags = set([t for _, _, t in data])\n",
    "tags"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L6sNHZGIl76w"
   },
   "source": [
    "## Training RoBERTa on MIDAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "executionInfo": {
     "elapsed": 16175,
     "status": "ok",
     "timestamp": 1645277709719,
     "user": {
      "displayName": "Thomas Bellucci",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "13668310691905895775"
     },
     "user_tz": -60
    },
    "id": "3zkgElM5mb8h"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install transformers\n",
    "\n",
    "import torch\n",
    "from transformers import RobertaTokenizer, RobertaForSequenceClassification\n",
    "\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "executionInfo": {
     "elapsed": 346,
     "status": "ok",
     "timestamp": 1645277710060,
     "user": {
      "displayName": "Thomas Bellucci",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "13668310691905895775"
     },
     "user_tz": -60
    },
    "id": "2XQ-rpK6mrJL"
   },
   "outputs": [],
   "source": [
    "class DialogTag:\n",
    "    def __init__(self, num_labels=23):\n",
    "        # Set up GPU if available\n",
    "        self._device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "\n",
    "        self._tokenizer = RobertaTokenizer.from_pretrained('roberta-base')\n",
    "        self._model = RobertaForSequenceClassification.from_pretrained('roberta-base', num_labels=num_labels)\n",
    "        self._model.to(self._device)\n",
    "\n",
    "        self._label2id = dict()\n",
    "        self._id2label = dict()\n",
    "\n",
    "    def _tokenize(self, strings):\n",
    "        return self._tokenizer(strings, padding=True, return_tensors='pt').to(self._device)\n",
    "\n",
    "    def _encode_labels(self, labels):\n",
    "        int_labels = []\n",
    "        for label in labels:\n",
    "            if label not in self._label2id:\n",
    "                self._label2id[label] = len(self._label2id)\n",
    "                self._id2label[len(self._id2label)] = label\n",
    "            int_labels.append(self._label2id[label])\n",
    "        return torch.LongTensor(int_labels).to(self._device)\n",
    "\n",
    "    def fit(self, data, epochs=4, batch_size=32, lrate=1e-5):\n",
    "        # Preprocess turns and index labels\n",
    "        strings = [t0 + self._tokenizer.sep_token + t1 for t0, t1, _ in data]\n",
    "        labels = [l for _, _, l in data]\n",
    "\n",
    "        X = [self._tokenize(strings[i:i + batch_size]) for i in range(0, len(strings), batch_size)]\n",
    "        y = [self._encode_labels(labels[i:i + batch_size]) for i in range(0, len(labels), batch_size)]\n",
    "\n",
    "        # Setup optimizer and objective function\n",
    "        optimizer = torch.optim.Adam(self._model.parameters(), lr=lrate)\n",
    "        criterion = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "        for epoch in range(epochs):\n",
    "            losses = []\n",
    "\n",
    "            for X_batch, y_batch in tqdm(zip(X, y)):\n",
    "                y_pred = self._model(**X_batch)\n",
    "                loss = criterion(y_pred.logits, y_batch)\n",
    "                losses.append(loss.item())\n",
    "\n",
    "                optimizer.zero_grad()\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "\n",
    "            print(np.mean(losses))\n",
    "\n",
    "    def predict(self, turn0, turn1):\n",
    "        string = turn0 + self._tokenizer.sep_token + turn1\n",
    "        X = self._tokenize([string])\n",
    "        y = self._model(**X).logits.cpu().detach().numpy()\n",
    "        return self._id2label[np.argmax(y[0])]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0d-VTuqUqpdB",
    "outputId": "0637dc73-9085-49b2-d945-5503b121e879"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "342it [02:00,  2.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.4480896588654546\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "342it [02:00,  2.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8076535619316045\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "121it [00:42,  3.40it/s]"
     ]
    }
   ],
   "source": [
    "dt = DialogTag()\n",
    "dt.fit(data)\n",
    "\n",
    "with open(model_dir + '/DialogTag.pkl', 'wb') as file:\n",
    "    pickle.dump(dt, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1645278245591,
     "user": {
      "displayName": "Thomas Bellucci",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "13668310691905895775"
     },
     "user_tz": -60
    },
    "id": "pK1Ls0xnvsiR",
    "outputId": "367393ac-ce60-40dd-afbd-87924ac41595"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'opening'"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt.predict('hello', 'hi')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CtAwI2F2yIpY"
   },
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 365,
     "status": "ok",
     "timestamp": 1645278299897,
     "user": {
      "displayName": "Thomas Bellucci",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "13668310691905895775"
     },
     "user_tz": -60
    },
    "id": "IsUq9olmyKCC",
    "outputId": "9ccc9fff-50d2-4546-d169-af69084f4e2d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('i think i heard wrong', 'top gun', 'statement'),\n",
       " (\"don't give up on me\", \"i'm not giving up on you\", 'statement'),\n",
       " ('do you mind tell me one more time',\n",
       "  'who is the piano',\n",
       "  'open_question_factual')]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_test = 'https://raw.githubusercontent.com/DianDYu/MIDAS_dialog_act/throw_exception_on_example_format_error/da_data/dev.txt'\n",
    "raw_test_data = pd.read_csv(url_test, sep='\\n', header=None).values.flatten().tolist()\n",
    "\n",
    "test_data = []\n",
    "for line in raw_test_data:\n",
    "    for item in split_line(line):\n",
    "        test_data.append(item)\n",
    "test_data[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 39739,
     "status": "ok",
     "timestamp": 1645278346226,
     "user": {
      "displayName": "Thomas Bellucci",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "13668310691905895775"
     },
     "user_tz": -60
    },
    "id": "VWtHV_hTzA7l",
    "outputId": "53121c00-e643-44ca-c0c2-36ed51d78767"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2529/2529 [00:39<00:00, 63.68it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7030446816923686"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "true_labels = [l[2] for l in test_data]\n",
    "pred_labels = []\n",
    "for t0, t1, _ in tqdm(test_data):\n",
    "    pred_labels.append(dt.predict(t0, t1))\n",
    "\n",
    "acc = np.mean([1 if y0 == y1 else 0 for y0, y1 in zip(true_labels, pred_labels)])\n",
    "acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5JPfLmPmMZH2"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyMfsxTJX6VSqeVIk5+tQzdd",
   "collapsed_sections": [],
   "name": "Dialogue Act Tagger (for dialogue analysis).ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}