Languange_Model_Embedding_.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Languange-Model-Embedding .ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gWarRRbl_Hi2",
        "outputId": "201c510d-ee89-495b-9f52-66fcdb2a5bf2"
      },
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F8xN0uId_Zo2",
        "outputId": "c2d042b9-1c42-4ba6-b833-cf806db273c1"
      },
      "source": [
        "!pip install transformers\n",
        "!pip install sentencepiece"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ed/d5/f4157a376b8a79489a76ce6cfe147f4f3be1e029b7144fa7b8432e8acb26/transformers-4.4.2-py3-none-any.whl (2.0MB)\n",
            "\u001b[K     |████████████████████████████████| 2.0MB 5.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 20.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/23/2ddc317b2121117bf34dd00f5b0de194158f2a44ee2bf5e47c7166878a97/tokenizers-0.10.1-cp37-cp37m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 29.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp37-none-any.whl size=893262 sha256=a64900263a09a299c11edaab97db38a9ba5ee58305637f50b820bdae8ad99e95\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.10.1 transformers-4.4.2\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 4.5MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.95\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vcSlIbIH_UkU",
        "outputId": "5c20a756-1019-42f9-fcf7-ea9aec902d4e"
      },
      "source": [
        "from transformers import BertModel, BertTokenizer, XLNetModel, XLNetTokenizer\n",
        "\n",
        "import torch\n",
        "\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "from nltk.corpus import stopwords\n",
        "from nltk import PorterStemmer, WordNetLemmatizer\n",
        "\n",
        "import pickle\n",
        "import re"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "12Nrmfqh_M6t"
      },
      "source": [
        "with open('./multilabelbinarizer.pickle', 'rb') as file:\n",
        "  mlb = pickle.load(file)\n",
        "with open('./train.pickle', 'rb') as file:\n",
        "  train_data = pickle.load(file)\n",
        "with open('./test.pickle', 'rb') as file:\n",
        "  test_data = pickle.load(file)\n",
        "  "
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XcTvP39s0M8P"
      },
      "source": [
        "#Text Cleaning Function"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kwkPzvPR0REH"
      },
      "source": [
        "##Full Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tZRSiDYNQLSr"
      },
      "source": [
        "stop = stopwords.words('english')\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "\n",
        "def preprocessingTextFull(text, stop=stop):\n",
        "  text = text.lower() #text to lowercase\n",
        "  text = re.sub(r'&lt;', '', text) #remove '&lt;' tag\n",
        "  text = re.sub(r'<.*?>', '', text) #remove html\n",
        "  text = re.sub(r'[0-9]+', '', text) #remove number\n",
        "  text = \" \".join([word for word in text.split() if word not in stop]) #remove stopwords\n",
        "  text = re.sub(r'[^\\w\\s]', '', text) #remove punctiation\n",
        "  text = re.sub(r'[^\\x00-\\x7f]', '', text) #remove non ASCII strings\n",
        "  for c in ['\\r', '\\n', '\\t'] :\n",
        "    text = re.sub(c, ' ', text) #replace newline and tab with tabs\n",
        "  text = re.sub('\\s+', ' ', text) #replace multiple spaces with one space\n",
        "  text = ' '.join([lemmatizer.lemmatize(word) for word in text.split()])\n",
        "  return text"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6PGpkk5c0fIA"
      },
      "source": [
        "##Minimum Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Utm8UO470lsQ"
      },
      "source": [
        "def preprocessingTextMinimum(text, stop=stop):\n",
        "  text = text.lower() #text to lowercase\n",
        "  text = re.sub(r'<.*?>', '', text) #remove html\n",
        "  text = re.sub(r'&lt;', '', text) #remove '&lt;' tag\n",
        "  text = re.sub(r'>', '', text) #remove < sign\n",
        "  text = re.sub(r'[0-9]+', '', text) #remove number\n",
        "  text = re.sub(r'[^\\x00-\\x7f]', '', text) #remove non ASCII strings\n",
        "  for c in ['\\r', '\\n', '\\t'] :\n",
        "    text = re.sub(c, ' ', text) #replace newline and tab with tabs\n",
        "  text = re.sub('\\s+', ' ', text) #replace multiple spaces with one space\n",
        "\n",
        "  return text"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YPU63GDo0nXC"
      },
      "source": [
        "## Minimum Preprocessing + SEP token"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z1f3q8FA1DQJ"
      },
      "source": [
        "def preprocessingTextMinimumWithSEP(text, sep, stop=stop):\n",
        "  text = text.lower() #text to lowercase\n",
        "  text = re.sub(r'<.*?>', '', text) #remove html\n",
        "  text = re.sub(r'&lt;', '', text) #remove '&lt;' tag\n",
        "  text = re.sub(r'>', '', text) #remove < sign\n",
        "  title_len = text.find('\\n')\n",
        "  text = text[:title_len] + ' ' + sep + ' ' + text[title_len+1:]\n",
        "  text = re.sub(r'\\.\\n', '. '+sep+' ',text)\n",
        "  text = re.sub(r'[0-9]+', '', text) #remove number\n",
        "  text = re.sub(r'[^\\x00-\\x7f]', '', text) #remove non ASCII strings\n",
        "  for c in ['\\r', '\\n', '\\t'] :\n",
        "    text = re.sub(c, ' ', text) #replace newline and tab with tabs\n",
        "  text = re.sub('\\s+', ' ', text) #replace multiple spaces with one space\n",
        "\n",
        "  return text"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qrAlLjp2z96e"
      },
      "source": [
        "#Embedding Function"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lpPjU_vo0A7l"
      },
      "source": [
        "##Embedding Text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mzpOV67NWJtf"
      },
      "source": [
        "def embedding_text(list_of_text, model, tokenizer, seq_len=128, mode='all'):\n",
        "\n",
        "  device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "  model.eval()\n",
        "  \n",
        "  model = model.to(device)\n",
        "  \n",
        "  embedding_features = torch.zeros(len(list_of_text), seq_len, 768)\n",
        "  print(len(list_of_text), seq_len)\n",
        "  \n",
        "  for index, text in enumerate(list_of_text):\n",
        "    token = tokenizer.encode_plus(text,\n",
        "                                  max_length=seq_len,\n",
        "                                  pad_to_multiple_of=seq_len,\n",
        "                                  padding=True,\n",
        "                                  truncation=True)\n",
        "\n",
        "    token_input = torch.tensor(token['input_ids']).unsqueeze(0).to(device)\n",
        "    token_mask = torch.tensor(token['attention_mask']).unsqueeze(0).to(device)\n",
        "    \n",
        "    with torch.no_grad():\n",
        "      if mode == 'all' :\n",
        "        result = model(token_input)['hidden_states'][1:]\n",
        "        result = torch.stack(result)\n",
        "        result = result.squeeze(1)\n",
        "        result = result.permute(1, 0, 2).sum(1)\n",
        "\n",
        "      elif mode == 'last4':\n",
        "        result = model(token_input, token_mask)['hidden_states'][-4:]\n",
        "        result = torch.stack(result)\n",
        "        result = result.squeeze(1)\n",
        "        result = result.permute(1, 0, 2).sum(1)\n",
        "      \n",
        "      elif mode == 'last':\n",
        "        result = model(token_input, token_mask)['hidden_states'][12]\n",
        "        result = result.squeeze(0)\n",
        "      \n",
        "      else:\n",
        "        print('The mode is not recognized')\n",
        "        break\n",
        "    \n",
        "    embedding_features[index] = result.cpu()\n",
        "  \n",
        "  return embedding_features\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jXnFjySl0FLF"
      },
      "source": [
        "##Embedding Label"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0L_rj3rgwVwT"
      },
      "source": [
        "def embedding_label(list_of_label, model, tokenizer, mode='all'):\n",
        "  device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "  model.eval()\n",
        "  \n",
        "  model = model.to(device)\n",
        "  \n",
        "  embedding_features = torch.zeros(len(list_of_label), 768)\n",
        "  \n",
        "  for index, text in enumerate(list_of_label):\n",
        "\n",
        "    text = ' '.join(text.split('-'))\n",
        "    token = tokenizer.encode(text)\n",
        "    token = torch.tensor(token).unsqueeze(0).to(device)\n",
        "    \n",
        "    with torch.no_grad():\n",
        "      if mode == 'all' :\n",
        "        result = model(token)['hidden_states'][1:]\n",
        "        result = torch.stack(result)\n",
        "        result = result.squeeze(1)\n",
        "        if model.name_or_path == 'bert-base-cased':\n",
        "          result = result.permute(1, 0, 2)[1:-1].sum(1).mean(0)\n",
        "        elif model.name_or_path == 'xlnet-base-cased':\n",
        "          result = result.permute(1, 0, 2)[:-2].sum(1).mean(0)\n",
        "\n",
        "      elif mode == 'last4':\n",
        "        result = model(token)['hidden_states'][-4:]\n",
        "        result = torch.stack(result)\n",
        "        result = result.squeeze(1)\n",
        "        if model.name_or_path == 'bert-base-cased':\n",
        "          result = result.permute(1, 0, 2)[1:-1].sum(1).mean(0)\n",
        "        elif model.name_or_path == 'xlnet-base-cased':\n",
        "          result = result.permute(1, 0, 2)[:-2].sum(1).mean(0)\n",
        "      \n",
        "      elif mode == 'last':\n",
        "        result = model(token)['hidden_states'][12]\n",
        "        result = result.squeeze(0)\n",
        "        if model.name_or_path == 'bert-base-cased':\n",
        "          result = result[1:-1].mean(0)\n",
        "        elif model.name_or_path == 'xlnet-base-cased':\n",
        "          result = result[:-2].mean(0)\n",
        "      \n",
        "      else:\n",
        "        print('The mode is not recognized')\n",
        "        break\n",
        "    \n",
        "    embedding_features[index] = result.cpu()\n",
        "  \n",
        "  return embedding_features"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7AaoT7Mc1HWT"
      },
      "source": [
        "#Initialize Language Model and Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BtXJA1KYEAoW",
        "outputId": "48870bce-e037-43d4-d308-87e1a36305e3"
      },
      "source": [
        "#Uncomment model configuration that you need\n",
        "\n",
        "# model = BertModel.from_pretrained('bert-base-cased', output_hidden_states=True)\n",
        "# tokenizer = BertTokenizer.from_pretrained('bert-base-cased')\n",
        "# sep = '[SEP]'\n",
        "\n",
        "model = XLNetModel.from_pretrained('xlnet-base-cased', output_hidden_states=True)\n",
        "tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')\n",
        "sep = '<sep>'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-05c292ece247>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# sep = '[SEP]'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mXLNetModel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'xlnet-base-cased'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_hidden_states\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0mtokenizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mXLNetTokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'xlnet-base-cased'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0msep\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'<sep>'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'XLNetModel' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lx8PcemY1806"
      },
      "source": [
        "#Clean Text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Met1KvcJE43D"
      },
      "source": [
        "#Uncomment one pair train and test data preprocessing that you need\n",
        "\n",
        "# train_text = [preprocessingTextFull(text) for text in train_data.text.values]\n",
        "# test_text = [preprocessingTextFull(text) for text in test_data.text.values]\n",
        "\n",
        "train_text = [preprocessingTextMinimum(text) for text in train_data.text.values]\n",
        "test_text = [preprocessingTextMinimum(text) for text in test_data.text.values]\n",
        "\n",
        "# train_text = [preprocessingTextMinimumWithSEP(text, sep=sep) for text in train_data.text.values]\n",
        "# test_text = [preprocessingTextMinimumWithSEP(text, sep=sep) for text in test_data.text.values]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R1uPjI5T2sMI"
      },
      "source": [
        "#Generate Text embedding and Label Embedding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QrtsWUYe21lk",
        "outputId": "a3451fb8-44da-4563-99c7-ef5ba5c3bf3b"
      },
      "source": [
        "seq_len = 128\n",
        "mode = 'all'\n",
        "save_path = './'\n",
        "preprocessing = 'full' #choose how to name your file indicating text cleaning process\n",
        "\n",
        "torch.save(embedding_text(train_text, model, tokenizer, seq_len=seq_len, mode=mode),\n",
        "           save_path+('train-' + model.name_or_path + '-' + str(seq_len) + '-' +preprocessing+'.pt'))\n",
        "torch.save(embedding_text(test_text, model, tokenizer, seq_len=seq_len, mode=mode),\n",
        "           save_path+('test-' + model.name_or_path + '-' + str(seq_len) + '-' +preprocessing+'.pt'))\n",
        "torch.save(embedding_label(mlb.classes_, model, tokenizer, mode=mode),\n",
        "           save_path+('label-embedding-' + model.name_or_path +'.pt'))\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "7769 128\n",
            "3019 128\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}