Open_CyKG_OIE_Model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Github Open CyKG: OIE Attention model ",
      "provenance": [],
      "collapsed_sections": [
        "ogjLEQVoBE4i"
      ],
      "toc_visible": true,
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rCoxTOMFhAAV"
      },
      "source": [
        "# Block 1: Installtions and drives"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XiWr4Nd8bIXt"
      },
      "source": [
        "block 1: installations and drives"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DFWCB5RT_hFl"
      },
      "source": [
        "!pip install -U -q PyDrive\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YNM0n6cu_mo0"
      },
      "source": [
        "your_module = drive.CreateFile({'id':'1YlZBlvViO4Iz0xU-6aPS5lUzL6O0nxHg'})\n",
        "\n",
        "your_module.GetContentFile('spacy_wrapper.py')\n",
        "\n",
        "your_module2= drive.CreateFile({'id':'1l8Sd_CrGY8Mk6hy9uhwGZnkrKRwSACX5'})\n",
        "\n",
        "your_module2.GetContentFile('symbols.py')\n",
        "\n",
        "your_module3= drive.CreateFile({'id':'1X6gc4BOll6iweDl7gFs4PmPi5w514Qhg'})\n",
        "\n",
        "your_module3.GetContentFile('load_pretrained_word_embeddings.py')\n",
        "\n",
        "your_module4= drive.CreateFile({'id':'1XhgGTsqee49it6xdLdAa69cWNVNGNdX5'})\n",
        "\n",
        "your_module4.GetContentFile('word_index.py')\n",
        "\n",
        "your_module5= drive.CreateFile({'id':'1NO71gAICxxpe3JfRcvTFicQIhUiHpIPZ'})\n",
        "\n",
        "your_module5.GetContentFile('elmo.py')\n",
        "\n",
        "your_module6= drive.CreateFile({'id':'1xUn888UMox08U2dxgBR1WmULPKQ0jyzB'})\n",
        "\n",
        "your_module6.GetContentFile('utils.py')\n",
        "\n",
        "your_module7= drive.CreateFile({'id':'1Y3pdrMF-5cha__aG3MKHkj0ydhBO0BDX'})\n",
        "\n",
        "your_module7.GetContentFile('Alayers.py')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fr1m72Nfq29k",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "23d8317d-b194-43ac-ec7b-89b0542a7532"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "import keras\n",
        "import os\n",
        "import sys\n",
        "import numpy as np\n",
        "import random\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Conv1D, GlobalMaxPooling1D, GlobalAveragePooling1D, Activation, Dropout\n",
        "from typing import List\n",
        "import torch\n",
        "import os\n",
        "import re\n",
        "torch.cuda.current_device(),torch.cuda.device_count(),torch.cuda.is_available()\n",
        "!pip install tiny-tokenizer flair\n",
        "!pip install flair\n",
        "import nltk\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "\n",
        "from flair.embeddings import XLNetEmbeddings\n",
        "from flair.data_fetcher import NLPTaskDataFetcher, NLPTask\n",
        "from flair.data import  Sentence, Token\n",
        "from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, \\\n",
        "    CharLMEmbeddings\n",
        "from typing import List\n",
        "import torch\n",
        "import os\n",
        "import re\n",
        "from flair.embeddings import BertEmbeddings\n",
        "from flair.embeddings import XLMRobertaEmbeddings\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "juP3_bavhVt5"
      },
      "source": [
        "# Block 2: Creating XLNet Embeddings Function"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I4rbqyi4bOmK"
      },
      "source": [
        "Block 2: Creating XLNet embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CL3tZkILVZen"
      },
      "source": [
        "xln = BertEmbeddings()\n",
        "emb_size= 2048\n",
        "# init embedding\n",
        "xln = XLMRobertaEmbeddings()\n",
        "emb_size= 1024 \n",
        "# init embedding\n",
        "from flair.embeddings import WordEmbeddings\n",
        "# init embedding\n",
        "xln = WordEmbeddings('glove')\n",
        "emb_size= 100"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ejv7vHoSY26t"
      },
      "source": [
        "def create_XLNET_embeddings(elmo, documents, max_sentences = 10):\n",
        "    embeddings = []\n",
        "    labels = []\n",
        "    xln = XLMRobertaEmbeddings()\n",
        "    emb_size= 1024 \n",
        "    while True:\n",
        "      for row in documents:\n",
        "          my_sent = row\n",
        "          sentence = Sentence(str(my_sent))\n",
        "          xln.embed(sentence)\n",
        "          \n",
        "          x = []\n",
        "          for token in sentence:\n",
        "            x.append(token.embedding.cpu().detach().numpy())\n",
        "            if len(x) == max_sentences:\n",
        "              break\n",
        "          \n",
        "          while len(x) < max_sentences:\n",
        "            x.append(np.zeros(emb_size))          \n",
        "          embeddings.append(x)            \n",
        "      return np.array(embeddings)\n",
        "    \n",
        "    return embeddings        "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zYv_ZntIiWKC"
      },
      "source": [
        "# Block 3: All functions + Hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gl0vhCO_wBCB"
      },
      "source": [
        "#-----------Disclaimer: Most of the functions here are originally from with some modifications --> Stanvosky et al. Please refer/cite his work: Stanovsky, Gabriel, et al. \"Supervised open information extraction.\" Proceedings of the 2018 Conference \n",
        "#of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers). 2018. ---------------\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn import preprocessing\n",
        "from keras.utils import np_utils\n",
        "import pandas as pd\n",
        "\n",
        "\n",
        "sent_maxlen = 20\n",
        "classes = None\n",
        "encoder =  LabelEncoder()\n",
        "batch_size = 5\n",
        "epochs=10\n",
        "emb_dropout = 0.1\n",
        "trainable_emb = True\n",
        "pos_tag_embedding_size = 5\n",
        "num_of_latent_layers = 2\n",
        "hidden_units = pow(2, 7)\n",
        "pred_dropout=0.1 \n",
        "class Sample:\n",
        "    \"\"\"\n",
        "    Single sample representation.\n",
        "    Containter which names spans in the input vector to simplify access\n",
        "    \"\"\"\n",
        "    def __init__ (self, word):\n",
        "        self.word = word\n",
        "\n",
        "    def encode(self):\n",
        "        \"\"\"\n",
        "        Encode this sample as vector as input for rnn,\n",
        "        Probably just concatenating members in the right order.\n",
        "        \"\"\"\n",
        "        return self.word\n",
        "      \n",
        "class Pad_sample(Sample):\n",
        "    \"\"\"\n",
        "    A dummy sample used for padding\n",
        "    \"\"\"\n",
        "    #Sample( word = 0 )\n",
        "    def __init__(self):\n",
        "        Sample.__init__(self, word = 0)    \n",
        "        \n",
        "def pad_sequences(sequences, pad_func, maxlen = None):\n",
        "    \"\"\"\n",
        "    Similar to keras.preprocessing.sequence.pad_sequence but using Sample as higher level\n",
        "    abstraction.\n",
        "    pad_func is a pad class generator.\n",
        "    \"\"\"\n",
        "    ret = []\n",
        "\n",
        "    max_value = max(map(len, sequences))\n",
        "    if maxlen is None:\n",
        "        maxlen = max_value\n",
        "\n",
        "    # Pad / truncate (done this way to deal with np.array)\n",
        "    for sequence in sequences:\n",
        "        cur_seq = list(sequence[:maxlen])\n",
        "        cur_seq.extend([pad_func()] * (maxlen - len(sequence)))\n",
        "        ret.append(cur_seq)\n",
        "    return ret\n",
        "\n",
        "\n",
        "def classes_():    \n",
        "        \"\"\"\n",
        "        Return the classes which are classified by this model\n",
        "        \"\"\"\n",
        "        return encoder.classes_     \n",
        "\n",
        "def num_of_classes():\n",
        "        \"\"\"\n",
        "        Return the number of ouput classes\n",
        "        \"\"\"\n",
        "        return len(classes_())\n",
        "        \n",
        "def load_dataset(fn,T):\n",
        "      \n",
        "      df = pandas.read_csv(fn,sep= \"\\t\",header=0,keep_default_na=False)\n",
        "\n",
        "      if T==\"train\":\n",
        "        encoder.fit(df.label.values)\n",
        "      else: \n",
        "            df = pandas.read_csv(fn,\n",
        "                         sep= \"\\t\",\n",
        "                         header=0,\n",
        "                         keep_default_na=False)\n",
        "      sents = get_sents_from_df(df)\n",
        "           \n",
        "      return (encode_outputs(sents))  \n",
        "    \n",
        "def load_datasetTestRE(fn,T):\n",
        "    \n",
        "      if T==\"test_RE\":\n",
        "          df = pandas.read_csv(fn,\n",
        "                         sep= \";\",\n",
        "                         header=0,\n",
        "                         keep_default_na=False)\n",
        "          df.drop(df.columns[df.columns.str.contains('unnamed',case = False)],axis = 1, inplace = True)\n",
        "          df.word_id = pd.to_numeric(df.word_id, errors='coerce').astype('Int64')\n",
        "          df.run_id = pd.to_numeric(df.run_id, errors='coerce').astype('Int64')\n",
        "          df.sent_id = pd.to_numeric(df.sent_id, errors='coerce').astype('Int64')\n",
        "          df.head_pred_id = pd.to_numeric(df.head_pred_id, errors='coerce').astype('Int64')\n",
        "          df = df[df.word.apply(lambda x : len(x)>0)] \n",
        "      else:\n",
        "            df = pandas.read_csv(fn,\n",
        "                         sep= \"\\t\",\n",
        "                         header=0,\n",
        "                         keep_default_na=False)\n",
        "     \n",
        "      if T==\"train\":\n",
        "        encoder.fit(df.label.values)\n",
        "        \n",
        "      sents = get_sents_from_df(df)    \n",
        "      return (encode_outputs(sents))\n",
        "  \n",
        "def load_dataset_encodeinputs(fn,T):\n",
        "      if \"RE\" in T:\n",
        "          df = pandas.read_csv(fn,\n",
        "                         sep= \";\",\n",
        "                         header=0,\n",
        "                         keep_default_na=False)\n",
        "          df.drop(df.columns[df.columns.str.contains('unnamed',case = False)],axis = 1, inplace = True)\n",
        "          df.word_id = pd.to_numeric(df.word_id, errors='coerce').astype('Int64')\n",
        "          df.run_id = pd.to_numeric(df.run_id, errors='coerce').astype('Int64')\n",
        "          df.sent_id = pd.to_numeric(df.sent_id, errors='coerce').astype('Int64')\n",
        "          df.head_pred_id = pd.to_numeric(df.head_pred_id, errors='coerce').astype('Int64')\n",
        "          df = df[df.word.apply(lambda x : len(x)>0)] #to skip null values\n",
        "\n",
        "      else:\n",
        "        df = pandas.read_csv(fn,sep= \"\\t\",header=0,keep_default_na=False)\n",
        "\n",
        "      if T==\"train\" or T==\"POS_train\" or T==\"train_xlnet_words\" or T==\"train_xlnet_pred\":\n",
        "        encoder.fit(df.label.values)\n",
        "      \n",
        "      \n",
        "      sents = get_sents_from_df(df)\n",
        "      if T==\"POS_train\" or T==\"POS\" or T== \"dev_xlnet_pos\" or T==\"test_xlnet_pos_RE\" or T==\"test_xlnet_pos_\": \n",
        "        return getPOS(sents)\n",
        "      if  T==\"train_xlnet_words\" or T==\"train_xlnet_pred\" or T== \"dev_xlnet_words\" or T== \"dev_xlnet_pred\" or T==\"test_xlnet_words_RE\" or T==\"test_xlnet_pred_RE\" or T==\"test_xlnet_words_\" or T==\"test_xlnet_pred_\"  : #13 july 2020 added this or T==\"test_xlnet_words_\" or T==\"test_xlnet_pred_\" :\n",
        "          return encodeXLnetInputs(sents,T)\n",
        "      return (encode_inputs(sents))    \n",
        "    \n",
        "def get_sents_from_df( df):\n",
        "      #Split a data frame by rows accroding to the sentences\n",
        "      return [df[df.run_id == run_id]\n",
        "            for run_id\n",
        "            in sorted(set(df.run_id.values))]  \n",
        "def transform_labels(labels):\n",
        "        \"\"\"\n",
        "        Encode a list of textual labels\n",
        "        \"\"\"\n",
        "        # Fallback:\n",
        "        classes  = list(classes_())\n",
        "        return [classes.index(label) for label in labels]\n",
        "\n",
        "  \n",
        "def get_fixed_size(sents):\n",
        "        \"\"\"\n",
        "        Partition sents into lists of sent_maxlen elements\n",
        "        (execept the last in each sentence, which might be shorter)\n",
        "        \"\"\"\n",
        "        return [sent[s_ind : s_ind + sent_maxlen]\n",
        "                for sent in sents\n",
        "                for s_ind in range(0, len(sent), sent_maxlen)]      \n",
        "\n",
        "    \n",
        "def encode_outputs(sents):\n",
        "        output_encodings = []\n",
        "        sents = get_fixed_size(sents)\n",
        "        # Encode outputs\n",
        "        for sent in sents:\n",
        "            output_encodings.append(list(np_utils.to_categorical(list(transform_labels(sent.label.values)),\n",
        "                                                                 num_classes = num_of_classes())))\n",
        "        \n",
        "        # Pad / truncate to maximum length\n",
        "      \n",
        "        return np.ndarray(shape = (len(sents),\n",
        "                                  sent_maxlen,\n",
        "                                  num_of_classes()),\n",
        "                          buffer = np.array(pad_sequences(output_encodings,\n",
        "                                                          lambda : \\\n",
        "                                                            np.zeros(num_of_classes()),\n",
        "                                                          maxlen = sent_maxlen)))\n",
        "def get_head_pred_word( full_sent):\n",
        "       \n",
        "        \"\"\"\n",
        "        Get the head predicate word from a full sentence conll.\n",
        "        \"\"\"\n",
        "        assert(len(set(full_sent.head_pred_id.values)) == 1) # Sanity check\n",
        "        pred_ind = full_sent.head_pred_id.values[0]\n",
        "        \n",
        "        return full_sent.word.values[pred_ind] \\\n",
        "            if pred_ind != -1 \\\n",
        "               else full_sent.pred.values[0].split(\" \")[0]\n",
        "def predict_classes():\n",
        "        \"\"\"\n",
        "        Predict to the number of classes\n",
        "        Named arguments are passed to the keras function\n",
        "        \"\"\"\n",
        "        return lambda x: stack(x,\n",
        "                                    [lambda : TimeDistributed(Dense(output_dim = num_of_classes(),\n",
        "                                                                    activation = \"softmax\"))] +\n",
        "                                    [lambda : TimeDistributed(Dense(hidden_units,\n",
        "                                                                    activation='relu'))] * 3)\n",
        "        \n",
        "def encode_inputs(sents):\n",
        "        word_inputs = []\n",
        "        pred_inputs = []\n",
        "        pos_inputs = []\n",
        "    \n",
        "        assert(all([len(set(sent.run_id.values)) == 1\n",
        "                    for sent in sents]))        \n",
        "         \n",
        "        run_id_to_pred = dict([(int(sent.run_id.values[0]),\n",
        "                                get_head_pred_word(sent))\n",
        "                               for sent in sents])\n",
        "        # Construct a mapping from running word index to pos\n",
        "        word_id_to_pos = {}\n",
        "        for sent in sents:\n",
        "            indices = sent.index.values\n",
        "            words = sent.word.values\n",
        "            \n",
        "            for index, word in zip(indices,\n",
        "                                    spacy_ws(\" \".join(words))):\n",
        "                word_id_to_pos[index] = word.tag_  #tag_ get the tag of the word by spacy\n",
        "        \n",
        "        \n",
        "        fixed_size_sents = get_fixed_size(sents)\n",
        "          \n",
        "\n",
        "        for sent in fixed_size_sents:\n",
        "\n",
        "            assert(len(set(sent.run_id.values)) == 1)\n",
        "\n",
        "            word_indices = sent.index.values\n",
        "            sent_words = sent.word.values\n",
        "\n",
        "            \n",
        "            pos_tags_encodings = [p for (x,p) in nltk.pos_tag(sent_words)]\n",
        "            word_encodings = [ w for w in sent_words]  \n",
        "            pred_word = run_id_to_pred[int(sent.run_id.values[0])]\n",
        "                                    \n",
        "            pred_word_encodings = [pred_word]\n",
        "            \n",
        "            word_inputs.append([Sample(w) for w in word_encodings])\n",
        "            pred_inputs.append([Sample(w) for w in pred_word_encodings])\n",
        "            pos_inputs.append([Sample(pos) for pos in pos_tags_encodings])\n",
        "          \n",
        "        ret = defaultdict(lambda: [])\n",
        "         \n",
        "        for name, sequence in zip([\"word_inputs\", \"predicate_inputs\",\"postags_inputs\"],\n",
        "                                  [word_inputs, pred_inputs,pos_inputs]): \n",
        "            for samples in pad_sequences(sequence,\n",
        "                                         pad_func = lambda : Pad_sample(),\n",
        "                                         maxlen = sent_maxlen):\n",
        "                ret[name].append([sample.encode() for sample in samples])\n",
        "        injy={k: np.array(v) for k, v in ret.items()} \n",
        "        return {k: np.array(v) for k, v in ret.items()} \n",
        "\n",
        "def getPOS(sents):\n",
        "        word_inputs = []\n",
        "        pred_inputs = []\n",
        "        pos_inputs = []\n",
        "        assert(all([len(set(sent.run_id.values)) == 1\n",
        "                    for sent in sents]))        \n",
        "         \n",
        "        run_id_to_pred = dict([(int(sent.run_id.values[0]),\n",
        "                                get_head_pred_word(sent))\n",
        "                               for sent in sents])\n",
        "        \n",
        "        # Construct a mapping from running word index to pos\n",
        "        word_id_to_pos = {}\n",
        "        for sent in sents:\n",
        "            indices = sent.index.values\n",
        "            words = sent.word.values\n",
        "       \n",
        "            for index, word in zip(indices,\n",
        "                                    spacy_ws(\" \".join(words))):\n",
        "                word_id_to_pos[index] = word.tag_  #tag_ get the tag of the word by spacy\n",
        "        \n",
        "        fixed_size_sents = get_fixed_size(sents)\n",
        "    \n",
        "        allPosEncoding=[] \n",
        "\n",
        "        for sent in fixed_size_sents:\n",
        "\n",
        "            assert(len(set(sent.run_id.values)) == 1)\n",
        "\n",
        "            word_indices = sent.index.values\n",
        "            sent_words = sent.word.values\n",
        "             \n",
        "            pos_tags_encodings = [p for (x,p) in nltk.pos_tag(sent_words)]\n",
        "            word_encodings = [ w for w in sent_words]  \n",
        "    \n",
        "            pred_word = run_id_to_pred[int(sent.run_id.values[0])]\n",
        "                                    \n",
        "            pred_word_encodings = [pred_word]\n",
        "            \n",
        "            word_inputs.append([Sample(w) for w in word_encodings])\n",
        "            pred_inputs.append([Sample(w) for w in pred_word_encodings])\n",
        "            pos_inputs.append([Sample(pos) for pos in pos_tags_encodings])\n",
        "            allPosEncoding.append(pos_tags_encodings) \n",
        "        return allPosEncoding \n",
        "\n",
        "def encodeXLnetInputs(sents,T):     \n",
        "        word_inputs = []\n",
        "        pred_inputs = []\n",
        "        pos_inputs = []\n",
        "        assert(all([len(set(sent.run_id.values)) == 1\n",
        "                    for sent in sents]))        \n",
        "         \n",
        "        run_id_to_pred = dict([(int(sent.run_id.values[0]),\n",
        "                                get_head_pred_word(sent))\n",
        "                               for sent in sents])\n",
        "        \n",
        "        # Construct a mapping from running word index to pos\n",
        "        word_id_to_pos = {}\n",
        "        for sent in sents:\n",
        "            indices = sent.index.values\n",
        "            words = sent.word.values\n",
        "   \n",
        "   \n",
        "            for index, word in zip(indices,\n",
        "                                    spacy_ws(\" \".join(words))):\n",
        "                word_id_to_pos[index] = word.tag_  #tag_ get the tag of the word by spacy\n",
        "        \n",
        "  \n",
        "        fixed_size_sents = get_fixed_size(sents)\n",
        "        allPosEncoding=[] #19 jan 2020\n",
        "        allWordsEncodingPartial=[]\n",
        "        allWordsEncoding=[]\n",
        "        allPredEncodingPartial=[]\n",
        "        allPredEncoding=[]\n",
        "        for sent in fixed_size_sents:\n",
        "\n",
        "            assert(len(set(sent.run_id.values)) == 1)\n",
        "\n",
        "            word_indices = sent.index.values\n",
        "            sent_words = sent.word.values\n",
        "       \n",
        "            pos_tags_encodings = [p for (x,p) in nltk.pos_tag(sent_words)]\n",
        "            word_encodings = [ w for w in sent_words]  \n",
        "    \n",
        "            pred_word = run_id_to_pred[int(sent.run_id.values[0])]\n",
        "                                    \n",
        "            pred_word_encodings = [pred_word]\n",
        "            \n",
        "            word_inputs.append([Sample(w) for w in word_encodings])\n",
        "            pred_inputs.append([Sample(w) for w in pred_word_encodings])\n",
        "            pos_inputs.append([Sample(pos) for pos in pos_tags_encodings])\n",
        "            allPosEncoding.append(pos_tags_encodings) \n",
        "            allWordsEncodingPartial.append(word_encodings) #for Xlnet output is ['a','b'] need to make it space seperated using join\n",
        "            allPredEncodingPartial.append(pred_word_encodings)\n",
        "       \n",
        "        if T==\"train_xlnet_words\" or T== \"dev_xlnet_words\" or T==\"test_xlnet_words_RE\" or T==\"test_xlnet_words_\" :  \n",
        "          for sentence in allWordsEncodingPartial:\n",
        "            x=[' '.join(sentence)]\n",
        "            allWordsEncoding.append(x)\n",
        "          return allWordsEncoding   \n",
        "        elif  T==\"train_xlnet_pred\" or T== \"dev_xlnet_pred\"  or T==\"test_xlnet_pred_RE\" or T==\"test_xlnet_pred_\":  \n",
        "          for sentence in allPredEncodingPartial:\n",
        "            x=[' '.join(sentence)]\n",
        "            allPredEncoding.append(x)\n",
        "          return allPredEncoding "
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zMAVyzT7crNj"
      },
      "source": [
        "#Block 4: Train File--> Adjust it according to your dataset and embeddings\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KoAFE0aJ14IR"
      },
      "source": [
        "!python -m spacy download en\n",
        "import spacy\n",
        "spacy.load('en')\n",
        "import pandas\n",
        "import numpy as np\n",
        "from keras.layers import Reshape, Flatten\n",
        "from keras.models import load_model\n",
        "from keras.layers.core import Dropout, Activation\n",
        "import spacy\n",
        "from symbols import SPACY_POS_TAGS\n",
        "from collections import defaultdict\n",
        "import nltk\n",
        "from spacy_wrapper import spacy_whitespace_parser as spacy_ws \n",
        "from keras.layers.merge import concatenate, Concatenate,multiply\n",
        "from keras.layers import TimeDistributed, Input, Bidirectional, Dense, Embedding, LSTM, Conv1D,Conv2D, Conv3D, GlobalMaxPooling1D, RepeatVector, MaxPooling1D\n",
        "from keras.models import Model\n",
        "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
        "from keras import optimizers\n",
        "from keras.layers import Masking, Activation, Input\n",
        "\n",
        "#-------Train File-------------\n",
        "#train_fn=\"add you file here\"\n",
        "\n",
        "dfE = pandas.read_csv(train_fn, sep= \"\\t\",\n",
        "                         header=0,\n",
        "                         keep_default_na=False)\n",
        "\n",
        "sents = get_sents_from_df(dfE)\n",
        "\n",
        "train_textEI = dfE.groupby(dfE['word_id'].eq(0).cumsum())['word'].apply(lambda x: [' '.join(x)]).tolist()\n",
        "train_wordsTokens=[]\n",
        "\n",
        "for i in train_textEI:\n",
        "  for j in i:\n",
        "    train_wordsTokens.append(j.split())\n",
        "\n",
        "train_wordsTokens_trunc=[]\n",
        "for i in  train_wordsTokens:\n",
        "  train_wordsTokens_trunc.append(i[:sent_maxlen])\n",
        "\n",
        "train_predIE =  dfE.groupby(dfE['word_id'].eq(0).cumsum())['pred'].apply(lambda x: [' '.join(x)]).tolist()\n",
        "\n",
        "train_predTokens=[]\n",
        "for i in train_predIE:\n",
        "  for j in i:\n",
        "    train_predTokens.append(j.split())\n",
        "\n",
        "train_predTokens_trunc=[]\n",
        "for i in  train_wordsTokens:\n",
        "  train_predTokens_trunc.append(i[:sent_maxlen])\n",
        "\n",
        "Label_train=load_dataset(train_fn,\"train\")\n",
        "X_train_elmo=load_dataset_encodeinputs(train_fn,\"train\")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F6IPIjWfl1ZC"
      },
      "source": [
        "# Block 4.2: Calling fucntions to get XLNet inputs+ shapes"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Oe2joP3_SIxt"
      },
      "source": [
        "**The snippet below is used for setting up the inputs for Embeddings**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1eSoAzB0wrl0"
      },
      "source": [
        "X_train_xlnet_Words=load_dataset_encodeinputs(train_fn,\"train_xlnet_words\")\n",
        "mydiffLenSet=set() \n",
        "for sentence in X_train_xlnet_Words:\n",
        "  mydiffLenSet.add(len(sentence[0].split()))\n",
        "X_train_xlnet_pred=load_dataset_encodeinputs(train_fn,\"train_xlnet_pred\")\n",
        "train_posIE =  load_dataset_encodeinputs(train_fn,\"POS_train\") \n",
        "train_posIE[0:5]\n",
        "newTrain_posIE=[]\n",
        "count=0\n",
        "for sentence in train_posIE:\n",
        "   count+=1\n",
        "   newTrain_posIE.append(' '.join(sentence))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FnKKDYKknx0c"
      },
      "source": [
        "# **Block 5:** Dealing with padded sentences and then calling create_xlnet_embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D-2Fx3f3F_CP"
      },
      "source": [
        "for x in X_train_elmo['predicate_inputs']:\n",
        "  x[x == '0'] =  x[0]\n",
        "train_PredicateInjyXLNET= create_XLNET_embeddings(\"elmo\", X_train_xlnet_pred, sent_maxlen)\n",
        "for y in X_train_elmo['word_inputs']:\n",
        "    y[y == '0'] = y[0]\n",
        "train_WordsInjyXLNET= create_XLNET_embeddings(\"elmo\", X_train_xlnet_Words, sent_maxlen)\n",
        "for z in X_train_elmo['postags_inputs']:\n",
        "    z[z == '0'] = z[0]\n",
        "train_POSInjyXLNET= create_XLNET_embeddings(\"elmo\", train_posIE, sent_maxlen)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nxoJZG_3s-D7"
      },
      "source": [
        "# Block 6: Dev File "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RlnDt9beVENm"
      },
      "source": [
        "In case you have a validation set; can be added as a part of test set by splitting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6gjyMeKxLiEL"
      },
      "source": [
        "#-------Dev File-------------\n",
        "#dev_fn=\"add you dev data here\"\n",
        "Label_dev=load_dataset(dev_fn,\"d\")\n",
        "X_dev_Xlnet_words=load_dataset_encodeinputs(dev_fn,\"dev_xlnet_words\")\n",
        "X_dev_Xlnet_pred=load_dataset_encodeinputs(dev_fn,\"dev_xlnet_pred\")\n",
        "X_dev_Xlnet_pos=load_dataset_encodeinputs(dev_fn,\"dev_xlnet_pos\")\n",
        "dev_PredicateInjyXLNET= create_XLNET_embeddings(\"elmo\", X_dev_Xlnet_pred, sent_maxlen)\n",
        "dev_WordsInjyXLNET= create_XLNET_embeddings(\"elmo\", X_dev_Xlnet_words, sent_maxlen)\n",
        "dev_POSInjyXLNET= create_XLNET_embeddings(\"elmo\", X_dev_Xlnet_pos, sent_maxlen)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xZRbpK4u1ual"
      },
      "source": [
        "# Block 7: The Model (stacking layers)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZaJo0IZ_G0Xh"
      },
      "source": [
        "!pip install keras-attention\n",
        "!pip install keras-self-attention\n",
        "import tensorflow.python.keras.layers\n",
        "from tensorflow.keras.layers import Attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yhwOyskBHWKY"
      },
      "source": [
        "import tensorflow as tf\n",
        "import os\n",
        "from tensorflow.python.keras.layers import Layer\n",
        "from tensorflow.python.keras import backend as K\n",
        "from keras import layers\n",
        "import pandas\n",
        "import numpy as np\n",
        "from keras.layers import Reshape, Flatten\n",
        "from keras.models import load_model\n",
        "from keras.layers.core import Dropout, Activation\n",
        "from collections import defaultdict\n",
        "from keras.layers.merge import concatenate, Concatenate,multiply\n",
        "from keras.layers import TimeDistributed, Input, Bidirectional, Dense, Embedding, LSTM, Conv1D,Conv2D, Conv3D, GlobalMaxPooling1D, RepeatVector, MaxPooling1D\n",
        "from keras.models import Model\n",
        "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
        "from keras import optimizers\n",
        "from keras.layers import Masking, Activation, Input\n",
        "from keras.layers import Conv1D, MaxPooling1D, Embedding, concatenate, Dropout, LSTM, GRU, Bidirectional, TimeDistributed, Masking\n",
        "\n",
        "class AttentionLayer(Layer):\n",
        "\n",
        "    def __init__(self, **kwargs):\n",
        "        super(AttentionLayer, self).__init__(**kwargs)\n",
        "\n",
        "    def build(self, input_shape):\n",
        "        assert isinstance(input_shape, list)\n",
        "        # Create a trainable weight variable for this layer.\n",
        "\n",
        "        self.W_a = self.add_weight(name='W_a',\n",
        "                                   shape=tf.TensorShape((input_shape[0][2], input_shape[0][2])),\n",
        "                                   initializer='uniform',\n",
        "                                   trainable=True)\n",
        "        self.U_a = self.add_weight(name='U_a',\n",
        "                                   shape=tf.TensorShape((input_shape[1][2], input_shape[0][2])),\n",
        "                                   initializer='uniform',\n",
        "                                   trainable=True)\n",
        "        self.V_a = self.add_weight(name='V_a',\n",
        "                                   shape=tf.TensorShape((input_shape[0][2], 1)),\n",
        "                                   initializer='uniform',\n",
        "                                   trainable=True)\n",
        "\n",
        "        super(AttentionLayer, self).build(input_shape)  # Be sure to call this at the end\n",
        "\n",
        "    def call(self, inputs, verbose=False):\n",
        "        \"\"\"\n",
        "        inputs: [encoder_output_sequence, decoder_output_sequence]\n",
        "        \"\"\"\n",
        "        assert type(inputs) == list\n",
        "        encoder_out_seq, decoder_out_seq = inputs\n",
        "        if verbose:\n",
        "            print('encoder_out_seq>', encoder_out_seq.shape)\n",
        "            print('decoder_out_seq>', decoder_out_seq.shape)\n",
        "\n",
        "        def energy_step(inputs, states):\n",
        "\n",
        "            assert_msg = \"States must be an iterable. Got {} of type {}\".format(states, type(states))\n",
        "            assert isinstance(states, list) or isinstance(states, tuple), assert_msg\n",
        "\n",
        "            \"\"\" Some parameters required for shaping tensors\"\"\"\n",
        "            en_seq_len, en_hidden = encoder_out_seq.shape[1], encoder_out_seq.shape[2]\n",
        "            de_hidden = inputs.shape[-1]\n",
        "\n",
        "            \"\"\" Computing S.Wa where S=[s0, s1, ..., si]\"\"\"\n",
        "            W_a_dot_s = K.dot(encoder_out_seq, self.W_a)\n",
        "\n",
        "            \"\"\" Computing hj.Ua \"\"\"\n",
        "            U_a_dot_h = K.expand_dims(K.dot(inputs, self.U_a), 1)  # <= batch_size, 1, latent_dim\n",
        "            if verbose:\n",
        "                print('Ua.h>', U_a_dot_h.shape)\n",
        "\n",
        "            \"\"\" tanh(S.Wa + hj.Ua) \"\"\"\n",
        "            Ws_plus_Uh = K.tanh(W_a_dot_s + U_a_dot_h)\n",
        "            if verbose:\n",
        "                print('Ws+Uh>', Ws_plus_Uh.shape)\n",
        "\n",
        "            \"\"\" softmax(va.tanh(S.Wa + hj.Ua)) \"\"\"\n",
        "            # <= batch_size, en_seq_len\n",
        "            e_i = K.squeeze(K.dot(Ws_plus_Uh, self.V_a), axis=-1)\n",
        "            # <= batch_size, en_seq_len\n",
        "            e_i = K.softmax(e_i)\n",
        "\n",
        "            if verbose:\n",
        "                print('ei>', e_i.shape)\n",
        "\n",
        "            return e_i, [e_i]\n",
        "\n",
        "        def context_step(inputs, states):\n",
        "            \"\"\" Step function for computing ci using ei \"\"\"\n",
        "\n",
        "            assert_msg = \"States must be an iterable. Got {} of type {}\".format(states, type(states))\n",
        "            assert isinstance(states, list) or isinstance(states, tuple), assert_msg\n",
        "\n",
        "            # <= batch_size, hidden_size\n",
        "            c_i = K.sum(encoder_out_seq * K.expand_dims(inputs, -1), axis=1)\n",
        "            if verbose:\n",
        "                print('ci>', c_i.shape)\n",
        "            return c_i, [c_i]\n",
        "\n",
        "        fake_state_c = K.sum(encoder_out_seq, axis=1)\n",
        "        fake_state_e = K.sum(encoder_out_seq, axis=2)  \n",
        "\n",
        "        \"\"\" Computing energy outputs \"\"\"\n",
        "        last_out, e_outputs, _ = K.rnn(\n",
        "            energy_step, decoder_out_seq, [fake_state_e],\n",
        "        )\n",
        "\n",
        "        \"\"\" Computing context vectors \"\"\"\n",
        "        last_out, c_outputs, _ = K.rnn(\n",
        "            context_step, e_outputs, [fake_state_c],\n",
        "        )\n",
        "\n",
        "        return c_outputs, e_outputs\n",
        "\n",
        "    def compute_output_shape(self, input_shape):\n",
        "        \"\"\" Outputs produced by the layer \"\"\"\n",
        "        return [\n",
        "            tf.TensorShape((input_shape[1][0], input_shape[1][1], input_shape[1][2])),\n",
        "            tf.TensorShape((input_shape[1][0], input_shape[1][1], input_shape[0][1]))\n",
        "        ]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z3XPE_NQ1UdQ"
      },
      "source": [
        "emb_size=1024\n",
        "def stack_latent_layers(n):\n",
        "        return lambda x: stack(x, [lambda : Bidirectional(GRU(hidden_units,\n",
        "                                                                    return_sequences = True))] * n )\n",
        "def stack(x, layers):\n",
        "        if not layers:\n",
        "            return x \n",
        "        else:\n",
        "            return layers[0]()(stack(x, layers[1:]))\n",
        "def predict_classes():\n",
        "        return lambda x: stack(x,\n",
        "                                    [lambda : TimeDistributed(Dense(units = num_of_classes(),\n",
        "                                                                    activation = \"softmax\"))] +\n",
        "                                    [lambda : TimeDistributed(Dense(hidden_units,\n",
        "                                                                    activation='relu'))] * 3)\n",
        "  \n",
        "inputs = [Input((sent_maxlen,emb_size), dtype='float32', name='word_inputs'),\n",
        "            Input((sent_maxlen,emb_size), dtype='float32', name='predicate_inputs'),\n",
        "          Input((sent_maxlen,emb_size), dtype='float32', name='POS_inputs'),]\n",
        "\n",
        "embeddingsCon = [inputs[0], \n",
        "            inputs[1],\n",
        "                inputs[2]]\n",
        "\n",
        "con11 = keras.layers.concatenate(embeddingsCon)\n",
        "\n",
        "latent_layers = stack_latent_layers(num_of_latent_layers)\n",
        "bigru1=Bidirectional(LSTM(hidden_units, return_sequences = True))(con11)\n",
        "bigru2=Bidirectional(LSTM(hidden_units, return_sequences = True))(bigru1)\n",
        "attn_layer = AttentionLayer(name='attention_layer')\n",
        "attn_out, attn_states = attn_layer([bigru1, bigru2])\n",
        "dropout = Dropout(0.1)(attn_out)\n",
        "td=TimeDistributed(Dense(hidden_units,activation='relu'))(dropout)\n",
        "outputI=TimeDistributed(Dense(units = num_of_classes(),activation = \"softmax\"))(td) \n",
        "model=Model(inputs, outputI)\n",
        "model.compile(optimizer='adam',\n",
        "                           loss='categorical_crossentropy',\n",
        "                           metrics=['categorical_accuracy'])\n",
        "model.summary()\n",
        "X=[train_WordsInjyXLNET,train_PredicateInjyXLNET,train_POSInjyXLNET]\n",
        "X_dev=[dev_WordsInjyXLNET,dev_POSInjyXLNET,dev_POSInjyXLNET] \n",
        "model.fit(X, Label_train,\n",
        "                       batch_size = 5,\n",
        "                       epochs = 20,\n",
        "                       validation_data = (X_dev, Label_dev),\n",
        "           )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1vpXh0qMAZRc"
      },
      "source": [
        "# **Block 8**: Testing-->load dataset for test set +  embedding + model.predict"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cS6tF1H9JNP4"
      },
      "source": [
        "import pandas as pd\n",
        "#test_fn=\"add your test file here\"\n",
        "Y=load_datasetTestRE(test_fn,\"test_\") \n",
        "X_test_Xlnet_words=load_dataset_encodeinputs(test_fn,\"test_xlnet_words_\") \n",
        "X_test_Xlnet_pred=load_dataset_encodeinputs(test_fn,\"test_xlnet_pred_\")\n",
        "X_test_Xlnet_pos=load_dataset_encodeinputs(test_fn,\"test_xlnet_pos_\")\n",
        "test_PredicateInjyXLNET= create_XLNET_embeddings(\"elmo\", X_test_Xlnet_pred, sent_maxlen)\n",
        "test_WordsInjyXLNET= create_XLNET_embeddings(\"elmo\", X_test_Xlnet_words, sent_maxlen)\n",
        "test_POSInjyXLNET= create_XLNET_embeddings(\"elmo\", X_test_Xlnet_pos, sent_maxlen)\n",
        "\n",
        "XTEST=[test_WordsInjyXLNET,test_PredicateInjyXLNET,test_POSInjyXLNET]\n",
        "y = model.predict(XTEST)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HPxXtkanA1H5"
      },
      "source": [
        "# **Block 9 :** considalte labels and evaluation metrics "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JyBRcwM-rsxj"
      },
      "source": [
        "from sklearn import metrics\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import accuracy_score\n",
        "import logging\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import matplotlib \n",
        "#labels=[Adapt based on your dataset labels]\n",
        "myLabelsDict={k:v for k,v in enumerate(labels)}\n",
        "for i,label in enumerate(y[10]):\n",
        "  maxVal=np.max(label)\n",
        "  index=np.where(label==maxVal)\n",
        "\n",
        "def sample_labels( y, num_of_sents = 5, num_of_samples = 10,\n",
        "                      num_of_classes = 3, start_index = 5, get_prob = True):\n",
        "        classes = classes_()\n",
        "        ret = []\n",
        "        x=list(y)\n",
        "        for sent in x[:num_of_sents]:\n",
        "            cur = []\n",
        "            for word in sent[start_index: start_index + num_of_samples]:\n",
        "                sorted_prob = am(word)\n",
        "                cur.append([(classes[ind], word[ind]) if get_prob \n",
        "                            else classes[ind]\n",
        "                            for ind in sorted_prob[:num_of_classes]])\n",
        "            ret.append(cur)\n",
        "        return ret\n",
        "def consolidate_labels(labels):\n",
        "        x=map(consolidate_label , labels)\n",
        "        return list(x)\n",
        "      \n",
        "def consolidate_label(label):\n",
        "        return label.split(\"-\")[0] if label.startswith(\"O\") else label     \n",
        "    \n",
        "def transform_output_probs(y, get_prob = False):\n",
        "        return np.array(sample_labels(y,\n",
        "                                  num_of_sents = len(list(y)), # all sentences\n",
        "                                  num_of_samples = max(list(map(len, y))), # all words\n",
        "                                  num_of_classes = 1, # Only top probability\n",
        "                                  start_index = 0, # all sentences  \n",
        "                                  get_prob = get_prob, # Indicate whether to get only labels\n",
        "\n",
        "               ))    \n",
        "# Get most probable predictions and flatten\n",
        "am = lambda myList: [i[0] for i in sorted(enumerate(myList), key=lambda x:x[1], reverse= True)]\n",
        "Y1 = consolidate_labels(transform_output_probs(Y).flatten()) \n",
        "y2 = consolidate_labels(transform_output_probs(y).flatten()) \n",
        "\n",
        "logging.basicConfig(level = logging.DEBUG)\n",
        "\n",
        "ret = []\n",
        "\n",
        "eval_metrics = [\n",
        "                (\"F1 (micro)\", lambda Y1, y2: metrics.f1_score(Y1, y2,average = 'micro')),\n",
        "                (\"Precision (micro)\",lambda Y1, y2: metrics.precision_score(Y1, y2,average = 'micro')),\n",
        "                (\"Recall (micro)\",lambda Y1, y2: metrics.recall_score(Y1, y2,average = 'micro')),\n",
        "                (\"Accuracy\", metrics.accuracy_score),  \n",
        "               ]\n",
        "\n",
        "eval_metrics2 = [\n",
        "                (\"F1 (wieghted)\", lambda Y1, y2: metrics.f1_score(Y1, y2,average = 'weighted')),\n",
        "                (\"Precision (wieghted)\",lambda Y1, y2: metrics.precision_score(Y1, y2,average = 'weighted')),\n",
        "                (\"Recall (wieghted)\",lambda Y1, y2: metrics.recall_score(Y1, y2,average = 'weighted')),\n",
        "                (\"Accuracy\", metrics.accuracy_score),  \n",
        "               ]\n",
        "for (metric_name, metric_func) in eval_metrics:\n",
        "        ret.append((metric_name, metric_func(Y1, y2)))\n",
        "        logging.debug(\"calculating {}\".format(ret[-1]))\n",
        "\n",
        "for (metric_name, metric_func) in eval_metrics2:\n",
        "        ret.append((metric_name, metric_func(Y1, y2)))\n",
        "        logging.debug(\"calculating {}\".format(ret[-1]))\n",
        "      \n",
        "for (metric_name, metric_val) in ret:\n",
        "     logging.info(\"{}: {:.4f}\".format(metric_name,\n",
        "                                             metric_val))\n",
        "\n",
        "def print_confusion_matrix(confusion_matrix, class_names, figsize = (10,7), fontsize=14):\n",
        "    \n",
        "    df_cm = pd.DataFrame(\n",
        "        confusion_matrix, index=class_names, columns=class_names, \n",
        "    )\n",
        "    fig = plt.figure(figsize=figsize)\n",
        "    try:\n",
        "        heatmap = sns.heatmap(df_cm, annot=True, fmt=\"d\")\n",
        "    except ValueError:\n",
        "        raise ValueError(\"Confusion matrix values must be integers.\")\n",
        "    heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=fontsize)\n",
        "    heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=45, ha='right', fontsize=fontsize)\n",
        "    plt.ylabel('True label')\n",
        "    plt.xlabel('Predicted label')\n",
        "    return fig\n",
        "from sklearn.metrics import confusion_matrix\n",
        "#xx=[#labels here]\n",
        "conM=confusion_matrix(Y1,y2)\n",
        "figg=print_confusion_matrix(conM, xx, figsize = (10,7), fontsize=14)  \n",
        "figg"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}