NLP_PROJECT_TASK_2_.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP_PROJECT_TASK_2_.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "mount_file_id": "1yGxHzBSIXN9q2rCdlxlyKtb8bPGyJ7Fa",
      "authorship_tag": "ABX9TyPRACKO9TC0vZQrMou8jY4o",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ParvaShah/Template_Extraction_NLP/blob/master/NLP_PROJECT_TASK_2_.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HfQNYZv5bJxr",
        "colab_type": "text"
      },
      "source": [
        "#**IMPORT LIBRARIES**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ey5mIHMUK8sL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install allennlp==1.0.0rc1 allennlp-models==1.0.0rc1\n",
        "!pip install spacy==2.2.0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xRu4DypbLD9b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import files\n",
        "print('pred')\n",
        "from allennlp.predictors.predictor import Predictor\n",
        "print('coref')\n",
        "import allennlp_models.coref\n",
        "print('ner')\n",
        "import allennlp_models.ner\n",
        "print('consti')\n",
        "import allennlp_models.syntax.constituency_parser\n",
        "print('srl')\n",
        "import allennlp_models.syntax.srl\n",
        "print('/srl')\n",
        "import spacy\n",
        "from spacy import displacy\n",
        "from collections import Counter\n",
        "import en_core_web_sm\n",
        "import os\n",
        "import math\n",
        "import csv\n",
        "import itertools\n",
        "import pandas as pd\n",
        "import re\n",
        "import json\n",
        "print('sp')\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "print('ner')\n",
        "predictor_ner = Predictor.from_path(\"https://storage.googleapis.com/allennlp-public-models/ner-model-2020.02.10.tar.gz\")\n",
        "print('coref')\n",
        "predictor_coref = Predictor.from_path(\"https://storage.googleapis.com/allennlp-public-models/coref-spanbert-large-2020.02.27.tar.gz\")\n",
        "print('consto')\n",
        "predictor_constituency = Predictor.from_path(\"https://storage.googleapis.com/allennlp-public-models/elmo-constituency-parser-2020.02.10.tar.gz\")\n",
        "print('srl')\n",
        "predictor_srl = Predictor.from_path(\"https://storage.googleapis.com/allennlp-public-models/bert-base-srl-2020.03.24.tar.gz\")\n",
        "print('import done')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PidSLFqvE_FJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "predictor_ner._dataset_reader._token_indexers['token_characters']._min_padding_length = 3"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pq7rTkANbZxY",
        "colab_type": "text"
      },
      "source": [
        "#**COREFERENCE RESOLUTION**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ng4Tll3pP84o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_batched_corpus(dir_path,filename):\n",
        "\n",
        "    record = 0\n",
        "    sp_ch = 0\n",
        "    corpus = []\n",
        "    batch = []\n",
        "    all_pr = []\n",
        "    spcy_pr = []\n",
        "    pronouns = ['all', 'another', 'any', 'anybody', 'anyone', 'anything', 'both', 'each', 'each', 'other', 'either', 'everybody', 'everyone', 'everything', 'few', 'he', 'her', 'hers', 'herself', 'him', 'himself', 'his', 'I', 'it', 'its', 'itself', 'little', 'many', 'me', 'mine', 'more', 'most', 'much', 'my', 'myself', 'neither', 'no one', 'nobody', 'none', 'nothing', 'one another', 'other', 'others', 'our', 'ours', 'ourselves', 'several', 'she', 'some', 'somebody', 'someone', 'something', 'that', 'their', 'theirs', 'them', 'themselves', 'these', 'they', 'this', 'those', 'us', 'we', 'what', 'whatever', 'which', 'whichever', 'who', 'whoever', 'whom', 'whomever', 'whose', 'you', 'your', 'yours', 'yourself', 'yourselves']\n",
        "    pronouns = set([x.lower() for x in pronouns])\n",
        "    with open(dir_path+filename, \"r\",encoding='UTF-8',errors='ignore') as f:\n",
        "        content = f.readlines()\n",
        "    ch = 0\n",
        "    batch_pointer = 0\n",
        "    for sentences in content:\n",
        "        sentences = sentences.strip() + ' '\n",
        "        if record < ch:\n",
        "            record = ch\n",
        "        length = len(sentences)\n",
        "        if ch + length < 2000:\n",
        "            batch.append(sentences)\n",
        "            ch += length\n",
        "            batch_pointer = 1\n",
        "            continue\n",
        "        wd_not_ch = sentences.split()\n",
        "        for words in wd_not_ch:\n",
        "            words = words.lower()\n",
        "            pron = []\n",
        "            for prons in pronouns:\n",
        "                if prons == words:\n",
        "                    pron.append(prons)\n",
        "        all_pr.append(pron)\n",
        "        if pron and ch + length < 3500 :\n",
        "            batch.append(sentences)\n",
        "            ch += length\n",
        "            batch_pointer = 1\n",
        "        else:\n",
        "            corpus.append(batch)\n",
        "            batch = []\n",
        "            batch.append(sentences)\n",
        "            ch = length\n",
        "            batch_pointer = 0\n",
        "    if batch_pointer == 1:\n",
        "        corpus.append(batch)\n",
        "    return corpus    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H-mScHJy1zZb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def merge_intervals(intervals):\n",
        "    intervals.sort(key=lambda x: x[0])\n",
        "    merged = []\n",
        "    for interval in intervals:\n",
        "        if not merged or merged[-1][1] < interval[0]:\n",
        "            merged.append(interval)\n",
        "        else:\n",
        "            merged[-1][1] = max(merged[-1][1], interval[1])\n",
        "    return merged"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XSogMIOb69AQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def do_coreference(coreferences,wds):\n",
        "    #coreferences = model_coref['clusters']\n",
        "    #wds = model_coref['document']\n",
        "    wds_inx = [x for x in range(len(wds))]\n",
        "    #input_sent = model_coref['document']\n",
        "    wds_res = []\n",
        "    wds_dict = {}\n",
        "    ct = -1\n",
        "    final_result = []\n",
        "    for i in range(len(wds_inx)):\n",
        "        wds_dict[i] = wds[i]\n",
        "   \n",
        "    for clusters in coreferences:\n",
        "        for words in clusters:\n",
        "            if wds[words[1]] == '\\n':\n",
        "                words[1] = words[1] - 1 \n",
        "            if wds[words[0]] == '\\n':\n",
        "                words[0] = words[0] + 1\n",
        "\n",
        "    #print('???????????????????????DICTIONARY',wds_dict)\n",
        "    for clusters in coreferences:\n",
        "        ner_word = ''\n",
        "        #print(clusters)\n",
        "        #print(' '.join(wds[clusters[0][0]:clusters[0][1]+1]))\n",
        "        length = float('inf')\n",
        "        final_head = (' '.join(wds[clusters[0][0]:clusters[0][1]+1])).strip()\n",
        "        for i in range(len(clusters)):\n",
        "            ner_word = (' '.join(wds[clusters[i][0]:clusters[i][1]+1])).strip()\n",
        "            doc = nlp(ner_word)\n",
        "            pred = [X.label_ for X in doc.ents]\n",
        "            \n",
        "            if pred:\n",
        "                #print('compete:',ner_word)\n",
        "                if len(ner_word) < length:\n",
        "                    length = len(ner_word)\n",
        "                    final_head = ner_word\n",
        "        #print('final_head: ',final_head)\n",
        "        wds_dict[ct]  = final_head\n",
        "        for refs in clusters:\n",
        "            for i in range(refs[0],refs[1]+1):\n",
        "                wds_inx[i] = ct\n",
        "        ct -= 1\n",
        "    rec_ct = 0\n",
        "    temp_wds = []\n",
        "    for i in range(len(wds_inx)-1):\n",
        "        #print(wds_inx[i],wds_inx[i+1])\n",
        "        if wds_inx[i] == wds_inx[i+1]:\n",
        "            temp_wds.append(i)\n",
        "            rec_ct += 1\n",
        "            continue\n",
        "        if rec_ct > 4:\n",
        "            for its in temp_wds:\n",
        "                wds_res.append(its)\n",
        "            wds_res.append(i)\n",
        "            rec_ct = 0\n",
        "            temp_wds = []\n",
        "            continue\n",
        "\n",
        "        wds_res.append(wds_inx[i])\n",
        "        #print('append done')\n",
        "        rec_ct = 0\n",
        "        temp_wds = []\n",
        "\n",
        "    if wds_inx[-1] != wds_inx[-2]:\n",
        "        wds_res.append(wds_inx[-1])\n",
        "    # print('IN MAIN DO COREF ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^6')\n",
        "    # print(wds_dict)\n",
        "    # print(wds_res)\n",
        "    result = ''\n",
        "    for ids in wds_res:\n",
        "        word = wds_dict[ids]\n",
        "        # print('word',word)\n",
        "        if word == '\\n':\n",
        "            print(result)\n",
        "            final_result.append(result.strip()+' ')\n",
        "            result = ''\n",
        "        result +=  word + ' '\n",
        "    return final_result"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AlnZxnLSIIX9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_Coref(batch,original_file,coreference_file):\n",
        "\n",
        "    coref_input = ''\n",
        "    original_batch = []\n",
        "    #print('Checking Original files!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')\n",
        "    for i,sentence in enumerate(batch):\n",
        "        #print(sentence)\n",
        "        original_batch.append(sentence.strip())\n",
        "        coref_input += sentence.strip()+'\\n'\n",
        "    model_coref = predictor_coref.predict(document=coref_input)   \n",
        "    clusters = model_coref['clusters']\n",
        "    words_coref = model_coref['document']\n",
        "    # print(words_coref)\n",
        "    word_range_dict = {}\n",
        "    word_start = 0\n",
        "    word_end = 0\n",
        "    line_number = 0\n",
        "    for words in words_coref:\n",
        "        if words != '\\n':\n",
        "            word_end += 1\n",
        "            continue\n",
        "        else:\n",
        "            word_range_dict[line_number] = [word_start,word_end-1]\n",
        "            line_number += 1\n",
        "            word_start = word_end + 1\n",
        "            word_end += 1\n",
        "    # print(word_range_dict)\n",
        "    # print('Clustures:',clusters)\n",
        "    merge_list = []\n",
        "    for cluster in clusters:\n",
        "        cluster_head = -1\n",
        "        line_number = -1\n",
        "        head_word = ''\n",
        "        for word in cluster:\n",
        "            start = word[0]\n",
        "            end = word[1]\n",
        "            for key,items in word_range_dict.items():\n",
        "                if end <= items[1] and start >= items[0]:\n",
        "                    line_number = key\n",
        "            ner_word = ' '.join(words_coref[word[0]:word[1]+1])\n",
        "            #print(ner_word)\n",
        "            doc = nlp(ner_word)\n",
        "            pred = [X.label_ for X in doc.ents if X.label_ == 'GPE' or  X.label_ == 'LOC' or X.label_ == 'PERSON'\\\n",
        "                    or  X.label_ == 'NORP' or  X.label_ == 'ORG' or  X.label_ == 'PRODUCT'or  X.label_ == 'EVENT'\\\n",
        "                    or X.label_ == 'LANGUAGE' or X.label_ == 'DATE']\n",
        "            \n",
        "            if len(ner_word) > 2 :\n",
        "               \n",
        "                model_ner = predictor_ner.predict(sentence=ner_word)\n",
        "                for ner in (model_ner['tags']):\n",
        "                    if len(ner) > 2:\n",
        "                        pred.append(str(ner)[2:])\n",
        "                        #print('ner_word####',ner_word)\n",
        "            #print('word :',ner_word,pred)\n",
        "            if pred or cluster_head == -1:\n",
        "                head_word = ner_word\n",
        "                cluster_head = line_number\n",
        "            else:\n",
        "                if cluster_head != line_number:\n",
        "                    #print('########## word:',ner_word,line_number,head_word,cluster_head)\n",
        "                    merge_list.append([cluster_head,line_number])\n",
        "    #print(merge_list,'\\n',len(merge_list))\n",
        "    merge_list = merge_intervals(merge_list)\n",
        "    #print('new',merge_list,'\\n',len(merge_list))\n",
        "\n",
        "        ### Merging list #####\n",
        "    ### input: merge_list, clusters, batch, coref_input, words_coref\n",
        "    coref_batch = do_coreference(clusters,words_coref)\n",
        "    # print('Original Output:',original_batch)\n",
        "    # print('Final_Output: ',coref_batch)\n",
        "    # print('--------------------------------')\n",
        "    # for i in range(len(coref_batch)):\n",
        "    #     print('Coref: ',coref_batch[i])\n",
        "    #     print('original: ',original_batch[i])\n",
        "\n",
        "    # print('################################')\n",
        "    # print(\"---------------------\",len(original_batch),len(coref_batch),\"--------------------\")\n",
        "    # if len(coref_batch) == len(original_batch):\n",
        "    #     print(\"--------------------------------------------------------------True----------------------------------------\")\n",
        "\n",
        "    j = 0\n",
        "    original_batch_merged = []\n",
        "    coreference_batch_merged = []\n",
        "    i = 0\n",
        "    if merge_list:\n",
        "        for sentence in (coref_batch):\n",
        "            if j < len(merge_list):\n",
        "                #print(merge_list[j])\n",
        "                start = merge_list[j][0]\n",
        "                end = merge_list[j][1]\n",
        "                if start == i:\n",
        "                    for k in range(start,end):\n",
        "                        original_batch_merged.append(original_batch[k])\n",
        "                        coreference_batch_merged.append(coref_batch[k])\n",
        "                        # print(original_batch[k])\n",
        "                        #if original_batch[k][-1] != '.':\n",
        "                        original_batch_merged[len(original_batch_merged)-1] += '. '\n",
        "                        coreference_batch_merged[len(coreference_batch_merged)-1] += '. '\n",
        "                        #else:\n",
        "                        #    original_batch_merged[len(original_batch_merged)-1] += ' '\n",
        "                        #    coreference_batch_merged[len(coreference_batch_merged)-1] += ' '\n",
        "\n",
        "                    j += 1\n",
        "                    i += end - start\n",
        "                    if i > len(coref_batch) - 1:\n",
        "                        break\n",
        "                original_batch_merged.append(original_batch[i]+'\\n')\n",
        "                coreference_batch_merged.append(coref_batch[i]+'\\n')\n",
        "                i += 1\n",
        "    else:\n",
        "        original_batch_merged = original_batch\n",
        "        coreference_batch_merged = coref_batch\n",
        "\n",
        "    # print(original_batch_merged)\n",
        "    # print(coreference_batch_merged)\n",
        "    # if len(original_batch_merged) == len(coreference_batch_merged):\n",
        "    #     print(\"--------------------------------------------------------------True merged----------------------------------------\")\n",
        "\n",
        "    ######## Appending outputs to files ######\n",
        "\n",
        "    og_file = open(original_file,\"a+\",encoding='UTF-8-sig')\n",
        "    co_file = open(coreference_file,\"a+\",encoding='UTF-8-sig')\n",
        "    length = len(coreference_batch_merged)\n",
        "    for i,sentence in enumerate(coreference_batch_merged):\n",
        "        og_file.write(original_batch_merged[i])\n",
        "        co_file.write(coreference_batch_merged[i])\n",
        "\n",
        "    og_file.close()\n",
        "    co_file.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "adH8rAslazEr",
        "colab_type": "text"
      },
      "source": [
        "#**WORK TEMPLATE**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y2zcOrgILoo0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_job_list():\n",
        "    rows = []\n",
        "    with open('/content/drive/My Drive/job_title.csv', 'r') as csvfile: \n",
        "        # creating a csv reader object \n",
        "        csvreader = csv.reader(csvfile) \n",
        "        # extracting each data row one by one \n",
        "        for row in csvreader: \n",
        "            rows.append(str(row).lower()[2:-2])\n",
        "    rows = set(rows)\n",
        "    #print(len(rows))\n",
        "    rows_copy = rows.copy()\n",
        "    for job in rows_copy:\n",
        "        if len(job) < 3:\n",
        "            rows.discard(job)\n",
        "        \n",
        "    #print(len(rows))\n",
        "    #print(rows)\n",
        "    position_list = rows\n",
        "    position_list.add('board member')\n",
        "    position_list.add('board of directors')\n",
        "    return position_list"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SOh9nSAlvtCm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def getNPS(children,parent,dict_parent_child):\n",
        "    \n",
        "    if 'children' in children.keys():\n",
        "        for sub_child in children['children']:\n",
        "            if children['nodeType'] in set(['NP','PP','NX','S']): \n",
        "                if parent not in dict_parent_child.keys():\n",
        "                    dict_parent_child[parent] = []\n",
        "                if children['word'] not in set(dict_parent_child[parent]):\n",
        "                    dict_parent_child[parent].append(children['word'])\n",
        "            getNPS(sub_child,children['word'],dict_parent_child)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S0p8tfI2HahU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def is_Sublist(l, s):\n",
        "  st = -1\n",
        "  en = -1\n",
        "  sub_set = False\n",
        "  if s == []:\n",
        "    return []\n",
        "  elif s == l:\n",
        "    return [0, len(s)-1]\n",
        "  elif len(s) > len(l):\n",
        "    sub_set = False\n",
        "\n",
        "  else:\n",
        "    for i in range(len(l)):\n",
        "      if l[i] == s[0] and not sub_set:\n",
        "        st = i\n",
        "        n = 1\n",
        "        while (n < len(s)) and (l[i+n] == s[n]):\n",
        "          n += 1\n",
        "        \n",
        "        if n == len(s):\n",
        "          en = i+n-1\n",
        "          sub_set = True\n",
        "  if st == -1 or en == -1:\n",
        "    sub_set = False\n",
        "    return []\n",
        "  return [st, en]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vXn17apaNW2s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def work_is_a(input_sentence,postion_list, original_words_list, original_tags_list):\n",
        "\n",
        "    pred = predictor_constituency.predict(sentence=input_sentence)\n",
        "    input_rec = pred['hierplane_tree']['root']\n",
        "    dict_parent_child = {}\n",
        "    \n",
        "    getNPS(input_rec,'root',dict_parent_child)\n",
        "    \n",
        "    position_org_list = []\n",
        "    for key, values in dict_parent_child.items():\n",
        "        # key = key.replace(' - ','-')\n",
        "        for vals in values:\n",
        "            # vals = vals.replace(' - ','-')\n",
        "            # vals = vals.replace('founderer','co-founder')\n",
        "            position_org_list.append(vals)\n",
        "            # print('CHILD: ',vals)\n",
        "        # key = key.replace('founderer','co-founder')\n",
        "        position_org_list.append(key)\n",
        "        # print('PARENT: ',key)\n",
        "    \n",
        "    position_org_dict = {}\n",
        "    location_org_dict = {}\n",
        "    for line in position_org_list:\n",
        "        sentences = line\n",
        "        doc = nlp(line)\n",
        "        # for token in doc:\n",
        "        #     if token.dep_ != 'det':\n",
        "        #         sentences += token.text + ' '\n",
        "        # sentences = sentences.strip()\n",
        "        # print(\"NP:- \",sentences)\n",
        "        wds = sentences.split()\n",
        "        if len(sentences) > 2 :\n",
        "            # print(\"np  - - - - - - - - - -\", line.split(' '))\n",
        "            ## optimize here\n",
        "            ## get model_ner['words'] and model_ner['tags']\n",
        "            ## get start and end index '\n",
        "            line_words_list = []\n",
        "            line_tags_list = []\n",
        "            list_index = is_Sublist(original_words_list, line.split(' '))\n",
        "            if list_index:\n",
        "                line_words_list = original_words_list[list_index[0]:list_index[1] + 1]\n",
        "                line_tags_list = original_tags_list[list_index[0]:list_index[1] + 1]\n",
        "            # model_ner = predictor_ner.predict(sentence=sentences)\n",
        "            # print(\"final_word_list: - - - - - - - - - - - - - \", line_words_list)\n",
        "            # print(\"final_tag_list: - - - - - - - - - - - - - \", line_tags_list)\n",
        "            lt = ''\n",
        "            organizations = []\n",
        "            flag = 0\n",
        "            combi_list =  ''\n",
        "            for wd,ner in zip(line_words_list,line_tags_list):\n",
        "                \n",
        "                if wd == ',' or wd == 'and' or 'ORG' in ner:\n",
        "                    flag = 1\n",
        "                    if len(ner) > 2 and 'ORG' in ner:\n",
        "                        if ner[0] == 'U':\n",
        "                            combi_list += wd+', '\n",
        "                            \n",
        "                        else:\n",
        "                            if ner[0] == 'L':\n",
        "                                if lt:\n",
        "                                    combi_list += lt+wd + ', '\n",
        "                                    lt = ''\n",
        "                            else:\n",
        "                                lt += wd + ' '\n",
        "                else:\n",
        "                    flag = 0\n",
        "                    if combi_list:\n",
        "                        organizations.append(combi_list[:-2])\n",
        "                        combi_list = ''\n",
        "            if combi_list:\n",
        "                organizations.append(combi_list[:-2])\n",
        "            ############ LOC##########\n",
        "            if not organizations:\n",
        "                doc_org = nlp(sentences)\n",
        "                for ent in doc_org.ents:\n",
        "                    if ent.label_ == 'ORG':\n",
        "                        organizations.append(ent.text)\n",
        "\n",
        "            organizations = list(set(organizations))\n",
        "            # print(\"organizations: \",organizations)\n",
        "            if len(organizations) == 1:\n",
        "                organizations = organizations[0]\n",
        "                if organizations not in set(position_org_dict.keys()):\n",
        "                    position_org_dict[organizations] = []\n",
        "                    location_org_dict[organizations] = []\n",
        "                \n",
        "                doc1 = nlp(sentences)\n",
        "                \n",
        "                for chunk in doc1.noun_chunks:\n",
        "                    ck = chunk.text\n",
        "                    ck = ck.replace(' - ','-')\n",
        "                    ck_list = ck.split('and')\n",
        "                    \n",
        "                    for small_chunks in ck_list:\n",
        "                        # print('small_chunk', small_chunks)\n",
        "                        small_chunks = small_chunks.strip()\n",
        "                        if small_chunks == 'founderer':\n",
        "                            small_chunks = 'co-founder'\n",
        "                        if small_chunks.lower() in position_list:\n",
        "                            \n",
        "                            if small_chunks not in set(position_org_dict[organizations]):\n",
        "                                position_org_dict[organizations].append(small_chunks)\n",
        "                    \n",
        "                    doc_loc = nlp(chunk.text)\n",
        "                    for ent in doc_loc.ents:\n",
        "                        \n",
        "                        if ent.label_ == 'LOC' or ent.label_ == 'GPE':\n",
        "                            \n",
        "                            if ent.text not in set(location_org_dict[organizations]):\n",
        "                                location_org_dict[organizations].append(ent.text)\n",
        "                \n",
        "    return position_org_dict,location_org_dict"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l3Qixn_82WOu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_work_template(doc_file,position_list):\n",
        "    ps_list_final = []\n",
        "    lc_list_final = []\n",
        "    person_list_final = []\n",
        "    sent_final = []\n",
        "    lc_template_final = []\n",
        "    # print(\"Main line: \",doc_file)\n",
        "    \n",
        "    for sentence in doc_file.sents:\n",
        "        # print(\"sentence: \", sentence)\n",
        "        # print(\"-----------------------------\")\n",
        "        # continue\n",
        "        ps_list = []\n",
        "        lc_list = []\n",
        "        person_list = []\n",
        "        lc_template = []\n",
        "        sent_temp = sentence\n",
        "        sentence = str(sentence).strip()\n",
        "        sentence = sentence.replace('co - founder','founderer')\n",
        "        sentence = sentence.replace('board of directors','board member')\n",
        "        sentence = sentence.replace('board of director','board member')\n",
        "        doc_words = nlp(sentence)\n",
        "        sentences = ''\n",
        "        #print(sentence)\n",
        "        for token in doc_words:\n",
        "            if token.dep_ != 'det':\n",
        "                sentences += token.text + ' '\n",
        "        sentence = sentences.strip()\n",
        "        ### Location Template\n",
        "        lc_template = get_location_teplate(doc_words)\n",
        "        # print('&*****&&&&&&&&*&&&&&&&&***********************',lc_template,'&*****&&&&&&&&*&&&&&&&&***********************')\n",
        "        pred = [X.label_ for X in doc_words.ents]\n",
        "        words = [X.text for X in doc_words.ents]\n",
        "        wds = sentence.split()\n",
        "        # print(\"pred: \",pred)\n",
        "        # print(\"words: \",words)\n",
        "        if len(sentence) > 2 :\n",
        "            model_ner = predictor_ner.predict(sentence=sentence)\n",
        "            # print(\"words_list - - - - - - - - - - - - - - - - - - - - - -  \", model_ner['words'])\n",
        "            # print(\"tags_list - - - - - - - - - - - - - - - - - - - - - -  \", model_ner['tags'])\n",
        "            lt = ''\n",
        "            for ner,word in zip(model_ner['tags'], model_ner['words']):\n",
        "                if len(ner) > 2 and 'PER' in ner:\n",
        "                    if ner[0] == 'U':\n",
        "                        person_list.append(word)\n",
        "                        pred.append('PER')\n",
        "                        \n",
        "                    else:\n",
        "                        if ner[0] == 'L':\n",
        "                            person_list.append(lt+' '+word)\n",
        "                            pred.append('PER')\n",
        "                            lt = ''\n",
        "                        else:\n",
        "                            lt += word + ' '\n",
        "        # print('person_list', person_list)\n",
        "        if 'PERSON' in pred or 'PER' in pred:\n",
        "            if 'ORG' in pred or 'NORP' in pred:\n",
        "                position,location = work_is_a(sentence,position_list, model_ner['words'], model_ner['tags'])\n",
        "                # print('---------################---------------')\n",
        "                # print(position)\n",
        "                # print('---------################---------------')\n",
        "                ps_list_final.append(position)\n",
        "                lc_list_final.append(location)\n",
        "                person_list_final.append(person_list)\n",
        "                sent_final.append(sentence)\n",
        "        lc_template_final.append(lc_template)\n",
        "    # print('&*****&&&&&&&&*&&&&&&&&***********************',lc_template,'&*****&&&&&&&&*&&&&&&&&***********************')\n",
        "    return ps_list_final,lc_list_final,person_list_final,sent_final,lc_template_final"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lhPLNPESaoHb",
        "colab_type": "text"
      },
      "source": [
        "#**LOCATION TEMPLATE**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hsBHrbmJRLxf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_location_list():\n",
        "    places = []\n",
        "    with open('/content/drive/My Drive/city_state_country.txt', 'r') as filehandle:\n",
        "        for line in filehandle:\n",
        "            currentPlace = line[:-1]\n",
        "            currentPlace = currentPlace.strip('][').split(', ')\n",
        "            currentPlace = [loc.strip(\"''\") for loc in currentPlace ]\n",
        "            places.append(currentPlace)\n",
        "    \n",
        "    preprocess_df = pd.read_csv(\"/content/drive/My Drive/city_state_country_preprocess.csv\",header=None)\n",
        "    codes = preprocess_df[0]\n",
        "    vals = preprocess_df[1]\n",
        "    preprocess_dict = {}\n",
        "\n",
        "    for key, vals in zip(codes,vals):\n",
        "        vals = vals.strip('][').split(', ')\n",
        "        vals = [loc.strip(\"''\") for loc in vals ]\n",
        "        preprocess_dict[key] = vals\n",
        "    preprocess_dict['uk'].append('united kingdom')\n",
        "    return preprocess_dict,places"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5ypi7qUAUidp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_location_teplate(doc):\n",
        "    \n",
        "    preprocess_dict,places = get_location_list()\n",
        "    test = [str(X.text).lower() for X in doc.ents if X.label_ == 'GPE' or  X.label_ == 'LOC' ]\n",
        "    final_keep = [str(X.text) for X in doc.ents if X.label_ == 'GPE' or  X.label_ == 'LOC' ]\n",
        "    book_keep = {}\n",
        "    punctuations = '''!()-[]{};:'\"\\,<>./?@#$%^&*_~'''\n",
        "    punctuations = punctuations+ '\"'\n",
        "    main_test = []\n",
        "    for items in test:\n",
        "        no_punct = \"\"\n",
        "        for char in items:\n",
        "            if char not in punctuations:\n",
        "                no_punct = no_punct + char\n",
        "        \n",
        "        if no_punct in preprocess_dict:\n",
        "\n",
        "            for abbrevations in preprocess_dict[no_punct]:\n",
        "                \n",
        "                main_test.append(''.join(abbrevations))\n",
        "                \n",
        "            if items not in book_keep:\n",
        "                book_keep[items] = preprocess_dict[no_punct]\n",
        "        else:\n",
        "            main_test.append(no_punct)\n",
        "    \n",
        "    main_test = (list(itertools.combinations(main_test,2)))\n",
        "    location_template = []\n",
        "    for loc_1,loc_2 in main_test:\n",
        "        for item in places:\n",
        "            if [x for x in item if loc_1 == x] and [x for x in item if loc_2 == x]:\n",
        "                if item.index(loc_1) > item.index(loc_2):\n",
        "                    loc_1,loc_2 = loc_2,loc_1\n",
        "                for key, val in book_keep.items():  \n",
        "                    if loc_1 in val:\n",
        "                        loc_1 = key\n",
        "                    if loc_2 in val:\n",
        "                        loc_2 = key\n",
        "                l_1_inx = test.index(loc_1)\n",
        "                l_2_inx = test.index(loc_2)\n",
        "                loc_1 = final_keep[l_1_inx]\n",
        "                loc_2 = final_keep[l_2_inx]\n",
        "                if loc_1 == loc_2:\n",
        "                    continue\n",
        "                location_template.append([loc_1,loc_2])\n",
        "    # print(\"template_loc:- \",location_template)\n",
        "    return location_template    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lpAkWLgmXZWD",
        "colab_type": "text"
      },
      "source": [
        "#**BUY TEMPLATE**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BcVuAoX1iqml",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def extract_buy_template(doc_input):\n",
        "\n",
        "    final_list = []\n",
        "    # print('################ In buy',doc_input)\n",
        "    for sentence in doc_input.sents:\n",
        "        input_sentence = str(sentence)\n",
        "        pred = predictor_srl.predict(sentence = input_sentence)\n",
        "        # print(\"---------------------------------\")\n",
        "        # print(\"sentence in Buy\", input_sentence)\n",
        "        buy_template_list = []\n",
        "        # print('pred', pred['verbs'])\n",
        "        for item in pred['verbs']:\n",
        "            verb_doc = nlp(item['verb'])\n",
        "            # print('verb:- ', verb_doc[0])\n",
        "            if verb_doc[0].lemma_ in ['buy', 'acquire', 'purchase', 'get', 'score', 'obtain', 'sell']:\n",
        "                verb_word = item['verb']\n",
        "                tags = item['tags']\n",
        "                # if verb_word in dict:\n",
        "                arg0 = []\n",
        "                arg1 = []\n",
        "                arg2 = []\n",
        "                other_args = []\n",
        "                outside_words = []\n",
        "                for tag,word in zip(tags, pred['words']):\n",
        "                # print(\"tag:- \",tag,\" word:- \", word)\n",
        "                    if tag[2:] == \"V\":\n",
        "                        continue\n",
        "                    if tag[2:] == \"ARG0\":\n",
        "                        arg0.append(word)\n",
        "                    elif tag[2:] == \"ARG1\":\n",
        "                        arg1.append(word)\n",
        "                    elif tag[2:] == \"ARG2\":\n",
        "                        arg2.append(word)\n",
        "                    elif tag == \"O\":\n",
        "                        outside_words.append(word)\n",
        "                    else:\n",
        "                        other_args.append(word)\n",
        "                # print(\"--------before-------\")\n",
        "                # print('args0:',arg0)\n",
        "                # print('args1',arg1)\n",
        "                # print('args2',arg2)\n",
        "                # print('other',other_args)\n",
        "                # print('outside',outside_words)\n",
        "                # print('VERB:',verb_word)\n",
        "\n",
        "                if verb_doc[0].lemma_ in [\"sell\"]:\n",
        "                    arg0, arg2 = arg2, arg0\n",
        "                    \n",
        "                # print(\"--------After-------\")\n",
        "                # print('args0:',arg0)\n",
        "                # print('args1',arg1)\n",
        "                # print('args2',arg2)\n",
        "                # print('other',other_args)\n",
        "                # print('outside',outside_words)\n",
        "            \n",
        "                # create key : label dictionary\n",
        "                \n",
        "                model_ner = predictor_ner.predict(sentence=input_sentence)\n",
        "                # print(model_ner)\n",
        "                lt = ''\n",
        "                keys = []\n",
        "                labels = []\n",
        "                for word,ner in zip(model_ner['words'],model_ner['tags']):\n",
        "                    if len(ner) > 2:\n",
        "                        if ner[0] == 'U':\n",
        "                            if word not in keys:\n",
        "                                keys.append(word)\n",
        "                                labels.append(str(ner)[2:])\n",
        "                            \n",
        "                        else:\n",
        "                            if ner[0] == 'L':\n",
        "                                if lt:\n",
        "                                    if (lt + word) not in keys:\n",
        "                                        keys.append(lt + word)\n",
        "                                        labels.append(str(ner)[2:])\n",
        "                                    lt = ''\n",
        "                            else:\n",
        "                                lt += word + ' '\n",
        "                # print('KEYS', keys)\n",
        "                # print('LABELS', labels)\n",
        "                \n",
        "                buyer = ''\n",
        "                item = ''\n",
        "                price = ''\n",
        "                quantity = ''\n",
        "                source = ''\n",
        "                # get buyer\n",
        "                buyer = get_arg0(arg0, keys, labels)\n",
        "                if buyer:\n",
        "                    if not arg1 and not arg2:\n",
        "                        # get item, source, price and quantity from other_args \n",
        "                        item, source, price, quantity = get_other_args(other_args, keys, labels, 1)\n",
        "                    elif arg0 and arg1 and not arg2:\n",
        "                        # get item from arg1\n",
        "                        item, price, quantity = get_arg1(arg1, keys, labels)\n",
        "                        # get source from other_args\n",
        "                        xyz, source, price, quantity = get_other_args(other_args, keys, labels, 2)\n",
        "                    elif arg0 and not arg1 and arg2:\n",
        "                        # get source, price and quantity from arg2\n",
        "                        source, price, quantity = get_arg2(arg2, keys, labels)\n",
        "                        # get price, quantity from other_args\n",
        "                        xyz, abc, price, quantity = get_other_args(other_args, keys, labels, 3)\n",
        "                    else:\n",
        "                        # get item from arg1\n",
        "                        item, price, quantity = get_arg1(arg1, keys, labels)\n",
        "                        # get source from arg2\n",
        "                        source, price, quantity = get_arg2(arg2, keys, labels)\n",
        "                        xyz, abc, price, quantity = get_other_args(other_args, keys, labels, 3)\n",
        "                \n",
        "                \n",
        "                source_list = source.split(\", \")\n",
        "                # print(\"###########source:- \", source_list)\n",
        "                for i in range(len(source_list)):\n",
        "                    if source_list[i] in buyer or source_list[i] in item:\n",
        "                        source_list[i] = ''\n",
        "                source = ' '.join(source_list)\n",
        "                final_list.append(buyer + \"||\" + item + \"||\" + price + \"||\" + quantity + \"||\" + source)\n",
        "        #print(\"template:- \", buy_template_list)\n",
        "        #final_list.append(buy_template_list)\n",
        "\n",
        "    return final_list"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PziYuZG5qWoj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_other_args(other_args, keys, labels, case):\n",
        "    # case 1 : item - ORG, NORP, PRODUCT, Source - ORG, NORP, PER\n",
        "    # case 2 : Source - ORG, NORP, PER\n",
        "    # case 3 : MONEY, CARDINAL\n",
        "    # print('other_args')\n",
        "    args = ' '.join(other_args)\n",
        "    item = ''\n",
        "    source = ''\n",
        "    price = ''\n",
        "    quantity = ''\n",
        "    # print(args)\n",
        "    doc = nlp(args)\n",
        "    for key in keys:\n",
        "        #print(ent.text)\n",
        "        if case == 1 :\n",
        "            if key in args:\n",
        "                if labels[keys.index(key)] in ['ORG','MISC']:\n",
        "                    item = key + ', '\n",
        "        if case == 2 :\n",
        "            if key in args:\n",
        "                if labels[keys.index(key)] in ['ORG', 'PER']:\n",
        "                    source = key + ', '\n",
        "        \n",
        "        if labels[keys.index(key)] in ['CARDINAL']:\n",
        "            #print(ent.text)\n",
        "            quantity += key+', '\n",
        "    for ent in doc.ents:\n",
        "        # print(\"######## ent:- \",ent)\n",
        "        if ent.label_ in ['MONEY']:\n",
        "            #print(ent.text)\n",
        "            price += ent.text+', '\n",
        "    if not doc.ents:\n",
        "        match = re.search(r'([A-Za-z]{0,3}?[\\s]?[^\\d\\.\\,\\s][\\s]?\\d[\\d,.]*(-\\d[\\d,.]*)?[\\s]?[KMB]?)', args)\n",
        "        if match:\n",
        "            price += str(match.group()) + ', '\n",
        "    item = item[:-2]\n",
        "    source = source[:-2]\n",
        "    price = price[:-2]\n",
        "    quantity = quantity[:-2]\n",
        "    # print('item: ', item)\n",
        "    # print('source: ',source)\n",
        "    # print('price',price)\n",
        "    # print('qunt',quantity)\n",
        "    return item, source, price, quantity"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Flqp15ZqR8g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_arg2(arg2, keys, labels):\n",
        "    ## arg2\n",
        "    # print('arg2')\n",
        "    args = ' '.join(arg2)\n",
        "    source = ''\n",
        "    price = ''\n",
        "    quantity = ''\n",
        "    #print(args)\n",
        "    doc = nlp(args)\n",
        "    for key in keys:\n",
        "        #print(ent.text)\n",
        "        if key in args:\n",
        "            if labels[keys.index(key)] in ['ORG','PER']:\n",
        "                #print(ent.text)\n",
        "                source += key+', '\n",
        "        if labels[keys.index(key)] in ['CARDINAL']:\n",
        "            #print(ent.text)\n",
        "            quantity += key+', '\n",
        "    for ent in doc.ents:\n",
        "        if ent.label_ in ['MONEY']:\n",
        "            #print(ent.text)\n",
        "            price += ent.text+', '\n",
        "    if not doc.ents:\n",
        "        match = re.search(r'([A-Za-z]{0,3}?[\\s]?[^\\d\\.\\,\\s][\\s]?\\d[\\d,.]*(-\\d[\\d,.]*)?[\\s]?[KMB]?)', args)\n",
        "        if match:\n",
        "            price += str(match.group()) + ', '\n",
        "    source = source[:-2]\n",
        "    price = price[:-2]\n",
        "    quantity = quantity[:-2]\n",
        "    # print('source: ',source)\n",
        "    # print('price',price)\n",
        "    # print('qunt',quantity)\n",
        "    return source, price, quantity"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OnJxQydMp85J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_arg1(arg1, keys, labels):\n",
        "    ## arg1\n",
        "    # print('arg1')\n",
        "    args = ' '.join(arg1)\n",
        "    item = ''\n",
        "    price = ''\n",
        "    quantity = ''\n",
        "    #print(args)\n",
        "    doc = nlp(args)\n",
        "    for key in keys:\n",
        "        if key in args:\n",
        "            # print(\"####################In item arg1:- \",key)\n",
        "            if labels[keys.index(key)] in ['ORG','MISC']:\n",
        "                #print(ent.text)\n",
        "                item += key + ', '\n",
        "        if labels[keys.index(key)] in ['CARDINAL']:\n",
        "            #print(ent.text)\n",
        "            quantity += key + ', '\n",
        "    for ent in doc.ents:\n",
        "        if ent.label_ in ['MONEY']:\n",
        "            #print(ent.text)\n",
        "            price += ent.text+', '\n",
        "    if not doc.ents:\n",
        "        match = re.search(r'([A-Za-z]{0,3}?[\\s]?[^\\d\\.\\,\\s][\\s]?\\d[\\d,.]*(-\\d[\\d,.]*)?[\\s]?[KMB]?)', args)\n",
        "        if match:\n",
        "            price += str(match.group()) + ', '\n",
        "    item = item[:-2]\n",
        "    price = price[:-2]\n",
        "    quantity = quantity[:-2]\n",
        "    # print('seller: ',seller)\n",
        "    # print('amount',amount)\n",
        "    # print('qunt',quantity)\n",
        "    return item, price, quantity"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LWHdq8fhkwSQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_arg0(arg0, keys, labels):\n",
        "    ## arg0\n",
        "    # print('arg0')\n",
        "    args = ' '.join(arg0)\n",
        "    buyer = ''\n",
        "    #print(args)\n",
        "    # doc = nlp(args)\n",
        "    for key in keys:\n",
        "        #print(ent.text)\n",
        "        if key in args:\n",
        "            if labels[keys.index(key)] in ['PER','ORG']:\n",
        "                #print(ent.text)\n",
        "                buyer += key+', '\n",
        "    buyer = buyer[:-2]\n",
        "    # print('Buyer: ',buyer)\n",
        "    return buyer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y8kAV1cFbAK4",
        "colab_type": "text"
      },
      "source": [
        "#**MAIN FUNCTION**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hOnde9DzbSSz",
        "colab_type": "text"
      },
      "source": [
        "### **GET SEGMENTED FILES**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FgbW27NFLRLF",
        "colab_type": "code",
        "outputId": "5af43ce9-7508-4970-b327-7053daa43c86",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        }
      },
      "source": [
        "!mkdir 'segmented'\n",
        "dir_path = '/content/drive/My Drive/WikipediaArticles/'\n",
        "out_path = '/content/segmented/'\n",
        "dir_in = '/content/segmented/'\n",
        "\n",
        "for filename in os.listdir(dir_path):\n",
        "    print(filename)\n",
        "    myfile = open(dir_path+filename, \"r\",encoding='UTF-8',errors='ignore').read()\n",
        "    doc_file = nlp(myfile)\n",
        "    f = open(out_path+'segmented_'+filename,'w')\n",
        "    for sentence in doc_file.sents:\n",
        "        f.write(str(sentence).strip() + \" \")\n",
        "        f.write('\\n')\n",
        "    f.close()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.txt\n",
            "2.txt\n",
            "3.txt\n",
            "4.txt\n",
            "5.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fti4sYP7jlUf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dir_path = '/content/segmented/'\n",
        "out_path = '/content/segmented_1/'\n",
        "!mkdir '/content/segmented_1/'\n",
        "for filename in os.listdir(dir_path):\n",
        "    with open(dir_path+filename, \"r\",encoding='UTF-8',errors='ignore') as file:\n",
        "        content = file.readlines()\n",
        "    f = open(out_path+filename,'w')\n",
        "    for sentence in content:\n",
        "        if len(sentence) > 1:\n",
        "            f.write(str(sentence).strip() + ' ')\n",
        "            f.write('\\n')\n",
        "    file.close()\n",
        "    f.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o-bMyhIPbgrF",
        "colab_type": "text"
      },
      "source": [
        "###**GENERATED COREFERENCE AND ORIGINAL FILES**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8F05rfl8HCR8",
        "colab_type": "code",
        "outputId": "0ee995b5-b704-4ce6-a459-141d4a95e244",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 171
        }
      },
      "source": [
        "dir_path = '/content/segmented_1/'\n",
        "\n",
        "!mkdir original_files\n",
        "!mkdir coreference_files\n",
        "for filename in os.listdir(dir_path):\n",
        "    filename = 'segmented_1.txt'\n",
        "    original_file = '/content/original_files/'+filename\n",
        "    coreference_file = '/content/coreference_files/'+filename\n",
        "    corpus = get_batched_corpus(dir_path,filename)\n",
        "    print(corpus)\n",
        "    for batch in corpus:\n",
        "        # print(batch)\n",
        "        coreference_batch = get_Coref(batch,original_file,coreference_file)\n",
        "    break    "
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "mkdir: cannot create directory ‘original_files’: File exists\n",
            "mkdir: cannot create directory ‘coreference_files’: File exists\n",
            "[['In 2017, Amazon acquired Whole Foods Market for US$13.4 billion. ', 'Jeff Bezos is the CEO of Amazon. ', 'Amazon was founded by Jeff Bezos in Bellevue, Washington, in July 1994. ']]\n",
            "In 2017 , Amazon acquired Whole Foods Market for US$ 13.4 billion . \n",
            "\n",
            " Jeff Bezos is the CEO of Amazon . \n",
            "\n",
            " Amazon was founded by Jeff Bezos in Bellevue , Washington , in July 1994 . \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZctNvsm7WcAO",
        "colab_type": "text"
      },
      "source": [
        "###**EXTRACTING TEMPLATES**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6F0M_EdDTEis",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "######### work and location template ###########\n",
        "file_coref = 'coreference_files/segmented_1.txt'\n",
        "file_ori = 'original_files/segmented_1.txt'\n",
        "final_result = []\n",
        "reader_coref = open(file_coref,'r',encoding = 'utf-8',errors='ignore')\n",
        "reader_ori = open(file_ori,'r',encoding = 'utf-8',errors='ignore')\n",
        "position_list = get_job_list()\n",
        "for line_coref,line_ori in zip(reader_coref,reader_ori):\n",
        "    # print(line_coref)\n",
        "    ## Work template\n",
        "    #line_coref = \"On October 3, 2017, reports indicated Powell Jobs had purchased stake in the ownership group Monumental Sports & Entertainment that includes the NBA's Washington Wizards, NHL's Washington Capitals, and Capital One Arena. Her approximately twenty percent stake makes her the second largest shareholder behind chairman Ted Leonsis.\"\n",
        "    line_coref = line_coref.replace('\\ufeff','')\n",
        "    line_ori = line_ori.replace('\\ufeff','')\n",
        "    line_ori = line_ori.strip()\n",
        "    doc_file_coref = nlp(line_coref)\n",
        "    ### Working on segmented sentences of 1 coref sentence and 1 original sentence\n",
        "    position_coref,location_coref,person_list,sent,loc_template =  get_work_template(doc_file_coref,position_list)\n",
        "    # print('&*****&&&&&&&&*&&&&&&&&***********************',loc_template,'))))')\n",
        "    ## removing duplicates\n",
        "    # person list\n",
        "    person_list_non_dups = []\n",
        "    for item in person_list:\n",
        "        if item:\n",
        "            person_list_non_dups.append(list(set(item)))\n",
        "    #position list\n",
        "    position_coref_non_null = []\n",
        "    for dd in position_coref:\n",
        "        position_coref_non_null.append({key:val for key, val in dd.items() if val})\n",
        "    \n",
        "    #Location list\n",
        "    location_coref_non_null = []\n",
        "    for dd in location_coref:\n",
        "        location_coref_non_null.append({key:val for key, val in dd.items() if val})\n",
        "    # print('----------------BEFORE FINAL',loc_template)\n",
        "    loc_template_non_null = [] \n",
        "    #list(k for k,_ in itertools.groupby(loc_template))\n",
        "    for locations in loc_template:\n",
        "        if locations:\n",
        "            for lcs in locations:\n",
        "                if lcs:\n",
        "                    loc_template_non_null.append(lcs)\n",
        "\n",
        "    loc_template_non_null.sort()\n",
        "    loc_template_non_null = list(k for k,_ in itertools.groupby(loc_template_non_null))\n",
        "    # print('IN LOCATION TEMPLATE',loc_template_non_null)\n",
        "    ######## Work Template ##########################\n",
        "    work_template_dict = []\n",
        "    for persons, positions, locs in zip(person_list_non_dups,position_coref_non_null,location_coref_non_null):\n",
        "        for person in persons:\n",
        "            pos_dict = {}\n",
        "            tmp_dict = {}\n",
        "            if person:\n",
        "                pos_dict['1'] = person\n",
        "                for org, position in positions.items():\n",
        "                    if position:\n",
        "                        pos_dict['2'] = org\n",
        "                        pos_dict['3'] = position\n",
        "                        if org in locs.keys():\n",
        "                            pos_dict['4'] = locs[org]\n",
        "                        tmp_dict['sentences'] = line_ori\n",
        "                        tmp_dict['template'] = 'WORK'\n",
        "                        pos_tmp_dict = pos_dict.copy()\n",
        "                        tmp_dict['arguments'] = pos_tmp_dict\n",
        "                        temp = tmp_dict.copy()\n",
        "                        work_template_dict.append(temp)\n",
        "    # print('WORK: ',work_template_dict)\n",
        "    \n",
        "    ################# Location Template #######################\n",
        "    location_template_dict = []\n",
        "    for location in loc_template_non_null:\n",
        "        loc_dict = {}\n",
        "        tmp_dict = {}\n",
        "        loc_dict['1'] = location[0]\n",
        "        loc_dict['2'] = location[1]\n",
        "        tmp_dict['sentence'] = line_ori\n",
        "        loc_tmp_dict = loc_dict.copy()\n",
        "        tmp_dict['argument'] = loc_tmp_dict\n",
        "        tmp_dict['template'] = 'PART'\n",
        "        temp = tmp_dict.copy()\n",
        "        location_template_dict.append(temp)\n",
        "    # print('######### LT: ',location_template_dict)\n",
        "     \n",
        "    ################# Buy Template ############################\n",
        "    list_buy = extract_buy_template(doc_file_coref) \n",
        "    # print(list_buy)\n",
        "    buy_dict = []\n",
        "    for items in list_buy:\n",
        "        buy = items.split('||')\n",
        "        # print(buy)\n",
        "        if buy[0]:\n",
        "            tmp_dict = {}\n",
        "            buy_sell_dict = {}\n",
        "            buy_sell_dict['buyer'] = buy[0]\n",
        "            buy_sell_dict['item'] = buy[1]\n",
        "            buy_sell_dict['price'] = buy[2]\n",
        "            buy_sell_dict['quantity'] = buy[3]\n",
        "            buy_sell_dict['source'] = buy[4]\n",
        "            tmp_dict['sentence'] = line_ori\n",
        "            tmp_dict['template'] = 'BUY'\n",
        "            buy_sell_tmp_dict = buy_sell_dict.copy()\n",
        "            tmp_dict['arguments'] = buy_sell_tmp_dict\n",
        "            temp = tmp_dict.copy()\n",
        "            buy_dict.append(temp)\n",
        "    # print('buy_template',buy_dict)\n",
        "    if work_template_dict:\n",
        "        final_result.append(work_template_dict)\n",
        "    if location_template_dict:\n",
        "        final_result.append(location_template_dict)\n",
        "    if buy_dict:\n",
        "        final_result.append(buy_dict)\n",
        "\n",
        "json_string = json.dumps(final_result)\n",
        "with open('result.json','w',encoding='utf-8') as file:\n",
        "    json.dump(final_result,file,ensure_ascii=False,indent=2)        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XKCt4JakoYIJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 675
        },
        "outputId": "f499ee38-1c55-4ad2-aeba-7ebc94d796cf"
      },
      "source": [
        "parsed = json.loads(json_string)\n",
        "print(json.dumps(parsed, indent=4, sort_keys=True))"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[\n",
            "    [\n",
            "        {\n",
            "            \"arguments\": {\n",
            "                \"1\": \"Jeff  Bezos\",\n",
            "                \"2\": \"Amazon\",\n",
            "                \"3\": [\n",
            "                    \"CEO\"\n",
            "                ]\n",
            "            },\n",
            "            \"sentences\": \"In 2017, Amazon acquired Whole Foods Market for US$13.4 billion.Jeff Bezos is the CEO of Amazon.Amazon was founded by Jeff Bezos in Bellevue, Washington, in July 1994.\",\n",
            "            \"template\": \"WORK\"\n",
            "        }\n",
            "    ],\n",
            "    [\n",
            "        {\n",
            "            \"argument\": {\n",
            "                \"1\": \"Bellevue\",\n",
            "                \"2\": \"Washington\"\n",
            "            },\n",
            "            \"sentence\": \"In 2017, Amazon acquired Whole Foods Market for US$13.4 billion.Jeff Bezos is the CEO of Amazon.Amazon was founded by Jeff Bezos in Bellevue, Washington, in July 1994.\",\n",
            "            \"template\": \"PART\"\n",
            "        }\n",
            "    ],\n",
            "    [\n",
            "        {\n",
            "            \"arguments\": {\n",
            "                \"buyer\": \"Amazon\",\n",
            "                \"item\": \"Whole Foods Market\",\n",
            "                \"price\": \"US$ 13.4 billion\",\n",
            "                \"quantity\": \"\",\n",
            "                \"source\": \"\"\n",
            "            },\n",
            "            \"sentence\": \"In 2017, Amazon acquired Whole Foods Market for US$13.4 billion.Jeff Bezos is the CEO of Amazon.Amazon was founded by Jeff Bezos in Bellevue, Washington, in July 1994.\",\n",
            "            \"template\": \"BUY\"\n",
            "        }\n",
            "    ]\n",
            "]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}