three authors complete code modified 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import glob\n",
    "import os.path \n",
    "from __future__ import division\n",
    "import math\n",
    "import re\n",
    "\n",
    "\n",
    "def matching( f ):\n",
    "    x=f.name.rsplit('\\\\')\n",
    "    y=x[3].rsplit('.')\n",
    "    match = re.match(r\"([a-z]+)([0-9]+)\", y[0], re.I)\n",
    "    if match:\n",
    "         items = match.groups()\n",
    "    return items\n",
    "\n",
    "\n",
    "def big_doc_c(replace):\n",
    "        list_of_files = glob.glob('corpus\\\\D\\\\'+replace+'\\\\train\\\\*.txt') \n",
    "        i=0\n",
    "\n",
    "        for file_name in list_of_files:\n",
    "            f2write = open('corpus\\\\D\\\\bigdoc_c\\\\'+replace+'%s.txt' % i,'w')\n",
    "            i+=1\n",
    "            with open(file_name,'r') as f:\n",
    "                for line in f:\n",
    "                    if(line!='\\n'):\n",
    "                        for word in line.split():\n",
    "                            f2write.write(word+\"\\n\")\n",
    "            f2write.close()\n",
    "def testdoc(replace):\n",
    "    list_of_files = glob.glob('corpus\\\\D\\\\test\\\\'+replace+'\\\\*.txt') \n",
    "    i=0\n",
    "    for file_name in list_of_files:\n",
    "        f2write = open('corpus\\\\D\\\\test_complete_doc\\\\'+replace+'%s.txt' % i,'w')\n",
    "        i+=1\n",
    "        with open(file_name,'r') as f:\n",
    "            for line in f:\n",
    "                if(line!='\\n'):\n",
    "#                 f1=open(\"unigram_tokens_of_english.txt\",\"a\")\n",
    "                    for word in line.split():\n",
    "#                 word_list.append(word)\n",
    "                        f2write.write(word+\"\\n\")\n",
    "        f2write.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vocab(replace):\n",
    "        value = 1\n",
    "        new_dict_words = []\n",
    "        total_count_distinct_words = 0\n",
    "        dict_word = []\n",
    "        \n",
    "        \n",
    "#         tokens in files are genareted\n",
    "        \n",
    "        \n",
    "        list_of_files = glob.glob('corpus\\\\D\\\\'+replace+'\\\\train\\\\*.txt') \n",
    "        i=0\n",
    "        for file_name in list_of_files:\n",
    "            f2write_pve = open('corpus\\\\D\\\\big_doc_D\\\\'+replace+'%s.txt' % i,'w')\n",
    "            i+=1\n",
    "            with open(file_name,'r') as f:\n",
    "                for line in f:\n",
    "                    if(line!='\\n'):\n",
    "                        for word in line.split():\n",
    "                            f2write_pve.write(word+\"\\n\")\n",
    "            f2write_pve.close()\n",
    "        \n",
    "#     for creating word-LIST for making dictionary\n",
    "    \n",
    "        word_seq = []\n",
    "        counts = []\n",
    "        list_of_files = glob.glob('corpus\\\\D\\\\big_doc_D\\\\*.txt') \n",
    "        for filename in (list_of_files):\n",
    "            with open(filename,'r') as f:  \n",
    "                for word in f:\n",
    "                    dict_word.append(word)\n",
    "        \n",
    "        ii=0\n",
    "        while ii < len(dict_word):\n",
    "            word = dict_word[ii]\n",
    "            i=0\n",
    "            while i < len(new_dict_words):\n",
    "                if(new_dict_words[i]==word):\n",
    "                    value=0\n",
    "                    break\n",
    "                else:\n",
    "                    value = 1\n",
    "                i+=1\n",
    "            if(value==1):\n",
    "                new_dict_words.append(word)\n",
    "                total_count_distinct_words+=1\n",
    "            ii+=1\n",
    "        print(\"Total number of vocab elements : \"+str(total_count_distinct_words))\n",
    "        return new_dict_words\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of vocab elements : 4688\n",
      "Total number of vocab elements : 4688\n",
      "Total number of vocab elements : 4688\n",
      "Tokens Count : 4034\n",
      "Vocab Length : 4688\n",
      "0\n",
      "Tokens Count : 3384\n",
      "Vocab Length : 4688\n",
      "1\n",
      "Tokens Count : 10495\n",
      "Vocab Length : 4688\n",
      "2\n",
      "[0.27967145790554415, 0.19014373716632443, 0.5301848049281315]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "ijk=0\n",
    "\n",
    "\n",
    "word_asad = []\n",
    "word_ghalib = []\n",
    "word_iqbal = []\n",
    "\n",
    "total_count_asad=0\n",
    "total_count_iqbal=0\n",
    "total_count_ghalib=0\n",
    "\n",
    "asad_livelihood=[]\n",
    "iqbal_livelihood=[]\n",
    "ghalib_livelihood=[]\n",
    "\n",
    "prior = []\n",
    "\n",
    "\n",
    "V = []\n",
    "    \n",
    "\n",
    "asad_files = glob.glob('corpus\\\\D\\\\AsadFolder\\\\train\\\\*.txt') \n",
    "iqbal_files = glob.glob('corpus\\\\D\\\\IqbalFolder\\\\train\\\\*.txt')  \n",
    "ghalib_files = glob.glob('corpus\\\\D\\\\GhalibFolder\\\\train\\\\*.txt')\n",
    "\n",
    "total_files = len(asad_files) + len(iqbal_files) + len(ghalib_files)\n",
    "ijk=0\n",
    "while(ijk<3):\n",
    "\n",
    "    if(ijk==0):\n",
    "        replace = \"AsadFolder\"\n",
    "    if(ijk==1):\n",
    "        replace = \"IqbalFolder\"\n",
    "    if(ijk==2):\n",
    "        replace = \"GhalibFolder\"\n",
    "#     prior\n",
    "    files = glob.glob('corpus\\\\D\\\\'+replace+'\\\\train\\\\*.txt') \n",
    "    prior.append((float(len(files)/total_files)))\n",
    "#     vocab\n",
    "    V = vocab(replace)\n",
    "#     big_document\n",
    "    big_doc_c(replace)\n",
    "\n",
    "    \n",
    "    ijk+=1\n",
    "ijk=0\n",
    "while(ijk<3):\n",
    "\n",
    "    if(ijk==0):\n",
    "        replace = \"AsadFolder\"\n",
    "    if(ijk==1):\n",
    "        replace = \"IqbalFolder\"\n",
    "    if(ijk==2):\n",
    "        replace = \"GhalibFolder\"\n",
    "        \n",
    "#     tokens count in documents\n",
    "\n",
    "    tokens_count = 0\n",
    "    list_of_files1  = glob.glob('corpus\\\\D\\\\bigdoc_c\\\\*.txt') \n",
    "    for file_name1 in list_of_files1:\n",
    "            with open(file_name1,'r') as f1: \n",
    "                items = matching(f1)\n",
    "                if(items[0]==replace):\n",
    "                    for line1 in f1:\n",
    "                        tokens_count = tokens_count + 1\n",
    "    print(\"Tokens Count : \"+str(tokens_count))\n",
    "\n",
    "#      Calculating Livelihood and appending word sequence \n",
    "    print(\"Vocab Length : \"+str(len(V)))\n",
    "    for w in V:\n",
    "                    list_of_files1  = glob.glob('corpus\\\\D\\\\bigdoc_c\\\\*.txt') \n",
    "                    c=0\n",
    "                    for file_name1 in list_of_files1:\n",
    "                        with open(file_name1,'r') as f1: \n",
    "                                        items = matching(f1)\n",
    "\n",
    "                                        if(items[0]==replace):\n",
    "                                            for line1 in f1:\n",
    "                                                if(w==line1):\n",
    "                                                    c = c + 1\n",
    "            \n",
    "            \n",
    "                    if(replace==\"AsadFolder\"):\n",
    "                        word_asad.append(w)\n",
    "                        asad_livelihood.append((float((c+1)/(((tokens_count))+len(V)))))\n",
    "                    if(replace==\"IqbalFolder\"):\n",
    "                        word_iqbal.append(w)\n",
    "                        iqbal_livelihood.append((float((c+1)/((tokens_count)+len(V)))))\n",
    "                    if(replace==\"GhalibFolder\"):\n",
    "                        word_ghalib.append(w)\n",
    "                        ghalib_livelihood.append((float((c+1)/((tokens_count)+len(V)))))\n",
    "    \n",
    "    print(ijk)    \n",
    "    ijk+=1\n",
    "# print(pve_livelihood)\n",
    "# print(nve_livelihood)\n",
    "# print(word_pve)\n",
    "# print(word_nve)\n",
    "# print(len(V))\n",
    "print(prior)\n",
    "\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "ijk=0\n",
    "while(ijk<3):\n",
    "    if(ijk==0):\n",
    "        replace = \"AsadFolder\"\n",
    "    if(ijk==1):\n",
    "        replace = \"IqbalFolder\"\n",
    "    if(ijk==2):\n",
    "        replace = \"GhalibFolder\"\n",
    "    testdoc(replace)\n",
    "    ijk+=1\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.279671457906\n",
      "0.190143737166\n",
      "0.530184804928\n"
     ]
    }
   ],
   "source": [
    "#################################################################################\n",
    "#              \t\tCaculating Contigency Matrix\n",
    "#################################################################################\n",
    "from __future__ import division\n",
    "import math\n",
    "i=0\n",
    "j=2\n",
    "tn=0\n",
    "tp=0\n",
    "fp=0\n",
    "fn=0\n",
    "sum = 1\n",
    "sum_concat = []\n",
    "import re \n",
    "list_of_files = glob.glob('corpus\\\\D\\\\test_complete_doc\\\\*.txt') \n",
    "# print(list_of_files)\n",
    "i=0\n",
    "j=3\n",
    "b=1\n",
    "c=1\n",
    "ta_a=0\n",
    "fa_i=0\n",
    "fa_g=0\n",
    "fi_a=0\n",
    "ti_i=0\n",
    "fi_g=0\n",
    "fg_a=0\n",
    "fg_i=0\n",
    "tg_g=0\n",
    "print(prior[0])\n",
    "print(prior[1])\n",
    "print(prior[2])\n",
    "for file_name in list_of_files:\n",
    "#     f2write_asad = open('corpus\\\\D\\\\bigdoc_c\\\\asad%s.txt' % i,'w')\n",
    "        sum_concat = []\n",
    "\n",
    "        i=0\n",
    "        while i < 3:\n",
    "            if(i==0):\n",
    "                sum=(sum*prior[0])\n",
    "            elif(i==1):\n",
    "                sum=(sum*prior[1])\n",
    "            elif(i==2):\n",
    "                sum=(sum*prior[2])\n",
    "            with open(file_name,'r') as f:\n",
    "                      for line in f:\n",
    "                        for words in line.split():\n",
    "                            if (words+'\\n') in V:\n",
    "                                    if(i==1):\n",
    "                                        if (words+'\\n') in word_asad:\n",
    "                                                x=word_asad.index(words+'\\n')\n",
    "                                                sum=(sum*asad_livelihood[x])\n",
    "                                    if(i==0):\n",
    "                                        if (words+'\\n') in word_iqbal:\n",
    "                                                x=word_iqbal.index(words+'\\n')\n",
    "                                                sum=(sum*iqbal_livelihood[x])\n",
    "                                    if(i==2):\n",
    "                                        if (words+'\\n') in word_ghalib:\n",
    "                                                x=word_ghalib.index(words+'\\n')\n",
    "                                                sum=float(sum*ghalib_livelihood[x])\n",
    "            \n",
    "            sum_concat.append(sum)\n",
    "            i+=1\n",
    "            sum=1\n",
    "#             print((sum_concat))\n",
    "            \n",
    "            if(len(sum_concat)>2):\n",
    "                items = matching(f)\n",
    "                if(sum_concat[1]<sum_concat[0] and sum_concat[2]<sum_concat[0]):#iqbal=0, asad=1\n",
    "                    if(items[0]==\"asad\"):# system=asad , gold=asad\n",
    "                        ta_a+=1\n",
    "                    elif(items[0]==\"iqbal\"):#  system=iqbal, gold=asad\n",
    "                        fa_i+=1\n",
    "                    elif(items[0]==\"ghalib\"):#  system=iqbal, gold=asad\n",
    "                        fa_g+=1\n",
    "                elif(sum_concat[0]<sum_concat[1] and sum_concat[2]<sum_concat[1]):#iqbal=0, asad=1\n",
    "                    if(items[0]==\"iqbal\"):# system=iqbal, gold=iqbal\n",
    "                        fi_a+=1\n",
    "                    elif(items[0]==\"asad\"):# system=asad, gold=iqbal\n",
    "                        ti_i+=1\n",
    "                    elif(items[0]==\"ghalib\"):# system=asad, gold=iqbal\n",
    "                        fi_g+=1\n",
    "                elif(sum_concat[1]<sum_concat[2] and sum_concat[0]<sum_concat[2]):#iqbal=0, asad=1\n",
    "                    if(items[0]==\"asad\"):# system=iqbal, gold=iqbal\n",
    "                        fg_a+=1\n",
    "                    elif(items[0]==\"iqbal\"):# system=asad, gold=iqbal\n",
    "                        fg_i+=1\n",
    "                    elif(items[0]==\"ghalib\"):# system=asad, gold=iqbal\n",
    "                        tg_g+=1\n",
    "# recall=float((tp)/(tp+fn))\n",
    "# precision=float((tp)/(tp+tn))\n",
    "accuracy=float((ta_a+ti_i+tg_g)/(ta_a+ti_i+tg_g+fa_i+fa_g+fi_a+fi_g+fg_a+fg_i))\n",
    "ra = (ta_a)/(ta_a+fi_a+fg_a)\n",
    "ri = (ti_i)/(fa_i+ti_i+fg_i)\n",
    "rg = (tg_g)/(fa_g+fi_g+tg_g)\n",
    "\n",
    "pa = (ta_a)/(ta_a+fa_i+fa_g)\n",
    "pi = (ti_i)/(fi_a+ti_i+fi_g)\n",
    "pg = (tg_g)/(fg_a+fg_i+tg_g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Asad , Iqbal,  Ghalib\n",
      "Asad 16, 53, 0\n",
      "Iqbal 10, 77, 1\n",
      "Ghalib 77, 53, 256\n",
      "Recall Asad: 15.5339805825\n",
      "Recall Iqbal: 42.0765027322\n",
      "Recall Ghalib: 99.6108949416\n",
      "Precision Asad: 23.1884057971\n",
      "Precision Iqbal: 87.5\n",
      "Precision Ghalib: 66.3212435233\n",
      "Accuracy: 64.2725598527\n"
     ]
    }
   ],
   "source": [
    "#################################################################################\n",
    "#              \t\tContigency Matrix\n",
    "#################################################################################\n",
    "# print(tp)\n",
    "# print(tn)\n",
    "# print(fp)\n",
    "# print(fn)\n",
    "from __future__ import division\n",
    "print(\"   Asad , Iqbal,  Ghalib\")\n",
    "print(\"Asad \"+str(ta_a)+\", \"+str(fa_i)+\", \"+str(fa_g))# TP,TN\n",
    "print(\"Iqbal \"+str(fi_a)+\", \"+str(ti_i)+\", \"+str(fi_g))# FP,FN\n",
    "print(\"Ghalib \"+str(fg_a)+\", \"+str(fg_i)+\", \"+str(tg_g))# FP,FN\n",
    "# print(float((tp+tn)/(tp+tn+fp+fn)*100))\n",
    "# print(\"Recall: \"+str(recall*100))\n",
    "# print(\"Precision: \"+str(precision*100))\n",
    "print(\"Recall Asad: \"+str(ra*100))\n",
    "print(\"Recall Iqbal: \"+str(ri*100))\n",
    "print(\"Recall Ghalib: \"+str(rg*100))\n",
    "\n",
    "print(\"Precision Asad: \"+str(pa*100))\n",
    "print(\"Precision Iqbal: \"+str(pi*100))\n",
    "print(\"Precision Ghalib: \"+str(pg*100))\n",
    "print(\"Accuracy: \"+str(accuracy*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}