Reading_Tokenizing And Storing Files.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "735e40fb-b8fc-47dc-b19b-43e2d15869ac"
    }
   },
   "source": [
    "# Reading ,Tokenizing And Storing Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "1d3675a9-b3e4-4ecb-a684-ae4919fdf5da"
    }
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "import os\n",
    "import re\n",
    "import operator\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import wordpunct_tokenize\n",
    "from nltk.corpus import wordnet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "93a1699a-cad7-4252-a0e9-443e47b3d1e3"
    }
   },
   "source": [
    "  &#x202b; \n",
    "  تتألف المدونة من مجلدين. يحوي المجلد الأول 607 من الملفات المشبوهة والثاني يضم 567 ملف أصلي(مصدر تمت سرقة النصوص منه).والمسارات التالية تعبر عنهما\n",
    " <br>\n",
    " <br>\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "f58b4d5e-d0c3-4f55-b582-2d04c2efccff"
    }
   },
   "outputs": [],
   "source": [
    "source_folder = 'source-documents'\n",
    "suspicious_folder = 'suspicious-documents'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "2462eaf3-af65-4c5c-9694-b28d280499f3"
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "\n",
    "from psycopg2 import connect\n",
    "from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "b462b3d1-1fa8-4480-94c1-a0becbe21fa4"
    }
   },
   "source": [
    "  <br>\n",
    "  &#x202b;نقوم بخلق قاعدة بيانات لمرة واحدة فقط. تضم قاعدة البيانات الملفات الأصلية والمشبوهة والجمل المكونة لكليهما بأشكالها الثلاثة كما ورد ذكرها سابقاً\n",
    " <br>\n",
    "  <br>\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "3ef92094-58d7-48e4-8f45-da12f285440a"
    }
   },
   "outputs": [],
   "source": [
    "dbname='plagiarisimdb2'\n",
    "con = None\n",
    "con = connect(dbname='postgres', user='postgres', host='localhost', password='ucv')\n",
    "\n",
    "con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)\n",
    "cur = con.cursor()\n",
    "cur.execute('CREATE DATABASE ' + dbname)\n",
    "cur.close()\n",
    "con.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "2fb0d590-0f06-4fc6-9de2-1fec8a1c8185"
    }
   },
   "outputs": [],
   "source": [
    "from sqlalchemy import Column, ForeignKey, Integer, String, UnicodeText\n",
    "from sqlalchemy.ext.declarative import declarative_base\n",
    "from sqlalchemy.orm import relationship,sessionmaker\n",
    "from sqlalchemy import create_engine"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "9e631218-f21f-4896-b68a-bbaac220d9ff"
    }
   },
   "source": [
    "<br>\n",
    ".&#x202b;نحن بحاجة لأربعة صفوف\n",
    "<br>\n",
    "1-صف لتخزين الملفات الأصلية\n",
    "<br>\n",
    "2-صف لتخزين الملفات المشبوهة\n",
    "<br>\n",
    "3-صف لتخزين الجمل والجمل النظيفة والجمل المجذعة للملف الأصلي\n",
    "<br>\n",
    "4-صف لتخزين الجمل والجمل النظيفة والجمل المجذعة للملف المشبوه\n",
    "\n",
    "<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Base = declarative_base()\n",
    "\n",
    "\n",
    "class source_file(Base):\n",
    "    __tablename__ = 'source_file'\n",
    "    # Here we define columns for the table source file\n",
    "    # Notice that each column is also a normal Python instance attribute.\n",
    "    Fname = Column(String(100),primary_key=True)\n",
    "    text = Column(UnicodeText, nullable=False)\n",
    "\n",
    "class source_sentence(Base):\n",
    "    __tablename__ = 'source_sentence'\n",
    "    # Here we define columns for the table source sentence.\n",
    "    id = Column(Integer, primary_key=True)\n",
    "    sentence = Column(UnicodeText, nullable=False)\n",
    "    cleaned_sentence = Column(UnicodeText, nullable=False)\n",
    "    stemmed_sentence = Column(UnicodeText, nullable=False)\n",
    "    offset_sentence = Column(Integer, nullable=False)\n",
    "    length_sentence = Column(Integer, nullable=False)\n",
    "    source_file_name = Column(String(100), ForeignKey('source_file.Fname'))\n",
    "    source_file = relationship(source_file)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "759c5c7e-71c0-4ceb-8635-decdb455b754"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "class suspicious_file(Base):\n",
    "    __tablename__ = 'suspicious_file'\n",
    "    # Here we define columns for the table suspicious file\n",
    "    #id = Column(Integer, )\n",
    "    Fname = Column(String(100),primary_key=True)\n",
    "    text = Column(UnicodeText, nullable=False)\n",
    "\n",
    "class suspicious_sentence(Base):\n",
    "    __tablename__ = 'suspicious_sentence'\n",
    "    # Here we define columns for the table suspicious sentence.\n",
    "    id = Column(Integer, primary_key=True)\n",
    "    sentence = Column(UnicodeText, nullable=False)\n",
    "    cleaned_sentence = Column(UnicodeText, nullable=False)\n",
    "    stemmed_sentence = Column(UnicodeText, nullable=False)\n",
    "    offset_sentence = Column(Integer, nullable=False)\n",
    "    length_sentence = Column(Integer, nullable=False)\n",
    "    suspicious_file_name = Column(String(100), ForeignKey('suspicious_file.Fname'))\n",
    "    suspicious_file = relationship(suspicious_file)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "91a7985b-33d4-4e41-919c-555bce66b375"
    }
   },
   "source": [
    "&#x202b;إنشاء المحرك الذي سيتعامل مع قاعدة البيانات وسيخزن المعطيات فيها.وإنشاء جميع الجداول عن طريق المحرك\n",
    "\n",
    "<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "fb7d3a07-4b59-4d8e-9659-a9c940fb38e3"
    }
   },
   "outputs": [],
   "source": [
    "engine = create_engine('postgresql+psycopg2://postgres:ucv@localhost/plagiarisimdb2')\n",
    "# This is equivalent to \"Create Table\" statements in raw SQL.\n",
    "Base.metadata.create_all(engine)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "74a1ac81-aeb3-4919-ac93-973cab1e1fc0"
    }
   },
   "source": [
    "&#x202b;DBSessionيقوم الصف \n",
    "بعقد كل المحادثات والمراسلات مع قاعدة البيانات ويشكل منطقة وسطى لاحتوائه على الأغراض المخزنة في غرض قاعدة البيانات. وأي تغيير على أغراض الجلسة المفتوحة حالياً لن يتم اعتماده حتى ننفذ إجرائية التأكيدcommit\n",
    "\n",
    "<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "863374ce-b03b-4f00-8daf-a9fc4391ae6d"
    }
   },
   "outputs": [],
   "source": [
    "# Bind the engine to the metadata of the Base class so that the\n",
    "# declaratives can be accessed through a DBSession instance\n",
    "Base.metadata.bind = engine\n",
    "DBSession = sessionmaker(bind=engine)\n",
    " \n",
    "session = DBSession()\n",
    "#If you're not happy about the changes, you can\n",
    "# revert all of them back to the last commit by calling\n",
    "# session.rollback()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "75ce3e41-6f42-4bf7-b818-76b7cea9a9f8"
    }
   },
   "source": [
    "ملخص سريع : \n",
    "في الخلايا الآتية<br> يتم قراءة الملفات واحد تلو الآخر وتخزينها في قاعدة البيانات ومن ثم تقطيع الملف لجمل وتنظيفها ورد كلماتها إلى الجذع المقابل وبالنهاية تخزين كل ما سبق ذكره\n",
    "\n",
    " ,Motaz SAADلصاحبه \n",
    " Arabic light stemming المجذّع المستخدم هو \n",
    "<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "634560b0-e662-49f7-ad91-4e9257c5f186"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\envs\\python27\\lib\\site-packages\\gensim\\utils.py:840: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "import codecs \n",
    "import re\n",
    "from nltk.tokenize import wordpunct_tokenize,sent_tokenize\n",
    "import imp\n",
    "\n",
    "tp = imp.load_source('textpro', 'textpro.py')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "cd5f4c31-ace9-476d-9bbc-ddf8ee0c6ad7"
    }
   },
   "outputs": [],
   "source": [
    "all_source_files = os.listdir(source_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def tokzr_SENT(_txt): \n",
    "    return ( re.findall(ur'(?ms)\\s*(.*?(?:\\.|؟|!)+)', _txt))  # split sentences\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "60db906c-f3c8-4339-b82a-c3f71536a2cc"
    }
   },
   "source": [
    "&#x202b;هنا يتم التعامل مع الملفات الأصلية<br />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "4032cd3c-0c9a-4c4a-b38d-e03be43618ab"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "m\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for file_name in all_source_files:\n",
    "    with codecs.open(source_folder + \"\\\\\" + file_name,'r',encoding = 'utf-8') as f: \n",
    "        txt = f.read()  #... 1 file\n",
    "        m = filter(None, re.split(r'(\\w.+)(\\.\\w+)',file_name ))[0]\n",
    "        s_file_object = source_file(Fname=m,text = unicode(txt))\n",
    "        session.add(s_file_object)   #... 2  store\n",
    "        session.commit()\n",
    "  \n",
    "        sent_tokens = sent_tokenize(txt)    #... 3  sentence\n",
    "        for A_sentence in sent_tokens: \n",
    "            clean_sentence = tp.process_text(A_sentence,removePunct=True, removeSW=False, removeNum=True)  #... 4  clean\n",
    "            stem_sentence = tp.lightStemAr(clean_sentence)    #.... 5 stem\n",
    "            offset = txt.find(A_sentence)\n",
    "            length = len(A_sentence)\n",
    "            sentence_object=source_sentence(offset_sentence=offset, length_sentence=length,sentence=unicode(A_sentence), cleaned_sentence=clean_sentence, stemmed_sentence=unicode(stem_sentence),source_file = s_file_object)\n",
    "            session.add(sentence_object)   #... 6  store\n",
    "            session.commit()\n",
    "    \n",
    "print \"m\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "c6a8c08b-f126-48ec-9273-c240b7ef3187"
    }
   },
   "source": [
    "&#x202b;هنا يتم التعامل مع الملفات المشبوهة<br />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "3de86b99-9fea-471f-8d9a-ea4e93fe9942"
    }
   },
   "outputs": [],
   "source": [
    "all_suspicious_files = os.listdir(suspicious_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "7fbcf7da-cbbd-485a-b585-4dc13b50cbd9"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "m\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for file_name in all_suspicious_files:\n",
    "    with codecs.open(suspicious_folder + \"\\\\\" + file_name,'r',encoding = 'utf-8') as f: \n",
    "        txt = f.read()  #... 1 file\n",
    "        #f_id = int(filter(None, re.split(r'(\\w.+)(\\d\\d\\d)',all_source_files[0] ))[1])\n",
    "        m=filter(None, re.split(r'(\\w.+)(\\.\\w+)',file_name ))[0]\n",
    "        s_file_object = suspicious_file(Fname =m,text = unicode(txt))\n",
    "        session.add(s_file_object)   #... 2  store\n",
    "        session.commit()\n",
    "        \n",
    "        sent_tokens = sent_tokenize(txt)    #... 3  sentence\n",
    "        for A_sentence in sent_tokens: \n",
    "            clean_sentence = tp.process_text(A_sentence,removePunct=True, removeSW=False, removeNum=True)  #... 4  clean\n",
    "            stem_sentence = tp.lightStemAr(clean_sentence)    #.... 5 stem\n",
    "            offset = txt.find(A_sentence)\n",
    "            length = len(A_sentence)\n",
    "            sentence_object=suspicious_sentence(offset_sentence=offset, length_sentence=length,sentence=unicode(A_sentence), cleaned_sentence=clean_sentence, stemmed_sentence=unicode(stem_sentence),suspicious_file =s_file_object)\n",
    "            session.add(sentence_object)   #... 6  store\n",
    "            session.commit()\n",
    "            \n",
    "print \"m\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:python27]",
   "language": "python",
   "name": "conda-env-python27-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}