Preprocess_Docs_IMDB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "data_path = \"/path_to_your_data_folder/\"\n",
    "\n",
    "\n",
    "positiveFiles = [data_path + 'positiveReviews/' + f for f in listdir(data_path + 'positiveReviews/') if isfile(join(data_path + 'positiveReviews/', f))]\n",
    "negativeFiles = [data_path + 'negativeReviews/' + f for f in listdir(data_path + 'negativeReviews/') if isfile(join(data_path + 'negativeReviews/', f))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def tokenFiles(files,maxNumWords,tk,maxNumFiles=None):\n",
    "    # tk: instance of keras tokenizer class\n",
    "    fileCounter = 0\n",
    "    \n",
    "    for fname in files:\n",
    "        with open(fname, \"r\", encoding='utf-8') as f:        \n",
    "            doc=f.readlines()\n",
    "            # fit the tokenizUsing TensorFlow backend.er on the documents\n",
    "            tk.fit_on_texts(doc)\n",
    "        \n",
    "        if (fileCounter % 100 == 0 and fileCounter > 0):\n",
    "            print(fileCounter, len(t.word_counts))\n",
    "        \n",
    "        if (maxNumFiles is not None and fileCounter == maxNumFiles-1):\n",
    "            break\n",
    "            \n",
    "        fileCounter += 1 \n",
    "            \n",
    "    return tk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def encodeFiles(files,maxSeqLength,tk,maxNumFiles):\n",
    "    # tk: instance of keras tokenizer class\n",
    "    ids = np.zeros((maxNumFiles, maxSeqLength), dtype='int32')\n",
    "    fileCounter = 0 \n",
    "    \n",
    "    for fname in files:\n",
    "        with open(fname, \"r\", encoding='utf-8') as f:        \n",
    "            doc=f.readlines()\n",
    "            # encode the document\n",
    "            id = tk.texts_to_sequences(doc)\n",
    "            # form a fixed length Using TensorFlow backend.sequence for each doc\n",
    "            idp = pad_sequences(id, maxlen=maxSeqLength, dtype='int32', padding='post', truncating='post', value=0.0) \n",
    "            # form an array for all documents\n",
    "            ids[fileCounter,:] = idp\n",
    "        \n",
    "        if (fileCounter % 100 == 0 and fileCounter > 0):\n",
    "            print(fileCounter)\n",
    "        \n",
    "        if fileCounter == maxNumFiles - 1:\n",
    "            break\n",
    "            \n",
    "        fileCounter += 1\n",
    "\n",
    "    return ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create the tokenizer\n",
    "MAX_NUM_WORDS = 40000 # only use top n words\n",
    "maxNumFiles = 10 # only apply to this numner of files\n",
    "maxSeqLength = 250 #Maximum number of words for each document Document level sentiment analysis obtains the sentiment of a complete document or paragraph.\n",
    "\n",
    "t = Tokenizer(num_words = MAX_NUM_WORDS, oov_token='UNK')\n",
    "#t = tf.keras.preprocessing.text.Tokenizer(num_words = MAX_NUM_WORDS, oov_token='UNK') # this api in tf is not working correctly for attribute \"document_count\"   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load all files to create wordsList\n",
    "\n",
    "# positive reviews\n",
    "t = tokenFiles(positiveFiles,MAX_NUM_WORDS,t)\n",
    "\n",
    "print('Positive files finished')\n",
    "# summarize what was learned\n",
    "print(len(t.word_counts))\n",
    "print(t.document_count)\n",
    "\n",
    "# negative reviews\n",
    "t = tokenFiles(negativeFiles,MAX_NUM_WORDS,t)\n",
    "\n",
    "print('All files finished') \n",
    "\n",
    "# summarize what was learned\n",
    "print(len(t.word_counts))\n",
    "print(t.document_count)\n",
    "\n",
    "# only save top MAX_NUM_WORDS-1 words, leave 0 for padding blanks\n",
    "t.word_index = {e:i for e,i in t.word_index.items() if i < MAX_NUM_WORDS} # tokenizer is 1 indexed \n",
    " \n",
    "# save Word index file\n",
    "np.save(\"wordIndex_all\", t.word_index)\n",
    "\n",
    "# encode all files based on tokenizer\n",
    "\n",
    "# positive reviews\n",
    "ids_p = encodeFiles(positiveFiles,maxSeqLength,t,len(positiveFiles))\n",
    "np.save(\"idsMatrix_pos\", ids_p)\n",
    "\n",
    "# negative reviews\n",
    "ids_n = encodeFiles(negativeFiles,maxSeqLength,t,len(negativetiveFiles))\n",
    "np.save(\"idsMatrix_neg\", ids_n)\n",
    "\n",
    "# merge all files\n",
    "all_ids = np.concatenate((ids_p, ids_n), axis=0)\n",
    "np.save(\"idsMatrix_all\", all_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load preprocessed data if wordIndex and idsMatrix files have been created.\n",
    "\n",
    "wordsList = np.load('wordIndex_all.npy')\n",
    "\n",
    "wordsList = wordsList.tolist() #Originally loaded as numpy array\n",
    "\n",
    "#index_to_word = {i: w for w, i in t.word_index.items()}\n",
    "# Reverse from integers to words using the DICTIONARY\n",
    "reverse_word_index = dict(\n",
    "[(value, key) for (key, value) in wordsList.items()])\n",
    "\n",
    "\n",
    "ids = np.load('idsMatrix_all.npy')\n",
    "\n",
    "#decoded_review = ' '.join([reverse_word_index.get(i, '?') for i in ids[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}