ch10_Sentiment Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 10: Sentiment Analysis\n",
    "This document contains python code details related to the Sentiment Analysis chapter.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "First, let's import useful libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pylab as plt\n",
    "#matplotlib inline \n",
    "#plt.style.use('seaborn-whitegrid')\n",
    "plt.rc('text', usetex=True)\n",
    "plt.rc('font', family='times')\n",
    "plt.rc('xtick', labelsize=10) \n",
    "plt.rc('ytick', labelsize=10) \n",
    "plt.rc('font', size=12) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Installation requirements:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are using pip version 6.0.8, however version 8.1.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\n",
      "Requirement already satisfied (use --upgrade to upgrade): unidecode in c:\\users\\sergio\\anaconda\\lib\\site-packages\n"
     ]
    }
   ],
   "source": [
    "!pip install unidecode"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before starting is important to download the nltk corpora. You can download individual data packages or you can download the entire collection (using “all”). Useful corpora for this notebook include wordnet, movie_reviews, and stopwords."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\Sergio\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "#nltk.download()\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example 1:\n",
    "Stemmer/lemmatizer example using NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "LookupError",
     "evalue": "\n**********************************************************************\n  Resource u'corpora/wordnet' not found.  Please use the NLTK\n  Downloader to obtain the resource:  >>> nltk.download()\n  Searched in:\n    - 'C:\\\\Users\\\\Sergio/nltk_data'\n    - 'C:\\\\nltk_data'\n    - 'D:\\\\nltk_data'\n    - 'E:\\\\nltk_data'\n    - 'C:\\\\Users\\\\Sergio\\\\Anaconda\\\\nltk_data'\n    - 'C:\\\\Users\\\\Sergio\\\\Anaconda\\\\lib\\\\nltk_data'\n    - 'C:\\\\Users\\\\Sergio\\\\AppData\\\\Roaming\\\\nltk_data'\n**********************************************************************",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mLookupError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-bc2d9c0b5be7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     25\u001b[0m \u001b[0msnowball\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mword\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     26\u001b[0m \u001b[0msnowball\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mword\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 27\u001b[1;33m \u001b[0mwordnet\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlemmatize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mword\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     28\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     29\u001b[0m \u001b[0mpreprocessed_docs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Sergio\\Anaconda\\lib\\site-packages\\nltk\\stem\\wordnet.pyc\u001b[0m in \u001b[0;36mlemmatize\u001b[1;34m(self, word, pos)\u001b[0m\n\u001b[0;32m     38\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mlemmatize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mword\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpos\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNOUN\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 40\u001b[1;33m         \u001b[0mlemmas\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mwordnet\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_morphy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mword\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpos\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     41\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlemmas\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mlemmas\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mword\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     42\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Sergio\\Anaconda\\lib\\site-packages\\nltk\\corpus\\util.pyc\u001b[0m in \u001b[0;36m__getattr__\u001b[1;34m(self, attr)\u001b[0m\n\u001b[0;32m     97\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"LazyCorpusLoader object has no attribute '__bases__'\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     98\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 99\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__load\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    100\u001b[0m         \u001b[1;31m# This looks circular, but its not, since __load() changes our\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m         \u001b[1;31m# __class__ to something new:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Sergio\\Anaconda\\lib\\site-packages\\nltk\\corpus\\util.pyc\u001b[0m in \u001b[0;36m__load\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     62\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     63\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mroot\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'corpora/%s'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mzip_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 64\u001b[1;33m                 \u001b[1;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     65\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m         \u001b[1;31m# Load the corpus.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mLookupError\u001b[0m: \n**********************************************************************\n  Resource u'corpora/wordnet' not found.  Please use the NLTK\n  Downloader to obtain the resource:  >>> nltk.download()\n  Searched in:\n    - 'C:\\\\Users\\\\Sergio/nltk_data'\n    - 'C:\\\\nltk_data'\n    - 'D:\\\\nltk_data'\n    - 'E:\\\\nltk_data'\n    - 'C:\\\\Users\\\\Sergio\\\\Anaconda\\\\nltk_data'\n    - 'C:\\\\Users\\\\Sergio\\\\Anaconda\\\\lib\\\\nltk_data'\n    - 'C:\\\\Users\\\\Sergio\\\\AppData\\\\Roaming\\\\nltk_data'\n**********************************************************************"
     ]
    }
   ],
   "source": [
    "raw_docs = [\"Here are some very simple basic sentences.\", \"They won't be very interesting, I'm afraid.\", \"The point of these examples is to _learn how basic text cleaning works_ on *very simple* data.\"]\n",
    "            \n",
    "from nltk.tokenize import word_tokenize\n",
    "tokenized_docs = [word_tokenize(doc) for doc in raw_docs]\n",
    "\n",
    "import re\n",
    "import string\n",
    "regex = re.compile('[%s]' % re.escape(string.punctuation))\n",
    "tokenized_docs_no_punctuation = []\n",
    "for review in tokenized_docs:\n",
    "    new_review = []\n",
    "    for token in review:\n",
    "        new_token = regex.sub(u'', token)\n",
    "        if not new_token == u'':\n",
    "            new_review.append(new_token)\n",
    "    tokenized_docs_no_punctuation.append(new_review)\n",
    "\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "porter = PorterStemmer()\n",
    "snowball = SnowballStemmer('english')\n",
    "wordnet = WordNetLemmatizer()\n",
    "\n",
    "snowball.stem(word)\n",
    "snowball.stem(word)\n",
    "wordnet.lemmatize(word)\n",
    "\n",
    "preprocessed_docs = []\n",
    "for doc in tokenized_docs_no_punctuation:\n",
    "    final_doc = []\n",
    "    for word in doc:\n",
    "        final_doc.append(porter.stem(word))\n",
    "        # final_doc.append(snowball.stem(word))\n",
    "        # requires 'corpora/wordnet' -> nltk.download()\n",
    "        # final_doc.append(wordnet.lemmatize(word))\n",
    "        # requires 'corpora/wordnet' -> nltk.download()\n",
    "    preprocessed_docs.append(final_doc)\n",
    "\n",
    "print tokenized_docs_no_punctuation\n",
    "print preprocessed_docs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These examples use functions of the modules \"PorterStemmer\", \"SnowballStemmer\", and \"WordNetLemmatizer\". Results of the three approaches are almost equivalent. Note that the script looks for all the items (words) in the list \"tokenized\\_docs\\_no\\_punctuation\", and for each of them one of the different approaches can be considered. Note that the last two approaches of \"SnowballStemmer\" and \"WordNetLemmatizer\" require to install additional corpora from NLTK, that can be download by running \"nltk.download()\". In this example we are running the first approach \"porter.stem(word)\" adding the new stemmer or lemmatizer results of \"word\" in the new list \"final\\_doc\" that is finally included in\n",
    "\"preprocessed\\_docs\"."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example 2: Word frequencies feature vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('me', 2), ('Mireia', 1), ('Hector', 1), ('loves', 2), ('than', 1), ('more', 1)]\n",
      "[('me', 2), ('Sergio', 1), ('Mireia', 1), ('likes', 1), ('loves', 1), ('than', 1), ('more', 1)]\n",
      "[('footbal', 1), ('basketball', 1), ('likes', 1), ('He', 1), ('than', 1), ('more', 1)]\n",
      "Our vocabulary vector is [me, footbal, basketball, Sergio, Mireia, Hector, likes, loves, He, than, more]\n",
      "The doc is \"Mireia loves me more than Hector loves me\"\n",
      "The tf vector for Document 1 is [2, 0, 0, 0, 1, 1, 0, 2, 0, 1, 1]\n",
      "The doc is \"Sergio likes me more than Mireia loves me\"\n",
      "The tf vector for Document 2 is [2, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1]\n",
      "The doc is \"He likes basketball more than footbal\"\n",
      "The tf vector for Document 3 is [0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1]\n",
      "All combined, here is our master document term matrix: \n",
      "[[2, 0, 0, 0, 1, 1, 0, 2, 0, 1, 1], [2, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1], [0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1]]\n"
     ]
    }
   ],
   "source": [
    "mydoclist = ['Mireia loves me more than Hector loves me', 'Sergio likes me more than Mireia loves me', 'He likes basketball more than footbal']\n",
    "from collections import Counter\n",
    "for doc in mydoclist:\n",
    "    tf = Counter()\n",
    "    for word in doc.split():\n",
    "        tf[word] +=1\n",
    "    print tf.items()\n",
    "    \n",
    "def build_lexicon(corpus):  # define a set with all possible words included\n",
    "                            # in all the sentences or \"corpus\"\n",
    "    lexicon = set()\n",
    "    for doc in corpus:\n",
    "        lexicon.update([word for word in doc.split()])\n",
    "    return lexicon\n",
    "def tf(term, document):\n",
    "    return freq(term, document)\n",
    "def freq(term, document):\n",
    "    return document.split().count(term)\n",
    "vocabulary = build_lexicon(mydoclist)\n",
    "doc_term_matrix = []\n",
    "print 'Our vocabulary vector is [' + ', '.join(list(vocabulary)) + ']'\n",
    "for doc in mydoclist:\n",
    "    print 'The doc is \"' + doc + '\"'\n",
    "    tf_vector = [tf(word, doc) for word in vocabulary]\n",
    "    tf_vector_string = ', '.join(format(freq, 'd') for freq in tf_vector)\n",
    "    print 'The tf vector for Document %d is [%s]' % ((mydoclist.index(doc)+1), tf_vector_string)\n",
    "    doc_term_matrix.append(tf_vector)\n",
    "    \n",
    "print 'All combined, here is our master document term matrix: '\n",
    "print doc_term_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on the previous script, each document is in the same feature space, meaning that we\n",
    "can represent the entire corpus in the same dimensional space\n",
    "without having lost too much information. Once we have the data in\n",
    "the same feature space, we can start applying some machine\n",
    "learning methods: classifying, clustering, and so on. But\n",
    "actually, we have a few problems. Words are not all equally\n",
    "informative. If words appear too frequently in a single document,\n",
    "they are going to muck up our analysis. We want to perform some\n",
    "scaling of each of these term frequency vectors into something a\n",
    "bit more representative.  In other words, we need to do some\n",
    "vector normalizing. One possibility is to ensure that the L2 norm\n",
    "of each vector is equal to 1. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example 3: L2 Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A regular old document term matrix: \n",
      "[[2 0 0 0 1 1 0 2 0 1 1]\n",
      " [2 0 0 1 1 0 1 1 0 1 1]\n",
      " [0 1 1 0 0 0 1 0 1 1 1]]\n",
      "\n",
      "A document term matrix with row-wise L2 norms of 1:\n",
      "[[ 0.57735027  0.          0.          0.          0.28867513  0.28867513\n",
      "   0.          0.57735027  0.          0.28867513  0.28867513]\n",
      " [ 0.63245553  0.          0.          0.31622777  0.31622777  0.\n",
      "   0.31622777  0.31622777  0.          0.31622777  0.31622777]\n",
      " [ 0.          0.40824829  0.40824829  0.          0.          0.\n",
      "   0.40824829  0.          0.40824829  0.40824829  0.40824829]]\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "import numpy as np\n",
    "def l2_normalizer(vec):\n",
    "    denom = np.sum([el**2 for el in vec])\n",
    "    return [(el / math.sqrt(denom)) for el in vec]\n",
    "doc_term_matrix_l2 = []\n",
    "for vec in doc_term_matrix:\n",
    "    doc_term_matrix_l2.append(l2_normalizer(vec))\n",
    "print 'A regular old document term matrix: '\n",
    "print np.matrix(doc_term_matrix)\n",
    "print '\\nA document term matrix with row-wise L2 norms of 1:'\n",
    "print np.matrix(doc_term_matrix_l2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can see that we have scaled down the vectors so that each element is between [0, 1], without losing too much\n",
    "valuable information. \n",
    "\n",
    "Finally, we have a remaining task to perform. Just as not all words\n",
    "are equally valuable within a document, not all words are\n",
    "valuable across all documents.  We can try reweighting every word\n",
    "by its inverse document frequency."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example 4: Feature weighting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our vocabulary vector is [me, footbal, basketball, Sergio, Mireia, Hector, likes, loves, He, than, more]\n",
      "The inverse document frequency vector is [0.405465, 1.098612, 1.098612, 1.098612, 0.405465, 1.098612, 0.405465, 0.405465, 1.098612, 0.000000, 0.000000]\n"
     ]
    }
   ],
   "source": [
    "def numDocsContaining(word, doclist):\n",
    "    doccount = 0\n",
    "    for doc in doclist:\n",
    "        if freq(word, doc) > 0:\n",
    "            doccount +=1\n",
    "    return doccount\n",
    "def idf(word, doclist):\n",
    "    n_samples = len(doclist)\n",
    "    df = numDocsContaining(word, doclist)\n",
    "    return np.log(n_samples / (float(df)) )\n",
    "my_idf_vector = [idf(word, mydoclist) for word in vocabulary]\n",
    "print 'Our vocabulary vector is [' + ', '.join(list(vocabulary)) + ']'\n",
    "print 'The inverse document frequency vector is [' + ', '.join(format(freq, 'f') for freq in my_idf_vector) + ']'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example 5: Film critics binary sentiment analysis recognition code \n",
    "\n",
    "In this example we apply the whole sentiment analysis process to the Large Movie reviews\n",
    "dataset (http://www.aclweb.org/anthology/P11-1015). This is one of the\n",
    "largest public available data sets for sentiment analysis, which\n",
    "includes more than 50.000 texts from movie reviews including the\n",
    "ground truth annotation related to positive and negative movie\n",
    "review. As a proof on concept for this example we use a subset of\n",
    "the dataset consisting in about 10% of the data. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can use the following commands in a Linux operating system to download the required data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\"wget\" no se reconoce como un comando interno o externo,\n",
      "programa o archivo por lotes ejecutable.\n"
     ]
    }
   ],
   "source": [
    "!wget http://ai.stanford.edu/~amaas//data/sentiment/aclImdb_v1.tar.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\"tar\" no se reconoce como un comando interno o externo,\n",
      "programa o archivo por lotes ejecutable.\n"
     ]
    }
   ],
   "source": [
    "!mkdir \"files\"\n",
    "!mkdir \"files/ch10\"\n",
    "!tar -xf aclImdb_v1.tar.gz -C files/ch10/\n",
    "!mkdir \"files/ch10/train\"\n",
    "!mkdir \"files/ch10/train/pos2\"\n",
    "!mkdir \"files/ch10/train/neg2\"\n",
    "!mkdir \"files/ch10/test\"\n",
    "!mkdir \"files/ch10/test/pos2\"\n",
    "!mkdir \"files/ch10/test/neg2\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For windows systems, you can download the data here: http://ai.stanford.edu/~amaas/data/sentiment/\n",
    "Within the compressed file you will find several film critics. Move them to the folder files/ch10/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next lines of code will select a subset of the critics of the dataset to run the next example for binary sentiment analysis recognition."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "WindowsError",
     "evalue": "[Error 3] El sistema no puede encontrar la ruta especificada: 'files/ch10/aclImdb/train/pos/*.*'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mWindowsError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-16-e1f520915779>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mfiles\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mcount\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mfile\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"files/ch10/aclImdb/train/pos/\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcount\u001b[0m \u001b[1;33m>\u001b[0m \u001b[0mfiles\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m         \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mWindowsError\u001b[0m: [Error 3] El sistema no puede encontrar la ruta especificada: 'files/ch10/aclImdb/train/pos/*.*'"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import shutil\n",
    "files=5\n",
    "count=0\n",
    "for file in os.listdir(\"files/ch10/aclImdb/train/pos/\"):\n",
    "    if count > files:\n",
    "        break\n",
    "    if file.endswith(\".txt\"):\n",
    "        os.rename('files/ch10/aclImdb/train/pos/' + file, 'files/ch10/train/pos2/' + file)\n",
    "    count=count+1\n",
    "count=0\n",
    "for file in os.listdir(\"files/ch10/aclImdb/train/neg/\"):\n",
    "    if count > files:\n",
    "        break\n",
    "    if file.endswith(\".txt\"):\n",
    "        os.rename('files/ch10/aclImdb/train/neg/' + file, 'files/ch10/train/neg2/' + file)\n",
    "    count=count+1\n",
    "count=0\n",
    "for file in os.listdir(\"files/ch10/aclImdb/test/pos/\"):\n",
    "    if count > files:\n",
    "        break\n",
    "    if file.endswith(\".txt\"):\n",
    "        os.rename('files/ch10/aclImdb/test/pos/' + file, 'files/ch10/test/pos2/' + file)\n",
    "    count=count+1\n",
    "count=0\n",
    "for file in os.listdir(\"files/ch10/aclImdb/test/neg/\"):\n",
    "    if count > files:\n",
    "        break\n",
    "    if file.endswith(\".txt\"):\n",
    "        os.rename('files/ch10/aclImdb/test/neg/' + file, 'files/ch10/test/neg2/' + file)\n",
    "    count=count+1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And the next script will perform the whole training and testing procedure on the selected subset of the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading the training data positive\n",
      "Reading the training data negative\n",
      "Defining dictionaries\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "empty vocabulary; perhaps the documents only contain stop words",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-17-ae06300210ac>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     56\u001b[0m \u001b[1;31m# Computing TIDF word space\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     57\u001b[0m \u001b[0mtfidf_vectorizer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTfidfVectorizer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmin_df\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 58\u001b[1;33m \u001b[0mtrainData\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtfidf_vectorizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpreprocessed_docs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     59\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     60\u001b[0m \u001b[1;31m# Reading the test data\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Sergio\\Anaconda\\lib\\site-packages\\sklearn\\feature_extraction\\text.pyc\u001b[0m in \u001b[0;36mfit_transform\u001b[1;34m(self, raw_documents, y)\u001b[0m\n\u001b[0;32m   1280\u001b[0m             \u001b[0mTf\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0midf\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mweighted\u001b[0m \u001b[0mdocument\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mterm\u001b[0m \u001b[0mmatrix\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1281\u001b[0m         \"\"\"\n\u001b[1;32m-> 1282\u001b[1;33m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mTfidfVectorizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mraw_documents\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1283\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_tfidf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1284\u001b[0m         \u001b[1;31m# X is already a transformed view of raw_documents so\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Sergio\\Anaconda\\lib\\site-packages\\sklearn\\feature_extraction\\text.pyc\u001b[0m in \u001b[0;36mfit_transform\u001b[1;34m(self, raw_documents, y)\u001b[0m\n\u001b[0;32m    815\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    816\u001b[0m         vocabulary, X = self._count_vocab(raw_documents,\n\u001b[1;32m--> 817\u001b[1;33m                                           self.fixed_vocabulary_)\n\u001b[0m\u001b[0;32m    818\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    819\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbinary\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Sergio\\Anaconda\\lib\\site-packages\\sklearn\\feature_extraction\\text.pyc\u001b[0m in \u001b[0;36m_count_vocab\u001b[1;34m(self, raw_documents, fixed_vocab)\u001b[0m\n\u001b[0;32m    758\u001b[0m             \u001b[0mvocabulary\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvocabulary\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    759\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mvocabulary\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 760\u001b[1;33m                 raise ValueError(\"empty vocabulary; perhaps the documents only\"\n\u001b[0m\u001b[0;32m    761\u001b[0m                                  \" contain stop words\")\n\u001b[0;32m    762\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: empty vocabulary; perhaps the documents only contain stop words"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from nltk.classify import NaiveBayesClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn import svm\n",
    "from unidecode import unidecode\n",
    "import time\n",
    "\n",
    "def BoW():\n",
    "    # Tokenizing text\n",
    "    text_tokenized = [word_tokenize(doc) for doc in text]\n",
    "    # Removing punctuation\n",
    "    regex = re.compile('[%s]' % re.escape(string.punctuation))\n",
    "    tokenized_docs_no_punctuation = []\n",
    "    for review in text_tokenized:\n",
    "        new_review = []\n",
    "        for token in review:\n",
    "            new_token = regex.sub(u'', token)\n",
    "            if not new_token == u'':\n",
    "                new_review.append(new_token)\n",
    "        tokenized_docs_no_punctuation.append(new_review)\n",
    "    # Stemming and Lemmatizing\n",
    "    porter = PorterStemmer()\n",
    "    preprocessed_docs = []\n",
    "    for doc in tokenized_docs_no_punctuation:\n",
    "        final_doc = ''\n",
    "        for word in doc:\n",
    "            final_doc = final_doc + ' ' + porter.stem(word)\n",
    "        preprocessed_docs.append(final_doc)\n",
    "    return preprocessed_docs\n",
    "\n",
    "print('Reading the training data positive')\n",
    "text = []\n",
    "for file in os.listdir(\"files/ch10/train/pos2/\"):\n",
    "    if file.endswith(\".txt\"):\n",
    "        infile = open('files/ch10/train/pos2/' + file, 'r')\n",
    "        text.append(unidecode(infile.read().decode('utf-8')))\n",
    "        infile.close()\n",
    "num_posTrain=len(text)\n",
    "\n",
    "print('Reading the training data negative')\n",
    "\n",
    "for file in os.listdir(\"files/ch10/train/neg2/\"):\n",
    "    if file.endswith(\".txt\"):\n",
    "        infile = open('files/ch10/train/neg2/' + file, 'r')\n",
    "        text.append(unidecode(infile.read().decode('utf-8')))\n",
    "        infile.close()\n",
    "num_Train=len(text)\n",
    "\n",
    "print('Defining dictionaries')\n",
    "\n",
    "preprocessed_docs=BoW()\n",
    "# Computing TIDF word space\n",
    "tfidf_vectorizer = TfidfVectorizer(min_df = 1)\n",
    "trainData = tfidf_vectorizer.fit_transform(preprocessed_docs)\n",
    "\n",
    "# Reading the test data\n",
    "\n",
    "print('Reading the test data positive')\n",
    "\n",
    "text = []\n",
    "for file in os.listdir(\"files/ch10/test/pos2/\"):\n",
    "    if file.endswith(\".txt\"):\n",
    "        infile = open('files/ch10/test/pos2/' + file, 'r')\n",
    "        text.append(unidecode(infile.read().decode('utf-8')))\n",
    "        infile.close()\n",
    "num_posTest=len(text)\n",
    "\n",
    "print('Reading the test data negative')\n",
    "\n",
    "for file in os.listdir(\"files/ch10/test/neg2/\"):\n",
    "    if file.endswith(\".txt\"):\n",
    "        infile = open('files/ch10/test/neg2/' + file, 'r')\n",
    "        text.append(unidecode(infile.read().decode('utf-8')))\n",
    "        infile.close()\n",
    "num_Test=len(text)\n",
    "\n",
    "print('Computing test feature vectors')\n",
    "start_time = time.time()\n",
    "\n",
    "preprocessed_docs=BoW()\n",
    "testData = tfidf_vectorizer.transform(preprocessed_docs)\n",
    "\n",
    "targetTrain = []\n",
    "for i in range(0,num_posTrain):\n",
    "    targetTrain.append(0)\n",
    "for i in range(0,num_Train-num_posTrain):\n",
    "    targetTrain.append(1)\n",
    "\n",
    "targetTest = []\n",
    "for i in range(0,num_posTest):\n",
    "    targetTest.append(0)\n",
    "for i in range(0,num_Test-num_posTest):\n",
    "    targetTest.append(1)\n",
    "\n",
    "print('Training and testing on training Naive Bayes')\n",
    "start_time = time.time()\n",
    "\n",
    "gnb = GaussianNB()\n",
    "testData.todense()\n",
    "y_pred = gnb.fit(trainData.todense(), targetTrain).predict(trainData.todense())\n",
    "print(\"Number of mislabeled training points out of a total %d points : %d\" % (trainData.shape[0],(targetTrain != y_pred).sum()))\n",
    "\n",
    "print('Training and testing on test Naive Bayes')\n",
    "\n",
    "y_pred = gnb.fit(trainData.todense(), targetTrain).predict(testData.todense())\n",
    "print(\"Number of mislabeled test points out of a total %d points : %d\" % (testData.shape[0],(targetTest != y_pred).sum()))\n",
    "\n",
    "print('Training and testing on train with SVM')\n",
    "clf = svm.SVC()\n",
    "clf.fit(trainData.todense(), targetTrain)\n",
    "y_pred = clf.predict(trainData.todense())\n",
    "print(\"Number of mislabeled test points out of a total %d points : %d\" % (trainData.shape[0],(targetTrain != y_pred).sum()))\n",
    "\n",
    "print('Testing on test with already trained SVM')\n",
    "y_pred = clf.predict(testData.todense())\n",
    "print(\"Number of mislabeled test points out of a total %d points : %d\" % (testData.shape[0],(targetTest != y_pred).sum()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The previous example uses one of the largest public available datasets for sentiment analysis, which\n",
    "includes more than 50,000 texts from movie reviews including the\n",
    "ground truth annotation related to positive and negative movie\n",
    "reviews. As a proof on concept, for this example we use a subset of\n",
    "the dataset consisting of about 30\\% of the data. Previous code re-uses part of the previous examples for data cleaning,\n",
    "reads training and test data from the folders as provided by the\n",
    "authors of the dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example 6: Tweet binary sentiment analysis recognition code (see book chapter reference to download data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, let us see another simple example of sentiment analysis\n",
    "based on tweets. Although there is some work using more \n",
    "tweet data\n",
    "(\\url{http://www.sananalytics.com/lab/twitter-sentiment/}) here we\n",
    "present a simple set of tweets which are analyzed as in the\n",
    "previous example of movie reviews. The main code remains the same \n",
    "except for the definition of the initial data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of mislabeled training points out of a total 10 points : 0\n",
      "Training and testing on test Naive Bayes\n",
      "Number of mislabeled test points out of a total 6 points : 2\n",
      "Training and testing on train with SVM\n",
      "Number of mislabeled test points out of a total 10 points : 0\n",
      "Testing on test with already trained SVM\n",
      "Number of mislabeled test points out of a total 6 points : 2\n"
     ]
    }
   ],
   "source": [
    "def BoW():\n",
    "    # Tokenizing text\n",
    "    text_tokenized = [word_tokenize(doc) for doc in text]\n",
    "    # Removing punctuation\n",
    "    regex = re.compile('[%s]' % re.escape(string.punctuation))\n",
    "    tokenized_docs_no_punctuation = []\n",
    "    for review in text_tokenized:\n",
    "        new_review = []\n",
    "        for token in review:\n",
    "            new_token = regex.sub(u'', token)\n",
    "            if not new_token == u'':\n",
    "                new_review.append(new_token)\n",
    "        tokenized_docs_no_punctuation.append(new_review)\n",
    "    # Stemming and Lemmatizing\n",
    "    porter = PorterStemmer()\n",
    "    preprocessed_docs = []\n",
    "    for doc in tokenized_docs_no_punctuation:\n",
    "        final_doc = ''\n",
    "        for word in doc:\n",
    "            final_doc = final_doc + ' ' + porter.stem(word)\n",
    "        preprocessed_docs.append(final_doc)\n",
    "    return preprocessed_docs\n",
    "\n",
    "text = ['I love this sandwich.', 'This is an amazing place!',\n",
    "        'I feel very good about these beers.',\n",
    "         'This is my best work.', 'What an awesome view', 'I do not like this restaurant',\n",
    "         'I am tired of this stuff.', 'I can not deal with this', 'He is my sworn enemy!',\n",
    "         'My boss is horrible.']\n",
    "\n",
    "targetTrain = [0,0,0,0,0,1,1,1,1,1]\n",
    "preprocessed_docs=BoW()\n",
    "tfidf_vectorizer = TfidfVectorizer(min_df = 1)\n",
    "trainData = tfidf_vectorizer.fit_transform(preprocessed_docs)\n",
    "\n",
    "text = ['The beer was good.', 'I do not enjoy my job', 'I aint feeling dandy today',\n",
    "        'I feel amazing!'\n",
    "        ,'Gary is a friend of mine.', 'I can not believe I am doing this.']\n",
    "targetTest = [0,1,1,0,0,1]\n",
    "preprocessed_docs=BoW()\n",
    "testData = tfidf_vectorizer.transform(preprocessed_docs)\n",
    "\n",
    "gnb = GaussianNB()\n",
    "testData.todense()\n",
    "y_pred = gnb.fit(trainData.todense(), targetTrain).predict(trainData.todense())\n",
    "print(\"Number of mislabeled training points out of a total %d points : %d\" % (trainData.shape[0],(targetTrain != y_pred).sum()))\n",
    "\n",
    "print('Training and testing on test Naive Bayes')\n",
    "\n",
    "y_pred = gnb.fit(trainData.todense(), targetTrain).predict(testData.todense())\n",
    "print(\"Number of mislabeled test points out of a total %d points : %d\" % (testData.shape[0],(targetTest != y_pred).sum()))\n",
    "\n",
    "print('Training and testing on train with SVM')\n",
    "clf = svm.SVC()\n",
    "clf.fit(trainData.todense(), targetTrain)\n",
    "y_pred = clf.predict(trainData.todense())\n",
    "print(\"Number of mislabeled test points out of a total %d points : %d\" % (trainData.shape[0],(targetTrain != y_pred).sum()))\n",
    "\n",
    "print('Testing on test with already trained SVM')\n",
    "y_pred = clf.predict(testData.todense())\n",
    "print(\"Number of mislabeled test points out of a total %d points : %d\" % (testData.shape[0],(targetTest != y_pred).sum()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this previous simple scenario both learning strategies achieve the same recognition rates in both training and test sets.\n",
    "Note that similar words are shared between tweets. In practice, with real examples, tweets will include unstructured sentences and\n",
    "abbreviations, making recognition harder. "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}