bow BrownCorpusMovies.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "documents: 2000\n"
     ]
    }
   ],
   "source": [
    "# use Brown corpus movie reviews\n",
    "from nltk.corpus import movie_reviews as mr\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import LancasterStemmer\n",
    "import operator\n",
    "import random\n",
    "ps = LancasterStemmer()\n",
    "stop = stopwords.words('english')\n",
    "# document [0] are the words list, [1] is 'pos' or 'neg' review\n",
    "documents = [([ps.stem(w.lower()) for w in mr.words(i) if w.lower() not in stop and w.lower() not in string.punctuation], i.split('/')[0]) for i in mr.fileids()]\n",
    "print('documents:', len(documents))\n",
    "MAX_WORDS = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "words: 21467\n",
      "# uncommon words: 19786\n"
     ]
    }
   ],
   "source": [
    "# organize unique words and their counts\n",
    "from collections import Counter\n",
    "words = []\n",
    "for d in documents:\n",
    "    for w in d[0]:\n",
    "        words.append(w)\n",
    "word_counts = Counter(words)\n",
    "print('words:', len(word_counts))\n",
    "uncommon_words = []\n",
    "for w in word_counts:\n",
    "    # focus on least common words\n",
    "    if 100 > word_counts[w] > 0 and len(w)>2:\n",
    "        uncommon_words.append(w)\n",
    "print('# uncommon words:', len(uncommon_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# map a document to a vector of word indeces\n",
    "def map_document(document):\n",
    "    map = [0] * MAX_WORDS\n",
    "    for i,w in enumerate(document[:MAX_WORDS]):\n",
    "        if w in uncommon_words:\n",
    "            map[i] = words.index(w)\n",
    "    return map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5435, 0, 0, 0, 0, 25023, 0, 0, 0, 0, 0, 165541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10476, 0, 9490, 0, 0, 0, 0, 0, 0, 36604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49454, 0, 0, 0, 0, 137, 0, 0, 0, 109263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189085, 0, 140500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127903, 0, 49516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39544, 39773, 0, 0, 2083, 0, 0, 0, 0, 0, 0, 0, 566056, 0, 0, 8819, 0, 189085, 0, 0, 0, 0, 0, 11723, 4872, 0, 0, 0, 0, 0, 0, 0, 189085, 566077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22458, 0, 0, 19254, 189085, 0, 0, 50854, 0, 0, 0, 0, 207038, 0, 19254, 0, 0, 0, 0, 189085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] pos\n"
     ]
    }
   ],
   "source": [
    "print(map_document(documents[0][0]), documents[0][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# build our data\n",
    "random.shuffle(documents)\n",
    "documents = documents[:200]\n",
    "training_subset = 0.8\n",
    "data = []\n",
    "labels = []\n",
    "for doc in documents[:int(len(documents)*training_subset)]:\n",
    "    data.append(map_document(doc[0]))\n",
    "    if doc[1] == 'pos':\n",
    "        labels.append([0, 1])\n",
    "    elif doc[1] == 'neg':\n",
    "        labels.append([1, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hdf5 is not supported on this machine (please install/reinstall h5py for optimal experience)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tflearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Step: 2999  | total loss: \u001b[1m\u001b[32m10.97868\u001b[0m\u001b[0m | time: 0.007s\n",
      "| Adam | epoch: 1000 | loss: 10.97868 - acc: 0.5232 -- iter: 32/40\n",
      "Training Step: 3000  | total loss: \u001b[1m\u001b[32m11.03211\u001b[0m\u001b[0m | time: 0.010s\n",
      "| Adam | epoch: 1000 | loss: 11.03211 - acc: 0.5209 -- iter: 40/40\n",
      "--\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "tf.reset_default_graph()\n",
    "# Build neural network\n",
    "net = tflearn.input_data(shape=[None, 200])\n",
    "net = tflearn.fully_connected(net, 100)\n",
    "net = tflearn.fully_connected(net, 100)\n",
    "net = tflearn.fully_connected(net, 2, activation='softmax')\n",
    "net = tflearn.regression(net)\n",
    "\n",
    "# Define model\n",
    "model = tflearn.DNN(net)\n",
    "# Start training (apply gradient descent algorithm)\n",
    "model.fit(data, labels, n_epoch=1000, batch_size=16, show_metric=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}