Emotion_Detector.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "import nltk \n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import *\n",
    "from textblob.classifiers import NaiveBayesClassifier\n",
    "from sklearn.cross_validation import KFold\n",
    "from nltk.classify.naivebayes import NaiveBayesClassifier\n",
    "from gensim import corpora, models, similarities\n",
    "from nltk.corpus import wordnet as wn\n",
    "import urllib\n",
    "import urllib2\n",
    "# Uncomment if using AlchemyAPI\n",
    "# from alchemyapi import AlchemyAPI\n",
    "# alchemyapi = AlchemyAPI()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n36 - Class Label\\n40 - Sentence\\n'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "Reading the Dataset (ISEAR Dataset)\n",
    "'''\n",
    "Data = pd.read_csv('ISEAR.csv',header=None)\n",
    "'''\n",
    "36 - Class Label\n",
    "40 - Sentence\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Emotion Labels\n",
    "'''\n",
    "emotion_labels = ['joy', 'fear', 'anger', 'sadness', 'disgust', 'shame', 'guilt']\n",
    "# emotion_labels = ['joy', 'fear', 'anger', 'sadness', 'disgust']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Negation words\n",
    "'''\n",
    "negation_words = ['not', 'neither', 'nor', 'but', 'however', 'although', 'nonetheless', 'despite', 'except', 'even though', 'yet']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Returns a list of all corresponding class labels\n",
    "'''\n",
    "def class_labels(emotions):\n",
    "    labels = []\n",
    "    labelset = []\n",
    "    exclude = []\n",
    "    for i in range(len(emotions)):\n",
    "#         labels.append(e)\n",
    "#         labelset.append([e])\n",
    "        if emotions[i] not in ['shame','guilt']:\n",
    "            labels.append(e)\n",
    "            labelset.append([e])\n",
    "        else:\n",
    "            exclude.append(i)\n",
    "    return labels, labelset, exclude"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Removes unnecessary characters from sentences\n",
    "'''\n",
    "def removal(sentences):\n",
    "    sentence_list = []\n",
    "    count = 0\n",
    "#     for sen in sentences:\n",
    "#         count += 1\n",
    "#         print count\n",
    "#         print sen\n",
    "#         print type(sen)\n",
    "    s = nltk.word_tokenize(sentences)\n",
    "    characters = [\"á\", \"\\xc3\", \"\\xa1\", \"\\n\", \",\", \".\", \"[\", \"]\", \"\"]\n",
    "    l = []\n",
    "    for t in s:\n",
    "        if t not in characters:\n",
    "            l.append(t)\n",
    "    return l\n",
    "#     new = ' '.join([i for i in s if not [e for e in characters if e in i]])\n",
    "#     print new\n",
    "#     sentence_list.append(new)\n",
    "#     return sentence_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''\n",
    "POS-TAGGER, returns NAVA words\n",
    "'''\n",
    "def pos_tag(sentences):\n",
    "    tags = [] #have the pos tag included\n",
    "    nava_sen = []\n",
    "    pt = nltk.pos_tag(sentences)\n",
    "#     for s in sentences:\n",
    "#     s_token = nltk.word_tokenize(sentences)\n",
    "#     pt = nltk.pos_tag(s_token)\n",
    "    nava = []\n",
    "    nava_words = []\n",
    "    for t in pt:\n",
    "        if t[1].startswith('NN') or t[1].startswith('JJ') or t[1].startswith('VB') or t[1].startswith('RB'):\n",
    "            nava.append(t)\n",
    "            nava_words.append(t[0])\n",
    "    return nava, nava_words\n",
    "#     tags.append(nava)\n",
    "#     nava_sen.append(nava_words)\n",
    "#     return tags, nava_sen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Performs stemming\n",
    "'''\n",
    "def stemming(sentences):\n",
    "    sentence_list = []\n",
    "    sen_string = []\n",
    "    sen_token = []\n",
    "    stemmer = PorterStemmer()\n",
    "    i = 0\n",
    "#     for sen in sentences:\n",
    "#         print i,\n",
    "    i += 1\n",
    "    st = \"\"\n",
    "    for word in sentences:\n",
    "        word_l = word.lower()\n",
    "        if len(word_l) >= 3:\n",
    "            st += stemmer.stem(word_l) + \" \"\n",
    "    sen_string.append(st)\n",
    "    w_set = nltk.word_tokenize(st)\n",
    "    sen_token.append(w_set)\n",
    "    w_text = nltk.Text(w_set)\n",
    "    sentence_list.append(w_text)\n",
    "    return w_text, st, w_set\n",
    "#     return sentence_list, sen_string, sen_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Write to file\n",
    "'''\n",
    "def write_to_file(filename, text):\n",
    "    o = open(filename,'w')\n",
    "    o.write(str(text))\n",
    "    o.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Creating the dataframe\n",
    "'''\n",
    "def create_frame(Data):\n",
    "    labels = []\n",
    "#     sentences = []\n",
    "#     sen_string = []\n",
    "#     sen_token =[]\n",
    "    sen = []\n",
    "    sen_s = []\n",
    "    sen_t = []\n",
    "    labelset = []\n",
    "    for i in range(len(Data)):\n",
    "        if i >= 0:\n",
    "#             print i,\n",
    "            emotion = Data[0][i]\n",
    "            sit = Data[1][i]\n",
    "#             if emotion not in ['shame', 'guilt']:\n",
    "            labels.append(emotion)\n",
    "            labelset.append([emotion])\n",
    "            sent = removal(sit)\n",
    "            nava, sent_pt = pos_tag(sent)\n",
    "            sentences, sen_string, sen_token = stemming(sent_pt)\n",
    "            sen.append(sentences)\n",
    "            sen_s.append(sen_string)\n",
    "            sen_t.append(sen_token)\n",
    "#     labels, labelset, exclude = class_labels(emotions[1:])\n",
    "#     sent = removal(sit[1:], exclude)\n",
    "#     nava, sent_pt = pos_tag(sent)\n",
    "#     sentences, sen_string, sen_token = stemming(sent_pt)\n",
    "    frame = pd.DataFrame({0 : labels,\n",
    "                          1 : sen,\n",
    "                          2 : sen_s,\n",
    "                          3 : sen_t,\n",
    "                          4 : labelset})\n",
    "    return frame, sen_t, labels, sen_s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "c, st, labels, senten = create_frame(Data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>joy</td>\n",
       "      <td>(day, feel, close, partner, other, friend, fee...</td>\n",
       "      <td>day feel close partner other friend feel peac ...</td>\n",
       "      <td>[day, feel, close, partner, other, friend, fee...</td>\n",
       "      <td>[joy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fear</td>\n",
       "      <td>(time, imagin, someon, love, contact, seriou, ...</td>\n",
       "      <td>time imagin someon love contact seriou ill eve...</td>\n",
       "      <td>[time, imagin, someon, love, contact, seriou, ...</td>\n",
       "      <td>[fear]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>anger</td>\n",
       "      <td>(had, been, obvious, unjustli, treat, had, pos...</td>\n",
       "      <td>had been obvious unjustli treat had possibl el...</td>\n",
       "      <td>[had, been, obvious, unjustli, treat, had, pos...</td>\n",
       "      <td>[anger]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sadness</td>\n",
       "      <td>(think, short, time, live, relat, period, life...</td>\n",
       "      <td>think short time live relat period life think ...</td>\n",
       "      <td>[think, short, time, live, relat, period, life...</td>\n",
       "      <td>[sadness]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>disgust</td>\n",
       "      <td>(gather, found, involuntarili, sit, next, peop...</td>\n",
       "      <td>gather found involuntarili sit next peopl expr...</td>\n",
       "      <td>[gather, found, involuntarili, sit, next, peop...</td>\n",
       "      <td>[disgust]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>shame</td>\n",
       "      <td>(realiz, wa, direct, feel, discont, partner, w...</td>\n",
       "      <td>realiz wa direct feel discont partner way wa t...</td>\n",
       "      <td>[realiz, wa, direct, feel, discont, partner, w...</td>\n",
       "      <td>[shame]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>guilt</td>\n",
       "      <td>(feel, guilti, realiz, consid, materi, thing, ...</td>\n",
       "      <td>feel guilti realiz consid materi thing more im...</td>\n",
       "      <td>[feel, guilti, realiz, consid, materi, thing, ...</td>\n",
       "      <td>[guilt]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>joy</td>\n",
       "      <td>(girlfriend, had, taken, exam, went, parent, p...</td>\n",
       "      <td>girlfriend had taken exam went parent place</td>\n",
       "      <td>[girlfriend, had, taken, exam, went, parent, p...</td>\n",
       "      <td>[joy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>fear</td>\n",
       "      <td>(first, time, realiz, mean, death)</td>\n",
       "      <td>first time realiz mean death</td>\n",
       "      <td>[first, time, realiz, mean, death]</td>\n",
       "      <td>[fear]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>anger</td>\n",
       "      <td>(car, overtak, forc, drive, road)</td>\n",
       "      <td>car overtak forc drive road</td>\n",
       "      <td>[car, overtak, forc, drive, road]</td>\n",
       "      <td>[anger]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>sadness</td>\n",
       "      <td>(recent, thought, hard, work, take, studi, wan...</td>\n",
       "      <td>recent thought hard work take studi want tri s...</td>\n",
       "      <td>[recent, thought, hard, work, take, studi, wan...</td>\n",
       "      <td>[sadness]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>disgust</td>\n",
       "      <td>(found, bristl, liver, past, tube)</td>\n",
       "      <td>found bristl liver past tube</td>\n",
       "      <td>[found, bristl, liver, past, tube]</td>\n",
       "      <td>[disgust]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>shame</td>\n",
       "      <td>(wa, tire, unmotiv, shout, girlfriend, brought...</td>\n",
       "      <td>wa tire unmotiv shout girlfriend brought neg s...</td>\n",
       "      <td>[wa, tire, unmotiv, shout, girlfriend, brought...</td>\n",
       "      <td>[shame]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>guilt</td>\n",
       "      <td>(think, not, studi, enough, weekend, think, ha...</td>\n",
       "      <td>think not studi enough weekend think have been...</td>\n",
       "      <td>[think, not, studi, enough, weekend, think, ha...</td>\n",
       "      <td>[guilt]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>joy</td>\n",
       "      <td>(pass, examin, did, not, think, did, well)</td>\n",
       "      <td>pass examin did not think did well</td>\n",
       "      <td>[pass, examin, did, not, think, did, well]</td>\n",
       "      <td>[joy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>fear</td>\n",
       "      <td>(one, ha, arrang, meet, someon, person, arriv,...</td>\n",
       "      <td>one ha arrang meet someon person arriv late me...</td>\n",
       "      <td>[one, ha, arrang, meet, someon, person, arriv,...</td>\n",
       "      <td>[fear]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>anger</td>\n",
       "      <td>(unjustli, accus, someth, ha, not, done)</td>\n",
       "      <td>unjustli accus someth ha not done</td>\n",
       "      <td>[unjustli, accus, someth, ha, not, done]</td>\n",
       "      <td>[anger]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>sadness</td>\n",
       "      <td>(one, studi, seem, hopelessli, difficult, unin...</td>\n",
       "      <td>one studi seem hopelessli difficult uninterest</td>\n",
       "      <td>[one, studi, seem, hopelessli, difficult, unin...</td>\n",
       "      <td>[sadness]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>disgust</td>\n",
       "      <td>(find, someon, know, not, had, thought, instan...</td>\n",
       "      <td>find someon know not had thought instanc frien...</td>\n",
       "      <td>[find, someon, know, not, had, thought, instan...</td>\n",
       "      <td>[disgust]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>shame</td>\n",
       "      <td>(one, ha, been, unjust, stupid, toward, someon...</td>\n",
       "      <td>one ha been unjust stupid toward someon els</td>\n",
       "      <td>[one, ha, been, unjust, stupid, toward, someon...</td>\n",
       "      <td>[shame]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>guilt</td>\n",
       "      <td>(one, ha, neglect, been, unjust, good, friend)</td>\n",
       "      <td>one ha neglect been unjust good friend</td>\n",
       "      <td>[one, ha, neglect, been, unjust, good, friend]</td>\n",
       "      <td>[guilt]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>joy</td>\n",
       "      <td>(pass, exam, did, not, expect, pass)</td>\n",
       "      <td>pass exam did not expect pass</td>\n",
       "      <td>[pass, exam, did, not, expect, pass]</td>\n",
       "      <td>[joy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>fear</td>\n",
       "      <td>(climb, tree, pick, appl, angl, ladder, wa, di...</td>\n",
       "      <td>climb tree pick appl angl ladder wa did not en...</td>\n",
       "      <td>[climb, tree, pick, appl, angl, ladder, wa, di...</td>\n",
       "      <td>[fear]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>guilt</td>\n",
       "      <td>(excus, are, necessari, get, do)</td>\n",
       "      <td>excus are necessari get do</td>\n",
       "      <td>[excus, are, necessari, get, do]</td>\n",
       "      <td>[guilt]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>joy</td>\n",
       "      <td>(had, children)</td>\n",
       "      <td>had children</td>\n",
       "      <td>[had, children]</td>\n",
       "      <td>[joy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>fear</td>\n",
       "      <td>(year, old, son, climb, sat, 7th, floor, balco...</td>\n",
       "      <td>year old son climb sat 7th floor balconi leg h...</td>\n",
       "      <td>[year, old, son, climb, sat, 7th, floor, balco...</td>\n",
       "      <td>[fear]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>anger</td>\n",
       "      <td>(partner, wa, attack, lost, teeth)</td>\n",
       "      <td>partner wa attack lost teeth</td>\n",
       "      <td>[partner, wa, attack, lost, teeth]</td>\n",
       "      <td>[anger]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sadness</td>\n",
       "      <td>(see, children, t.v, area, devast, drought, war)</td>\n",
       "      <td>see children t.v area devast drought war</td>\n",
       "      <td>[see, children, t.v, area, devast, drought, war]</td>\n",
       "      <td>[sadness]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>disgust</td>\n",
       "      <td>(nearli, walk, blindworm, then, saw, crawl, away)</td>\n",
       "      <td>nearli walk blindworm then saw crawl away</td>\n",
       "      <td>[nearli, walk, blindworm, then, saw, crawl, away]</td>\n",
       "      <td>[disgust]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>shame</td>\n",
       "      <td>(saw, year, old, son, grab, oxygen, mask, had,...</td>\n",
       "      <td>saw year old son grab oxygen mask had breath d...</td>\n",
       "      <td>[saw, year, old, son, grab, oxygen, mask, had,...</td>\n",
       "      <td>[shame]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7486</th>\n",
       "      <td>shame</td>\n",
       "      <td>(forgot, zip, trouser, wa, not, notic, anyon)</td>\n",
       "      <td>forgot zip trouser wa not notic anyon</td>\n",
       "      <td>[forgot, zip, trouser, wa, not, notic, anyon]</td>\n",
       "      <td>[shame]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7487</th>\n",
       "      <td>guilt</td>\n",
       "      <td>(peep)</td>\n",
       "      <td>peep</td>\n",
       "      <td>[peep]</td>\n",
       "      <td>[guilt]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7488</th>\n",
       "      <td>joy</td>\n",
       "      <td>(had, picnic, old, classmat, chat, play, game)</td>\n",
       "      <td>had picnic old classmat chat play game</td>\n",
       "      <td>[had, picnic, old, classmat, chat, play, game]</td>\n",
       "      <td>[joy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7489</th>\n",
       "      <td>fear</td>\n",
       "      <td>(night, wa, alon, home, famili, member, usual,...</td>\n",
       "      <td>night wa alon home famili member usual get tog...</td>\n",
       "      <td>[night, wa, alon, home, famili, member, usual,...</td>\n",
       "      <td>[fear]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7490</th>\n",
       "      <td>anger</td>\n",
       "      <td>(saw, bed, hostel, wa, mess, guess, someon, el...</td>\n",
       "      <td>saw bed hostel wa mess guess someon els had us...</td>\n",
       "      <td>[saw, bed, hostel, wa, mess, guess, someon, el...</td>\n",
       "      <td>[anger]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7491</th>\n",
       "      <td>sadness</td>\n",
       "      <td>(physic, experi, session, did, not, understand...</td>\n",
       "      <td>physic experi session did not understand conte...</td>\n",
       "      <td>[physic, experi, session, did, not, understand...</td>\n",
       "      <td>[sadness]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7492</th>\n",
       "      <td>disgust</td>\n",
       "      <td>(man, sexual, aggress, small, girl, bu, girl, ...</td>\n",
       "      <td>man sexual aggress small girl bu girl did not ...</td>\n",
       "      <td>[man, sexual, aggress, small, girl, bu, girl, ...</td>\n",
       "      <td>[disgust]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7493</th>\n",
       "      <td>shame</td>\n",
       "      <td>(be, unabl, stop, urin, bu)</td>\n",
       "      <td>be unabl stop urin bu</td>\n",
       "      <td>[be, unabl, stop, urin, bu]</td>\n",
       "      <td>[shame]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7494</th>\n",
       "      <td>guilt</td>\n",
       "      <td>(were, old, peopl, crowd, bu, did, not, have, ...</td>\n",
       "      <td>were old peopl crowd bu did not have courag gi...</td>\n",
       "      <td>[were, old, peopl, crowd, bu, did, not, have, ...</td>\n",
       "      <td>[guilt]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7495</th>\n",
       "      <td>joy</td>\n",
       "      <td>(wa, first, time, gave, birthday, present, fri...</td>\n",
       "      <td>wa first time gave birthday present friend wro...</td>\n",
       "      <td>[wa, first, time, gave, birthday, present, fri...</td>\n",
       "      <td>[joy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7496</th>\n",
       "      <td>fear</td>\n",
       "      <td>(wa, week, higher, level, result, were, announ...</td>\n",
       "      <td>wa week higher level result were announc had t...</td>\n",
       "      <td>[wa, week, higher, level, result, were, announ...</td>\n",
       "      <td>[fear]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7497</th>\n",
       "      <td>anger</td>\n",
       "      <td>(surnam, brother, wa, differ, often, grin, hom...</td>\n",
       "      <td>surnam brother wa differ often grin home provo...</td>\n",
       "      <td>[surnam, brother, wa, differ, often, grin, hom...</td>\n",
       "      <td>[anger]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7498</th>\n",
       "      <td>sadness</td>\n",
       "      <td>(wa, primari, father, die, wa, veri, young, th...</td>\n",
       "      <td>wa primari father die wa veri young then did n...</td>\n",
       "      <td>[wa, primari, father, die, wa, veri, young, th...</td>\n",
       "      <td>[sadness]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7499</th>\n",
       "      <td>guit</td>\n",
       "      <td>(hostel, roommat, wa, veri, selfish, person, a...</td>\n",
       "      <td>hostel roommat wa veri selfish person avoid do...</td>\n",
       "      <td>[hostel, roommat, wa, veri, selfish, person, a...</td>\n",
       "      <td>[guit]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7500</th>\n",
       "      <td>shame</td>\n",
       "      <td>(friend, had, mani, femal, friend, thought, we...</td>\n",
       "      <td>friend had mani femal friend thought were love...</td>\n",
       "      <td>[friend, had, mani, femal, friend, thought, we...</td>\n",
       "      <td>[shame]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7501</th>\n",
       "      <td>guilt</td>\n",
       "      <td>(past, use, think, mother, wa, veri, nag, pers...</td>\n",
       "      <td>past use think mother wa veri nag person start...</td>\n",
       "      <td>[past, use, think, mother, wa, veri, nag, pers...</td>\n",
       "      <td>[guilt]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7502</th>\n",
       "      <td>joy</td>\n",
       "      <td>(august,1983, long, await, big, envelop, docum...</td>\n",
       "      <td>august,1983 long await big envelop document c....</td>\n",
       "      <td>[august,1983, long, await, big, envelop, docum...</td>\n",
       "      <td>[joy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7503</th>\n",
       "      <td>fear</td>\n",
       "      <td>(christma, eve,1984, had, just, finish, exam, ...</td>\n",
       "      <td>christma eve,1984 had just finish exam wa afra...</td>\n",
       "      <td>[christma, eve,1984, had, just, finish, exam, ...</td>\n",
       "      <td>[fear]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7504</th>\n",
       "      <td>anger</td>\n",
       "      <td>(septemb, wa, forc, live, someon, did, not, fi...</td>\n",
       "      <td>septemb wa forc live someon did not first week...</td>\n",
       "      <td>[septemb, wa, forc, live, someon, did, not, fi...</td>\n",
       "      <td>[anger]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7505</th>\n",
       "      <td>fear</td>\n",
       "      <td>(issu, worri, rather, sadden, mid, septemb, we...</td>\n",
       "      <td>issu worri rather sadden mid septemb went guan...</td>\n",
       "      <td>[issu, worri, rather, sadden, mid, septemb, we...</td>\n",
       "      <td>[fear]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7506</th>\n",
       "      <td>anger</td>\n",
       "      <td>(roommat, like, listen, meaningless, song, had...</td>\n",
       "      <td>roommat like listen meaningless song had melod...</td>\n",
       "      <td>[roommat, like, listen, meaningless, song, had...</td>\n",
       "      <td>[anger]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7507</th>\n",
       "      <td>shame</td>\n",
       "      <td>(last, summer, went, camp, c.u, student, wa, w...</td>\n",
       "      <td>last summer went camp c.u student wa work then...</td>\n",
       "      <td>[last, summer, went, camp, c.u, student, wa, w...</td>\n",
       "      <td>[shame]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7508</th>\n",
       "      <td>shame</td>\n",
       "      <td>(lie, best, friend)</td>\n",
       "      <td>lie best friend</td>\n",
       "      <td>[lie, best, friend]</td>\n",
       "      <td>[shame]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7509</th>\n",
       "      <td>joy</td>\n",
       "      <td>(receiv, letter, distant, friend)</td>\n",
       "      <td>receiv letter distant friend</td>\n",
       "      <td>[receiv, letter, distant, friend]</td>\n",
       "      <td>[joy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7510</th>\n",
       "      <td>fear</td>\n",
       "      <td>(parent, were, wa, eldest, home, midnight, mal...</td>\n",
       "      <td>parent were wa eldest home midnight male stran...</td>\n",
       "      <td>[parent, were, wa, eldest, home, midnight, mal...</td>\n",
       "      <td>[fear]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7511</th>\n",
       "      <td>shame</td>\n",
       "      <td>(year, back, someon, invit, tutor, grand-daugh...</td>\n",
       "      <td>year back someon invit tutor grand-daught gran...</td>\n",
       "      <td>[year, back, someon, invit, tutor, grand-daugh...</td>\n",
       "      <td>[shame]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7512</th>\n",
       "      <td>shame</td>\n",
       "      <td>(had, taken, respons, someth, had, prepar, how...</td>\n",
       "      <td>had taken respons someth had prepar howev fail...</td>\n",
       "      <td>[had, taken, respons, someth, had, prepar, how...</td>\n",
       "      <td>[shame]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7513</th>\n",
       "      <td>fear</td>\n",
       "      <td>(wa, home, heard, loud, sound, spit, door, tho...</td>\n",
       "      <td>wa home heard loud sound spit door thought fam...</td>\n",
       "      <td>[wa, home, heard, loud, sound, spit, door, tho...</td>\n",
       "      <td>[fear]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7514</th>\n",
       "      <td>guilt</td>\n",
       "      <td>(did, not, homework, teacher, had, ask, wa, sc...</td>\n",
       "      <td>did not homework teacher had ask wa scold immedi</td>\n",
       "      <td>[did, not, homework, teacher, had, ask, wa, sc...</td>\n",
       "      <td>[guilt]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7515</th>\n",
       "      <td>fear</td>\n",
       "      <td>(had, shout, younger, brother, wa, alway, afra...</td>\n",
       "      <td>had shout younger brother wa alway afraid call...</td>\n",
       "      <td>[had, shout, younger, brother, wa, alway, afra...</td>\n",
       "      <td>[fear]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7516 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0                                                  1  \\\n",
       "0         joy  (day, feel, close, partner, other, friend, fee...   \n",
       "1        fear  (time, imagin, someon, love, contact, seriou, ...   \n",
       "2       anger  (had, been, obvious, unjustli, treat, had, pos...   \n",
       "3     sadness  (think, short, time, live, relat, period, life...   \n",
       "4     disgust  (gather, found, involuntarili, sit, next, peop...   \n",
       "5       shame  (realiz, wa, direct, feel, discont, partner, w...   \n",
       "6       guilt  (feel, guilti, realiz, consid, materi, thing, ...   \n",
       "7         joy  (girlfriend, had, taken, exam, went, parent, p...   \n",
       "8        fear                 (first, time, realiz, mean, death)   \n",
       "9       anger                  (car, overtak, forc, drive, road)   \n",
       "10    sadness  (recent, thought, hard, work, take, studi, wan...   \n",
       "11    disgust                 (found, bristl, liver, past, tube)   \n",
       "12      shame  (wa, tire, unmotiv, shout, girlfriend, brought...   \n",
       "13      guilt  (think, not, studi, enough, weekend, think, ha...   \n",
       "14        joy         (pass, examin, did, not, think, did, well)   \n",
       "15       fear  (one, ha, arrang, meet, someon, person, arriv,...   \n",
       "16      anger           (unjustli, accus, someth, ha, not, done)   \n",
       "17    sadness  (one, studi, seem, hopelessli, difficult, unin...   \n",
       "18    disgust  (find, someon, know, not, had, thought, instan...   \n",
       "19      shame  (one, ha, been, unjust, stupid, toward, someon...   \n",
       "20      guilt     (one, ha, neglect, been, unjust, good, friend)   \n",
       "21        joy               (pass, exam, did, not, expect, pass)   \n",
       "22       fear  (climb, tree, pick, appl, angl, ladder, wa, di...   \n",
       "23      guilt                   (excus, are, necessari, get, do)   \n",
       "24        joy                                    (had, children)   \n",
       "25       fear  (year, old, son, climb, sat, 7th, floor, balco...   \n",
       "26      anger                 (partner, wa, attack, lost, teeth)   \n",
       "27    sadness   (see, children, t.v, area, devast, drought, war)   \n",
       "28    disgust  (nearli, walk, blindworm, then, saw, crawl, away)   \n",
       "29      shame  (saw, year, old, son, grab, oxygen, mask, had,...   \n",
       "...       ...                                                ...   \n",
       "7486    shame      (forgot, zip, trouser, wa, not, notic, anyon)   \n",
       "7487    guilt                                             (peep)   \n",
       "7488      joy     (had, picnic, old, classmat, chat, play, game)   \n",
       "7489     fear  (night, wa, alon, home, famili, member, usual,...   \n",
       "7490    anger  (saw, bed, hostel, wa, mess, guess, someon, el...   \n",
       "7491  sadness  (physic, experi, session, did, not, understand...   \n",
       "7492  disgust  (man, sexual, aggress, small, girl, bu, girl, ...   \n",
       "7493    shame                        (be, unabl, stop, urin, bu)   \n",
       "7494    guilt  (were, old, peopl, crowd, bu, did, not, have, ...   \n",
       "7495      joy  (wa, first, time, gave, birthday, present, fri...   \n",
       "7496     fear  (wa, week, higher, level, result, were, announ...   \n",
       "7497    anger  (surnam, brother, wa, differ, often, grin, hom...   \n",
       "7498  sadness  (wa, primari, father, die, wa, veri, young, th...   \n",
       "7499     guit  (hostel, roommat, wa, veri, selfish, person, a...   \n",
       "7500    shame  (friend, had, mani, femal, friend, thought, we...   \n",
       "7501    guilt  (past, use, think, mother, wa, veri, nag, pers...   \n",
       "7502      joy  (august,1983, long, await, big, envelop, docum...   \n",
       "7503     fear  (christma, eve,1984, had, just, finish, exam, ...   \n",
       "7504    anger  (septemb, wa, forc, live, someon, did, not, fi...   \n",
       "7505     fear  (issu, worri, rather, sadden, mid, septemb, we...   \n",
       "7506    anger  (roommat, like, listen, meaningless, song, had...   \n",
       "7507    shame  (last, summer, went, camp, c.u, student, wa, w...   \n",
       "7508    shame                                (lie, best, friend)   \n",
       "7509      joy                  (receiv, letter, distant, friend)   \n",
       "7510     fear  (parent, were, wa, eldest, home, midnight, mal...   \n",
       "7511    shame  (year, back, someon, invit, tutor, grand-daugh...   \n",
       "7512    shame  (had, taken, respons, someth, had, prepar, how...   \n",
       "7513     fear  (wa, home, heard, loud, sound, spit, door, tho...   \n",
       "7514    guilt  (did, not, homework, teacher, had, ask, wa, sc...   \n",
       "7515     fear  (had, shout, younger, brother, wa, alway, afra...   \n",
       "\n",
       "                                                      2  \\\n",
       "0     day feel close partner other friend feel peac ...   \n",
       "1     time imagin someon love contact seriou ill eve...   \n",
       "2     had been obvious unjustli treat had possibl el...   \n",
       "3     think short time live relat period life think ...   \n",
       "4     gather found involuntarili sit next peopl expr...   \n",
       "5     realiz wa direct feel discont partner way wa t...   \n",
       "6     feel guilti realiz consid materi thing more im...   \n",
       "7          girlfriend had taken exam went parent place    \n",
       "8                         first time realiz mean death    \n",
       "9                          car overtak forc drive road    \n",
       "10    recent thought hard work take studi want tri s...   \n",
       "11                        found bristl liver past tube    \n",
       "12    wa tire unmotiv shout girlfriend brought neg s...   \n",
       "13    think not studi enough weekend think have been...   \n",
       "14                  pass examin did not think did well    \n",
       "15    one ha arrang meet someon person arriv late me...   \n",
       "16                   unjustli accus someth ha not done    \n",
       "17      one studi seem hopelessli difficult uninterest    \n",
       "18    find someon know not had thought instanc frien...   \n",
       "19         one ha been unjust stupid toward someon els    \n",
       "20              one ha neglect been unjust good friend    \n",
       "21                       pass exam did not expect pass    \n",
       "22    climb tree pick appl angl ladder wa did not en...   \n",
       "23                          excus are necessari get do    \n",
       "24                                        had children    \n",
       "25    year old son climb sat 7th floor balconi leg h...   \n",
       "26                        partner wa attack lost teeth    \n",
       "27            see children t.v area devast drought war    \n",
       "28           nearli walk blindworm then saw crawl away    \n",
       "29    saw year old son grab oxygen mask had breath d...   \n",
       "...                                                 ...   \n",
       "7486             forgot zip trouser wa not notic anyon    \n",
       "7487                                              peep    \n",
       "7488            had picnic old classmat chat play game    \n",
       "7489  night wa alon home famili member usual get tog...   \n",
       "7490  saw bed hostel wa mess guess someon els had us...   \n",
       "7491  physic experi session did not understand conte...   \n",
       "7492  man sexual aggress small girl bu girl did not ...   \n",
       "7493                             be unabl stop urin bu    \n",
       "7494  were old peopl crowd bu did not have courag gi...   \n",
       "7495  wa first time gave birthday present friend wro...   \n",
       "7496  wa week higher level result were announc had t...   \n",
       "7497  surnam brother wa differ often grin home provo...   \n",
       "7498  wa primari father die wa veri young then did n...   \n",
       "7499  hostel roommat wa veri selfish person avoid do...   \n",
       "7500  friend had mani femal friend thought were love...   \n",
       "7501  past use think mother wa veri nag person start...   \n",
       "7502  august,1983 long await big envelop document c....   \n",
       "7503  christma eve,1984 had just finish exam wa afra...   \n",
       "7504  septemb wa forc live someon did not first week...   \n",
       "7505  issu worri rather sadden mid septemb went guan...   \n",
       "7506  roommat like listen meaningless song had melod...   \n",
       "7507  last summer went camp c.u student wa work then...   \n",
       "7508                                   lie best friend    \n",
       "7509                      receiv letter distant friend    \n",
       "7510  parent were wa eldest home midnight male stran...   \n",
       "7511  year back someon invit tutor grand-daught gran...   \n",
       "7512  had taken respons someth had prepar howev fail...   \n",
       "7513  wa home heard loud sound spit door thought fam...   \n",
       "7514  did not homework teacher had ask wa scold immedi    \n",
       "7515  had shout younger brother wa alway afraid call...   \n",
       "\n",
       "                                                      3          4  \n",
       "0     [day, feel, close, partner, other, friend, fee...      [joy]  \n",
       "1     [time, imagin, someon, love, contact, seriou, ...     [fear]  \n",
       "2     [had, been, obvious, unjustli, treat, had, pos...    [anger]  \n",
       "3     [think, short, time, live, relat, period, life...  [sadness]  \n",
       "4     [gather, found, involuntarili, sit, next, peop...  [disgust]  \n",
       "5     [realiz, wa, direct, feel, discont, partner, w...    [shame]  \n",
       "6     [feel, guilti, realiz, consid, materi, thing, ...    [guilt]  \n",
       "7     [girlfriend, had, taken, exam, went, parent, p...      [joy]  \n",
       "8                    [first, time, realiz, mean, death]     [fear]  \n",
       "9                     [car, overtak, forc, drive, road]    [anger]  \n",
       "10    [recent, thought, hard, work, take, studi, wan...  [sadness]  \n",
       "11                   [found, bristl, liver, past, tube]  [disgust]  \n",
       "12    [wa, tire, unmotiv, shout, girlfriend, brought...    [shame]  \n",
       "13    [think, not, studi, enough, weekend, think, ha...    [guilt]  \n",
       "14           [pass, examin, did, not, think, did, well]      [joy]  \n",
       "15    [one, ha, arrang, meet, someon, person, arriv,...     [fear]  \n",
       "16             [unjustli, accus, someth, ha, not, done]    [anger]  \n",
       "17    [one, studi, seem, hopelessli, difficult, unin...  [sadness]  \n",
       "18    [find, someon, know, not, had, thought, instan...  [disgust]  \n",
       "19    [one, ha, been, unjust, stupid, toward, someon...    [shame]  \n",
       "20       [one, ha, neglect, been, unjust, good, friend]    [guilt]  \n",
       "21                 [pass, exam, did, not, expect, pass]      [joy]  \n",
       "22    [climb, tree, pick, appl, angl, ladder, wa, di...     [fear]  \n",
       "23                     [excus, are, necessari, get, do]    [guilt]  \n",
       "24                                      [had, children]      [joy]  \n",
       "25    [year, old, son, climb, sat, 7th, floor, balco...     [fear]  \n",
       "26                   [partner, wa, attack, lost, teeth]    [anger]  \n",
       "27     [see, children, t.v, area, devast, drought, war]  [sadness]  \n",
       "28    [nearli, walk, blindworm, then, saw, crawl, away]  [disgust]  \n",
       "29    [saw, year, old, son, grab, oxygen, mask, had,...    [shame]  \n",
       "...                                                 ...        ...  \n",
       "7486      [forgot, zip, trouser, wa, not, notic, anyon]    [shame]  \n",
       "7487                                             [peep]    [guilt]  \n",
       "7488     [had, picnic, old, classmat, chat, play, game]      [joy]  \n",
       "7489  [night, wa, alon, home, famili, member, usual,...     [fear]  \n",
       "7490  [saw, bed, hostel, wa, mess, guess, someon, el...    [anger]  \n",
       "7491  [physic, experi, session, did, not, understand...  [sadness]  \n",
       "7492  [man, sexual, aggress, small, girl, bu, girl, ...  [disgust]  \n",
       "7493                        [be, unabl, stop, urin, bu]    [shame]  \n",
       "7494  [were, old, peopl, crowd, bu, did, not, have, ...    [guilt]  \n",
       "7495  [wa, first, time, gave, birthday, present, fri...      [joy]  \n",
       "7496  [wa, week, higher, level, result, were, announ...     [fear]  \n",
       "7497  [surnam, brother, wa, differ, often, grin, hom...    [anger]  \n",
       "7498  [wa, primari, father, die, wa, veri, young, th...  [sadness]  \n",
       "7499  [hostel, roommat, wa, veri, selfish, person, a...     [guit]  \n",
       "7500  [friend, had, mani, femal, friend, thought, we...    [shame]  \n",
       "7501  [past, use, think, mother, wa, veri, nag, pers...    [guilt]  \n",
       "7502  [august,1983, long, await, big, envelop, docum...      [joy]  \n",
       "7503  [christma, eve,1984, had, just, finish, exam, ...     [fear]  \n",
       "7504  [septemb, wa, forc, live, someon, did, not, fi...    [anger]  \n",
       "7505  [issu, worri, rather, sadden, mid, septemb, we...     [fear]  \n",
       "7506  [roommat, like, listen, meaningless, song, had...    [anger]  \n",
       "7507  [last, summer, went, camp, c.u, student, wa, w...    [shame]  \n",
       "7508                                [lie, best, friend]    [shame]  \n",
       "7509                  [receiv, letter, distant, friend]      [joy]  \n",
       "7510  [parent, were, wa, eldest, home, midnight, mal...     [fear]  \n",
       "7511  [year, back, someon, invit, tutor, grand-daugh...    [shame]  \n",
       "7512  [had, taken, respons, someth, had, prepar, how...    [shame]  \n",
       "7513  [wa, home, heard, loud, sound, spit, door, tho...     [fear]  \n",
       "7514  [did, not, homework, teacher, had, ask, wa, sc...    [guilt]  \n",
       "7515  [had, shout, younger, brother, wa, alway, afra...     [fear]  \n",
       "\n",
       "[7516 rows x 5 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Reads the emotion representative words file\n",
    "'''\n",
    "def readfile(filename):\n",
    "    f = open(filename,'r')\n",
    "    representative_words = []\n",
    "    for line in f.readlines():\n",
    "        characters = [\"\\n\", \" \", \"\\r\", \"\\t\"]\n",
    "        new = ''.join([i for i in line if not [e for e in characters if e in i]])\n",
    "        representative_words.append(new)\n",
    "    return representative_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Makes a list of all words semantically related to an emotion and Stemming\n",
    "'''\n",
    "def affect_wordlist(words):\n",
    "    affect_words = []\n",
    "    stemmer = PorterStemmer()\n",
    "    for w in words:\n",
    "        w_l = w.lower()\n",
    "        word_stem = stemmer.stem(w_l)\n",
    "        if word_stem not in affect_words:\n",
    "            affect_words.append(word_stem)\n",
    "    return affect_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Creating an emotion wordnet\n",
    "'''\n",
    "def emotion_word_set(emotions):\n",
    "    word_set = {}\n",
    "    for e in emotions:\n",
    "        representative_words = readfile(e)\n",
    "        wordlist = affect_wordlist(representative_words)\n",
    "        word_set[e] = wordlist\n",
    "    return word_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Lexicon based approach - Check for lexicons\n",
    "'''\n",
    "def lexicon_based(sentences, word_set):\n",
    "    text_vector = []\n",
    "    for sen in sentences:\n",
    "        s_vector = []\n",
    "        for word in sen:\n",
    "            w_vector = {}\n",
    "            for emo in word_set:\n",
    "                if word in word_set[emo]:\n",
    "#                     print word\n",
    "                    try:\n",
    "                        if emo not in w_vector[word]:\n",
    "                            w_vector[word].append(emo)\n",
    "                    except KeyError:\n",
    "                        w_vector[word] = [emo]\n",
    "            if w_vector:\n",
    "                s_vector.append(w_vector)\n",
    "        if not s_vector:\n",
    "            text_vector.append(s_vector)\n",
    "        else:\n",
    "            text_vector.append(s_vector)\n",
    "    return text_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Lexicon based approach - Classify based on lexicons\n",
    "'''\n",
    "def classify_lexicon(text_vector, labels, emotion_labels):\n",
    "    count = 0\n",
    "    total = 0\n",
    "    for j in range(len(text_vector)):\n",
    "        sen = text_vector[j]\n",
    "        sen_emo = np.empty(len(emotion_labels))\n",
    "        sen_emo.fill(0)\n",
    "        if sen:\n",
    "            total += 1\n",
    "            w_emo = []\n",
    "            for word in sen:\n",
    "                emotions =  word.values()[0][0]\n",
    "#                 print emotions, type(emotions), j\n",
    "                w_emo.append(emotions)\n",
    "                i = emotion_labels.index(emotions)\n",
    "                sen_emo[i] += 1\n",
    "#             print sen_emo\n",
    "            winner = np.argwhere(sen_emo == np.amax(sen_emo))\n",
    "            indices = winner.flatten().tolist()\n",
    "            for i in indices:\n",
    "                if emotion_labels[i] == labels[j]:\n",
    "                    count += 1\n",
    "                    break\n",
    "#                 else:\n",
    "#                     print j, text_vector[j]\n",
    "    accuracy = count/len(text_vector)\n",
    "    tot_accuracy = count/total\n",
    "    return accuracy, tot_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.227514635444 0.540113708149\n"
     ]
    }
   ],
   "source": [
    "e = emotion_word_set(emotion_labels)\n",
    "l = lexicon_based(c[1],e) \n",
    "a, b = classify_lexicon(l, c[0], emotion_labels)\n",
    "print a, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Calculate pmi\n",
    "'''\n",
    "def pmi(x, y, sentences):\n",
    "    count_x = 1\n",
    "    count_y = 1\n",
    "    count_xy = 1\n",
    "    for sen in sentences:\n",
    "        if x and y in sentences:\n",
    "            count_xy += 1\n",
    "            count_x += 1\n",
    "            count_y += 1\n",
    "        if x in sentences:\n",
    "            count_x += 1\n",
    "        if y in sentences:\n",
    "            count_y += 1\n",
    "        result = count_xy/(count_x * count_y)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22.7514635444 %\n",
      "54.0113708149 %\n"
     ]
    }
   ],
   "source": [
    "print a*100, '%'\n",
    "print b*100, \"%\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nEmotion Detector\\n'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "Emotion Detector\n",
    "'''\n",
    "# c = create_frame(Data)\n",
    "# emo_word_net = emotion_word_set(emotion_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Getting synonyms from wordnet synsets\n",
    "'''\n",
    "def get_synonyms():\n",
    "    syn = {}\n",
    "    for e in emotion_labels:\n",
    "        jw = wn.synsets(e)\n",
    "        for s in jw:\n",
    "            v = s.name()\n",
    "            try:\n",
    "                syn[e].append(wn.synset(v).lemma_names())\n",
    "            except KeyError:\n",
    "                syn[e] = wn.synset(v).lemma_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Creating training/testing set for Naive Bayes classifier TextBlob -- Not used\n",
    "'''\n",
    "def create_dataset_textblob(sentences, emotions):\n",
    "    train = []\n",
    "    sen = []\n",
    "    emo = []\n",
    "    for s in sentences:\n",
    "        sen.append(s)\n",
    "    for e in emotions:\n",
    "        emo.append(e)\n",
    "    for i in range(len(sen)):\n",
    "        s = sen[i]\n",
    "        e = emo[i]\n",
    "        train.append((str(s), e))\n",
    "    return train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Testing for Naive Bayes Classifier\n",
    "'''\n",
    "def testing(cl, test):\n",
    "    for s, e in test:\n",
    "        r = cl.classify(s)\n",
    "        print s, e, r\n",
    "        if r == e:\n",
    "            print \"*\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Create dataset for nltk Naive Bayes\n",
    "'''\n",
    "def create_data(sentence, emotion):\n",
    "    data = []\n",
    "    for i in range(len(sentence)):\n",
    "        sen = []\n",
    "        for s in sentence[i]:\n",
    "            sen.append(str(s))\n",
    "        emo = emotion[i]\n",
    "        data.append((sen, emo))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Get all words in dataset\n",
    "'''\n",
    "def get_words_in_dataset(dataset):\n",
    "    all_words = []\n",
    "    for (words, sentiment) in dataset:\n",
    "        all_words.extend(words)\n",
    "    return all_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Getting frequency dist of words\n",
    "'''\n",
    "def get_word_features(wordlist):\n",
    "    wordlist = nltk.FreqDist(wordlist)\n",
    "    word_features = wordlist.keys()\n",
    "    return word_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Extacting features\n",
    "'''\n",
    "def extract_features(document):\n",
    "    document_words = set(document)\n",
    "    features = {}\n",
    "    for word in word_features:\n",
    "        features['contains(%s)' % word] = (word in document_words)\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Create test data\n",
    "'''\n",
    "def create_test(sentence, emotion):\n",
    "    data = []\n",
    "    sen = []\n",
    "    emo = []\n",
    "    for s in sentence:\n",
    "        sen.append(str(s))\n",
    "    for e in emotion:\n",
    "        emo.append(e)\n",
    "    for i in range(len(sen)):\n",
    "        temp = []\n",
    "        temp.append(sen[i])\n",
    "        temp.append(emo[i])\n",
    "        data.append(temp)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Classifier\n",
    "'''\n",
    "def classify_dataset(data):\n",
    "    return \\\n",
    "        classifier.classify(extract_features(nltk.word_tokenize(data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Get accuracy\n",
    "'''\n",
    "def get_accuracy(test_data, classifier):\n",
    "    total = accuracy = float(len(test_data))\n",
    "    for data in test_data:\n",
    "        if classify_dataset(data[0]) != data[1]:\n",
    "#             print data, classify_dataset(data[0]), data[1]\n",
    "            accuracy -= 1\n",
    "    print('Total accuracy: %f%% (%d/20).' % (accuracy / total * 100, accuracy))\n",
    "    final = accuracy / total * 100\n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create training and testing data\n",
    "sen = c[3]\n",
    "emo = c[0]\n",
    "l = len(c[3])\n",
    "limit = (9*l)//10\n",
    "sente = c[2]\n",
    "Data = create_data(sen[:limit], emo[:limit])\n",
    "test_data = create_test(sente[limit:], emo[limit:]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# extract the word features out from the training data\n",
    "word_features = get_word_features(\\\n",
    "                    get_words_in_dataset(Data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get the training set and train the Naive Bayes Classifier\n",
    "training_set = nltk.classify.util.apply_features(extract_features, Data)\n",
    "classifier = NaiveBayesClassifier.train(training_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total accuracy: 63.164894% (475/20).\n"
     ]
    }
   ],
   "source": [
    "Naive_accu = get_accuracy(test_data, classifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "length = len(st)\n",
    "train_size = (9 * length) // 10\n",
    "train_data = st[:train_size]\n",
    "test_data = senten[train_size:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gensim_dict = corpora.Dictionary(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gensim_dict.save('corpus.dict')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gen_token = gensim_dict.token2id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "corpusmm = [gensim_dict.doc2bow(text) for text in train_data]\n",
    "corpora.MmCorpus.serialize('corpus.mm', corpusmm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gen_c = corpora.MmCorpus('corpus.mm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tfidf = models.TfidfModel(gen_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corpus_tfidf = tfidf[gen_c]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dictionary = corpora.Dictionary.load('corpus.dict')\n",
    "lsi = models.LsiModel(corpus_tfidf, id2word=gensim_dict, num_topics=7) # initialize an LSI transformation\n",
    "corpus_lsi = lsi[corpus_tfidf]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:gensim.similarities.docsim:scanning corpus to determine the number of features (consider setting `num_features` explicitly)\n"
     ]
    }
   ],
   "source": [
    "index = similarities.MatrixSimilarity(lsi[gen_c])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lsi.save('corpus.lsi') \n",
    "index.save('corpus.index')\n",
    "# lsi = models.LsiModel.load('corpus.lsi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# length = len(corpus_lsi)\n",
    "# train_size = (9 * length) // 10\n",
    "# train_data = corpus_lsi[:train_size]\n",
    "# test_data = corpus_lsi[train_size:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_labels = labels[train_size:]\n",
    "train_labels = labels[:train_size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def semantic_sim(test_data):\n",
    "    total = 0\n",
    "    for count, doc in enumerate(test_data):\n",
    "        vec_bow = dictionary.doc2bow(doc.lower().split())\n",
    "        vec_lsi = lsi[vec_bow]\n",
    "        sims = index[vec_lsi]\n",
    "        ans = sorted(enumerate(sims), key=lambda item: -item[1])[0]\n",
    "        ind = ans[0] \n",
    "        if train_labels[ind] == test_labels[count]:\n",
    "            total += 2\n",
    "    avg = total/count\n",
    "    return avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sem_accu = semantic_sim(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Experimenting with Alchemy API\n",
    "url = 'http://gateway-a.watsonplatform.net/calls/text/TextGetEmotion'\n",
    "params = urllib.urlencode({\n",
    "  'apikey': '15ce4bd07b66f9e000a15383777870c0afb383fb',\n",
    "  'text': 'I am excited',\n",
    "  'outputMode': 'json'\n",
    "})\n",
    "response = urllib2.urlopen(url, params).read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy using Lexical Component - First Phase   22.7514635444 %\n",
      "Accuracy using Lexical Component - Second Phase   54.0113708149 %\n"
     ]
    }
   ],
   "source": [
    "print \"Accuracy using Lexical Component - First Phase  \", a*100, '%'\n",
    "print \"Accuracy using Lexical Component - Second Phase  \", b*100, \"%\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy using Naive Bayes Component   63.164893617 %\n"
     ]
    }
   ],
   "source": [
    "print \"Accuracy using Naive Bayes Component  \", Naive_accu, \"%\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy using semantic similarity component  53.7949400799 %\n"
     ]
    }
   ],
   "source": [
    "print \"Accuracy using semantic similarity component \", sem_accu*100, \"%\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}