corpus_wilcoxon_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.stats import wilcoxon\n",
    "corpus = pd.read_csv('corpus.csv.gz', compression='gzip')\n",
    "corpus_liwc = pd.read_csv('corpus_liwc_mtx.csv.gz', compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "corpus_liwc['class'] = corpus['qual_a_melhor_classificao_para_esse_texto']\n",
    "corpus_liwc['confidence'] = corpus['qual_a_melhor_classificao_para_esse_texto:confidence']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">wc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>class</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>diario</th>\n",
       "      <td>371.153179</td>\n",
       "      <td>346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>outro</th>\n",
       "      <td>477.047872</td>\n",
       "      <td>188</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                wc      \n",
       "              mean count\n",
       "class                   \n",
       "diario  371.153179   346\n",
       "outro   477.047872   188"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "def wc(x): \n",
    "    try:\n",
    "        return len(re.findall(r'\\w+', x['content']))\n",
    "    except:\n",
    "        return 0\n",
    "    \n",
    "corpus['wc'] = corpus.apply(wc,axis=1)\n",
    "corpus_liwc['wc'] = corpus['wc']\n",
    "corpus_liwc = corpus_liwc[corpus_liwc['confidence'] == 1]\n",
    "corpus_liwc[['class','wc']].groupby(['class']).agg(['mean','count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "outro = corpus_liwc[corpus_liwc['class'] == 'outro']\n",
    "diario = corpus_liwc[corpus_liwc['class'] == 'diario'].sample(len(outro))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "columns = ['funct','pronoun','ppron','i','we','you','shehe','they','ipron','article','verb','auxverb','past','present','future','adverb','preps','conj','negate','quant','number','swear','social','family','friend','humans','affect','posemo','negemo','anx','anger','sad','cogmech','insight','cause','discrep','tentat','certain','inhib','incl','excl','percept','see','hear','feel','bio','body','health','sexual','ingest','relativ','motion','space','time','work','achieve','leisure','home','money','relig','death','assent','nonfl','filler']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def diff(x): \n",
    "    return (x['diario_mean']) - (x['outro_mean'])\n",
    "    #return 100 * ((x['diario_mean']/diario.wc.mean()) - (x['outro_mean']/outro.wc.mean()))\n",
    "    \n",
    "stats = pd.DataFrame(data={'diario_mean': diario.mean(axis=0)}, index=columns)\n",
    "stats['outro_mean'] = outro.mean(axis=0)\n",
    "\n",
    "stats['diff'] = stats.apply(diff,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "significance = []\n",
    "for column in list(stats.index.values):\n",
    "    a = diario[column]\n",
    "    b = outro[column]\n",
    "    t, p = wilcoxon(a, b)\n",
    "    significance.append(p)\n",
    "stats['significance'] = significance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Wilcoxon Not Rejected Categories\n",
    "#### two populations are equally interested"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>significance</th>\n",
       "      <th>diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>negate</th>\n",
       "      <td>0.918024</td>\n",
       "      <td>0.037234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>adverb</th>\n",
       "      <td>0.843622</td>\n",
       "      <td>-0.175532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>past</th>\n",
       "      <td>0.414383</td>\n",
       "      <td>1.361702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>future</th>\n",
       "      <td>0.128246</td>\n",
       "      <td>-0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>quant</th>\n",
       "      <td>0.104887</td>\n",
       "      <td>-2.569149</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        significance      diff\n",
       "negate      0.918024  0.037234\n",
       "adverb      0.843622 -0.175532\n",
       "past        0.414383  1.361702\n",
       "future      0.128246 -0.500000\n",
       "quant       0.104887 -2.569149"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "liguistic_columns = ['funct','pronoun','ppron','i','we','you','shehe','they','ipron','article','verb','auxverb','past','present','future','adverb','preps','conj','negate','quant','number']\n",
    "liguistic_stats = stats.ix[liguistic_columns]\n",
    "liguistic_stats[['significance','diff']][liguistic_stats.significance > 0.05].sort_values('significance',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>significance</th>\n",
       "      <th>diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>anx</th>\n",
       "      <td>0.920032</td>\n",
       "      <td>0.239362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sexual</th>\n",
       "      <td>0.853533</td>\n",
       "      <td>-0.255319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>friend</th>\n",
       "      <td>0.806944</td>\n",
       "      <td>0.063830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>home</th>\n",
       "      <td>0.790528</td>\n",
       "      <td>-0.085106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feel</th>\n",
       "      <td>0.660014</td>\n",
       "      <td>-0.239362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>see</th>\n",
       "      <td>0.294309</td>\n",
       "      <td>-0.569149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>assent</th>\n",
       "      <td>0.276611</td>\n",
       "      <td>-0.207447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>certain</th>\n",
       "      <td>0.271694</td>\n",
       "      <td>-1.473404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sad</th>\n",
       "      <td>0.238550</td>\n",
       "      <td>-0.446809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>family</th>\n",
       "      <td>0.223343</td>\n",
       "      <td>-0.590426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>humans</th>\n",
       "      <td>0.163131</td>\n",
       "      <td>-3.026596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leisure</th>\n",
       "      <td>0.113809</td>\n",
       "      <td>-1.319149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percept</th>\n",
       "      <td>0.072740</td>\n",
       "      <td>-2.361702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>discrep</th>\n",
       "      <td>0.059690</td>\n",
       "      <td>-3.281915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ingest</th>\n",
       "      <td>0.056383</td>\n",
       "      <td>-2.771277</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         significance      diff\n",
       "anx          0.920032  0.239362\n",
       "sexual       0.853533 -0.255319\n",
       "friend       0.806944  0.063830\n",
       "home         0.790528 -0.085106\n",
       "feel         0.660014 -0.239362\n",
       "see          0.294309 -0.569149\n",
       "assent       0.276611 -0.207447\n",
       "certain      0.271694 -1.473404\n",
       "sad          0.238550 -0.446809\n",
       "family       0.223343 -0.590426\n",
       "humans       0.163131 -3.026596\n",
       "leisure      0.113809 -1.319149\n",
       "percept      0.072740 -2.361702\n",
       "discrep      0.059690 -3.281915\n",
       "ingest       0.056383 -2.771277"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "psychological_columns = ['swear','social','family','friend','humans','affect','posemo','negemo','anx','anger','sad','cogmech','insight','cause','discrep','tentat','certain','inhib','incl','excl','percept','see','hear','feel','bio','body','health','sexual','ingest','relativ','motion','space','time','work','achieve','leisure','home','money','relig','death','assent','nonfl','filler']\n",
    "psychological_stats = stats.ix[psychological_columns]\n",
    "psychological_stats[['significance','diff']][psychological_stats.significance > 0.05].sort_values('significance',ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Wilcoxon Rejected Categories\n",
    "#### two populations has different interests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>significance</th>\n",
       "      <th>diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>auxverb</th>\n",
       "      <td>3.437545e-02</td>\n",
       "      <td>-4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>verb</th>\n",
       "      <td>3.075465e-02</td>\n",
       "      <td>-7.601064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>number</th>\n",
       "      <td>1.374029e-02</td>\n",
       "      <td>-1.047872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pronoun</th>\n",
       "      <td>2.403478e-03</td>\n",
       "      <td>-13.590426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>conj</th>\n",
       "      <td>1.211852e-03</td>\n",
       "      <td>-7.925532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>funct</th>\n",
       "      <td>7.842559e-04</td>\n",
       "      <td>-38.867021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ipron</th>\n",
       "      <td>2.667456e-04</td>\n",
       "      <td>-11.643617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>we</th>\n",
       "      <td>1.235138e-04</td>\n",
       "      <td>-1.579787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ppron</th>\n",
       "      <td>3.375651e-05</td>\n",
       "      <td>-12.111702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>present</th>\n",
       "      <td>2.071924e-05</td>\n",
       "      <td>-9.994681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>article</th>\n",
       "      <td>5.446126e-07</td>\n",
       "      <td>-10.893617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>preps</th>\n",
       "      <td>2.041736e-07</td>\n",
       "      <td>-20.808511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>they</th>\n",
       "      <td>4.573661e-08</td>\n",
       "      <td>-4.021277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>i</th>\n",
       "      <td>1.473191e-08</td>\n",
       "      <td>4.590426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shehe</th>\n",
       "      <td>4.115445e-10</td>\n",
       "      <td>-11.920213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>you</th>\n",
       "      <td>3.427160e-10</td>\n",
       "      <td>-13.377660</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         significance       diff\n",
       "auxverb  3.437545e-02  -4.000000\n",
       "verb     3.075465e-02  -7.601064\n",
       "number   1.374029e-02  -1.047872\n",
       "pronoun  2.403478e-03 -13.590426\n",
       "conj     1.211852e-03  -7.925532\n",
       "funct    7.842559e-04 -38.867021\n",
       "ipron    2.667456e-04 -11.643617\n",
       "we       1.235138e-04  -1.579787\n",
       "ppron    3.375651e-05 -12.111702\n",
       "present  2.071924e-05  -9.994681\n",
       "article  5.446126e-07 -10.893617\n",
       "preps    2.041736e-07 -20.808511\n",
       "they     4.573661e-08  -4.021277\n",
       "i        1.473191e-08   4.590426\n",
       "shehe    4.115445e-10 -11.920213\n",
       "you      3.427160e-10 -13.377660"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "liguistic_columns = ['funct','pronoun','ppron','i','we','you','shehe','they','ipron','article','verb','auxverb','past','present','future','adverb','preps','conj','negate','quant','number']\n",
    "liguistic_stats = stats.ix[liguistic_columns]\n",
    "liguistic_stats[['significance','diff']][liguistic_stats.significance <= 0.05].sort_values('significance',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>significance</th>\n",
       "      <th>diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>nonfl</th>\n",
       "      <td>2.648187e-08</td>\n",
       "      <td>-4.489362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>space</th>\n",
       "      <td>4.476228e-06</td>\n",
       "      <td>-12.361702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relig</th>\n",
       "      <td>7.700756e-06</td>\n",
       "      <td>-2.409574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>incl</th>\n",
       "      <td>1.430851e-05</td>\n",
       "      <td>-15.505319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>social</th>\n",
       "      <td>5.259275e-05</td>\n",
       "      <td>-20.803191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work</th>\n",
       "      <td>1.695272e-04</td>\n",
       "      <td>-4.510638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>swear</th>\n",
       "      <td>3.120360e-04</td>\n",
       "      <td>-6.281915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>achieve</th>\n",
       "      <td>4.190609e-04</td>\n",
       "      <td>-3.877660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cogmech</th>\n",
       "      <td>6.824609e-04</td>\n",
       "      <td>-33.340426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>anger</th>\n",
       "      <td>1.280878e-03</td>\n",
       "      <td>-1.643617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bio</th>\n",
       "      <td>2.171747e-03</td>\n",
       "      <td>-4.239362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>affect</th>\n",
       "      <td>2.872561e-03</td>\n",
       "      <td>-7.244681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>body</th>\n",
       "      <td>3.560716e-03</td>\n",
       "      <td>-2.069149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>filler</th>\n",
       "      <td>5.353296e-03</td>\n",
       "      <td>0.207447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relativ</th>\n",
       "      <td>5.721526e-03</td>\n",
       "      <td>-14.382979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hear</th>\n",
       "      <td>9.077359e-03</td>\n",
       "      <td>-1.531915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>money</th>\n",
       "      <td>1.021516e-02</td>\n",
       "      <td>-2.117021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>posemo</th>\n",
       "      <td>1.253129e-02</td>\n",
       "      <td>-4.239362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inhib</th>\n",
       "      <td>1.431834e-02</td>\n",
       "      <td>-2.968085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>insight</th>\n",
       "      <td>1.556139e-02</td>\n",
       "      <td>-4.180851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>negemo</th>\n",
       "      <td>1.636996e-02</td>\n",
       "      <td>-2.260638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>death</th>\n",
       "      <td>1.655279e-02</td>\n",
       "      <td>-0.531915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time</th>\n",
       "      <td>1.719307e-02</td>\n",
       "      <td>-5.319149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>excl</th>\n",
       "      <td>2.745114e-02</td>\n",
       "      <td>-4.441489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>health</th>\n",
       "      <td>2.756701e-02</td>\n",
       "      <td>-1.117021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tentat</th>\n",
       "      <td>2.761175e-02</td>\n",
       "      <td>-6.393617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cause</th>\n",
       "      <td>3.001133e-02</td>\n",
       "      <td>-3.420213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>motion</th>\n",
       "      <td>3.695266e-02</td>\n",
       "      <td>-2.537234</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         significance       diff\n",
       "nonfl    2.648187e-08  -4.489362\n",
       "space    4.476228e-06 -12.361702\n",
       "relig    7.700756e-06  -2.409574\n",
       "incl     1.430851e-05 -15.505319\n",
       "social   5.259275e-05 -20.803191\n",
       "work     1.695272e-04  -4.510638\n",
       "swear    3.120360e-04  -6.281915\n",
       "achieve  4.190609e-04  -3.877660\n",
       "cogmech  6.824609e-04 -33.340426\n",
       "anger    1.280878e-03  -1.643617\n",
       "bio      2.171747e-03  -4.239362\n",
       "affect   2.872561e-03  -7.244681\n",
       "body     3.560716e-03  -2.069149\n",
       "filler   5.353296e-03   0.207447\n",
       "relativ  5.721526e-03 -14.382979\n",
       "hear     9.077359e-03  -1.531915\n",
       "money    1.021516e-02  -2.117021\n",
       "posemo   1.253129e-02  -4.239362\n",
       "inhib    1.431834e-02  -2.968085\n",
       "insight  1.556139e-02  -4.180851\n",
       "negemo   1.636996e-02  -2.260638\n",
       "death    1.655279e-02  -0.531915\n",
       "time     1.719307e-02  -5.319149\n",
       "excl     2.745114e-02  -4.441489\n",
       "health   2.756701e-02  -1.117021\n",
       "tentat   2.761175e-02  -6.393617\n",
       "cause    3.001133e-02  -3.420213\n",
       "motion   3.695266e-02  -2.537234"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "psychological_columns = ['swear','social','family','friend','humans','affect','posemo','negemo','anx','anger','sad','cogmech','insight','cause','discrep','tentat','certain','inhib','incl','excl','percept','see','hear','feel','bio','body','health','sexual','ingest','relativ','motion','space','time','work','achieve','leisure','home','money','relig','death','assent','nonfl','filler']\n",
    "psychological_stats = stats.ix[psychological_columns]\n",
    "print(len(psychological_stats[psychological_stats.significance <= 0.05]))\n",
    "psychological_stats[['significance','diff']][psychological_stats.significance <= 0.05].sort_values('significance',ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>diario_mean</th>\n",
       "      <th>outro_mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>significance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>filler</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.292553</td>\n",
       "      <td>0.207447</td>\n",
       "      <td>0.005353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>death</th>\n",
       "      <td>0.941489</td>\n",
       "      <td>1.473404</td>\n",
       "      <td>-0.531915</td>\n",
       "      <td>0.016553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>health</th>\n",
       "      <td>3.930851</td>\n",
       "      <td>5.047872</td>\n",
       "      <td>-1.117021</td>\n",
       "      <td>0.027567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hear</th>\n",
       "      <td>4.845745</td>\n",
       "      <td>6.377660</td>\n",
       "      <td>-1.531915</td>\n",
       "      <td>0.009077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>anger</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.643617</td>\n",
       "      <td>-1.643617</td>\n",
       "      <td>0.001281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>body</th>\n",
       "      <td>9.255319</td>\n",
       "      <td>11.324468</td>\n",
       "      <td>-2.069149</td>\n",
       "      <td>0.003561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>money</th>\n",
       "      <td>7.276596</td>\n",
       "      <td>9.393617</td>\n",
       "      <td>-2.117021</td>\n",
       "      <td>0.010215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>negemo</th>\n",
       "      <td>9.601064</td>\n",
       "      <td>11.861702</td>\n",
       "      <td>-2.260638</td>\n",
       "      <td>0.016370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relig</th>\n",
       "      <td>2.239362</td>\n",
       "      <td>4.648936</td>\n",
       "      <td>-2.409574</td>\n",
       "      <td>0.000008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>motion</th>\n",
       "      <td>23.851064</td>\n",
       "      <td>26.388298</td>\n",
       "      <td>-2.537234</td>\n",
       "      <td>0.036953</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        diario_mean  outro_mean      diff  significance\n",
       "filler     0.500000    0.292553  0.207447      0.005353\n",
       "death      0.941489    1.473404 -0.531915      0.016553\n",
       "health     3.930851    5.047872 -1.117021      0.027567\n",
       "hear       4.845745    6.377660 -1.531915      0.009077\n",
       "anger      3.000000    4.643617 -1.643617      0.001281\n",
       "body       9.255319   11.324468 -2.069149      0.003561\n",
       "money      7.276596    9.393617 -2.117021      0.010215\n",
       "negemo     9.601064   11.861702 -2.260638      0.016370\n",
       "relig      2.239362    4.648936 -2.409574      0.000008\n",
       "motion    23.851064   26.388298 -2.537234      0.036953"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## story categories\n",
    "psychological_stats[psychological_stats.significance <= 0.05].sort_values('diff',ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>diario_mean</th>\n",
       "      <th>outro_mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>significance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cogmech</th>\n",
       "      <td>141.526596</td>\n",
       "      <td>174.867021</td>\n",
       "      <td>-33.340426</td>\n",
       "      <td>0.000682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>social</th>\n",
       "      <td>68.010638</td>\n",
       "      <td>88.813830</td>\n",
       "      <td>-20.803191</td>\n",
       "      <td>0.000053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>incl</th>\n",
       "      <td>52.010638</td>\n",
       "      <td>67.515957</td>\n",
       "      <td>-15.505319</td>\n",
       "      <td>0.000014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relativ</th>\n",
       "      <td>79.946809</td>\n",
       "      <td>94.329787</td>\n",
       "      <td>-14.382979</td>\n",
       "      <td>0.005722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>space</th>\n",
       "      <td>35.723404</td>\n",
       "      <td>48.085106</td>\n",
       "      <td>-12.361702</td>\n",
       "      <td>0.000004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>affect</th>\n",
       "      <td>30.707447</td>\n",
       "      <td>37.952128</td>\n",
       "      <td>-7.244681</td>\n",
       "      <td>0.002873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tentat</th>\n",
       "      <td>34.930851</td>\n",
       "      <td>41.324468</td>\n",
       "      <td>-6.393617</td>\n",
       "      <td>0.027612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>swear</th>\n",
       "      <td>19.196809</td>\n",
       "      <td>25.478723</td>\n",
       "      <td>-6.281915</td>\n",
       "      <td>0.000312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time</th>\n",
       "      <td>29.579787</td>\n",
       "      <td>34.898936</td>\n",
       "      <td>-5.319149</td>\n",
       "      <td>0.017193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work</th>\n",
       "      <td>9.053191</td>\n",
       "      <td>13.563830</td>\n",
       "      <td>-4.510638</td>\n",
       "      <td>0.000170</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         diario_mean  outro_mean       diff  significance\n",
       "cogmech   141.526596  174.867021 -33.340426      0.000682\n",
       "social     68.010638   88.813830 -20.803191      0.000053\n",
       "incl       52.010638   67.515957 -15.505319      0.000014\n",
       "relativ    79.946809   94.329787 -14.382979      0.005722\n",
       "space      35.723404   48.085106 -12.361702      0.000004\n",
       "affect     30.707447   37.952128  -7.244681      0.002873\n",
       "tentat     34.930851   41.324468  -6.393617      0.027612\n",
       "swear      19.196809   25.478723  -6.281915      0.000312\n",
       "time       29.579787   34.898936  -5.319149      0.017193\n",
       "work        9.053191   13.563830  -4.510638      0.000170"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## non-story categories\n",
    "psychological_stats[psychological_stats.significance <= 0.05].sort_values('diff',ascending=True).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}