corpus_prop_liwc_wilcoxon.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(534, 68)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.stats import wilcoxon\n",
    "\n",
    "# load corpus\n",
    "corpus = pd.read_csv('corpus.csv.gz', compression='gzip')\n",
    "corpus = corpus[corpus['qual_a_melhor_classificao_para_esse_texto:confidence'] == 1]\n",
    "corpus = corpus.reset_index()\n",
    "\n",
    "corpus_feat = pd.read_csv('corpus_liwc_mtx.csv.gz', compression='gzip')\n",
    "corpus_feat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def wc(x): \n",
    "    try:\n",
    "        return len(re.findall(r'\\w+', x['content']))\n",
    "    except:\n",
    "        return 0\n",
    "    \n",
    "corpus['wc'] = corpus.apply(wc,axis=1)\n",
    "corpus_feat['wc'] = corpus['wc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "corpus_feat.drop('Unnamed: 0', axis=1,inplace=True)\n",
    "corpus_feat.drop('confidence', axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "wc_vector = corpus_feat['wc']\n",
    "class_vector = corpus_feat['class']\n",
    "\n",
    "corpus_feat.drop('class',axis=1,inplace=True)\n",
    "corpus_feat.drop('wc',axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel/__main__.py:1: RuntimeWarning: invalid value encountered in true_divide\n",
      "  if __name__ == '__main__':\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(534, 64)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = corpus_feat.as_matrix().astype(float) / wc_vector.as_matrix().astype(float)[:, np.newaxis]\n",
    "data[np.isnan(data)] = 0\n",
    "data[data >= 1E308] = 0\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "columns = ['funct','pronoun','ppron','i','we','you','shehe','they','ipron','article','verb','auxverb','past','present','future','adverb','preps','conj','negate','quant','number','swear','social','family','friend','humans','affect','posemo','negemo','anx','anger','sad','cogmech','insight','cause','discrep','tentat','certain','inhib','incl','excl','percept','see','hear','feel','bio','body','health','sexual','ingest','relativ','motion','space','time','work','achieve','leisure','home','money','relig','death','assent','nonfl','filler']\n",
    "prop_liwc = pd.DataFrame(data, columns=columns)\n",
    "prop_liwc['class'] = class_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "outro = prop_liwc[prop_liwc['class'] == 'outro']\n",
    "diario = prop_liwc[prop_liwc['class'] == 'diario']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def diff(x): \n",
    "    return (x['diario_mean']) - (x['outro_mean'])\n",
    "    #return 100 * ((x['diario_mean']/diario.wc.mean()) - (x['outro_mean']/outro.wc.mean()))\n",
    "    \n",
    "stats = pd.DataFrame(data={'diario_mean': diario.mean(axis=0)}, index=columns)\n",
    "stats['diario_std'] = diario.std(axis=0)\n",
    "stats['outro_mean'] = outro.mean(axis=0)\n",
    "stats['outro_std'] = outro.std(axis=0)\n",
    "stats['diff'] = stats.apply(diff,axis=1)\n",
    "stats = stats * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "outro = prop_liwc[prop_liwc['class'] == 'outro']\n",
    "diario = prop_liwc[prop_liwc['class'] == 'diario'].sample(len(outro))\n",
    "\n",
    "significance = []\n",
    "for column in list(stats.index.values):\n",
    "    a = diario[column]\n",
    "    b = outro[column]\n",
    "    t, p = wilcoxon(a, b)\n",
    "    significance.append(p)\n",
    "stats['significance'] = significance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Wilcoxon Rejected Linguistic Categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>diario_mean</th>\n",
       "      <th>diario_std</th>\n",
       "      <th>outro_mean</th>\n",
       "      <th>outro_std</th>\n",
       "      <th>diff</th>\n",
       "      <th>significance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>i</th>\n",
       "      <td>3.443419</td>\n",
       "      <td>2.105991</td>\n",
       "      <td>1.939779</td>\n",
       "      <td>1.614582</td>\n",
       "      <td>1.503640</td>\n",
       "      <td>3.526405e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>you</th>\n",
       "      <td>6.368339</td>\n",
       "      <td>2.258465</td>\n",
       "      <td>7.655838</td>\n",
       "      <td>2.295980</td>\n",
       "      <td>-1.287499</td>\n",
       "      <td>9.123177e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shehe</th>\n",
       "      <td>5.729460</td>\n",
       "      <td>2.008560</td>\n",
       "      <td>6.969947</td>\n",
       "      <td>2.039647</td>\n",
       "      <td>-1.240487</td>\n",
       "      <td>2.155176e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>adverb</th>\n",
       "      <td>3.801350</td>\n",
       "      <td>1.571370</td>\n",
       "      <td>2.972749</td>\n",
       "      <td>1.349097</td>\n",
       "      <td>0.828601</td>\n",
       "      <td>2.016648e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>funct</th>\n",
       "      <td>48.074812</td>\n",
       "      <td>6.240477</td>\n",
       "      <td>44.914592</td>\n",
       "      <td>6.389209</td>\n",
       "      <td>3.160221</td>\n",
       "      <td>3.090001e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>verb</th>\n",
       "      <td>14.640012</td>\n",
       "      <td>3.383896</td>\n",
       "      <td>13.058287</td>\n",
       "      <td>3.319171</td>\n",
       "      <td>1.581725</td>\n",
       "      <td>3.090001e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>past</th>\n",
       "      <td>4.339875</td>\n",
       "      <td>2.268508</td>\n",
       "      <td>3.251727</td>\n",
       "      <td>1.721237</td>\n",
       "      <td>1.088148</td>\n",
       "      <td>8.725385e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>preps</th>\n",
       "      <td>13.385029</td>\n",
       "      <td>3.045543</td>\n",
       "      <td>14.666125</td>\n",
       "      <td>3.216046</td>\n",
       "      <td>-1.281096</td>\n",
       "      <td>2.731157e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>they</th>\n",
       "      <td>1.613463</td>\n",
       "      <td>1.174379</td>\n",
       "      <td>2.020093</td>\n",
       "      <td>1.167320</td>\n",
       "      <td>-0.406630</td>\n",
       "      <td>1.434357e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>article</th>\n",
       "      <td>7.246217</td>\n",
       "      <td>2.025399</td>\n",
       "      <td>7.858596</td>\n",
       "      <td>2.066490</td>\n",
       "      <td>-0.612378</td>\n",
       "      <td>5.163599e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>quant</th>\n",
       "      <td>5.374879</td>\n",
       "      <td>1.811825</td>\n",
       "      <td>4.521710</td>\n",
       "      <td>1.636805</td>\n",
       "      <td>0.853169</td>\n",
       "      <td>7.216904e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pronoun</th>\n",
       "      <td>18.011798</td>\n",
       "      <td>4.372227</td>\n",
       "      <td>16.738534</td>\n",
       "      <td>4.048724</td>\n",
       "      <td>1.273263</td>\n",
       "      <td>1.398070e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>negate</th>\n",
       "      <td>1.697326</td>\n",
       "      <td>1.182546</td>\n",
       "      <td>1.352419</td>\n",
       "      <td>0.931901</td>\n",
       "      <td>0.344907</td>\n",
       "      <td>3.475731e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>auxverb</th>\n",
       "      <td>5.849782</td>\n",
       "      <td>2.112848</td>\n",
       "      <td>5.321220</td>\n",
       "      <td>2.042196</td>\n",
       "      <td>0.528562</td>\n",
       "      <td>4.450120e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>conj</th>\n",
       "      <td>9.588038</td>\n",
       "      <td>2.314788</td>\n",
       "      <td>8.958689</td>\n",
       "      <td>2.294111</td>\n",
       "      <td>0.629350</td>\n",
       "      <td>6.144815e-03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         diario_mean  diario_std  outro_mean  outro_std      diff  \\\n",
       "i           3.443419    2.105991    1.939779   1.614582  1.503640   \n",
       "you         6.368339    2.258465    7.655838   2.295980 -1.287499   \n",
       "shehe       5.729460    2.008560    6.969947   2.039647 -1.240487   \n",
       "adverb      3.801350    1.571370    2.972749   1.349097  0.828601   \n",
       "funct      48.074812    6.240477   44.914592   6.389209  3.160221   \n",
       "verb       14.640012    3.383896   13.058287   3.319171  1.581725   \n",
       "past        4.339875    2.268508    3.251727   1.721237  1.088148   \n",
       "preps      13.385029    3.045543   14.666125   3.216046 -1.281096   \n",
       "they        1.613463    1.174379    2.020093   1.167320 -0.406630   \n",
       "article     7.246217    2.025399    7.858596   2.066490 -0.612378   \n",
       "quant       5.374879    1.811825    4.521710   1.636805  0.853169   \n",
       "pronoun    18.011798    4.372227   16.738534   4.048724  1.273263   \n",
       "negate      1.697326    1.182546    1.352419   0.931901  0.344907   \n",
       "auxverb     5.849782    2.112848    5.321220   2.042196  0.528562   \n",
       "conj        9.588038    2.314788    8.958689   2.294111  0.629350   \n",
       "\n",
       "         significance  \n",
       "i        3.526405e-14  \n",
       "you      9.123177e-08  \n",
       "shehe    2.155176e-07  \n",
       "adverb   2.016648e-06  \n",
       "funct    3.090001e-06  \n",
       "verb     3.090001e-06  \n",
       "past     8.725385e-06  \n",
       "preps    2.731157e-05  \n",
       "they     1.434357e-04  \n",
       "article  5.163599e-04  \n",
       "quant    7.216904e-04  \n",
       "pronoun  1.398070e-03  \n",
       "negate   3.475731e-03  \n",
       "auxverb  4.450120e-03  \n",
       "conj     6.144815e-03  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linguistic_columns = ['funct','pronoun','ppron','i','we','you','shehe','they','ipron','article','verb','auxverb','past','present','future','adverb','preps','conj','negate','quant','number']\n",
    "linguistic_stats = stats.ix[linguistic_columns]\n",
    "linguistic_stats.sort_values('diff',ascending=False)\n",
    "linguistic_stats[linguistic_stats.significance <= 0.05].sort_values('significance',ascending=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Wilcoxon Not Rejected Linguistic Categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>diario_mean</th>\n",
       "      <th>diario_std</th>\n",
       "      <th>outro_mean</th>\n",
       "      <th>outro_std</th>\n",
       "      <th>diff</th>\n",
       "      <th>significance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ppron</th>\n",
       "      <td>11.286512</td>\n",
       "      <td>3.014084</td>\n",
       "      <td>11.265673</td>\n",
       "      <td>3.006212</td>\n",
       "      <td>0.020839</td>\n",
       "      <td>0.616648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>present</th>\n",
       "      <td>7.815691</td>\n",
       "      <td>2.640534</td>\n",
       "      <td>7.912399</td>\n",
       "      <td>2.738010</td>\n",
       "      <td>-0.096709</td>\n",
       "      <td>0.481394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>future</th>\n",
       "      <td>0.594461</td>\n",
       "      <td>0.529025</td>\n",
       "      <td>0.549943</td>\n",
       "      <td>0.557398</td>\n",
       "      <td>0.044517</td>\n",
       "      <td>0.418611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>we</th>\n",
       "      <td>0.426263</td>\n",
       "      <td>0.555506</td>\n",
       "      <td>0.553866</td>\n",
       "      <td>0.739920</td>\n",
       "      <td>-0.127603</td>\n",
       "      <td>0.417790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ipron</th>\n",
       "      <td>11.785252</td>\n",
       "      <td>3.022925</td>\n",
       "      <td>11.517185</td>\n",
       "      <td>2.908371</td>\n",
       "      <td>0.268067</td>\n",
       "      <td>0.404338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>number</th>\n",
       "      <td>1.495991</td>\n",
       "      <td>0.971771</td>\n",
       "      <td>1.311089</td>\n",
       "      <td>0.785835</td>\n",
       "      <td>0.184902</td>\n",
       "      <td>0.052191</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         diario_mean  diario_std  outro_mean  outro_std      diff  \\\n",
       "ppron      11.286512    3.014084   11.265673   3.006212  0.020839   \n",
       "present     7.815691    2.640534    7.912399   2.738010 -0.096709   \n",
       "future      0.594461    0.529025    0.549943   0.557398  0.044517   \n",
       "we          0.426263    0.555506    0.553866   0.739920 -0.127603   \n",
       "ipron      11.785252    3.022925   11.517185   2.908371  0.268067   \n",
       "number      1.495991    0.971771    1.311089   0.785835  0.184902   \n",
       "\n",
       "         significance  \n",
       "ppron        0.616648  \n",
       "present      0.481394  \n",
       "future       0.418611  \n",
       "we           0.417790  \n",
       "ipron        0.404338  \n",
       "number       0.052191  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linguistic_stats[linguistic_stats.significance > 0.05].sort_values('significance',ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Wilcoxon Rejected Psychological Categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>diario_mean</th>\n",
       "      <th>diario_std</th>\n",
       "      <th>outro_mean</th>\n",
       "      <th>outro_std</th>\n",
       "      <th>diff</th>\n",
       "      <th>significance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>relativ</th>\n",
       "      <td>21.787661</td>\n",
       "      <td>4.222960</td>\n",
       "      <td>19.606526</td>\n",
       "      <td>3.610765</td>\n",
       "      <td>2.181135</td>\n",
       "      <td>0.000005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feel</th>\n",
       "      <td>2.224723</td>\n",
       "      <td>1.191583</td>\n",
       "      <td>1.829541</td>\n",
       "      <td>1.156732</td>\n",
       "      <td>0.395182</td>\n",
       "      <td>0.000033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>humans</th>\n",
       "      <td>6.877624</td>\n",
       "      <td>2.503676</td>\n",
       "      <td>5.898336</td>\n",
       "      <td>2.514503</td>\n",
       "      <td>0.979288</td>\n",
       "      <td>0.000044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percept</th>\n",
       "      <td>5.688852</td>\n",
       "      <td>2.211484</td>\n",
       "      <td>4.992647</td>\n",
       "      <td>2.107234</td>\n",
       "      <td>0.696204</td>\n",
       "      <td>0.000048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sexual</th>\n",
       "      <td>1.398003</td>\n",
       "      <td>1.058580</td>\n",
       "      <td>1.066595</td>\n",
       "      <td>0.823168</td>\n",
       "      <td>0.331408</td>\n",
       "      <td>0.000056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>filler</th>\n",
       "      <td>0.155947</td>\n",
       "      <td>0.344473</td>\n",
       "      <td>0.083384</td>\n",
       "      <td>0.270106</td>\n",
       "      <td>0.072563</td>\n",
       "      <td>0.000116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ingest</th>\n",
       "      <td>6.800587</td>\n",
       "      <td>2.385995</td>\n",
       "      <td>5.814962</td>\n",
       "      <td>1.987943</td>\n",
       "      <td>0.985625</td>\n",
       "      <td>0.000283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>certain</th>\n",
       "      <td>2.486801</td>\n",
       "      <td>1.156612</td>\n",
       "      <td>2.145316</td>\n",
       "      <td>1.098878</td>\n",
       "      <td>0.341485</td>\n",
       "      <td>0.000431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>discrep</th>\n",
       "      <td>6.122851</td>\n",
       "      <td>2.199861</td>\n",
       "      <td>5.371300</td>\n",
       "      <td>1.797836</td>\n",
       "      <td>0.751551</td>\n",
       "      <td>0.000639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tentat</th>\n",
       "      <td>9.366380</td>\n",
       "      <td>2.865459</td>\n",
       "      <td>8.321216</td>\n",
       "      <td>2.464374</td>\n",
       "      <td>1.045164</td>\n",
       "      <td>0.001075</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         diario_mean  diario_std  outro_mean  outro_std      diff  \\\n",
       "relativ    21.787661    4.222960   19.606526   3.610765  2.181135   \n",
       "feel        2.224723    1.191583    1.829541   1.156732  0.395182   \n",
       "humans      6.877624    2.503676    5.898336   2.514503  0.979288   \n",
       "percept     5.688852    2.211484    4.992647   2.107234  0.696204   \n",
       "sexual      1.398003    1.058580    1.066595   0.823168  0.331408   \n",
       "filler      0.155947    0.344473    0.083384   0.270106  0.072563   \n",
       "ingest      6.800587    2.385995    5.814962   1.987943  0.985625   \n",
       "certain     2.486801    1.156612    2.145316   1.098878  0.341485   \n",
       "discrep     6.122851    2.199861    5.371300   1.797836  0.751551   \n",
       "tentat      9.366380    2.865459    8.321216   2.464374  1.045164   \n",
       "\n",
       "         significance  \n",
       "relativ      0.000005  \n",
       "feel         0.000033  \n",
       "humans       0.000044  \n",
       "percept      0.000048  \n",
       "sexual       0.000056  \n",
       "filler       0.000116  \n",
       "ingest       0.000283  \n",
       "certain      0.000431  \n",
       "discrep      0.000639  \n",
       "tentat       0.001075  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "psychological_columns = ['swear','social','family','friend','humans','affect','posemo','negemo','anx','anger','sad','cogmech','insight','cause','discrep','tentat','certain','inhib','incl','excl','percept','see','hear','feel','bio','body','health','sexual','ingest','relativ','motion','space','time','work','achieve','leisure','home','money','relig','death','assent','nonfl','filler']\n",
    "psychoProc_stats = stats.ix[psychological_columns]\n",
    "psychoProc_stats[psychoProc_stats.significance <= 0.05].sort_values('significance',ascending=True).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Wilcoxon Not Rejected Psychological Categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>diario_mean</th>\n",
       "      <th>diario_std</th>\n",
       "      <th>outro_mean</th>\n",
       "      <th>outro_std</th>\n",
       "      <th>diff</th>\n",
       "      <th>significance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>money</th>\n",
       "      <td>1.953920</td>\n",
       "      <td>1.125584</td>\n",
       "      <td>1.963778</td>\n",
       "      <td>1.227436</td>\n",
       "      <td>-0.009858</td>\n",
       "      <td>0.755287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>health</th>\n",
       "      <td>1.047675</td>\n",
       "      <td>0.991947</td>\n",
       "      <td>0.968300</td>\n",
       "      <td>0.923305</td>\n",
       "      <td>0.079375</td>\n",
       "      <td>0.575502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>swear</th>\n",
       "      <td>5.179338</td>\n",
       "      <td>1.927663</td>\n",
       "      <td>5.353678</td>\n",
       "      <td>2.051963</td>\n",
       "      <td>-0.174340</td>\n",
       "      <td>0.493140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sad</th>\n",
       "      <td>1.034851</td>\n",
       "      <td>1.010062</td>\n",
       "      <td>0.919221</td>\n",
       "      <td>0.898489</td>\n",
       "      <td>0.115630</td>\n",
       "      <td>0.460145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>social</th>\n",
       "      <td>18.581261</td>\n",
       "      <td>4.463031</td>\n",
       "      <td>18.555436</td>\n",
       "      <td>4.583370</td>\n",
       "      <td>0.025824</td>\n",
       "      <td>0.446288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>incl</th>\n",
       "      <td>13.843956</td>\n",
       "      <td>2.971938</td>\n",
       "      <td>13.900658</td>\n",
       "      <td>2.923731</td>\n",
       "      <td>-0.056702</td>\n",
       "      <td>0.414216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hear</th>\n",
       "      <td>1.342451</td>\n",
       "      <td>1.059902</td>\n",
       "      <td>1.368052</td>\n",
       "      <td>1.056469</td>\n",
       "      <td>-0.025601</td>\n",
       "      <td>0.385011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>negemo</th>\n",
       "      <td>2.493855</td>\n",
       "      <td>1.572730</td>\n",
       "      <td>2.424255</td>\n",
       "      <td>1.580562</td>\n",
       "      <td>0.069600</td>\n",
       "      <td>0.371968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>family</th>\n",
       "      <td>0.426120</td>\n",
       "      <td>0.766946</td>\n",
       "      <td>0.401683</td>\n",
       "      <td>0.805169</td>\n",
       "      <td>0.024437</td>\n",
       "      <td>0.312460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>death</th>\n",
       "      <td>0.229088</td>\n",
       "      <td>0.382567</td>\n",
       "      <td>0.304890</td>\n",
       "      <td>0.491463</td>\n",
       "      <td>-0.075802</td>\n",
       "      <td>0.271313</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        diario_mean  diario_std  outro_mean  outro_std      diff  significance\n",
       "money      1.953920    1.125584    1.963778   1.227436 -0.009858      0.755287\n",
       "health     1.047675    0.991947    0.968300   0.923305  0.079375      0.575502\n",
       "swear      5.179338    1.927663    5.353678   2.051963 -0.174340      0.493140\n",
       "sad        1.034851    1.010062    0.919221   0.898489  0.115630      0.460145\n",
       "social    18.581261    4.463031   18.555436   4.583370  0.025824      0.446288\n",
       "incl      13.843956    2.971938   13.900658   2.923731 -0.056702      0.414216\n",
       "hear       1.342451    1.059902    1.368052   1.056469 -0.025601      0.385011\n",
       "negemo     2.493855    1.572730    2.424255   1.580562  0.069600      0.371968\n",
       "family     0.426120    0.766946    0.401683   0.805169  0.024437      0.312460\n",
       "death      0.229088    0.382567    0.304890   0.491463 -0.075802      0.271313"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "psychoProc_stats[psychoProc_stats.significance > 0.05].sort_values('significance',ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}