post_compute_pol.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "178060"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "posts = pd.read_csv('posts_count_me.csv.gz', compression='gzip', error_bad_lines=False)\n",
    "posts = posts[pd.isnull(posts['labels']) == False]\n",
    "len(posts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from liwc import LIWC\n",
    "liwc_class = LIWC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def buildPolarity(x): \n",
    "    try:\n",
    "        texts = []\n",
    "        texts.append(x['content'])\n",
    "        mtx = liwc_class.build_features(texts)\n",
    "        return mtx[0][27] + mtx[0][28]\n",
    "    except:\n",
    "        return 0\n",
    "\n",
    "posts['polarity'] = posts.apply(buildPolarity,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "posts.to_csv('posts_count_me_pol.csv.gz', compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fd7a01dfd30>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAD8CAYAAAC/1zkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFCxJREFUeJzt3X/wZXV93/Hny0UQjQoIpZSFLCRbM5u0UdwgHWOaSgML\nRCA/tDC2bC0j7QSnOmknLpoJNgkz0DYx0jEmRHYEaoL4K2wCDlnwR6Yz5cfyQ35K+IpYdl1hAwga\nDQR894/72fXy5fvjLnzu934v+3zM3Nlz3uece9/3fL+7rz3nfO65qSokSerhJZNuQJL04mGoSJK6\nMVQkSd0YKpKkbgwVSVI3hookqRtDRZLUjaEiSerGUJEkdbPXpBtYagceeGCtWrVq0m1I0tS4+eab\n/7aqDhpl3T0uVFatWsWWLVsm3YYkTY0k3xh1XU9/SZK6MVQkSd0YKpKkbgwVSVI3hookqRtDRZLU\njaEiSerGUJEkdWOoSJK62eM+Uf9CrNpw1bzLHjj/pCXsRJKWJ49UJEndGCqSpG4MFUlSN4aKJKkb\nQ0WS1I2hIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiS\nujFUJEndGCqSpG4MFUlSN4aKJKmbsYdKkhVJbk3yl23+iCQ3JJlJ8skke7f6Pm1+pi1fNfQc57T6\nvUmOH6qva7WZJBvG/V4kSQtbiiOV9wD3DM1fAHyoqn4ceAw4s9XPBB5r9Q+19UiyBjgN+ElgHfCH\nLahWAB8BTgDWAKe3dSVJEzLWUEmyEjgJ+FibD/AW4NNtlUuAU9v0KW2etvzYtv4pwOVV9WRVfR2Y\nAY5uj5mqur+qngIub+tKkiZk3EcqfwD8BvCDNv8a4NtV9XSb3woc2qYPBR4EaMsfb+vvqs/aZr66\nJGlCxhYqSX4ReLiqbh7Xa+xGL2cl2ZJky44dOybdjiS9aI3zSOVNwMlJHmBwauotwIeB/ZLs1dZZ\nCWxr09uAwwDa8lcDjwzXZ20zX/05quqiqlpbVWsPOuigF/7OJElzGluoVNU5VbWyqlYxuND+hap6\nB/BF4FfbauuBK9v0pjZPW/6FqqpWP62NDjsCWA3cCNwErG6jyfZur7FpXO9HkrS4vRZfpbv3AZcn\n+V3gVuDiVr8YuCzJDPAog5Cgqu5KcgVwN/A0cHZVPQOQ5N3ANcAKYGNV3bWk70SS9CxLEipV9SXg\nS236fgYjt2av8/fA2+bZ/jzgvDnqVwNXd2xVkvQC+Il6SVI3hookqRtDRZLUjaEiSerGUJEkdWOo\nSJK6MVQkSd0YKpKkbgwVSVI3hookqRtDRZLUjaEiSerGUJEkdWOoSJK6MVQkSd0YKpKkbgwVSVI3\nhookqRtDRZLUjaEiSerGUJEkdWOoSJK6MVQkSd0YKpKkbgwVSVI3hookqRtDRZLUjaEiSerGUJEk\ndWOoSJK6MVQkSd0YKpKkbgwVSVI3hookqRtDRZLUjaEiSerGUJEkdTO2UEnysiQ3JvlKkruS/LdW\nPyLJDUlmknwyyd6tvk+bn2nLVw091zmtfm+S44fq61ptJsmGcb0XSdJoxnmk8iTwlqr6aeB1wLok\nxwAXAB+qqh8HHgPObOufCTzW6h9q65FkDXAa8JPAOuAPk6xIsgL4CHACsAY4va0rSZqQsYVKDXy3\nzb60PQp4C/DpVr8EOLVNn9LmacuPTZJWv7yqnqyqrwMzwNHtMVNV91fVU8DlbV1J0oSM9ZpKO6K4\nDXgY2Ax8Dfh2VT3dVtkKHNqmDwUeBGjLHwdeM1yftc189bn6OCvJliRbduzY0eOtSZLmMNZQqapn\nqup1wEoGRxY/Mc7XW6CPi6pqbVWtPeiggybRgiTtEZZk9FdVfRv4IvAvgP2S7NUWrQS2teltwGEA\nbfmrgUeG67O2ma8uSZqQcY7+OijJfm16X+AXgHsYhMuvttXWA1e26U1tnrb8C1VVrX5aGx12BLAa\nuBG4CVjdRpPtzeBi/qZxvR9J0uL2WnwVSPLPquqO3XzuQ4BL2iitlwBXVNVfJrkbuDzJ7wK3Ahe3\n9S8GLksyAzzKICSoqruSXAHcDTwNnF1Vz7S+3g1cA6wANlbVXbvZoySpo5FChcEw3n2AjwOfqKrH\nF9ugqm4HXj9H/X4G11dm1/8eeNs8z3UecN4c9auBqxfrRZK0NEY6/VVVbwbeweAaxs1J/jTJL4y1\nM0nS1Bn5mkpV3Qf8JvA+4F8CFyb5apJfHldzkqTpMuo1lX8OvBM4icHnTd5aVbck+SfA/wU+O74W\np8OqDVctuPyB809aok4kaXJGvabyv4CPAe+vqu/vLFbVN5P85lg6kyRNnVFD5STg+0Ojrl4CvKyq\nvldVl42tO0nSVBn1msq1wL5D8y9vNUmSdhk1VF42dHNI2vTLx9OSJGlajRoqf5fkqJ0zSd4AfH+B\n9SVJe6BRr6m8F/hUkm8CAf4x8G/G1pUkaSqNFCpVdVOSnwBe20r3VtU/jK8tSdI0GvVIBeBngFVt\nm6OSUFWXjqUrSdJUGvXDj5cBPwbcBjzTygUYKpKkXUY9UlkLrGm3opckaU6jjv66k8HFeUmS5jXq\nkcqBwN1JbgSe3FmsqpPH0pUkaSqNGiofHGcTkqQXh1GHFH85yY8Cq6vq2iQvZ/Bti5Ik7TLSNZUk\n7wI+DfxxKx0K/Pm4mpIkTadRL9SfDbwJeAJ2fWHXPxpXU5Kk6TRqqDxZVU/tnEmyF4PPqUiStMuo\nofLlJO8H9m3fTf8p4C/G15YkaRqNGiobgB3AHcB/BK5m8H31kiTtMurorx8Af9IekiTNadR7f32d\nOa6hVNWR3TuSJE2t3bn3104vA94GHNC/HUnSNBvpmkpVPTL02FZVfwCcNObeJElTZtTTX0cNzb6E\nwZHL7nwXiyRpDzBqMPze0PTTwAPA27t3I0maaqOO/vpX425EkjT9Rj399esLLa+q3+/TjiRpmu3O\n6K+fATa1+bcCNwL3jaMpSdJ0GjVUVgJHVdV3AJJ8ELiqqv7tuBqTJE2fUW/TcjDw1ND8U60mSdIu\nox6pXArcmORzbf5U4JLxtCRJmlajjv46L8nngTe30jur6tbxtSVJmkajnv4CeDnwRFV9GNia5Igx\n9SRJmlKjfp3wucD7gHNa6aXA/15km8OSfDHJ3UnuSvKeVj8gyeYk97U/92/1JLkwyUyS24c/xZ9k\nfVv/viTrh+pvSHJH2+bCJNm9ty9J6mnUI5VfAk4G/g6gqr4JvHKRbZ4G/ktVrQGOAc5OsobBd7Nc\nV1WrgevaPMAJwOr2OAv4KAxCCDgXeCNwNHDuziBq67xraLt1I74fSdIYjBoqT1VV0W5/n+QVi21Q\nVdur6pY2/R3gHuBQ4BR+eJH/EgYX/Wn1S2vgemC/JIcAxwObq+rRqnoM2Aysa8teVVXXt94uHXou\nSdIEjBoqVyT5Ywb/0L8LuJbd+MKuJKuA1wM3AAdX1fa26Fv8cGjyocCDQ5ttbbWF6lvnqEuSJmTU\n0V//s303/RPAa4HfqqrNo2yb5EeAzwDvraonhi97VFUlec6Xf/WW5CwGp9Q4/PDDx/1ykrTHWjRU\nkqwArm03lRwpSIa2fSmDQPlEVX22lR9KckhVbW+nsB5u9W3AYUObr2y1bcDPz6p/qdVXzrH+c1TV\nRcBFAGvXrh17iEnSnmrR019V9QzwgySv3p0nbiOxLgbumXXDyU3AzhFc64Erh+pntFFgxwCPt9Nk\n1wDHJdm/XaA/DrimLXsiyTHttc4Yei5J0gSM+on67wJ3JNlMGwEGUFX/eYFt3gT8u7bdba32fuB8\nBtdozgS+wQ+/l+Vq4ERgBvge8M72Go8m+R3gprbeb1fVo23614CPA/sCn28PSdKEjBoqn22PkVXV\n/wHm+9zIsXOsX8DZ8zzXRmDjHPUtwE/tTl+SpPFZMFSSHF5V/6+qvM+XJGlRi11T+fOdE0k+M+Ze\nJElTbrFQGT59deQ4G5EkTb/FQqXmmZYk6TkWu1D/00meYHDEsm+bps1XVb1qrN1JkqbKgqFSVSuW\nqhFJ0vTbne9TkSRpQYaKJKkbQ0WS1I2hIknqxlCRJHVjqEiSuhn1hpJ6gVZtuGrB5Q+cf9ISdSJJ\n4+ORiiSpG0NFktSNoSJJ6sZQkSR144X6ZWKhC/lexJc0LTxSkSR1Y6hIkroxVCRJ3RgqkqRuDBVJ\nUjeGiiSpG0NFktSNoSJJ6sZQkSR1Y6hIkroxVCRJ3RgqkqRuDBVJUjeGiiSpG0NFktSNoSJJ6sZQ\nkSR1Y6hIkroxVCRJ3YwtVJJsTPJwkjuHagck2Zzkvvbn/q2eJBcmmUlye5KjhrZZ39a/L8n6ofob\nktzRtrkwScb1XiRJoxnnkcrHgXWzahuA66pqNXBdmwc4AVjdHmcBH4VBCAHnAm8EjgbO3RlEbZ13\nDW03+7UkSUtsbKFSVX8NPDqrfApwSZu+BDh1qH5pDVwP7JfkEOB4YHNVPVpVjwGbgXVt2auq6vqq\nKuDSoeeSJE3IUl9TObiqtrfpbwEHt+lDgQeH1tvaagvVt85Rn1OSs5JsSbJlx44dL+wdSJLmtdek\nXriqKkkt0WtdBFwEsHbt2iV5zaW0asNV8y574PyTlrATSXu6pQ6Vh5IcUlXb2ymsh1t9G3DY0Hor\nW20b8POz6l9q9ZVzrP+itFBoSNJystSnvzYBO0dwrQeuHKqf0UaBHQM83k6TXQMcl2T/doH+OOCa\ntuyJJMe0UV9nDD2XJGlCxnakkuTPGBxlHJhkK4NRXOcDVyQ5E/gG8Pa2+tXAicAM8D3gnQBV9WiS\n3wFuauv9dlXtvPj/awxGmO0LfL49JEkTNLZQqarT51l07BzrFnD2PM+zEdg4R30L8FMvpEdJUl9+\nol6S1I2hIknqxlCRJHVjqEiSujFUJEndTOwT9Voai31w0k/cS+rJIxVJUjeGiiSpG0NFktSNoSJJ\n6sZQkSR1Y6hIkroxVCRJ3RgqkqRuDBVJUjeGiiSpG2/TogUtdJsXb/EiaTZDZQ+32L3BJGl3ePpL\nktSNoSJJ6sZQkSR1Y6hIkroxVCRJ3Tj6S2PjcGRpz2Oo6HlzOLKk2Tz9JUnqxlCRJHXj6S9NxGKn\nzrzmIk0nj1QkSd0YKpKkbgwVSVI3XlPRVPIzMNLyZKhoWfIzMNJ08vSXJKkbj1T0ouNwZWlyDBXt\ncQwdaXymPlSSrAM+DKwAPlZV50+4JU05BwFIz99UX1NJsgL4CHACsAY4PcmayXYlSXuuaT9SORqY\nqar7AZJcDpwC3D3RrvSi9UJGpXmUoz3BtIfKocCDQ/NbgTdOqBdpQZMcJm2gaalMe6iMJMlZwFlt\n9rtJ7t3NpzgQ+Nu+XXU3DT3CdPQ5DT3CbvSZC8bcyfymYV9OQ48w2T5/dNQVpz1UtgGHDc2vbLVn\nqaqLgIue74sk2VJVa5/v9kthGnqE6ehzGnqE6ejTHvuZlj6n+kI9cBOwOskRSfYGTgM2TbgnSdpj\nTfWRSlU9neTdwDUMhhRvrKq7JtyWJO2xpjpUAKrqauDqMb/M8z51toSmoUeYjj6noUeYjj7tsZ+p\n6DNVNekeJEkvEtN+TUWStIwYKotIsi7JvUlmkmyYdD8ASQ5L8sUkdye5K8l7Wv2DSbYlua09Tpxw\nnw8kuaP1sqXVDkiyOcl97c/9J9zja4f2121Jnkjy3knvyyQbkzyc5M6h2pz7LgMXtt/R25McNcEe\n/0eSr7Y+Ppdkv1ZfleT7Q/vzj5aixwX6nPfnm+Scti/vTXL8BHv85FB/DyS5rdUnti9HUlU+5nkw\nuPj/NeBIYG/gK8CaZdDXIcBRbfqVwN8wuE3NB4H/Oun+hvp8ADhwVu2/Axva9Abggkn3Oevn/S0G\nY/Inui+BnwOOAu5cbN8BJwKfBwIcA9wwwR6PA/Zq0xcM9bhqeL1lsC/n/Pm2v0dfAfYBjmh//1dM\nosdZy38P+K1J78tRHh6pLGzXbWCq6ilg521gJqqqtlfVLW36O8A9DO4uMA1OAS5p05cAp06wl9mO\nBb5WVd+YdCNV9dfAo7PK8+27U4BLa+B6YL8kh0yix6r6q6p6us1ez+CzYxM1z76czynA5VX1ZFV9\nHZhh8O/AWC3UY5IAbwf+bNx99GCoLGyu28Asq3+8k6wCXg/c0ErvbqceNk761BJQwF8lubnd1QDg\n4Kra3qa/BRw8mdbmdBrP/ou7nPYlzL/vluvv6X9gcAS10xFJbk3y5SRvnlRTQ+b6+S7Hfflm4KGq\num+ottz25S6GyhRL8iPAZ4D3VtUTwEeBHwNeB2xncMg8ST9bVUcxuIv02Ul+bnhhDY7ll8Xww/bh\n2ZOBT7XSctuXz7Kc9t1cknwAeBr4RCttBw6vqtcDvw78aZJXTao/lvnPd5bTefZ/dpbbvnwWQ2Vh\nI90GZhKSvJRBoHyiqj4LUFUPVdUzVfUD4E9YgsP2hVTVtvbnw8DnWj8P7Tw10/58eHIdPssJwC1V\n9RAsv33ZzLfvltXvaZJ/D/wi8I4WfrTTSY+06ZsZXKv4p5PqcYGf73Lbl3sBvwx8cmdtue3L2QyV\nhS3L28C0c6wXA/dU1e8P1YfPo/8ScOfsbZdKklckeeXOaQYXcO9ksP/Wt9XWA1dOpsPneNb/BpfT\nvhwy377bBJzRRoEdAzw+dJpsSWXwpXm/AZxcVd8bqh+UwfcfkeRIYDVw/yR6bD3M9/PdBJyWZJ8k\nRzDo88al7m/Ivwa+WlVbdxaW2758jkmPFFjuDwYja/6Gwf8GPjDpflpPP8vg1MftwG3tcSJwGXBH\nq28CDplgj0cyGEXzFeCunfsOeA1wHXAfcC1wwDLYn68AHgFePVSb6L5kEHDbgX9gcF7/zPn2HYNR\nXx9pv6N3AGsn2OMMg2sSO38v/6it+yvt9+A24BbgrRPel/P+fIEPtH15L3DCpHps9Y8D/2nWuhPb\nl6M8/ES9JKkbT39JkroxVCRJ3RgqkqRuDBVJUjeGiiSpG0NFktSNoSJJ6sZQkSR18/8BkymmyxWP\nP2EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd7a01df208>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "polHist = posts['polarity']\n",
    "polHist = polHist[polHist > 0]\n",
    "polHist = polHist[polHist < 300]\n",
    "polHist.plot.hist(bins=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "161017"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(polHist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18.518640888850246"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(polHist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18.581631648462473"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.std(polHist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}