corpus_tfidf_liwc_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn import svm\n",
    "from sklearn import preprocessing\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "corpus_feat = pd.read_csv('corpus_liwc_mtx.csv.gz', compression='gzip')\n",
    "corpus = pd.read_csv('corpus.csv.gz', compression='gzip')\n",
    "stopwords = stopwords.words(\"portuguese\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# filter corpus\n",
    "corpus = corpus[corpus['qual_a_melhor_classificao_para_esse_texto:confidence'] == 1]\n",
    "corpus = corpus.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['diario' 'outro']\n",
      "0    1\n",
      "1   -1\n",
      "Name: class, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# fix labels to binary\n",
    "def classFit(x):\n",
    "    if x['qual_a_melhor_classificao_para_esse_texto'] == \"diario\":\n",
    "        return 1\n",
    "    else:\n",
    "        return -1\n",
    "    \n",
    "corpus['class'] = corpus.apply(classFit,axis=1)\n",
    "target = corpus['class'].values\n",
    "\n",
    "print(corpus['qual_a_melhor_classificao_para_esse_texto'].values[:2])\n",
    "print(corpus['class'][:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(534, 964)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.sparse import hstack\n",
    "\n",
    "corpus_feat.drop('Unnamed: 0', axis=1,inplace=True)\n",
    "corpus_feat.drop('confidence', axis=1,inplace=True)\n",
    "corpus_feat.drop('wc', axis=1,inplace=True)\n",
    "liwc_data = corpus_feat.drop('class', 1).values\n",
    "\n",
    "tfidf_data = TfidfVectorizer(max_features=900, strip_accents='unicode', stop_words=stopwords).fit_transform(corpus.content)\n",
    "\n",
    "data = hstack((tfidf_data, liwc_data))\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.822620154495\n",
      "0.751122130839\n",
      "0.785966386555\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "model = MultinomialNB()\n",
    "\n",
    "precision = cross_val_score(model, data, target, cv=10, scoring='precision').mean()\n",
    "acc = cross_val_score(model, data, target, cv=10, scoring='accuracy').mean()\n",
    "recall = cross_val_score(model, data, target, cv=10, scoring='recall').mean()\n",
    "print(precision)\n",
    "print(acc)\n",
    "print(recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.742708974284\n",
      "0.633075310434\n",
      "0.814369747899\n"
     ]
    }
   ],
   "source": [
    "model = svm.LinearSVC(C=50)\n",
    "\n",
    "precision = cross_val_score(model, data, target, cv=10, scoring='precision').mean()\n",
    "acc = cross_val_score(model, data, target, cv=10, scoring='accuracy').mean()\n",
    "recall = cross_val_score(model, data, target, cv=10, scoring='recall').mean()\n",
    "print(precision)\n",
    "print(acc)\n",
    "print(recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "### confidence-weighted linear classifier (Dredze et al., 2008)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P: acc(0.651685393258), prec(0.652326615077), rec(0.988023952096)\n",
      "AP: acc(0.691011235955), prec(0.693067596974), rec(0.960291707089)\n",
      "PA: acc(0.64606741573), prec(0.647236067697), rec(0.997005988024)\n",
      "PA1: acc(0.64606741573), prec(0.647236067697), rec(0.997005988024)\n",
      "PA2: acc(0.64606741573), prec(0.647236067697), rec(0.997005988024)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1113: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PAK: acc(0.352059925094), prec(0.0), rec(0.0)\n",
      "CW: acc(0.698501872659), prec(0.771896722939), rec(0.772304552905)\n",
      "AL: acc(0.679775280899), prec(0.676242062606), rec(0.976850767738)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score\n",
    "import numpy as np\n",
    "import oll\n",
    "\n",
    "## manual 10-fold cross-validation\n",
    "kf = KFold(n_splits=2, random_state=None, shuffle=False)\n",
    "\n",
    "methods = [\"P\" ,\"AP\" ,\"PA\" ,\"PA1\",\"PA2\" ,\"PAK\",\"CW\" ,\"AL\"]\n",
    "\n",
    "for m in methods:\n",
    "    model = oll.oll(m, C=1)\n",
    "\n",
    "    accuracy = []\n",
    "    precision = []\n",
    "    recall = []\n",
    "    \n",
    "    for train_index, test_index in kf.split(data):\n",
    "\n",
    "        X_train, X_test = data.toarray()[train_index], data.toarray()[test_index]\n",
    "        y_train, y_test = target[train_index], target[test_index]\n",
    "        model.fit(X_train, y_train)\n",
    "        predicted = model.predict(X_test)\n",
    "\n",
    "        accuracy.append(accuracy_score(y_test, predicted))\n",
    "        precision.append(precision_score(y_test, predicted))\n",
    "        recall.append(recall_score(y_test, predicted))\n",
    "\n",
    "    print(m + ': acc(' + str(np.mean(accuracy)) \n",
    "          + '), prec(' + str(np.mean(precision))\n",
    "          + '), rec(' + str(np.mean(recall)) + ')'\n",
    "         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.77768426797\n",
      "0.808712052435\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "model = GradientBoostingClassifier()\n",
    "\n",
    "precision = cross_val_score(model, data.toarray(), target, cv=10, scoring='precision').mean()\n",
    "acc = cross_val_score(model, data.toarray(), target, cv=10, scoring='accuracy').mean()\n",
    "recall = cross_val_score(model, data.toarray(), target, cv=10, scoring='recall').mean()\n",
    "fscore = cross_val_score(model, data.toarray(), target, cv=10, scoring='f1').mean()\n",
    "print(precision)\n",
    "print(fscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}