corpus_topics_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.decomposition import NMF, LatentDirichletAllocation\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn import svm\n",
    "\n",
    "corpus = pd.read_csv('corpus.csv.gz', compression='gzip')\n",
    "corpus = corpus[corpus['qual_a_melhor_classificao_para_esse_texto:confidence'] == 1]\n",
    "stopwords = stopwords.words(\"portuguese\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_features = 1600\n",
    "\n",
    "tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2, strip_accents='unicode',\n",
    "                                   max_features=n_features,\n",
    "                                   stop_words=stopwords)\n",
    "tfidf = tfidf_vectorizer.fit_transform(corpus.content)\n",
    "\n",
    "tf_vectorizer = CountVectorizer(max_df=0.95, min_df=2, strip_accents='unicode',\n",
    "                                max_features=n_features,\n",
    "                                stop_words=stopwords)\n",
    "tf = tf_vectorizer.fit_transform(corpus.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['diario' 'outro']\n",
      "466    1\n",
      "467   -1\n",
      "Name: class, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# fix labels to binary\n",
    "def classFit(x):\n",
    "    if x['qual_a_melhor_classificao_para_esse_texto'] == \"diario\":\n",
    "        return 1\n",
    "    else:\n",
    "        return -1\n",
    "    \n",
    "corpus['class'] = corpus.apply(classFit,axis=1)\n",
    "target = corpus['class'].values\n",
    "\n",
    "print(corpus['qual_a_melhor_classificao_para_esse_texto'].values[:2])\n",
    "print(corpus['class'][:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10: f1(0.8094), acc(0.7004), precision(0.6903), recall(0.98)\n",
      "20: f1(0.8353), acc(0.7548), precision(0.7425), recall(0.9566)\n",
      "30: f1(0.835), acc(0.7566), precision(0.747), recall(0.9482)\n",
      "40: f1(0.7844), acc(0.6481), precision(0.6502), recall(0.9886)\n",
      "50: f1(0.8103), acc(0.7022), precision(0.691), recall(0.9798)\n"
     ]
    }
   ],
   "source": [
    "model = MultinomialNB()\n",
    "\n",
    "for n_topics in range(10,60,10):\n",
    "    lda = LatentDirichletAllocation(n_topics=n_topics, max_iter=5,\n",
    "                                    learning_method='online',\n",
    "                                    learning_offset=50.,\n",
    "                                    random_state=0)\n",
    "    data = lda.fit_transform(tf)\n",
    "\n",
    "    f1 = cross_val_score(model, data, target, cv=10, scoring='f1').mean()\n",
    "    acc = cross_val_score(model, data, target, cv=10, scoring='accuracy').mean()\n",
    "    recall = cross_val_score(model, data, target, cv=10, scoring='recall').mean()\n",
    "    precision = cross_val_score(model, data, target, cv=10, scoring='precision').mean()\n",
    "    \n",
    "    print(str(n_topics) + ': ' + 'f1(' + str(round(f1,4)) \n",
    "          + '), acc(' + str(round(acc,4)) \n",
    "          + '), precision(' + str(round(precision,4)) \n",
    "          + '), recall(' + str(round(recall,4)) + ')')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.722667577845\n",
      "0.730255872709\n",
      "0.951092436975\n"
     ]
    }
   ],
   "source": [
    "model = svm.LinearSVC()\n",
    "\n",
    "precision = cross_val_score(model, data, target, cv=10, scoring='precision').mean()\n",
    "acc = cross_val_score(model, data, target, cv=10, scoring='accuracy').mean()\n",
    "recall = cross_val_score(model, data, target, cv=10, scoring='recall').mean()\n",
    "print(precision)\n",
    "print(acc)\n",
    "print(recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 14 candidates, totalling 140 fits\n",
      "[CV] C=0.001, kernel=rbf .............................................\n",
      "[CV] C=0.001, kernel=rbf .............................................\n",
      "[CV] C=0.001, kernel=rbf .............................................\n",
      "[CV] C=0.001, kernel=rbf .............................................\n",
      "[CV] C=0.001, kernel=rbf .............................................\n",
      "[CV] C=0.001, kernel=rbf .............................................\n",
      "[CV] C=0.001, kernel=rbf .............................................\n",
      "[CV] .............. C=0.001, kernel=rbf, score=0.648148, total=   0.0s\n",
      "[CV] .............. C=0.001, kernel=rbf, score=0.648148, total=   0.0s\n",
      "[CV] C=0.001, kernel=rbf .............................................\n",
      "[CV] .............. C=0.001, kernel=rbf, score=0.648148, total=   0.0s\n",
      "[CV] C=0.001, kernel=rbf .............................................\n",
      "[CV] .............. C=0.001, kernel=rbf, score=0.648148, total=   0.0s\n",
      "[CV] C=0.001, kernel=rbf .............................................\n",
      "[CV] C=0.001, kernel=linear ..........................................\n",
      "[CV] C=0.001, kernel=linear ..........................................\n",
      "[CV] .............. C=0.001, kernel=rbf, score=0.648148, total=   0.0s\n",
      "[CV] C=0.001, kernel=linear ..........................................\n",
      "[CV] C=0.001, kernel=linear ..........................................\n",
      "[CV] C=0.001, kernel=linear ..........................................\n",
      "[CV] .............. C=0.001, kernel=rbf, score=0.641509, total=   0.1s\n",
      "[CV] ........... C=0.001, kernel=linear, score=0.648148, total=   0.1s\n",
      "[CV] .............. C=0.001, kernel=rbf, score=0.648148, total=   0.0s\n",
      "[CV] ........... C=0.001, kernel=linear, score=0.648148, total=   0.1s\n",
      "[CV] ........... C=0.001, kernel=linear, score=0.648148, total=   0.1s\n",
      "[CV] C=0.001, kernel=linear ..........................................\n",
      "[CV] C=0.001, kernel=linear ..........................................\n",
      "[CV] ........... C=0.001, kernel=linear, score=0.648148, total=   0.0s\n",
      "[CV] C=0.001, kernel=linear ..........................................\n",
      "[CV] C=0.001, kernel=linear ..........................................\n",
      "[CV] .............. C=0.001, kernel=rbf, score=0.641509, total=   0.1s\n",
      "[CV] C=0.001, kernel=linear ..........................................\n",
      "[CV] C=0.01, kernel=rbf ..............................................\n",
      "[CV] C=0.01, kernel=rbf ..............................................\n",
      "[CV] ........... C=0.001, kernel=linear, score=0.648148, total=   0.0s\n",
      "[CV] .............. C=0.001, kernel=rbf, score=0.653846, total=   0.0s\n",
      "[CV] .............. C=0.001, kernel=rbf, score=0.653846, total=   0.1s\n",
      "[CV] ........... C=0.001, kernel=linear, score=0.641509, total=   0.1s\n",
      "[CV] C=0.01, kernel=rbf ..............................................\n",
      "[CV] C=0.01, kernel=rbf ..............................................\n",
      "[CV] C=0.01, kernel=rbf ..............................................\n",
      "[CV] ........... C=0.001, kernel=linear, score=0.641509, total=   0.1s\n",
      "[CV] ........... C=0.001, kernel=linear, score=0.648148, total=   0.1s\n",
      "[CV] ........... C=0.001, kernel=linear, score=0.653846, total=   0.1s\n",
      "[CV] C=0.01, kernel=linear ...........................................\n",
      "[CV] C=0.01, kernel=linear ...........................................\n",
      "[CV] ........... C=0.001, kernel=linear, score=0.653846, total=   0.1s\n",
      "[CV] C=0.01, kernel=linear ...........................................\n",
      "[CV] C=0.01, kernel=linear ...........................................\n",
      "[CV] ............... C=0.01, kernel=rbf, score=0.648148, total=   0.1s\n",
      "[CV] C=0.01, kernel=linear ...........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Done  12 tasks      | elapsed:    0.3s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............... C=0.01, kernel=rbf, score=0.641509, total=   0.1s\n",
      "[CV] C=0.01, kernel=rbf ..............................................\n",
      "[CV] C=0.01, kernel=rbf ..............................................\n",
      "[CV] ............... C=0.01, kernel=rbf, score=0.648148, total=   0.1s\n",
      "[CV] ............... C=0.01, kernel=rbf, score=0.648148, total=   0.1s\n",
      "[CV] ............ C=0.01, kernel=linear, score=0.641509, total=   0.1s\n",
      "[CV] C=0.01, kernel=rbf ..............................................\n",
      "[CV] ............ C=0.01, kernel=linear, score=0.648148, total=   0.0s\n",
      "[CV] ............ C=0.01, kernel=linear, score=0.648148, total=   0.0s\n",
      "[CV] C=0.01, kernel=linear ...........................................\n",
      "[CV] ............... C=0.01, kernel=rbf, score=0.653846, total=   0.1s\n",
      "[CV] C=0.01, kernel=rbf ..............................................\n",
      "[CV] ............ C=0.01, kernel=linear, score=0.648148, total=   0.0s\n",
      "[CV] C=0.01, kernel=linear ...........................................\n",
      "[CV] C=0.01, kernel=rbf ..............................................\n",
      "[CV] C=0.01, kernel=linear ...........................................\n",
      "[CV] C=0.01, kernel=linear ...........................................\n",
      "[CV] ............ C=0.01, kernel=linear, score=0.653846, total=   0.0s\n",
      "[CV] C=0.01, kernel=linear ...........................................\n",
      "[CV] ............... C=0.01, kernel=rbf, score=0.648148, total=   0.1s\n",
      "[CV] C=0.1, kernel=rbf ...............................................\n",
      "[CV] ............ C=0.01, kernel=linear, score=0.648148, total=   0.0s\n",
      "[CV] ............... C=0.01, kernel=rbf, score=0.648148, total=   0.0s\n",
      "[CV] ............... C=0.01, kernel=rbf, score=0.648148, total=   0.1s\n",
      "[CV] ............ C=0.01, kernel=linear, score=0.648148, total=   0.0s\n",
      "[CV] C=0.1, kernel=rbf ...............................................\n",
      "[CV] C=0.1, kernel=rbf ...............................................\n",
      "[CV] C=0.1, kernel=rbf ...............................................\n",
      "[CV] ............... C=0.01, kernel=rbf, score=0.641509, total=   0.1s\n",
      "[CV] C=0.1, kernel=rbf ...............................................\n",
      "[CV] ............ C=0.01, kernel=linear, score=0.648148, total=   0.1s\n",
      "[CV] C=0.1, kernel=linear ............................................\n",
      "[CV] ............... C=0.01, kernel=rbf, score=0.653846, total=   0.1s\n",
      "[CV] C=0.1, kernel=linear ............................................\n",
      "[CV] C=0.1, kernel=linear ............................................\n",
      "[CV] ............ C=0.01, kernel=linear, score=0.641509, total=   0.0s\n",
      "[CV] C=0.1, kernel=linear ............................................\n",
      "[CV] ............ C=0.01, kernel=linear, score=0.653846, total=   0.0s\n",
      "[CV] C=0.1, kernel=linear ............................................\n",
      "[CV] ................ C=0.1, kernel=rbf, score=0.648148, total=   0.1s\n",
      "[CV] ................ C=0.1, kernel=rbf, score=0.648148, total=   0.1s\n",
      "[CV] ............. C=0.1, kernel=linear, score=0.722222, total=   0.0s\n",
      "[CV] C=0.1, kernel=rbf ...............................................\n",
      "[CV] ............. C=0.1, kernel=linear, score=0.685185, total=   0.1s\n",
      "[CV] ............. C=0.1, kernel=linear, score=0.722222, total=   0.1s\n",
      "[CV] ............. C=0.1, kernel=linear, score=0.754717, total=   0.0s\n",
      "[CV] C=0.1, kernel=linear ............................................\n",
      "[CV] ................ C=0.1, kernel=rbf, score=0.648148, total=   0.1s\n",
      "[CV] ................ C=0.1, kernel=rbf, score=0.653846, total=   0.1s\n",
      "[CV] C=0.1, kernel=linear ............................................\n",
      "[CV] C=0.1, kernel=rbf ...............................................\n",
      "[CV] C=0.1, kernel=rbf ...............................................\n",
      "[CV] C=0.1, kernel=linear ............................................\n",
      "[CV] ................ C=0.1, kernel=rbf, score=0.641509, total=   0.1s\n",
      "[CV] C=0.1, kernel=rbf ...............................................\n",
      "[CV] C=0.1, kernel=linear ............................................\n",
      "[CV] ............. C=0.1, kernel=linear, score=0.750000, total=   0.0s\n",
      "[CV] C=0.1, kernel=rbf ...............................................\n",
      "[CV] C=0.1, kernel=linear ............................................\n",
      "[CV] ............. C=0.1, kernel=linear, score=0.666667, total=   0.0s\n",
      "[CV] C=1.0, kernel=rbf ...............................................\n",
      "[CV] ............. C=0.1, kernel=linear, score=0.711538, total=   0.0s\n",
      "[CV] C=1.0, kernel=rbf ...............................................\n",
      "[CV] ............. C=0.1, kernel=linear, score=0.679245, total=   0.0s\n",
      "[CV] C=1.0, kernel=rbf ...............................................\n",
      "[CV] ................ C=0.1, kernel=rbf, score=0.648148, total=   0.1s\n",
      "[CV] ............. C=0.1, kernel=linear, score=0.722222, total=   0.1s\n",
      "[CV] ............. C=0.1, kernel=linear, score=0.666667, total=   0.0s\n",
      "[CV] ................ C=0.1, kernel=rbf, score=0.641509, total=   0.1s\n",
      "[CV] C=1.0, kernel=rbf ...............................................\n",
      "[CV] C=1.0, kernel=rbf ...............................................\n",
      "[CV] ................ C=0.1, kernel=rbf, score=0.648148, total=   0.1s\n",
      "[CV] ................ C=0.1, kernel=rbf, score=0.648148, total=   0.1s\n",
      "[CV] C=1.0, kernel=linear ............................................\n",
      "[CV] C=1.0, kernel=linear ............................................\n",
      "[CV] C=1.0, kernel=linear ............................................\n",
      "[CV] ................ C=0.1, kernel=rbf, score=0.653846, total=   0.1s\n",
      "[CV] C=1.0, kernel=linear ............................................\n",
      "[CV] C=1.0, kernel=linear ............................................\n",
      "[CV] ............. C=1.0, kernel=linear, score=0.703704, total=   0.0s\n",
      "[CV] C=1.0, kernel=linear ............................................\n",
      "[CV] ................ C=1.0, kernel=rbf, score=0.648148, total=   0.1s\n",
      "[CV] C=1.0, kernel=rbf ...............................................\n",
      "[CV] ................ C=1.0, kernel=rbf, score=0.673077, total=   0.1s\n",
      "[CV] ............. C=1.0, kernel=linear, score=0.722222, total=   0.0s\n",
      "[CV] C=1.0, kernel=rbf ...............................................\n",
      "[CV] ............. C=1.0, kernel=linear, score=0.769231, total=   0.0s\n",
      "[CV] ................ C=1.0, kernel=rbf, score=0.722222, total=   0.1s\n",
      "[CV] ............. C=1.0, kernel=linear, score=0.735849, total=   0.0s\n",
      "[CV] ............. C=1.0, kernel=linear, score=0.722222, total=   0.1s\n",
      "[CV] ................ C=1.0, kernel=rbf, score=0.685185, total=   0.1s\n",
      "[CV] C=1.0, kernel=linear ............................................\n",
      "[CV] C=1.0, kernel=rbf ...............................................\n",
      "[CV] C=1.0, kernel=linear ............................................\n",
      "[CV] C=1.0, kernel=linear ............................................\n",
      "[CV] C=1.0, kernel=linear ............................................\n",
      "[CV] ............. C=1.0, kernel=linear, score=0.722222, total=   0.0s\n",
      "[CV] ................ C=1.0, kernel=rbf, score=0.698113, total=   0.0s\n",
      "[CV] C=1.0, kernel=rbf ...............................................\n",
      "[CV] C=1.0, kernel=rbf ...............................................\n",
      "[CV] C=10.0, kernel=rbf ..............................................\n",
      "[CV] ............. C=1.0, kernel=linear, score=0.759259, total=   0.1s\n",
      "[CV] ............. C=1.0, kernel=linear, score=0.698113, total=   0.0s\n",
      "[CV] ............. C=1.0, kernel=linear, score=0.685185, total=   0.0s\n",
      "[CV] C=10.0, kernel=rbf ..............................................\n",
      "[CV] C=10.0, kernel=rbf ..............................................\n",
      "[CV] C=10.0, kernel=rbf ..............................................\n",
      "[CV] ............. C=1.0, kernel=linear, score=0.711538, total=   0.1s\n",
      "[CV] C=10.0, kernel=rbf ..............................................\n",
      "[CV] ................ C=1.0, kernel=rbf, score=0.666667, total=   0.1s\n",
      "[CV] ................ C=1.0, kernel=rbf, score=0.703704, total=   0.1s\n",
      "[CV] ................ C=1.0, kernel=rbf, score=0.648148, total=   0.1s\n",
      "[CV] ............... C=10.0, kernel=rbf, score=0.703704, total=   0.1s\n",
      "[CV] ................ C=1.0, kernel=rbf, score=0.692308, total=   0.1s\n",
      "[CV] C=10.0, kernel=rbf ..............................................\n",
      "[CV] ................ C=1.0, kernel=rbf, score=0.679245, total=   0.1s\n",
      "[CV] C=10.0, kernel=linear ...........................................\n",
      "[CV] C=10.0, kernel=linear ...........................................\n",
      "[CV] C=10.0, kernel=linear ...........................................\n",
      "[CV] C=10.0, kernel=linear ...........................................\n",
      "[CV] C=10.0, kernel=linear ...........................................\n",
      "[CV] ............ C=10.0, kernel=linear, score=0.685185, total=   0.0s\n",
      "[CV] ............... C=10.0, kernel=rbf, score=0.722222, total=   0.1s\n",
      "[CV] ............... C=10.0, kernel=rbf, score=0.722222, total=   0.1s\n",
      "[CV] ............... C=10.0, kernel=rbf, score=0.735849, total=   0.1s\n",
      "[CV] C=10.0, kernel=linear ...........................................\n",
      "[CV] C=10.0, kernel=rbf ..............................................\n",
      "[CV] C=10.0, kernel=rbf ..............................................\n",
      "[CV] ............ C=10.0, kernel=linear, score=0.735849, total=   0.0s\n",
      "[CV] ............ C=10.0, kernel=linear, score=0.740741, total=   0.0s\n",
      "[CV] ............ C=10.0, kernel=linear, score=0.777778, total=   0.0s\n",
      "[CV] C=10.0, kernel=rbf ..............................................\n",
      "[CV] ............ C=10.0, kernel=linear, score=0.750000, total=   0.1s\n",
      "[CV] C=10.0, kernel=linear ...........................................\n",
      "[CV] C=10.0, kernel=linear ...........................................\n",
      "[CV] C=10.0, kernel=linear ...........................................\n",
      "[CV] ............... C=10.0, kernel=rbf, score=0.769231, total=   0.1s\n",
      "[CV] C=10.0, kernel=linear ...........................................\n",
      "[CV] C=10.0, kernel=rbf ..............................................\n",
      "[CV] ............... C=10.0, kernel=rbf, score=0.703704, total=   0.1s\n",
      "[CV] C=100.0, kernel=rbf .............................................\n",
      "[CV] ............ C=10.0, kernel=linear, score=0.759259, total=   0.0s\n",
      "[CV] ............ C=10.0, kernel=linear, score=0.685185, total=   0.1s\n",
      "[CV] C=100.0, kernel=rbf .............................................\n",
      "[CV] C=100.0, kernel=rbf .............................................\n",
      "[CV] ............ C=10.0, kernel=linear, score=0.692308, total=   0.0s\n",
      "[CV] ............ C=10.0, kernel=linear, score=0.698113, total=   0.0s\n",
      "[CV] ............... C=10.0, kernel=rbf, score=0.759259, total=   0.1s\n",
      "[CV] ............ C=10.0, kernel=linear, score=0.777778, total=   0.1s\n",
      "[CV] ............... C=10.0, kernel=rbf, score=0.703704, total=   0.1s\n",
      "[CV] C=100.0, kernel=rbf .............................................\n",
      "[CV] C=100.0, kernel=rbf .............................................\n",
      "[CV] ............... C=10.0, kernel=rbf, score=0.679245, total=   0.1s\n",
      "[CV] C=100.0, kernel=linear ..........................................\n",
      "[CV] C=100.0, kernel=linear ..........................................\n",
      "[CV] C=100.0, kernel=linear ..........................................\n",
      "[CV] C=100.0, kernel=linear ..........................................\n",
      "[CV] ............... C=10.0, kernel=rbf, score=0.711538, total=   0.1s\n",
      "[CV] .............. C=100.0, kernel=rbf, score=0.685185, total=   0.1s\n",
      "[CV] C=100.0, kernel=linear ..........................................\n",
      "[CV] .............. C=100.0, kernel=rbf, score=0.777778, total=   0.1s\n",
      "[CV] ........... C=100.0, kernel=linear, score=0.777778, total=   0.0s\n",
      "[CV] C=100.0, kernel=rbf .............................................\n",
      "[CV] C=100.0, kernel=rbf .............................................\n",
      "[CV] ........... C=100.0, kernel=linear, score=0.754717, total=   0.0s\n",
      "[CV] C=100.0, kernel=linear ..........................................\n",
      "[CV] ........... C=100.0, kernel=linear, score=0.740741, total=   0.0s\n",
      "[CV] .............. C=100.0, kernel=rbf, score=0.740741, total=   0.1s\n",
      "[CV] C=100.0, kernel=linear ..........................................\n",
      "[CV] C=100.0, kernel=linear ..........................................\n",
      "[CV] ........... C=100.0, kernel=linear, score=0.685185, total=   0.0s\n",
      "[CV] C=100.0, kernel=rbf .............................................\n",
      "[CV] .............. C=100.0, kernel=rbf, score=0.750000, total=   0.1s\n",
      "[CV] C=100.0, kernel=linear ..........................................\n",
      "[CV] ........... C=100.0, kernel=linear, score=0.750000, total=   0.0s\n",
      "[CV] .............. C=100.0, kernel=rbf, score=0.735849, total=   0.1s\n",
      "[CV] C=100.0, kernel=rbf .............................................\n",
      "[CV] C=100.0, kernel=linear ..........................................\n",
      "[CV] C=100.0, kernel=rbf .............................................\n",
      "[CV] ........... C=100.0, kernel=linear, score=0.722222, total=   0.0s\n",
      "[CV] .............. C=100.0, kernel=rbf, score=0.740741, total=   0.1s\n",
      "[CV] ........... C=100.0, kernel=linear, score=0.759259, total=   0.0s\n",
      "[CV] C=1000.0, kernel=rbf ............................................\n",
      "[CV] C=1000.0, kernel=rbf ............................................\n",
      "[CV] ........... C=100.0, kernel=linear, score=0.777778, total=   0.0s\n",
      "[CV] ........... C=100.0, kernel=linear, score=0.679245, total=   0.1s\n",
      "[CV] C=1000.0, kernel=rbf ............................................\n",
      "[CV] .............. C=100.0, kernel=rbf, score=0.711538, total=   0.0s\n",
      "[CV] .............. C=100.0, kernel=rbf, score=0.698113, total=   0.0s\n",
      "[CV] C=1000.0, kernel=rbf ............................................\n",
      "[CV] C=1000.0, kernel=linear .........................................\n",
      "[CV] .............. C=100.0, kernel=rbf, score=0.685185, total=   0.1s\n",
      "[CV] C=1000.0, kernel=rbf ............................................\n",
      "[CV] .............. C=100.0, kernel=rbf, score=0.759259, total=   0.1s\n",
      "[CV] C=1000.0, kernel=linear .........................................\n",
      "[CV] C=1000.0, kernel=linear .........................................\n",
      "[CV] C=1000.0, kernel=linear .........................................\n",
      "[CV] ........... C=100.0, kernel=linear, score=0.692308, total=   0.0s\n",
      "[CV] C=1000.0, kernel=linear .........................................\n",
      "[CV] ............. C=1000.0, kernel=rbf, score=0.685185, total=   0.1s\n",
      "[CV] ............. C=1000.0, kernel=rbf, score=0.759259, total=   0.1s\n",
      "[CV] ............. C=1000.0, kernel=rbf, score=0.754717, total=   0.1s\n",
      "[CV] ............. C=1000.0, kernel=rbf, score=0.750000, total=   0.1s\n",
      "[CV] C=1000.0, kernel=rbf ............................................\n",
      "[CV] ............. C=1000.0, kernel=rbf, score=0.777778, total=   0.1s\n",
      "[CV] C=1000.0, kernel=rbf ............................................\n",
      "[CV] C=1000.0, kernel=rbf ............................................\n",
      "[CV] C=1000.0, kernel=rbf ............................................\n",
      "[CV] C=1000.0, kernel=rbf ............................................\n",
      "[CV] .......... C=1000.0, kernel=linear, score=0.796296, total=   0.1s\n",
      "[CV] .......... C=1000.0, kernel=linear, score=0.722222, total=   0.1s\n",
      "[CV] C=1000.0, kernel=linear .........................................\n",
      "[CV] C=1000.0, kernel=linear .........................................\n",
      "[CV] .......... C=1000.0, kernel=linear, score=0.754717, total=   0.1s\n",
      "[CV] C=1000.0, kernel=linear .........................................\n",
      "[CV] .......... C=1000.0, kernel=linear, score=0.666667, total=   0.2s\n",
      "[CV] ............. C=1000.0, kernel=rbf, score=0.777778, total=   0.1s\n",
      "[CV] ............. C=1000.0, kernel=rbf, score=0.692308, total=   0.0s\n",
      "[CV] ............. C=1000.0, kernel=rbf, score=0.641509, total=   0.0s\n",
      "[CV] C=1000.0, kernel=linear .........................................\n",
      "[CV] .......... C=1000.0, kernel=linear, score=0.750000, total=   0.2s\n",
      "[CV] ............. C=1000.0, kernel=rbf, score=0.703704, total=   0.1s\n",
      "[CV] C=1000.0, kernel=linear .........................................\n",
      "[CV] ............. C=1000.0, kernel=rbf, score=0.759259, total=   0.1s\n",
      "[CV] .......... C=1000.0, kernel=linear, score=0.722222, total=   0.1s\n",
      "[CV] .......... C=1000.0, kernel=linear, score=0.692308, total=   0.0s\n",
      "[CV] .......... C=1000.0, kernel=linear, score=0.679245, total=   0.1s\n",
      "[CV] .......... C=1000.0, kernel=linear, score=0.796296, total=   0.1s\n",
      "[CV] .......... C=1000.0, kernel=linear, score=0.759259, total=   0.1s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Done 140 out of 140 | elapsed:    2.3s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise',\n",
       "       estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape=None, degree=3, gamma='auto', kernel='rbf',\n",
       "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False),\n",
       "       fit_params={}, iid=True, n_jobs=10,\n",
       "       param_grid=[{'kernel': ['rbf', 'linear'], 'C': array([  1.00000e-03,   1.00000e-02,   1.00000e-01,   1.00000e+00,\n",
       "         1.00000e+01,   1.00000e+02,   1.00000e+03])}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring=None, verbose=3)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c_range = np.logspace(-3,3,7)\n",
    "param_grid = [\n",
    "    {'kernel': ['rbf','linear'], 'C': c_range},\n",
    "]\n",
    "grid_search = GridSearchCV(svm.SVC(), param_grid, cv=10, verbose=3, n_jobs=10)\n",
    "grid_search.fit(data, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVC(C=100.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape=None, degree=3, gamma='auto', kernel='linear',\n",
      "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False)\n",
      "0.734082397004\n",
      "{'C': 100.0, 'kernel': 'linear'}\n"
     ]
    }
   ],
   "source": [
    "print(grid_search.best_estimator_)\n",
    "print(grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P: acc(0.696629213483), prec(0.687305237548), rec(0.973254608102)\n",
      "AP: acc(0.717228464419), prec(0.70695997115), rec(0.962482855518)\n",
      "PA: acc(0.705992509363), prec(0.694784545795), rec(0.973656039876)\n",
      "PA1: acc(0.707865168539), prec(0.696164813961), rec(0.973656039876)\n",
      "PA2: acc(0.707865168539), prec(0.695473251029), rec(0.976650051852)\n",
      "PAK: acc(0.717228464419), prec(0.702645228216), rec(0.976650051852)\n",
      "CW: acc(0.677902621723), prec(0.689977300506), rec(0.908189208176)\n",
      "AL: acc(0.705992509363), prec(0.696661664107), rec(0.964473288061)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score\n",
    "import oll\n",
    "\n",
    "## manual 10-fold cross-validation\n",
    "kf = KFold(n_splits=2, random_state=None, shuffle=False)\n",
    "\n",
    "methods = [\"P\" ,\"AP\" ,\"PA\" ,\"PA1\",\"PA2\" ,\"PAK\",\"CW\" ,\"AL\"]\n",
    "\n",
    "for m in methods:\n",
    "    model = oll.oll(m, C=1)\n",
    "\n",
    "    accuracy = []\n",
    "    precision = []\n",
    "    recall = []\n",
    "    \n",
    "    for train_index, test_index in kf.split(data):\n",
    "\n",
    "        X_train, X_test = data[train_index], data[test_index]\n",
    "        y_train, y_test = target[train_index], target[test_index]\n",
    "        model.fit(X_train, y_train)\n",
    "        predicted = model.predict(X_test)\n",
    "\n",
    "        accuracy.append(accuracy_score(y_test, predicted))\n",
    "        precision.append(precision_score(y_test, predicted))\n",
    "        recall.append(recall_score(y_test, predicted))\n",
    "\n",
    "    print(m + ': acc(' + str(np.mean(accuracy)) \n",
    "          + '), prec(' + str(np.mean(precision))\n",
    "          + '), rec(' + str(np.mean(recall)) + ')'\n",
    "         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.726359577115\n",
      "0.776806315693\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "model = GradientBoostingClassifier()\n",
    "\n",
    "precision = cross_val_score(model, data, target, cv=10, scoring='precision').mean()\n",
    "acc = cross_val_score(model, data, target, cv=10, scoring='accuracy').mean()\n",
    "recall = cross_val_score(model, data, target, cv=10, scoring='recall').mean()\n",
    "fscore = cross_val_score(model, data, target, cv=10, scoring='f1').mean()\n",
    "print(precision)\n",
    "print(fscore)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tfidf_vectorizer = TfidfVectorizer(strip_accents='unicode', stop_words=stopwords)\n",
    "data = tfidf_vectorizer.fit_transform(corpus['labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.664332695342\n",
      "0.665087620276\n",
      "0.976974789916\n"
     ]
    }
   ],
   "source": [
    "model = MultinomialNB()\n",
    "\n",
    "precision = cross_val_score(model, data, target, cv=10, scoring='precision').mean()\n",
    "acc = cross_val_score(model, data, target, cv=10, scoring='accuracy').mean()\n",
    "recall = cross_val_score(model, data, target, cv=10, scoring='recall').mean()\n",
    "print(precision)\n",
    "print(acc)\n",
    "print(recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.668333100874\n",
      "0.649990592915\n",
      "0.913193277311\n"
     ]
    }
   ],
   "source": [
    "model = svm.LinearSVC()\n",
    "\n",
    "precision = cross_val_score(model, data, target, cv=10, scoring='precision').mean()\n",
    "acc = cross_val_score(model, data, target, cv=10, scoring='accuracy').mean()\n",
    "recall = cross_val_score(model, data, target, cv=10, scoring='recall').mean()\n",
    "print(precision)\n",
    "print(acc)\n",
    "print(recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tfidf_vectorizer = TfidfVectorizer(strip_accents='unicode', stop_words=stopwords)\n",
    "data = tfidf_vectorizer.fit_transform(corpus['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.649580447176\n",
      "0.646036929527\n",
      "0.985462184874\n"
     ]
    }
   ],
   "source": [
    "model = MultinomialNB()\n",
    "\n",
    "precision = cross_val_score(model, data, target, cv=10, scoring='precision').mean()\n",
    "acc = cross_val_score(model, data, target, cv=10, scoring='accuracy').mean()\n",
    "recall = cross_val_score(model, data, target, cv=10, scoring='recall').mean()\n",
    "print(precision)\n",
    "print(acc)\n",
    "print(recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.643274977858\n",
      "0.613986991345\n",
      "0.907226890756\n"
     ]
    }
   ],
   "source": [
    "model = svm.LinearSVC()\n",
    "\n",
    "precision = cross_val_score(model, data, target, cv=10, scoring='precision').mean()\n",
    "acc = cross_val_score(model, data, target, cv=10, scoring='accuracy').mean()\n",
    "recall = cross_val_score(model, data, target, cv=10, scoring='recall').mean()\n",
    "print(precision)\n",
    "print(acc)\n",
    "print(recall)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}