Relevance Detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "\n",
    "# Relevance Detection\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dannyyang/anaconda3/lib/python3.6/site-packages/sklearn/ensemble/weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.\n",
      "  from numpy.core.umath_tests import inner1d\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.insert(0, './preprocessing/')\n",
    "import pandas as pd\n",
    "import nltk\n",
    "import numpy as np\n",
    "import preprocessing.main\n",
    "import preprocessing.helpers\n",
    "import preprocessing.utils\n",
    "import preprocessing.feature_engineering\n",
    "import preprocessing.word_embeddings\n",
    "import utils\n",
    "import importlib\n",
    "from collections import Counter\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import score\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "importlib.reload(preprocessing.main)\n",
    "importlib.reload(preprocessing.utils)\n",
    "importlib.reload(preprocessing.helpers)\n",
    "importlib.reload(preprocessing.feature_engineering)\n",
    "importlib.reload(preprocessing.word_embeddings)\n",
    "preprocess = preprocessing.main.Preprocessing()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(49972, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Headline</th>\n",
       "      <th>Body ID</th>\n",
       "      <th>Stance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Police find mass graves with at least '15 bodi...</td>\n",
       "      <td>712</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hundreds of Palestinians flee floods in Gaza a...</td>\n",
       "      <td>158</td>\n",
       "      <td>agree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Christian Bale passes on role of Steve Jobs, a...</td>\n",
       "      <td>137</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HBO and Apple in Talks for $15/Month Apple TV ...</td>\n",
       "      <td>1034</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Spider burrowed through tourist's stomach and ...</td>\n",
       "      <td>1923</td>\n",
       "      <td>disagree</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Headline  Body ID     Stance\n",
       "0  Police find mass graves with at least '15 bodi...      712  unrelated\n",
       "1  Hundreds of Palestinians flee floods in Gaza a...      158      agree\n",
       "2  Christian Bale passes on role of Steve Jobs, a...      137  unrelated\n",
       "3  HBO and Apple in Talks for $15/Month Apple TV ...     1034  unrelated\n",
       "4  Spider burrowed through tourist's stomach and ...     1923   disagree"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_stances = pd.read_csv(\"fn_data/train_stances.csv\")\n",
    "print(train_stances.shape)\n",
    "train_stances.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1683, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Body ID</th>\n",
       "      <th>articleBody</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>Last week we hinted at what was to come as Ebo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>(NEWSER) – Wonder how long a Quarter Pounder w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>Posting photos of a gun-toting child online, I...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>At least 25 suspected Boko Haram insurgents we...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Body ID                                        articleBody\n",
       "0        0  A small meteorite crashed into a wooded area i...\n",
       "1        4  Last week we hinted at what was to come as Ebo...\n",
       "2        5  (NEWSER) – Wonder how long a Quarter Pounder w...\n",
       "3        6  Posting photos of a gun-toting child online, I...\n",
       "4        7  At least 25 suspected Boko Haram insurgents we..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_bodies = pd.read_csv(\"fn_data/train_bodies.csv\")\n",
    "print(train_bodies.shape)\n",
    "train_bodies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((40635, 3), (9337, 3))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stances_tr, stances_val = preprocess.train_test_split(train_bodies, train_stances)\n",
    "stances_tr.shape, stances_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stances_tr.to_csv(\"saved_data/stances_tr.csv\")\n",
    "stances_val.to_csv(\"saved_data/stances_val.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# this one takes a while!\n",
    "idf = preprocess.build_idf(train_bodies, stances_tr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('saved_data/idf.json', 'w') as fp:\n",
    "    json.dump(idf, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'agree': 3678, 'disagree': 840, 'discuss': 8909, 'unrelated': 36545})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(train_stances['Stance'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['burger', 'year', 'friend', 'news', 'report'] ['say', 'bought', 'showed', 'started', 'wonder']\n",
      "['burger', 'year', 'friend', 'charity', 'mcjordan'] ['bought', 'started', 'showed', 'dissuaded', 'sauce']\n"
     ]
    }
   ],
   "source": [
    "#this is just a comparison between using IDF score and not using IDF score - not related to the model\n",
    "#change the body id to see\n",
    "body = preprocess.get_body(5, train_bodies)\n",
    "#no IDF\n",
    "processed2 = preprocess.process_body(body)\n",
    "print(processed2['common_nouns'],processed2['common_verbs'])\n",
    "\n",
    "#with IDF\n",
    "processed = preprocess.process_body(body, idf)\n",
    "print(processed['common_nouns'],processed['common_verbs'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['unconfirmed', 'report', 'circulating', 'social', 'medium', 'islamic', 'state', 'group', 'carried', 'chemical', 'attack', 'battling', 'kurdish', 'force', 'kobani']\n",
      "['symptom', 'chlorine', 'attack', 'include', 'teary', 'eye', 'burning', 'sensation', 'throat', 'sensation', 'suffocation', 'headache']\n",
      "[] []\n",
      "['unconfirmed', 'social', 'islamic', 'chemical', 'kurdish'] ['symptom', 'teary']\n",
      "['circulating', 'carried', 'battling'] ['include', 'burning']\n"
     ]
    }
   ],
   "source": [
    "body = preprocess.get_body(1369, train_bodies)\n",
    "processed = preprocess.process_body(body, idf)\n",
    "print(processed['first_sentence']['tokens'])\n",
    "print(processed['significant_sentence']['tokens'])\n",
    "print(processed['first_sentence']['adverbs'],processed['significant_sentence']['adverbs'])\n",
    "print(processed['first_sentence']['adjectives'],processed['significant_sentence']['adjectives'])\n",
    "print(processed['first_sentence']['verbs'],processed['significant_sentence']['verbs'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed 100\n",
      "processed 200\n",
      "processed 300\n",
      "processed 400\n",
      "processed 500\n",
      "processed 600\n",
      "processed 700\n",
      "processed 800\n",
      "processed 900\n",
      "processed 1000\n",
      "processed 1100\n",
      "processed 1200\n",
      "processed 1300\n",
      "processed 1400\n",
      "processed 1500\n",
      "processed 1600\n",
      "done! processed 1683\n"
     ]
    }
   ],
   "source": [
    "#this takes a while!\n",
    "body_info = preprocess.process_bodies(train_bodies, idf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "json_body_info = {}\n",
    "for k in body_info:\n",
    "    body_info[k]['vocabulary'] = list(body_info[k]['vocabulary'])\n",
    "    json_body_info[str(k)] = body_info[k]\n",
    "with open('saved_data/body_info.json', 'w') as fp:\n",
    "    json.dump(json_body_info, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feats_list = [\n",
    "    'shared_nouns',\n",
    "    'shared_verbs',\n",
    "    'shared_bigrams',\n",
    "    'shared_tokens',\n",
    "\n",
    "    'shared_nouns_fst',\n",
    "    'shared_verbs_fst',\n",
    "    'shared_bigrams_fst',\n",
    "    'shared_tokens_fst',\n",
    "\n",
    "    'shared_nouns_sig',\n",
    "    'shared_verbs_sig',\n",
    "    'shared_bigrams_sig',\n",
    "    'shared_tokens_sig',\n",
    "\n",
    "    'svo_s_fst',\n",
    "    'svo_v_fst',\n",
    "    'svo_o_fst',\n",
    "    \n",
    "    'svo_s_sig',\n",
    "    'svo_v_sig',\n",
    "    'svo_o_sig',\n",
    "    \n",
    "    'cos_nouns_sig',\n",
    "    'cos_bigrams_sig',\n",
    "    'cos_tokens_sig',\n",
    "\n",
    "    'cos_nouns_fst',\n",
    "    'cos_bigrams_fst',\n",
    "    'cos_tokens_fst',\n",
    "    \n",
    "    'sentiment_pos',\n",
    "    'sentiment_neg',\n",
    "    'sentiment_neu',\n",
    "    'sentiment_compound',\n",
    "    \n",
    "    'sentiment_pos_fst',\n",
    "    'sentiment_neg_fst',\n",
    "    'sentiment_neu_fst',\n",
    "    'sentiment_compound_fst',\n",
    "    \n",
    "    'sentiment_pos_sig',\n",
    "    'sentiment_neg_sig',\n",
    "    'sentiment_neu_sig',\n",
    "    'sentiment_compound_sig',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "641\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "# this one takes a while also! ~10 mins\n",
    "start = time.time()\n",
    "#train data\n",
    "data_feats = [preprocess.get_feats(i, body_info) for i in stances_tr.values]\n",
    "val_feats = [preprocess.get_feats(i, body_info) for i in stances_val.values]\n",
    "end = time.time()\n",
    "print(int(end-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('saved_data/train_feats.json', 'w') as fp:\n",
    "    json.dump(data_feats, fp)\n",
    "with open('saved_data/val_feats.json', 'w') as fp:\n",
    "    json.dump(val_feats, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shared_nouns\n",
      "shared_verbs\n",
      "shared_bigrams\n",
      "shared_tokens\n",
      "shared_nouns_fst\n",
      "shared_verbs_fst\n",
      "shared_bigrams_fst\n",
      "shared_tokens_fst\n",
      "shared_nouns_sig\n",
      "shared_verbs_sig\n",
      "shared_bigrams_sig\n",
      "shared_tokens_sig\n",
      "svo_s_fst\n",
      "svo_v_fst\n",
      "svo_o_fst\n",
      "svo_s_sig\n",
      "svo_v_sig\n",
      "svo_o_sig\n",
      "cos_nouns_sig\n",
      "cos_bigrams_sig\n",
      "cos_tokens_sig\n",
      "cos_nouns_fst\n",
      "cos_bigrams_fst\n",
      "cos_tokens_fst\n",
      "sentiment_pos\n",
      "sentiment_neg\n",
      "sentiment_neu\n",
      "sentiment_compound\n",
      "sentiment_pos_fst\n",
      "sentiment_neg_fst\n",
      "sentiment_neu_fst\n",
      "sentiment_compound_fst\n",
      "sentiment_pos_sig\n",
      "sentiment_neg_sig\n",
      "sentiment_neu_sig\n",
      "sentiment_compound_sig\n"
     ]
    }
   ],
   "source": [
    "#training data\n",
    "train_df = pd.DataFrame()\n",
    "for i in feats_list:\n",
    "    print(i)\n",
    "    train_df[i] = [x[i] for x in data_feats]\n",
    "\n",
    "#val data\n",
    "val_df = pd.DataFrame()\n",
    "for i in feats_list:\n",
    "    val_df[i] = [x[i] for x in val_feats]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shared_nouns</th>\n",
       "      <th>shared_verbs</th>\n",
       "      <th>shared_bigrams</th>\n",
       "      <th>shared_tokens</th>\n",
       "      <th>shared_nouns_fst</th>\n",
       "      <th>shared_verbs_fst</th>\n",
       "      <th>shared_bigrams_fst</th>\n",
       "      <th>shared_tokens_fst</th>\n",
       "      <th>shared_nouns_sig</th>\n",
       "      <th>shared_verbs_sig</th>\n",
       "      <th>...</th>\n",
       "      <th>sentiment_neu</th>\n",
       "      <th>sentiment_compound</th>\n",
       "      <th>sentiment_pos_fst</th>\n",
       "      <th>sentiment_neg_fst</th>\n",
       "      <th>sentiment_neu_fst</th>\n",
       "      <th>sentiment_compound_fst</th>\n",
       "      <th>sentiment_pos_sig</th>\n",
       "      <th>sentiment_neg_sig</th>\n",
       "      <th>sentiment_neu_sig</th>\n",
       "      <th>sentiment_compound_sig</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.125444</td>\n",
       "      <td>-0.703411</td>\n",
       "      <td>-0.098</td>\n",
       "      <td>0.194</td>\n",
       "      <td>-0.096</td>\n",
       "      <td>-0.8167</td>\n",
       "      <td>-0.124</td>\n",
       "      <td>0.194</td>\n",
       "      <td>-0.070</td>\n",
       "      <td>-0.8167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.098000</td>\n",
       "      <td>0.372425</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.186</td>\n",
       "      <td>0.186</td>\n",
       "      <td>0.8300</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.100029</td>\n",
       "      <td>-0.110026</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.198</td>\n",
       "      <td>0.198</td>\n",
       "      <td>0.4939</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.254083</td>\n",
       "      <td>-0.785625</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.246</td>\n",
       "      <td>-0.246</td>\n",
       "      <td>-0.2320</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.343</td>\n",
       "      <td>-0.343</td>\n",
       "      <td>-0.6908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.029455</td>\n",
       "      <td>-0.634673</td>\n",
       "      <td>-0.069</td>\n",
       "      <td>0.102</td>\n",
       "      <td>-0.033</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>-0.313</td>\n",
       "      <td>0.198</td>\n",
       "      <td>0.115</td>\n",
       "      <td>-1.2855</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   shared_nouns  shared_verbs  shared_bigrams  shared_tokens  \\\n",
       "0             0             0               0              0   \n",
       "1             0             0               0              0   \n",
       "2             1             0               0              1   \n",
       "3             0             0               0              0   \n",
       "4             0             1               0              0   \n",
       "\n",
       "   shared_nouns_fst  shared_verbs_fst  shared_bigrams_fst  shared_tokens_fst  \\\n",
       "0                 0                 0                   0                  0   \n",
       "1                 0                 0                   0                  0   \n",
       "2                 0                 0                   0                  0   \n",
       "3                 0                 0                   0                  0   \n",
       "4                 0                 1                   0                  2   \n",
       "\n",
       "   shared_nouns_sig  shared_verbs_sig           ...            sentiment_neu  \\\n",
       "0                 0                 0           ...                -0.125444   \n",
       "1                 0                 0           ...                 0.098000   \n",
       "2                 1                 0           ...                 0.100029   \n",
       "3                 0                 0           ...                -0.254083   \n",
       "4                 0                 0           ...                -0.029455   \n",
       "\n",
       "   sentiment_compound  sentiment_pos_fst  sentiment_neg_fst  \\\n",
       "0           -0.703411             -0.098              0.194   \n",
       "1            0.372425              0.000             -0.186   \n",
       "2           -0.110026              0.000             -0.198   \n",
       "3           -0.785625              0.000              0.246   \n",
       "4           -0.634673             -0.069              0.102   \n",
       "\n",
       "   sentiment_neu_fst  sentiment_compound_fst  sentiment_pos_sig  \\\n",
       "0             -0.096                 -0.8167             -0.124   \n",
       "1              0.186                  0.8300              0.000   \n",
       "2              0.198                  0.4939              0.000   \n",
       "3             -0.246                 -0.2320              0.000   \n",
       "4             -0.033                  0.0000             -0.313   \n",
       "\n",
       "   sentiment_neg_sig  sentiment_neu_sig  sentiment_compound_sig  \n",
       "0              0.194             -0.070                 -0.8167  \n",
       "1              0.000              0.000                  0.0000  \n",
       "2              0.000              0.000                  0.0000  \n",
       "3              0.343             -0.343                 -0.6908  \n",
       "4              0.198              0.115                 -1.2855  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shared_nouns</th>\n",
       "      <th>shared_verbs</th>\n",
       "      <th>shared_bigrams</th>\n",
       "      <th>shared_tokens</th>\n",
       "      <th>shared_nouns_fst</th>\n",
       "      <th>shared_verbs_fst</th>\n",
       "      <th>shared_bigrams_fst</th>\n",
       "      <th>shared_tokens_fst</th>\n",
       "      <th>shared_nouns_sig</th>\n",
       "      <th>shared_verbs_sig</th>\n",
       "      <th>...</th>\n",
       "      <th>sentiment_compound</th>\n",
       "      <th>sentiment_pos_fst</th>\n",
       "      <th>sentiment_neg_fst</th>\n",
       "      <th>sentiment_neu_fst</th>\n",
       "      <th>sentiment_compound_fst</th>\n",
       "      <th>sentiment_pos_sig</th>\n",
       "      <th>sentiment_neg_sig</th>\n",
       "      <th>sentiment_neu_sig</th>\n",
       "      <th>sentiment_compound_sig</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.703411</td>\n",
       "      <td>-0.098</td>\n",
       "      <td>0.194</td>\n",
       "      <td>-0.096</td>\n",
       "      <td>-0.8167</td>\n",
       "      <td>-0.124</td>\n",
       "      <td>0.194</td>\n",
       "      <td>-0.070</td>\n",
       "      <td>-0.8167</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.372425</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.186</td>\n",
       "      <td>0.186</td>\n",
       "      <td>0.8300</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.110026</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.198</td>\n",
       "      <td>0.198</td>\n",
       "      <td>0.4939</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.785625</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.246</td>\n",
       "      <td>-0.246</td>\n",
       "      <td>-0.2320</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.343</td>\n",
       "      <td>-0.343</td>\n",
       "      <td>-0.6908</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.634673</td>\n",
       "      <td>-0.069</td>\n",
       "      <td>0.102</td>\n",
       "      <td>-0.033</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>-0.313</td>\n",
       "      <td>0.198</td>\n",
       "      <td>0.115</td>\n",
       "      <td>-1.2855</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   shared_nouns  shared_verbs  shared_bigrams  shared_tokens  \\\n",
       "0             0             0               0              0   \n",
       "1             0             0               0              0   \n",
       "2             1             0               0              1   \n",
       "3             0             0               0              0   \n",
       "4             0             1               0              0   \n",
       "\n",
       "   shared_nouns_fst  shared_verbs_fst  shared_bigrams_fst  shared_tokens_fst  \\\n",
       "0                 0                 0                   0                  0   \n",
       "1                 0                 0                   0                  0   \n",
       "2                 0                 0                   0                  0   \n",
       "3                 0                 0                   0                  0   \n",
       "4                 0                 1                   0                  2   \n",
       "\n",
       "   shared_nouns_sig  shared_verbs_sig  ...    sentiment_compound  \\\n",
       "0                 0                 0  ...             -0.703411   \n",
       "1                 0                 0  ...              0.372425   \n",
       "2                 1                 0  ...             -0.110026   \n",
       "3                 0                 0  ...             -0.785625   \n",
       "4                 0                 0  ...             -0.634673   \n",
       "\n",
       "   sentiment_pos_fst  sentiment_neg_fst  sentiment_neu_fst  \\\n",
       "0             -0.098              0.194             -0.096   \n",
       "1              0.000             -0.186              0.186   \n",
       "2              0.000             -0.198              0.198   \n",
       "3              0.000              0.246             -0.246   \n",
       "4             -0.069              0.102             -0.033   \n",
       "\n",
       "   sentiment_compound_fst  sentiment_pos_sig  sentiment_neg_sig  \\\n",
       "0                 -0.8167             -0.124              0.194   \n",
       "1                  0.8300              0.000              0.000   \n",
       "2                  0.4939              0.000              0.000   \n",
       "3                 -0.2320              0.000              0.343   \n",
       "4                  0.0000             -0.313              0.198   \n",
       "\n",
       "   sentiment_neu_sig  sentiment_compound_sig  label  \n",
       "0             -0.070                 -0.8167      0  \n",
       "1              0.000                  0.0000      0  \n",
       "2              0.000                  0.0000      1  \n",
       "3             -0.343                 -0.6908      0  \n",
       "4              0.115                 -1.2855      1  \n",
       "\n",
       "[5 rows x 37 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['label'] = [0 if x == \"unrelated\" else 1 for x in list(stances_tr['Stance'])]\n",
    "val_df['label'] = [0 if x == \"unrelated\" else 1 for x in list(stances_val['Stance'])]\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_df.to_csv(\"saved_data/train_data.csv\")\n",
    "val_df.to_csv(\"saved_data/val_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dannyyang/anaconda3/lib/python3.6/site-packages/sklearn/utils/validation.py:444: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shared_nouns</th>\n",
       "      <th>shared_verbs</th>\n",
       "      <th>shared_bigrams</th>\n",
       "      <th>shared_tokens</th>\n",
       "      <th>shared_nouns_fst</th>\n",
       "      <th>shared_verbs_fst</th>\n",
       "      <th>shared_bigrams_fst</th>\n",
       "      <th>shared_tokens_fst</th>\n",
       "      <th>shared_nouns_sig</th>\n",
       "      <th>shared_verbs_sig</th>\n",
       "      <th>...</th>\n",
       "      <th>sentiment_compound</th>\n",
       "      <th>sentiment_pos_fst</th>\n",
       "      <th>sentiment_neg_fst</th>\n",
       "      <th>sentiment_neu_fst</th>\n",
       "      <th>sentiment_compound_fst</th>\n",
       "      <th>sentiment_pos_sig</th>\n",
       "      <th>sentiment_neg_sig</th>\n",
       "      <th>sentiment_neu_sig</th>\n",
       "      <th>sentiment_compound_sig</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.398302</td>\n",
       "      <td>-0.213156</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.439242</td>\n",
       "      <td>-0.495243</td>\n",
       "      <td>-0.264359</td>\n",
       "      <td>-0.34473</td>\n",
       "      <td>-0.543312</td>\n",
       "      <td>-0.372422</td>\n",
       "      <td>-0.168255</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.527420</td>\n",
       "      <td>-0.766033</td>\n",
       "      <td>0.771855</td>\n",
       "      <td>-0.280880</td>\n",
       "      <td>-1.510595</td>\n",
       "      <td>-0.307755</td>\n",
       "      <td>0.669070</td>\n",
       "      <td>-0.345443</td>\n",
       "      <td>-1.224380</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.398302</td>\n",
       "      <td>-0.213156</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.439242</td>\n",
       "      <td>-0.495243</td>\n",
       "      <td>-0.264359</td>\n",
       "      <td>-0.34473</td>\n",
       "      <td>-0.543312</td>\n",
       "      <td>-0.372422</td>\n",
       "      <td>-0.168255</td>\n",
       "      <td>...</td>\n",
       "      <td>1.298953</td>\n",
       "      <td>0.071125</td>\n",
       "      <td>-1.189612</td>\n",
       "      <td>1.042358</td>\n",
       "      <td>1.598287</td>\n",
       "      <td>0.387839</td>\n",
       "      <td>-0.226392</td>\n",
       "      <td>-0.077032</td>\n",
       "      <td>0.393939</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.180317</td>\n",
       "      <td>-0.213156</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.894055</td>\n",
       "      <td>-0.495243</td>\n",
       "      <td>-0.264359</td>\n",
       "      <td>-0.34473</td>\n",
       "      <td>-0.543312</td>\n",
       "      <td>1.119214</td>\n",
       "      <td>-0.168255</td>\n",
       "      <td>...</td>\n",
       "      <td>0.031487</td>\n",
       "      <td>0.071125</td>\n",
       "      <td>-1.251553</td>\n",
       "      <td>1.098666</td>\n",
       "      <td>0.963748</td>\n",
       "      <td>0.387839</td>\n",
       "      <td>-0.226392</td>\n",
       "      <td>-0.077032</td>\n",
       "      <td>0.393939</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.398302</td>\n",
       "      <td>-0.213156</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.439242</td>\n",
       "      <td>-0.495243</td>\n",
       "      <td>-0.264359</td>\n",
       "      <td>-0.34473</td>\n",
       "      <td>-0.543312</td>\n",
       "      <td>-0.372422</td>\n",
       "      <td>-0.168255</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.743408</td>\n",
       "      <td>0.071125</td>\n",
       "      <td>1.040266</td>\n",
       "      <td>-0.984730</td>\n",
       "      <td>-0.406712</td>\n",
       "      <td>0.387839</td>\n",
       "      <td>1.356822</td>\n",
       "      <td>-1.392247</td>\n",
       "      <td>-0.974905</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.398302</td>\n",
       "      <td>4.092797</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.439242</td>\n",
       "      <td>-0.495243</td>\n",
       "      <td>2.884222</td>\n",
       "      <td>-0.34473</td>\n",
       "      <td>0.599421</td>\n",
       "      <td>-0.372422</td>\n",
       "      <td>-0.168255</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.346835</td>\n",
       "      <td>-0.518303</td>\n",
       "      <td>0.296973</td>\n",
       "      <td>0.014737</td>\n",
       "      <td>0.031291</td>\n",
       "      <td>-1.367974</td>\n",
       "      <td>0.687534</td>\n",
       "      <td>0.363930</td>\n",
       "      <td>-2.153323</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   shared_nouns  shared_verbs  shared_bigrams  shared_tokens  \\\n",
       "0     -0.398302     -0.213156             0.0      -0.439242   \n",
       "1     -0.398302     -0.213156             0.0      -0.439242   \n",
       "2      1.180317     -0.213156             0.0       0.894055   \n",
       "3     -0.398302     -0.213156             0.0      -0.439242   \n",
       "4     -0.398302      4.092797             0.0      -0.439242   \n",
       "\n",
       "   shared_nouns_fst  shared_verbs_fst  shared_bigrams_fst  shared_tokens_fst  \\\n",
       "0         -0.495243         -0.264359            -0.34473          -0.543312   \n",
       "1         -0.495243         -0.264359            -0.34473          -0.543312   \n",
       "2         -0.495243         -0.264359            -0.34473          -0.543312   \n",
       "3         -0.495243         -0.264359            -0.34473          -0.543312   \n",
       "4         -0.495243          2.884222            -0.34473           0.599421   \n",
       "\n",
       "   shared_nouns_sig  shared_verbs_sig  ...    sentiment_compound  \\\n",
       "0         -0.372422         -0.168255  ...             -1.527420   \n",
       "1         -0.372422         -0.168255  ...              1.298953   \n",
       "2          1.119214         -0.168255  ...              0.031487   \n",
       "3         -0.372422         -0.168255  ...             -1.743408   \n",
       "4         -0.372422         -0.168255  ...             -1.346835   \n",
       "\n",
       "   sentiment_pos_fst  sentiment_neg_fst  sentiment_neu_fst  \\\n",
       "0          -0.766033           0.771855          -0.280880   \n",
       "1           0.071125          -1.189612           1.042358   \n",
       "2           0.071125          -1.251553           1.098666   \n",
       "3           0.071125           1.040266          -0.984730   \n",
       "4          -0.518303           0.296973           0.014737   \n",
       "\n",
       "   sentiment_compound_fst  sentiment_pos_sig  sentiment_neg_sig  \\\n",
       "0               -1.510595          -0.307755           0.669070   \n",
       "1                1.598287           0.387839          -0.226392   \n",
       "2                0.963748           0.387839          -0.226392   \n",
       "3               -0.406712           0.387839           1.356822   \n",
       "4                0.031291          -1.367974           0.687534   \n",
       "\n",
       "   sentiment_neu_sig  sentiment_compound_sig  label  \n",
       "0          -0.345443               -1.224380      0  \n",
       "1          -0.077032                0.393939      0  \n",
       "2          -0.077032                0.393939      1  \n",
       "3          -1.392247               -0.974905      0  \n",
       "4           0.363930               -2.153323      1  \n",
       "\n",
       "[5 rows x 37 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaler = StandardScaler()\n",
    "\n",
    "for i in feats_list:\n",
    "    train_df[i] = scaler.fit_transform(train_df[i].values.reshape(-1,1))\n",
    "    val_df[i] = scaler.fit_transform(val_df[i].values.reshape(-1,1))\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=6, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=5, min_samples_split=10,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=1,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression, RidgeClassifier, Lasso\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "#adjust params as you see fit\n",
    "model = DecisionTreeClassifier(min_samples_split = 10, min_samples_leaf = 5, max_depth = 15)\n",
    "model2 = RandomForestClassifier(n_estimators = 100, min_samples_split = 10, min_samples_leaf = 5, max_depth = 6)\n",
    "# model2 = LogisticRegression()\n",
    "model.fit(train_df.iloc[:,:-1], train_df.iloc[:,-1])\n",
    "model2.fit(train_df.iloc[:,:-1], train_df.iloc[:,-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "97.26% training accuracy\n",
      "94.22% validation accuracy\n",
      "95.35% training accuracy\n",
      "95.44% validation accuracy\n"
     ]
    }
   ],
   "source": [
    "tr_acc = model.score(train_df.iloc[:,:-1], train_df.iloc[:,-1].values.reshape(-1))\n",
    "print('{0:.2f}% training accuracy'.format(tr_acc*100))\n",
    "val_acc = model.score(val_df.iloc[:,:-1], val_df.iloc[:,-1].values.reshape(-1))\n",
    "print('{0:.2f}% validation accuracy'.format(val_acc*100))\n",
    "\n",
    "tr_acc = model2.score(train_df.iloc[:,:-1], train_df.iloc[:,-1].values.reshape(-1))\n",
    "print('{0:.2f}% training accuracy'.format(tr_acc*100))\n",
    "val_acc = model2.score(val_df.iloc[:,:-1], val_df.iloc[:,-1].values.reshape(-1))\n",
    "print('{0:.2f}% validation accuracy'.format(val_acc*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get coefficients - logistic\n",
    "# [(feats_list[i],model2.coef_[0][i]) for i in list(range(len(feats_list)))]\n",
    "\n",
    "#coefficients - lasso\n",
    "#model.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# #usage example for json dump\n",
    "utils.rf_json_dump(model2, list(train_df.iloc[:,:-1]), \"saved_models/random_forest_dump.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#dump validation data to CSV\n",
    "val_df.to_csv('saved_data/test_val_dump.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def score_model(predictions):\n",
    "    true_label = [(1 if x[-1] == \"discuss\" else 0) for x in stances_val.values]\n",
    "    matrix = confusion_matrix(true_label,predictions)\n",
    "    print('confusion matrix: \\n{}\\n'.format(matrix))\n",
    "    #use FNC scorer to generate score report\n",
    "    label_prediction = [(\"discuss\" if x == 1 else \"unrelated\") for x in predictions]\n",
    "    label_actual = pd.DataFrame(stances_val)['Stance']\n",
    "    score.report_score(label_actual, label_prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix: \n",
      "[[7437 1008]\n",
      " [ 219 1492]]\n",
      "\n",
      "-------------------------------------------------------------\n",
      "|           |   agree   | disagree  |  discuss  | unrelated |\n",
      "-------------------------------------------------------------\n",
      "|   agree   |     0     |     0     |    604    |    73     |\n",
      "-------------------------------------------------------------\n",
      "| disagree  |     0     |     0     |    132    |    23     |\n",
      "-------------------------------------------------------------\n",
      "|  discuss  |     0     |     0     |   1492    |    219    |\n",
      "-------------------------------------------------------------\n",
      "| unrelated |     0     |     0     |    272    |   7341    |\n",
      "-------------------------------------------------------------\n",
      "Score: 3511.25 out of 4446.25\t(78.97104301377566%)\n"
     ]
    }
   ],
   "source": [
    "true_label = val_df.iloc[:,-1]\n",
    "prediction = model.predict(val_df.iloc[:,:-1])\n",
    "score_model(prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['saved_models/rf_trained.joblib']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from joblib import dump, load\n",
    "dump(model, 'saved_models/rf_trained.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}