Stance Detection RNN WIP.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stance Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dannyyang/anaconda3/lib/python3.6/site-packages/sklearn/ensemble/weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.\n",
      "  from numpy.core.umath_tests import inner1d\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.insert(0, './preprocessing/')\n",
    "import torch\n",
    "import pandas as pd\n",
    "import nltk\n",
    "import numpy as np\n",
    "import preprocessing.main\n",
    "import preprocessing.helpers\n",
    "import preprocessing.utils\n",
    "import preprocessing.feature_engineering\n",
    "import preprocessing.word_embeddings\n",
    "import importlib\n",
    "from collections import Counter\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import score\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "import scipy\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import time\n",
    "import itertools\n",
    "import utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#preprocess.extract_word_embeddings(\"glove.6B.50d\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "importlib.reload(preprocessing.main)\n",
    "importlib.reload(preprocessing.utils)\n",
    "importlib.reload(preprocessing.helpers)\n",
    "importlib.reload(preprocessing.feature_engineering)\n",
    "importlib.reload(preprocessing.word_embeddings)\n",
    "preprocess = preprocessing.main.Preprocessing()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13427, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Headline</th>\n",
       "      <th>Body ID</th>\n",
       "      <th>Stance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hundreds of Palestinians flee floods in Gaza a...</td>\n",
       "      <td>158</td>\n",
       "      <td>agree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Spider burrowed through tourist's stomach and ...</td>\n",
       "      <td>1923</td>\n",
       "      <td>disagree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>'Nasa Confirms Earth Will Experience 6 Days of...</td>\n",
       "      <td>154</td>\n",
       "      <td>agree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Banksy 'Arrested &amp; Real Identity Revealed' Is ...</td>\n",
       "      <td>1739</td>\n",
       "      <td>agree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Gateway Pundit</td>\n",
       "      <td>2327</td>\n",
       "      <td>discuss</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Headline  Body ID    Stance\n",
       "1   Hundreds of Palestinians flee floods in Gaza a...      158     agree\n",
       "4   Spider burrowed through tourist's stomach and ...     1923  disagree\n",
       "5   'Nasa Confirms Earth Will Experience 6 Days of...      154     agree\n",
       "8   Banksy 'Arrested & Real Identity Revealed' Is ...     1739     agree\n",
       "10                                     Gateway Pundit     2327   discuss"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_stances = pd.read_csv(\"fn_data/train_stances.csv\")\n",
    "train_stances = train_stances.loc[lambda x: x.Stance != \"unrelated\"]\n",
    "print(train_stances.shape)\n",
    "train_stances.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1683, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Body ID</th>\n",
       "      <th>articleBody</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>Last week we hinted at what was to come as Ebo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>(NEWSER) – Wonder how long a Quarter Pounder w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>Posting photos of a gun-toting child online, I...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>At least 25 suspected Boko Haram insurgents we...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Body ID                                        articleBody\n",
       "0        0  A small meteorite crashed into a wooded area i...\n",
       "1        4  Last week we hinted at what was to come as Ebo...\n",
       "2        5  (NEWSER) – Wonder how long a Quarter Pounder w...\n",
       "3        6  Posting photos of a gun-toting child online, I...\n",
       "4        7  At least 25 suspected Boko Haram insurgents we..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_bodies = pd.read_csv(\"fn_data/train_bodies.csv\")\n",
    "print(train_bodies.shape)\n",
    "train_bodies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10589, 3), (2838, 3))"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stances_tr, stances_val = preprocess.train_test_split(train_bodies, train_stances)\n",
    "stances_tr.shape, stances_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({'discuss': 1926, 'agree': 733, 'disagree': 179}) Counter({'discuss': 6983, 'agree': 2945, 'disagree': 661})\n",
      "0.678646934460888\n"
     ]
    }
   ],
   "source": [
    "ct,ct2 = Counter(stances_val['Stance']),Counter(stances_tr['Stance'])\n",
    "print(ct, ct2)\n",
    "print(ct.most_common(1)[0][1]/len(list(stances_val[\"Stance\"])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "glove_dict = preprocess.get_glove_dict(\"glove.6B.50d\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# [(nltk.pos_tag([x]),preprocess.get_sentiment(x)) for x in preprocess.get_clean_tokens(list(stances_tr.iloc[2,:])[0], False)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# preprocess.cosine_similarity(glove_dict['reveal'], glove_dict['revealed'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "disagrees = stances_tr[stances_tr[\"Stance\"]==\"disagree\"]\n",
    "stances_tr = pd.concat([stances_tr, disagrees, disagrees]).sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'agree': 2945, 'disagree': 1983, 'discuss': 6983})"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(stances_tr['Stance'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_word_stance(word, glove_dict):\n",
    "    #50d word vector\n",
    "    if word in glove_dict:\n",
    "        wv = glove_dict[word]\n",
    "    else:\n",
    "        wv = np.zeros((50, ))\n",
    "    #4d sentiment\n",
    "    sent = preprocess.get_sentiment(word)\n",
    "    #16d one-hot encoding of part of speech (shortened)\n",
    "    pos = nltk.pos_tag(word)[1][0]\n",
    "    pos_encoding = [(1 if tag == pos else 0) for tag in preprocess.pos_short]\n",
    "    #boolean flag for negating word\n",
    "    stemmed_word = preprocess.stem_word(word)\n",
    "    is_neg = (1 if stemmed_word in preprocess.negating_words_stemmed else 0)\n",
    "    is_refuting = (1 if stemmed_word in preprocess.refuting_words_stemmed else 0)\n",
    "    embedding = np.concatenate([wv, [sent[\"pos\"], sent[\"neg\"], sent[\"neu\"], sent[\"compound\"], is_neg, is_refuting], pos_encoding])\n",
    "    return embedding\n",
    "\n",
    "def process_text_stance(text, glove_dict, n_words = 20):\n",
    "    tokens = preprocess.get_clean_tokens(text, False)\n",
    "    if len(tokens)>=n_words:\n",
    "        tokens = tokens[:n_words]\n",
    "        encoding = np.array([process_word_stance(token, glove_dict) for token in tokens])\n",
    "    elif len(tokens)<n_words:\n",
    "        padding = [np.zeros((72,))]*(n_words-len(tokens))\n",
    "        encoding = np.array([process_word_stance(token, glove_dict) for token in tokens]+padding)\n",
    "    return encoding\n",
    "\n",
    "def process_bodies_stance(df, glove_dict):\n",
    "    body_info = {}\n",
    "    ids = list(df[\"Body ID\"])\n",
    "    for i in range(len(ids)):\n",
    "        if i % 100 == 0 and i != 0:\n",
    "            print(\"processed \"+str(i))\n",
    "        body_info[ids[i]] = process_text_stance(preprocess.get_body(ids[i],df), glove_dict, 40)\n",
    "    print(\"done! processed \" + str(len(ids)))\n",
    "    return body_info\n",
    "\n",
    "def process_feats_stance(data, body_dict, glove_dict):\n",
    "    headline, body_id = data[0], int(data[1])\n",
    "    padding = [np.zeros((72,))]*(1)\n",
    "    return np.concatenate([process_text_stance(headline, glove_dict), np.array(padding), body_dict[body_id]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed 100\n",
      "processed 200\n",
      "processed 300\n",
      "processed 400\n",
      "processed 500\n",
      "processed 600\n",
      "processed 700\n",
      "processed 800\n",
      "processed 900\n",
      "processed 1000\n",
      "processed 1100\n",
      "processed 1200\n",
      "processed 1300\n",
      "processed 1400\n",
      "processed 1500\n",
      "processed 1600\n",
      "done! processed 1683\n"
     ]
    }
   ],
   "source": [
    "body_dict = process_bodies_stance(train_bodies, glove_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "train_feats = [process_feats_stance(i, body_dict, glove_dict) for i in stances_tr.values]\n",
    "val_feats = [process_feats_stance(i, body_dict, glove_dict) for i in stances_val.values]\n",
    "end = time.time()\n",
    "print(int(end-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_batch(data, targets, i,batch_size):\n",
    "    batches = data[i*batch_size:i*batch_size+batch_size]\n",
    "    results = targets[i*batch_size:i*batch_size+batch_size]\n",
    "    results = [(2.0 if result == \"agree\" else (1.0 if result == \"discuss\" else 0.0)) for result in results]\n",
    "     batches = np.array(batches)\n",
    "    return np.swapaxes(batches, 0, 1), np.array(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Test the Model\n",
    "def eval_model(model):\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    batch_x_test,batch_y_test = get_batch(val_feats,[str(x[-1]) for x in stances_val.values],0,len(stances_val))\n",
    "    model.eval()\n",
    "    predicted = None\n",
    "    with torch.no_grad():\n",
    "        inputs = Variable(torch.FloatTensor(batch_x_test))\n",
    "        labels = torch.LongTensor(batch_y_test)\n",
    "        outputs = model(inputs)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum()\n",
    "\n",
    "    print('Accuracy: %d %%' % (100 * correct / total))\n",
    "    return predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def score_model(predictions):    \n",
    "    #use FNC scorer to generate score report\n",
    "    label_prediction = [(\"agree\" if x == 2 else (\"discuss\" if x == 1 else \"disagree\")) for x in predictions]\n",
    "    label_actual = pd.DataFrame(stances_val)['Stance']\n",
    "    matrix = confusion_matrix(label_actual,label_prediction)\n",
    "    print('confusion matrix: \\n{}\\n'.format(matrix))\n",
    "    score.report_score(label_actual, label_prediction)If \n",
    "    return matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def binary_accuracy(preds, y):\n",
    "    \"\"\"\n",
    "    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n",
    "    \"\"\"\n",
    "    correct = (preds == y).float() #convert into float for division \n",
    "    acc = correct.sum()/len(correct)\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# LSTM\n",
    "class RNN_LSTM(nn.Module):\n",
    "    def __init__(self, embedding_dim, hidden_dim, output_dim, n_layers, dropout):\n",
    "        super().__init__()\n",
    "        self.rnn = nn.LSTM(embedding_dim, hidden_dim, num_layers=n_layers, bidirectional=True, dropout=dropout)\n",
    "        self.fc = nn.Linear(hidden_dim*2, hidden_dim)\n",
    "        self.fc2 = nn.Linear(hidden_dim, output_dim)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        #x = [sent len, batch size, emb dim]\n",
    "        output, (hidden, cell) = self.rnn(x)\n",
    "        #output = [sent len, batch size, hid dim * num directions]\n",
    "        #hidden = [num layers * num directions, batch size, hid dim]\n",
    "        #cell = [num layers * num directions, batch size, hid dim]\n",
    "        #concat the final forward (hidden[-2,:,:]) and backward (hidden[-1,:,:]) hidden layers\n",
    "        #and apply dropout\n",
    "        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1))\n",
    "        #hidden = [batch size, hid dim * num directions]\n",
    "        fc = self.fc(hidden.squeeze(0))\n",
    "        fc2 = self.fc2(F.relu(fc))\n",
    "        return fc2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# GRU\n",
    "class RNN_GRU(nn.Module):\n",
    "    def __init__(self, embedding_dim, hidden_dim, output_dim, n_layers, dropout):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.rnn = nn.GRU(embedding_dim, hidden_dim, num_layers=n_layers, bidirectional=True, dropout=dropout)\n",
    "        self.fc = nn.Linear(hidden_dim*2, hidden_dim)\n",
    "        self.fc2 = nn.Linear(hidden_dim, output_dim)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        output, hidden = self.rnn(x)\n",
    "        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1))\n",
    "        fc = self.fc(hidden.squeeze(0))\n",
    "        fc2 = self.fc2(F.relu(fc))\n",
    "        return fc2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 437,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#CNN, GRU\n",
    "class Combined(nn.Module):\n",
    "    def __init__(self, embedding_dim, n_filters, filter_size, hidden_dim, n_layers, output_dim, dropout):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.conv = nn.Conv2d(in_channels=1, out_channels=n_filters, kernel_size=(filter_size,embedding_dim))\n",
    "        self.rnn = nn.GRU(embedding_dim, hidden_dim, num_layers=n_layers, bidirectional=True, dropout=dropout)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        h = hidden_dim*2 + n_filters\n",
    "        self.fc = nn.Linear(h, int(h/2))\n",
    "        self.fc2 = nn.Linear(int(h/2), output_dim)\n",
    "        \n",
    "    def forward(self, x): \n",
    "        # RNN\n",
    "        output, hidden = self.rnn(x)\n",
    "        cat1 = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1))\n",
    "        \n",
    "        # CNN\n",
    "        x_ = torch.transpose(x, 0, 1).unsqueeze(1)\n",
    "        conved = F.relu(self.conv(x_)).squeeze(3)\n",
    "        pooled = F.max_pool1d(conved, conved.shape[2]).squeeze(2)\n",
    "        \n",
    "        cat2 = torch.cat((cat1, pooled), dim=1)\n",
    "        fc = F.relu(self.fc(cat2))\n",
    "        fc = self.fc2(fc)\n",
    "        return fc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 438,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_epochs = 8\n",
    "batch_size = 250\n",
    "\n",
    "EMBEDDING_DIM = 72\n",
    "OUTPUT_DIM = 3\n",
    "DROPOUT = 0.2\n",
    "N_FILTERS = 100\n",
    "FILTER_SIZE = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dannyyang/anaconda3/lib/python3.6/site-packages/torch/nn/modules/rnn.py:38: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n"
     ]
    }
   ],
   "source": [
    "# model1 = RNN_GRU(EMBEDDING_DIM, 128, OUTPUT_DIM, 1, DROPOUT)\n",
    "# opt1 = torch.optim.Adam(model1.parameters(), lr=2e-4)\n",
    "# model2 = RNN_GRU(EMBEDDING_DIM, 256, OUTPUT_DIM, 1, DROPOUT)\n",
    "# opt2 = torch.optim.Adam(model2.parameters(), lr=2e-4)\n",
    "# model3 = RNN_GRU(EMBEDDING_DIM, 128, OUTPUT_DIM, 2, DROPOUT)\n",
    "# opt3 = torch.optim.Adam(model3.parameters(), lr=2e-4)\n",
    "# model4 = RNN_GRU(EMBEDDING_DIM, 256, OUTPUT_DIM, 2, DROPOUT)\n",
    "# opt4 = torch.optim.Adam(model4.parameters(), lr=2e-4)\n",
    "model1 = Combined(EMBEDDING_DIM, N_FILTERS, FILTER_SIZE, 128, 1, OUTPUT_DIM, DROPOUT)\n",
    "opt1 = torch.optim.Adam(model1.parameters(), lr=2e-4)\n",
    "\n",
    "m1 = model1, opt1\n",
    "# m2 = model2, opt2\n",
    "# m3 = model3, opt3\n",
    "# m4 = model4, opt4\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "queue = [m1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train(model, total_batch, optimizer, criterion):\n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    model.train()\n",
    "    for i in range(total_batch):\n",
    "        batch_x,batch_y = get_batch(train_feats, [str(x[-1]) for x in stances_tr.values],i,batch_size)\n",
    "        inputs = Variable(torch.FloatTensor(batch_x))\n",
    "        labels = Variable(torch.LongTensor(batch_y))\n",
    "        optimizer.zero_grad()\n",
    "        predictions = model(inputs)\n",
    "        loss = criterion(predictions, labels)\n",
    "        _, predicted = torch.max(predictions.data, 1)\n",
    "        acc = binary_accuracy(predicted, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "    return epoch_loss / total_batch, epoch_acc / total_batch\n",
    "\n",
    "def evaluate(model, total_batch, criterion):\n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for i in range(total_batch):\n",
    "            batch_x,batch_y = get_batch(val_feats, [str(x[-1]) for x in stances_val.values],i,batch_size)\n",
    "            inputs = Variable(torch.FloatTensor(batch_x))\n",
    "            labels = Variable(torch.LongTensor(batch_y))\n",
    "            predictions = model(inputs)\n",
    "            loss = criterion(predictions, labels)\n",
    "            _, predicted = torch.max(predictions.data, 1)\n",
    "            acc = binary_accuracy(predicted, labels)\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "    return epoch_loss / total_batch, epoch_acc / total_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 445,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "| Epoch: 01 | Train Loss: 0.929 | Train Acc: 58.57% | Val. Loss: 0.770 | Val. Acc: 68.11% |\n",
      "| Epoch: 02 | Train Loss: 0.852 | Train Acc: 60.89% | Val. Loss: 0.710 | Val. Acc: 72.25% |\n",
      "| Epoch: 03 | Train Loss: 0.777 | Train Acc: 65.96% | Val. Loss: 0.686 | Val. Acc: 71.89% |\n",
      "| Epoch: 04 | Train Loss: 0.691 | Train Acc: 70.60% | Val. Loss: 0.649 | Val. Acc: 72.58% |\n",
      "| Epoch: 05 | Train Loss: 0.603 | Train Acc: 75.18% | Val. Loss: 0.621 | Val. Acc: 73.35% |\n",
      "| Epoch: 06 | Train Loss: 0.522 | Train Acc: 78.97% | Val. Loss: 0.599 | Val. Acc: 74.91% |\n",
      "| Epoch: 07 | Train Loss: 0.453 | Train Acc: 82.52% | Val. Loss: 0.584 | Val. Acc: 75.20% |\n",
      "| Epoch: 08 | Train Loss: 0.398 | Train Acc: 84.93% | Val. Loss: 0.572 | Val. Acc: 75.71% |\n",
      "237\n"
     ]
    }
   ],
   "source": [
    "batches_train= int(len(train_feats)/batch_size)\n",
    "batches_val = int(len(val_feats)/batch_size)\n",
    "\n",
    "for x in queue:\n",
    "    model = x[0]\n",
    "    optimizer = x[1]\n",
    "    print(\"\\n\")\n",
    "    start = time.time()\n",
    "    for epoch in range(num_epochs):\n",
    "\n",
    "        train_loss, train_acc = train(model, batches_train, optimizer, criterion)\n",
    "        valid_loss, valid_acc = evaluate(model, batches_val, criterion)\n",
    "\n",
    "        print(f'| Epoch: {epoch+1:02} | Train Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}% | Val. Loss: {valid_loss:.3f} | Val. Acc: {valid_acc*100:.2f}% |')\n",
    "    end = time.time()\n",
    "    print(int(end-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 446,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 75 %\n"
     ]
    }
   ],
   "source": [
    "# Test the Model\n",
    "predicted = eval_model(model1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 447,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({0: 204, 1: 1967, 2: 667})"
      ]
     },
     "execution_count": 447,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted_l = [i.item() for i in list(predicted)]\n",
    "Counter(predicted_l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 448,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix: \n",
      "[[ 422   50  261]\n",
      " [  60   67   52]\n",
      " [ 185   87 1654]]\n",
      "\n",
      "-------------------------------------------------------------\n",
      "|           |   agree   | disagree  |  discuss  | unrelated |\n",
      "-------------------------------------------------------------\n",
      "|   agree   |    422    |    50     |    261    |     0     |\n",
      "-------------------------------------------------------------\n",
      "| disagree  |    60     |    67     |    52     |     0     |\n",
      "-------------------------------------------------------------\n",
      "|  discuss  |    185    |    87     |   1654    |     0     |\n",
      "-------------------------------------------------------------\n",
      "| unrelated |     0     |     0     |     0     |     0     |\n",
      "-------------------------------------------------------------\n",
      "Score: 2316.75 out of 2838.0\t(81.63319238900634%)\n",
      "Normalized confusion matrix\n",
      "[[0.57571623 0.06821282 0.35607094]\n",
      " [0.33519553 0.37430168 0.29050279]\n",
      " [0.096054   0.04517134 0.85877466]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAEYCAYAAAApuP8NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VNXWx/HvLwkdpIYWpCpdqXZUUBGRIir23rjy2rte\nC17s117vVaxYwa7YFVHhioCAUhQV6UgHRXrCev84J2ESQmaAYWaSrI/PPM7MObPP3pOwsvc+5+wl\nM8M550qztGRXwDnnks0DoXOu1PNA6Jwr9TwQOudKPQ+EzrlSzwOhc67U80BYiki6VdJL4fOGkv6W\nlB7nY8yWdEQ8y4zhmAMlLQ7bU3MnyvlbUtN41i1ZJE2T1DXZ9SguPBDGURgEFkuqFPHe+ZJGJbFa\nhTKzuWZW2cxykl2XnSGpDPAAcGTYnuU7Wlb4+d/jV7v4k/S8pNuj7WdmbcxsVAKqVCJ4IIy/DOCy\nnS1EAf/5RFcHKA9MS3ZFUoGkjGTXoTjyf2jxdy9wtaRqhW2UdKCk8ZL+DP9/YMS2UZLukDQGWAs0\nDd+7XdL/wqHb+5JqSnpZ0l9hGY0jynhY0rxw2/eSDt5GPRpLMkkZkg4Iy859rJc0O9wvTdL1kmZK\nWi5puKQaEeWcIWlOuO3Gor4YSRUk3R/u/6ek0ZIqhNv6hsO5VWGbW0V8brakqyX9GH5umKTykpoD\nM8LdVkkaGdmuAt/r+eHzPSR9FZazTNKwiP1M0h7h86qShkpaGtb3ptw/TJLODut+n6SVkmZJ6llE\nu2dLuias/xpJz0iqI+kjSaslfS6pesT+r0taFNbxa0ltwvcHAKcB1+b+LkSUf52kH4E14c80b4pC\n0oeS7o8of5ikZ4v6WZU6ZuaPOD2A2cARwFvA7eF75wOjwuc1gJXAGQQ9x1PC1zXD7aOAuUCbcHuZ\n8L3fgGZAVWA68Et4nAxgKPBcRB1OB2qG264CFgHlw223Ai+FzxsDBmQUaEPuMe8KX18OjAUaAOWA\nJ4FXw22tgb+BQ8JtDwDZwBHb+H4eD8vOAtKBA8PPNQfWAN3D418btrlsxPc6Dqgffoc/ARcW1o7C\n2hUe8/zw+avAjQSdgPJAl4j9DNgjfD4UeBeoEpb5C3BeuO1sYBNwQdiOgcBCQEX8Xowl6L1mAUuA\niUCHsP0jgUER+58bHrcc8BAwOWLb84S/WwXKnwzsDlSI/F0Mn9cNj3kYQSD9HaiS7H8vqfRIegVK\n0oMtgbAt8CeQSf5AeAYwrsBnvgXODp+PAgYX2D4KuDHi9f3ARxGv+0T+QymkTiuBduHzW4keCP8D\nfACkha9/Ag6P2F4vDAIZwC3AaxHbKgEbKSQQhoFnXW5dCmy7GRheYN8FQNeI7/X0iO3/Bv5bWDsK\naxf5A+FQ4CmgQSH1MGAPguC2AWgdse0fET/Hs4HfIrZVDD9bt4jfi9MiXr8J/Cfi9SXAO9v4bLWw\n7Krh6+cpPBCeW9jvYsTr44B5wDIigr8/gocPjXcBM5sKjACuL7CpPjCnwHtzCHoJueYVUuTiiOfr\nCnldOfeFpKsk/RQOq1YR9CJrxVJvSf8AugKnmtnm8O1GwNvhkHUVQWDMIejd1I+sr5mtAbZ1sqIW\nQQ9sZiHb8n0v4bHnkf97WRTxfC0Rbd5O1wICxoVD8XO3Udey5P9ZFfw55dXHzNaGT4uqU0w/Q0np\nku4OpyL+IghouXUqSmG/N5FGEAT4GWY2Osq+pY4Hwl1nEMHQKfIfz0KCwBKpIUHvJ9cOLwcUzgde\nB5wIVDezagQ9U8X42duAY8zsz4hN84CeZlYt4lHezBYAfxAMx3LLqEgwLC/MMmA9wRC/oHzfiySF\n5S4oZN9o1oT/rxjxXt3cJ2a2yMwuMLP6BL28J3LnBQvUdRP5f1YFf067yqnAMQQji6oEPVzY8jPc\n1u9HtN+bOwj+iNWTdMpO1rHE8UC4i5jZb8Aw4NKItz8Emks6NZzQPolgnm1EnA5bhWCObimQIekW\nYLdoH5K0e1jXM83slwKb/wvcIalRuG+mpGPCbW8AvSV1kVQWGMw2fqfCXt6zwAOS6oc9nwMklQOG\nA70kHa7gcpirCIam/9uu1gfHWUoQsE4Pj3EuEcFX0gmSGoQvVxIEkJwCZeSEdbpDUpWw7VcCL21v\nfXZAFYK2LycI5ncW2L4Y2K5rHSUdApwDnBk+HpWUVfSnShcPhLvWYIJ5MwAsuMatN8E/9OUEw7Te\nZrYsTsf7BPiIYGJ/DkEPLNqQCeBwgl7TG9py5jj3cpSHgfeATyWtJpj03y9szzTgIuAVgt7hSmB+\nEce5GpgCjAdWAPcQzEXOIDjJ8yhBb6wP0MfMNsbY7oIuAK4h+I7bkD+g7gN8J+nvsF2XmdmsQsq4\nhKB3+TswOmxjIs60DiX42S0gODE2tsD2Z4DW4VTFO9EKk7RbWObFZrYgHBY/AzwX9rwd4Vku55wr\nzbxH6Jwr9TwQOudKPQ+EzrlSzwOhc67U8xu0Y1CmUjUrV6Nu9B1LqD1r7+i1y8XfyrU7euK6ZJjz\n85RlZpYZj7LSd2tklr0u6n62buknZnZUPI4ZKw+EMShXoy7tLxuS7GokzYcXH5TsKiTN8B9iufqo\n5Lpg/8YF74TaYZa9jnItToy63/rJj8d0J1Q8eSB0ziWGBGlxXQc4bjwQOucSJ0WX2PRA6JxLnBS9\nmcUDoXMuQXxo7Jwr7YQPjZ1zpZ18aOyccz40ds6VcvKhsXOulBMpOzROzfDsnCuBBGkZ0R+xlCQd\nJWmGpN8kFcwNhKSGkr6UNClMo3p0UeV5IHTOJU6aoj+ikJROkBq2J0Gqi1MktS6w200EmRE7ACcD\nTxRZrR1qjHPOba/cy2eiPaLblyCd6u9hOofXCBJeRTK25OupSpAgbJt8jtA5lyAxX1BdS9KEiNdP\nmdlTEa+zyJ+LZz5hHp0ItxLk2bmEIG/QEUUd0AOhcy5xYjtZsszMOhdVSiHvFUy+dArwvJndL+kA\n4EVJbSPydefjgdA5lzjxuXxmPhH5tIEGbD30PQ84CsDMvpVUHqgFLCmsQJ8jdM4lRu4yXNEe0Y0H\n9pTUJMynfTJBatZIcwnS1CKpFVCeIN93obxH6JxLnDhcR2hm2ZIuJsjjnQ48a2bTJA0GJpjZewS5\nw4dIuoJg2Hy2FZG72AOhcy5B4ndniZl9CHxY4L1bIp5PB2JeWt0DoXMuMYTfa+ycK+38XmPnnPMe\noXPOpeqiCx4InXOJIR8au9B+jatz2WFNSZMYMWURL42bn297zza1+b9Dm7Ls7w0AvDlpISOmLKbD\n7lW5tFvTvP0a1qjIrSN+5pvflgMwoEsjujWvRY7BO5P/4I1JRd5amTSff/ox1119BTk5OZx59nlc\nec11+bZv2LCBf5x3FpMnTaRGjZo899KrNGrUGICpU37k8osHsnr1X6SlpfHl6O8oX748bwx7lfvv\nvRtJ1K1XjyHPvkjNWglPjRvV1G9H8dqDg9m8OYeD+55EzzP/L9/2UW+9xKg3X0RpaZSvUIkzbriL\n+k32ZNnCedxyyhHUaRj8/Ju27cAZ190JQPamjbxy3yBmTBxLWpro949r6HRYz4S3LVZK80BY6qUJ\nrjyiGVe8PpUlqzfw9OntGT1zBbOXr82338gZS3nwi5n53ps070/OGToJgCrlMxh2XmfGzV4JwNFt\n61C7SjlOffZ7DKhWsUxC2rO9cnJyuOryS3jng0/IympAty77cXTvPrRstWXhkKHPP0u16tWZPO0X\n3hj+GoNuvJ7nX3qN7OxsBpx7Jk8+8wJ77d2OFcuXU6ZMGbKzs7numisYN3EqNWvV4uZ/XsdT/32c\nG24alMSWbm1zTg6v3HcLVzzyEtVr1+WOc/rS7uDu1G+yZ94++/U4hq7HnQ7A5K8/Y/jDt3H5Q0MB\nyMxqxKAXP9qq3A+ef4wq1Wtyx+tfsnnzZtb8tSoxDdoBwXKEqTk0Ts3wXEK1qluF+SvXs/DP9WRv\nNj7/eSldmtXY7nK6Na/F2Fkr2ZAd3DbZr109nvt2bt7NlqvWbopjrePn+/HjaNqsGU2aNKVs2bIc\nd8JJfDAi/w0BH454l1NPOxOAfsf156tRIzEzRn7+KW3a7sVee7cDoEbNmqSnp2NmmBlr1qzBzFi9\n+i/q1quX8LZFM2v6ZDIbNCIzqyEZZcqyT/c+TP7603z7VKhUJe/5hvVrUaG31OY35v3XOfqsoGeZ\nlpZGlWrb//uUMIrxkQTeI0ygzCrlWLJ6Q97rpX9vpHW9Klvtd+ietWjXoCrzVq7j0S9nsmT1xnzb\nD2+ZybAJC/JeZ1Urz+EtMjlkz5qsWreJh76YyfxV63ddQ3bQwoULyGqw5RbRrKwsJowbl2+fPxYu\nzNsnIyOD3Xaryorly/nt11+RxLF9jmLZsmUc3/8kLr/qGsqUKcMDDz/Ogfu0o2KlSjRrtif3P/RY\nQtsVi1VLF1Ojdv2819Vr12PWtMlb7fflG0P57NWnyd60iaseeyXv/WUL5zH4zKOpUKkyx/zjapq3\n35e1q/8E4J0n7+eXiWPJbNCIU6/6F7vVzNz1DdohIi1Fh8apWStA0rGSTFLLZNclXgpdMqPATT9j\nZq7ghCHjOPuFiUyYs5Ibe7bIt71mpTI0rVWJ78JhMUCZ9DQ25mzm/Jcm896Pi7jhqOa7oPY7r7A7\nnAoOlba1T3Z2Nt/+bwxPP/cSn3zxNSPee4dRX37Bpk2beGbIk3w99ntm/D6fNm334oF7795lbdhR\nhd/dtfVvRLf+Z3Lnm19z/EXX88HzjwJQtVZt7nn3f9wy9ENOvOxmnr7lMtatWU1OTg4rl/zBHnt3\n5uahH9C0bUdef/TOXdySnSMp6iMZUjYQEiyjM5rghuqYhavXpqQlqzdQu0q5vNeZlcvmnRTJ9df6\nbDblBP9o3v9xES3qVM63/bAWmXzz6zJyNm/5h7V09QZG/bIMgK9/XU6zzEq7qgk7JSurAQvmb1lG\nbsGCBdStXz/fPvWzsvL2yc7O5q+//qR6jRrUz8qiy8GHULNWLSpWrMiRR/Xkh0mT+PGHoFfVtGmz\noMfY/wS+G/u/xDUqRtVr12XFki0nsFYu+YNqmbW3uf8+3fsw+avPAChTthyVq1YHoFHLvcjMasji\nubOoXLU6ZctXoEPXHgB0Pvxo5syYugtbsfM8EG4HSZUJ7hM8jzAQSkqT9ISkaZJGSPpQUv9w22xJ\nt0gaDZwgqZmkjyV9L+mb3F6lpExJb0oaHz5ivhcxHn5etJrdq5enXtVyZKSJI1pmMmbminz71Ky0\n5URHl2Y1mVPgRMoRLTP57Of8i2h889tyOjWsBkCH3YMhdSrq2HkfZv72G7Nnz2Ljxo289fowju7V\nJ98+R/fqyysvBycI3nnrDQ45tBuSOLx7D6ZOncLatWvJzs5m9Ddf07JVK+rXz2LGz9NZtjT4Tr78\n4nNatGiV8LZF07hVO5bMm83ShfPI3rSR8Z+9T7uDu+fbZ/HcWXnPp4wZSe3dGwOweuVyNufkALB0\nwVyWzJ9NZv2GSKJdl8OZMXEsAD+NH5Pv5EuqkYTSoj+SIVXnCPsBH5vZL5JWSOoINAUaA3sBtYGf\ngGcjPrPezLoASPoCuNDMfpW0H0G+gsOAh4EHzWy0pIYEq1cU+q9G0gBgAEC5anXi0qgcgwe+mMkD\nx7clLU18MGUxs5av5byDGvHzotWMmbmC/h2z6NKsBjmbjb/WZ3PHx7/kfb7ubuWoXaUck+f9ma/c\nl8bN45ZeLTmxUxbrNuVwzye/xqW+8ZaRkcF9Dz7CcX16kpOTw+lnnUOr1m24Y/AgOnTsxNG9+3LG\n2ecy4Nwzad+mOdWr1+DZF4N5surVq3PxpZfTrct+SKJ7j5706NkLgOv+eTM9u3elTJky7N6wIf95\n6rlkNrNQ6RkZnHr1YB667Exscw4H9T6RrKbNefepB2jUci/aH9KdL994genjx5CekUGlKlU555b7\nAfhl0jjeHfIA6enppKWlc/q1d1CpavCH7/iLrueZf13JsAcHU6V6Dc6+6d5kNjOqVD1rrCJWpkka\nSR8AD5nZZ5IuJViEsQzwg5k9F+7zFvCKmb0haTZwqJnNCXuTS4EZEUWWM7NWkpaQfwHHTKClma0u\nqj6Vd29p7S8bErf2FTcfXpzQjnNKGf7DvOg7lWAX7N/4+yirRccso2ZT2+3o26Put/Kl0+J2zFil\nXI9QUk2C3ltbSUaw3pgBb0f56Jrw/2nAKjNrX8g+acABZpaaY0fnSjIRt6GvpKMIRnjpwNNmdneB\n7Q8C3cKXFYHaZlZtW+Wl4hxhf2ComTUys8ZmtjswC1gGHB/OFdYBuhb2YTP7C5gl6QQABdqFmz8F\nLs7dV1JhwdI5t4vE42RJLOk8zewKM2sfdogeBd4qqsxUDISnsHXv702gPkGugqnAk8B3wJ8U7jTg\nPEk/ANPYkurvUqBzmPB5OnBhnOvunNsGET0IxjiHGEs6z0inAK8WVWDKDY3NrGsh7z0CwdlkM/s7\nHD6PA6aE2xsX2H8WYeKWAu8vA06Kf62dc7GI09A4lnSewfGkRkATYGRRBaZcIIxihKRqQFngNjNb\nlOwKOedipJjPGkfLaxxLOs9cJwNvmFlOUQcsVoGwsN6ic674iDEQRstrHEs6z1wnAxdFO2CxCoTO\nueJL8bvXOC+dJ7CAINidutXxpBZAdeDbaAWm4skS51xJFYfVZ8wsm+Dqj08IbqwYnpvOU1LfiF1P\nAV4rKo1nLu8ROucSI/Y5wqiipfMMX98aa3keCJ1zCZOqy3B5IHTOJU5q3mrsgdA5lzipuuiCB0Ln\nXEJIqbtCtQdC51zCeI/QOedSMw56IHTOJYj8rLFzrpQL8honuxaF80DonEuQ5CVnisYDoXMuYdKS\nlJwpGg+EzrnEkA+NnXOlnPAeoXPOeSB0zpVyPjR2zpV2weUzqRkJU/PqRudcCSTS0qI/YipJOkrS\nDEm/Sbp+G/ucKGm6pGmSXimqPO8ROucSJh49woi8xt0J8peMl/SemU2P2GdP4AbgIDNbKal2UWV6\nIHTOJYQUt5MleXmNg3KVm9d4esQ+FwCPm9lKADNbUlSBPjR2ziWMFP1BmM4z4jGgQDGF5TXOKrBP\nc6C5pDGSxkraKs95JO8ROucSJk7pPGPJa5wB7Al0JUj3+Y2ktma2qrACPRA65xIjfkPjWPIazwfG\nmtkmYJakGQSBcXxhBXogjEHtKmW5sFuTZFcjaV6YMCfZVUiaZtUqJbsKJUYcV5+JJa/xOwTpPJ+X\nVItgqPz7tgr0QOicS5D4rD5jZtmScvMapwPP5uY1BiaY2XvhtiMlTQdygGvMbPm2yvRA6JxLmHjd\nYhctr3GY1P3K8BGVB0LnXGL4LXbOudIulW+x80DonEsYX33GOVfqeY/QOVe6+Ryhc660E7GvLpNo\n2wyEknYr6oNm9lf8q+OcK8nSUrRLWFSPcBrB/XuRNc99bUDDXVgv51wJlKJxcNuB0Mx239Y255zb\nXhKkp+jQOKZluCSdLOmf4fMGkjrt2mo550oiSVEfyRA1EEp6DOgGnBG+tRb4766slHOuZIpxPcKE\ni+Ws8YFm1lHSJAAzWyGp7C6ul3OuhBGQnqKThLEEwk2S0ggXPpRUE9i8S2vlnCt5kjj0jSaWOcLH\ngTeBTEn/AkYD9+zSWjnnSqRUHRpHDYRmNhS4CbgPWAGcYGav7eqKOedKFhGcNY72iKmsKOk8JZ0t\naamkyeHj/KLKi/XOknRgE8Hw2BM+Oed2SKLSeYaGmdnFsZQZy1njG4FXgfoEuQFekXTDdtXcOVfq\nxTIsjjFO5qXzNLONQG46zx0WS4/wdKCTma0FkHQH8D1w184c2DlX+sR41riWpAkRr58ys6ciXheW\nznO/Qso5XtIhwC/AFWY2r5B9gNgC4ZwC+2VQRBIU55zblgSm83wfeNXMNki6EHgBOGxbBRa16MKD\nYeFrgWmSPglfH0lw5tg552ImIE532EVN51kgUdMQolzpUlSPcGr4/2nABxHvj41aTeecK0hxW4Yr\najpPSfXM7I/wZV/gp6IKLGrRhWd2rq7OOZdfAtN5XiqpL5BNcNnf2UWVGXWOUFIz4A6gNVA+ojLN\nd7QhpdmP//uSF++7lc2bc+ja7xT6nH1Rvu1fvPEin7/+Amnp6ZSvUIlzb7ybrKbNmTl1Es/eGVwu\nZWYcN+AKOnfrCcAVfQ6gfMVKpKWnk56ezuAXP9zquKli+ndf8dbDg9m8eTMH9D6R7qcPzLd99Dsv\n883bL5KWlk65ChU56Zo7qddkT8Z/+g4jXx2St9/CmT9zzTPv02DP1nnvPXX9BSxfOI8bhn6csPZs\njwmjR/LUPTexOSeHI487jRPPvzTf9rdf+C+fvPUy6enpVK1Rk8sHP0Tt+sEI8NkHbmPCN58BcPI/\nruSQo/oB8P4rz/DuS0/xx7zZvPL1dKpWr5nYRm2HOA6NY0nneQMQ89UtsZwseR64neCC6p7AOfgt\ndjtkc04OL9xzE9c9/go16tTjljN70/GQ7mQ13fI35cCj+nF4/2B9i4lffcrLDw7m2kdfosEeLRk8\n9APSMzJYtWwx/zylBx0O7k56RvAj/OeTw6lSrUZS2hWrzTk5vP7AIC56cCjVMuty3wX9aHvQEdRr\nsmfePp2696VLv9MAmDL6c95+7A7+7/7n2efIfuxzZPCPf+HMnxlywz/yBcEfvvqYchUqJrZB2yEn\nJ4f/3HE9tz81nFp163PFyT3Yv1sPGjZrkbdP01Zteei1TyhfoSIfDHueZx8YzPX3DWHc158x86cf\nefT1kWzauIHrzjmWzl0Op2LlKrTusC/7Htqd6889Lomti12qLsway8XRFc3sEwAzm2lmNxGsRuO2\n08xpk6mze2NqN2hERpmy7H9kX77/6tN8+1SoXCXv+YZ1a/OGEuXKV8gLehs3bEjZezaLMuenH8jM\nakSt+g3JKFOWjof3Zsroz/LtU6HSlvZvXL+20HZ+//n7dDqiT97rDWvX8OWwZzjyzJiunU2KX6ZM\npH7DJtTbvTFlypTlkJ79GPtl/p5ru327UD4M5i337sSyxcEU17yZv7BX5wNIz8igfMVKNGnRmu9H\njwSgWau9qJNVPNZIloJAGO2RDLH0CDco+G2cGZ6GXgDU3rXVKplWLllEjTr1817XqF2PmVMnbbXf\nZ8Of5+OXh5CdvYkb/jMs7/3fpk7i6cFXs+yP+Vw4+KG8wIjEPRedhiS6HXcahx132i5vy45YtXQR\n1WrXy3tdLbMec36avNV+X781lC+HPUtO9iYufuilrbZPHPkBF9z1ZN7rD55+gG4nn0/Z8hV2TcXj\nYPmSRdSqu+VnX6tOfWb8OHGb+3/61it07hJc7dGkRRte+c999DvzQjasX8eP48bk60kWJ6n69zuW\nHuEVQGXgUuAg4ALg3GgfkpQT3uM3TdIPkq4MV7FBUmdJj+xMxYsj2+pSp8Inj7ufeDb3vzuGky65\ngXef2fI17dG2A3cP/4J/DR3B+889zsYN6wG45Zm3uP3lj7j6kaF8/voL/Dyx+JzYVyGXhB1y3JkM\nGjaKvhdey6dDH8+3bfa0yZQtX576TYNAMP/X6SxdMId2h/RISH13lNnWP/tCr4YDRr7/Br9On8zx\n5wTzxx0P7Erngw/n6jN68+9rL6RVu86kp6fvwtruOmlpivpISr2i7WBm35nZajOba2ZnmFlfMxsT\nQ9nrzKy9mbUhuCfwaGBQWOYEM7u0yE/vBEkpmZ2vRu16rFi85XKnFUv+oFpmnW3uv/+Rx/D9qE+2\nej+ryZ6Uq1CR+TNnAFA9sy4AVWvUonPXo5g5beteViqollmXVUv+yHu9aukf7FZr24OLjof34cdv\n8k8dTPzifTodvmVYPGvqRObNmMqtJxzMQxedyJJ5s3jkklPiX/mdVKtOPZYt2vKzX7Z4ITVr191q\nv0nffsWwIQ9xyyNDKVO2XN77Jw+4gsfeGMkdQ17HzKjfsGlC6h1PIvqwOFlD420GQklvS3prW4/t\nOYiZLQEGABcr0FXSiPA4h0asEDFJUhVJlSV9IWmipCmS8u4jlHSzpJ8lfSbpVUlXh++PknSnpK+A\nyyRlSnpT0vjwcVC4XyVJz4bvTYose1dr2rodi+bNZsmCuWRv2sjYT9+j4yHd8+2zaO6svOeTR39B\n3YaNAViyYC452dkALPtjPn/MmUlm/d1Zv24t69b8DcD6dWuZ8t3X7J6iw6aGLfdm6fzZLF84j+xN\nG5n4xQj26nJEvn2WzNvS/mnffklmg8Z5rzdv3sykUR/RMWJ+8OBjT+f2d8Zy6+vfcPnjw6m9exMu\nffTVXd6W7dW8bQcWzPmdRfPnsGnTRr7+6B3265q/Fzvzpyk8Nvgabnl0KNVqZua9n5OTw1+rVgAw\na8Y0Zv86nY4Hdk1k9eNDqdsjLKrn9Fg8D2Rmv4dD44JdgKuBi8xsjKTKwPrw/WPN7C9JtYCxkt4D\nOgHHAx3Cuk8kuO85VzUzOxRA0ivAg2Y2WlJDgmuOWgE3AiPN7FxJ1YBxkj43szWRlZI0gCB4U7Nu\nVly+g/SMDM685jbuveR0NufkcEjfk2jQrAVv/vc+mrTam46HHslnw59n2rjRpGdkUKlKVQbc+iAA\nv0wez4gXniA9IwMpjbOuv4Mq1WqwZP4cHrrmAiA4K3tAj2PY+8DUPJeVnpFB/ytu5YmrzmLz5s3s\n3+sE6jVpzgdPP0jDlnuxV5cj+OatF5kxYQzpGRlUqFKV02+8L+/zM38YR7XMutSqXzxODkRKz8hg\n4D/v4uYLT2ZzTg7djz2FRnu05MXH7mHPNu3Yv9tRPHP/v1i/dg13XRWsGJVZL4tBj75ITvYmrj0r\n+HtdsXJlrrrribz54fdeHsIbzz7OyuVLuPj4bnQ++HAu+9eDSWtnNKm6dJUKnbuIR8HS32ZWucB7\nq4AWBAHpajPrHa4ldizwMvCWmc2XVAZ4EDiE4FKdFkATgivIq5vZoLC8B4CFZnafpFHAIDP7Kty2\nhPy33WQCLYEvCa6HzA7frwH0MLNtXnnetPXelsrX5u1qK9dtSnYVkqZZtUrJrkJS9dqrzvdR7vuN\nWZ092tpF8SmQAAAeMklEQVRJ970Rdb9Hj20Vt2PGKmFzaZKaAjnAEoJACICZ3S3pA4I5xLGSjgD2\nJwhcncxsk6TZBMErWr85sleXBhxgZusK1EPA8WY2Yyeb5JzbThkp2iVMSLUkZRJkvnvMCnRBJTUz\nsylmdg8wgaDXVhVYEgbBbkCjcPfRQB9J5cNhdK8iDvspkHdhmaT24dNPgEvCgIikDjvfQudcNMF6\ng6mZzjPmHqGkcma2YTvKriBpMlCGYBj6IvBAIftdHga7HGA68BFQBXg/XJNsMvAzgJmND+cKfyBY\nHmwC8Oc2jn8p8LikHwna+TVwIXAb8BDwYxgMZwO9t6NdzrkdlKL53WO613hf4BmCXlpDSe2A883s\nkqI+Z2bbvNDJzEYBo8LnhZWzAThgGx+/z8xulVSRILjdH5bTtcAxlgEnFXLsdcA/iqq7cy7+cnOW\npKJYhsaPEPSYlgOY2Q8k9xa7p8Ke5kTgTTPb9uX5zrmUkhbDIxliGRqnmdmcAmP3nF1Un6jM7NTo\neznnUlGq3mIXSyCcFw6PTUH2qEsIcgA451zMpNjTdSZaLD3RgcCVQENgMcGlLQOL/IRzzhUiTdEf\nsVCUvMYR+/WXZJKKvC4xao8wvD3u5Niq55xzhQsWZk1cXmNJVQiuHvkuWpmxnDUewtYZojCzATHW\n2znnQJAen7MheXmNASTl5jUumOD9NuDfBLfxFimWan0OfBE+xhDcK7w91xM65xwQrEAT7T/CvMYR\nj4KdrsLyGudbECC8UWJ3MxsRS71iGRoPi3wt6UXgs23s7pxzhdqOnCU7ldc4XNzlQaIkbIq0I/ca\nN2HLLW/OORezOJ01jpbXuArQFhgVXvZXF3hPUl8zm1BYgbHMEa5kS7RNI0iNt82zNM45V5g4ZrEr\nMq+xmf0J1Mo7brAy1dXbCoIQJRCG9+K2Cw8GsLngognOORcTxeeC6hjzGm+XIgOhmZmkt82s045V\n2TnnAgIy4tQljJbXuMD7XaOVF8tZ43GSOsZUO+ecK4IU/ZEM2+wRSsows2ygC3CBpJkEC5+KoLPo\nwdE5tx1EWtS1lZOjqKHxOKAj0C9BdXHOlWCK3wXVcVdUIBSAmc1MUF2ccyVcstJ1RlNUIMyUdOW2\nNppZYatNO+dcoUTxXIYrHahM9IRJzjkXk1RdhquoQPiHmQ1OWE2ccyWaSN28xlHnCJ1zLi7CLHap\nqKhAeHjCauGcK/EEpBe3QGhmKxJZEedcyZeaYXDHVp9xzrkdkqIdQg+EzrnEECp+Q2PnnIu34niy\nxIV2K1+GHs3rJLsaSVOlQplkVyFpqu9zcbKrUKKkZhhM3ct6nHMljBScNY72iK2sotN5SrpQ0hRJ\nkyWNltS6qPI8EDrnEkZS1EcMZeSm8+wJtAZOKSTQvWJme5lZe4JMdkXeEuyB0DmXMIrhEYO8dJ5m\nthHITeeZx8z+inhZiUJSEkfyOULnXEJsxwXVtSRF5hd5ysyeinhdWDrP/bY6nnQRcCVQFjisqAN6\nIHTOJUyMU4A7lc4z7w2zx4HHJZ0K3AScta0CPRA65xJE8VqPMFo6z4JeA/5TVIE+R+icS4hg9RlF\nfcQgL52npLIE6TzzZa6TtGfEy17Ar0UV6D1C51xiJDad58WSjgA2ASspYlgMHgidcwkUr6X6o6Xz\nNLPLtqc8D4TOuYQQkKILVHsgdM4ljlL0JjsPhM65hCmOWeyccy5ufGjsnHPIh8bOuVJO3iN0zpVy\nwdA4NSOhB0LnXMKkZhj0QOicSyBfqt85V+qlaBz0QOicS5wUjYMeCJ1ziSF8aOycK+3itPrMruCB\n0DmXMCkaBz0QOucSJbYsdcngK1Qn2MjPPuGAjm3Yt10rHnng31tt37BhAxecfSr7tmvFUd0OYu6c\n2QBs3LiRSweez6H7d6DrgZ0Y881XeZ/ZuHEjV106kP07tObATm15/923EtWc7fbpJx+zd5sWtGm5\nB/f+++6ttm/YsIHTTz2JNi334OAD92PO7Nn5ts+dO5da1Srz4AP35b3XYo/GdG6/F/t1as9B+xWV\n6iK5uh/Yih/evpmp7w7i6nO6b7V997rV+fipS/n21esYN+wGenQJMlQ2rFeDFd8+wNjXrmfsa9fz\nyI0n532m/5EdGTfsBr5/40buuOyYrcpMNVL0R2zlRM1rfKWk6ZJ+lPSFpEZFlec9wgTKycnhuqsu\n4/V3P6R+VgOO7HoAPY7uTYuWW1Kyvjz0OapWq864H37i7TeGcdugfzLk+Vd48flnAPhq7CSWLl3C\nKcf34dNR35KWlsaD995FrVqZjJ00nc2bN7Ny5YpkNbFIOTk5XH7pRXzw0WdkNWhAl/33oXfvvrRq\nvaX9zz/7DNWrVWfaz78xfNhr3PjP63jplWF526+9+gqOPKrnVmV//PmX1KpVKyHt2BFpaeKh60+k\n18DHWLB4FaNfvoYRX03h598X5e1z3flH8eZnExny+mhaNq3LO48OpGWvQQD8Pn8Z+5+c/w9HjaqV\nuPPyfhx42r9ZtvJvhgw+g677NmfUuF8S2rZYbUe6zqLL2ZLXuDtB/pLxkt4zs+kRu00COpvZWkkD\nCXIbn7StMr1HmEATJ4ynSdNmNG7SlLJly3Ls8Sfy8Qfv59vn4w/e56RTzgCgT7/j+WbUl5gZv/z8\nEwcf2g2AzMzaVK1ajckTvwfg1Zde4NKrrgMgLS2NmjVTMyCMHzeOZs32oEnToP0nnHQyI95/N98+\nI95/l9POCFZVP+74/owa+QVmQYKy9959hyZNmtK6dZuE131n7dO2MTPnLWP2guVsys7h9U8m0rvr\n3vn2MTN2q1QegKqVK/DH0j+LLLNJVk1+nbuEZSv/BmDkdz/T7/D2u6YBcRKPBO/Eltf4SzNbG74c\nS5DgaZs8ECbQoj8WkNVgy8+jXv0s/li4cJv7ZGRkUGW3qqxYsZw2e+3Nxx++T3Z2NnNmz+KHyRNZ\nsGAef65aBcDdt9/K4Qfvy3lnnsySJYsT1qbtsXDhAho02JJ8LCurAQsWLNh6n92DfTIyMtitalWW\nL1/OmjVruP/ee7jx5kFblSuJPj2P5MB9O/HMkKe22p4K6teuyvzFK/NeL1i8kqzMqvn2uePJDzn5\n6H357ePbePvRgVx5z+t52xpn1eTbV6/j06cv46AOzQCYOW8pLRrXoWG9GqSnp9G3Wzsa1KmemAbt\noBiHxrUkTYh4DChQTGF5jbOKOOx5wEdF1SspQ2NJOcAUoAyQDbwAPGRmmyV1Bs40s0uTUbddKbdn\nE6ngX8BC90GcesbZ/DrjZ7ofuj+7796QffY9gIyMDLJzslm4YD777n8At911L/957CFuvfE6nhjy\n/K5qxg7b4fZL3PavQVxy2RVUrlx5q+0jvxpD/fr1WbJkCb2P6k6Lli3pcvAh8at4HBS2/FTBlp54\nVGdeen8sD784kv32bsIzt59Jp/53smjZXzTveQsr/lxDh1a7M/yBAXTsfwerVq/j0juH8dI957LZ\njLE//E6TrNQcDeSKcWgcl7zGAJJOBzoDhxZ1wGTNEa4zs/YAkmoDrwBVgUFmNgGYUNSHi6t69Ruw\nYP78vNd/LFxA3Xr1Ct2nflYDsrOzWf3Xn1SvUSMIBndvOUFw9BGH0LTZHtSoUZOKFSvSq08/APr2\nO55Xhj6XmAZtp6ysBsyfv+UP+YIF86lfv/7W+8ybR4MGQfv/+vNPatSowfhx3/H2W29w4w3X8ueq\nVaSlpVG+XHkGXnRxXhm1a9emb79jGT9+XMoFwgVLVuXrrWXVqc7CAkPfs/odwDEXPQ7Adz/OonzZ\nMtSqVomlK/9mxZ/ZAEz6aR6/z1/Gno1qM3H6XD78eioffj0VgHOPO4icnM0JatEOUNwuqI4pr3GY\nxe5G4FAz21BUgUkfGpvZEmAAQfo9SeoqaQSApEMlTQ4fkyRVCd+/VtIUST9Iujt8b1TYm0RSLUmz\nw+dtJI0Ly/hR0p6SKkn6IPz8VEnbnESNpw6dOvP7778xZ/YsNm7cyNtvDqfH0b3z7dPj6N4Me/VF\nAN5/5026HNoVSaxdu5Y1a9YAMGrk52RkZNCiZWskceRRvfLOIn/z1Zc0b9kqEc3Zbp332YfffvuV\n2bOC9r8+7DV69e6bb59evfvy8osvAPDWm29waLfDkMQXo75hxm+zmfHbbC6+9HKuuf6fDLzoYtas\nWcPq1asBWLNmDZ9/9ilt2rRNeNuimTBtDns0zKRR/ZqUyUjnhB4d+WDUj/n2mbdoBV33bQFAiyZ1\nKF+uDEtX/k2t6pVJCxfya5xVkz0aZjJr/jIAMqsHPeRqVSow4MSDee7tbxPYqu0T3FkSl7PGseQ1\n7gA8CfQNY0yRUuKssZn9LikNqF1g09XARWY2RlJlYL2knkA/YL/wjFCNKMVfCDxsZi+HX1o6cDSw\n0Mx6AUiqWvBD4bzEAIAGuzfcmeblycjI4O57H+KkY3uRk7OZU884i5at2nD37bfSvmMnjjq6D6ed\neQ4XDTibfdu1onr16jz53EsALFu6hJOO7UVaWhp162fx+FNben03D76Tiwacw03XX0WtWpk8/MSQ\nuNQ33jIyMnjw4cfo06sHOTk5nHX2ubRu04bBt95Cx06d6d2nL2efex7nnn0GbVruQfXqNXjx5deK\nLHPJ4sWc1P9YALJzsjnp5FM5ssdRiWjOdsnJ2cwV9wzn/ScuIj1NvPDuWH76fRE3D+zFxOlz+eCr\nKVz/wNs8cfMpXHJ6N8zggluCP4hdOu7BzQN7kZ2TQ06Occkdr7Hyr+A8wH3X9mev5sH02F1Pfcxv\nc6P+m0+qePQHY8xrfC9QGXg97IXONbO+2ypThc3J7GqS/jazygXeWwW0AFoBV5tZ7/D6oGOBl4G3\nzGy+pPuBn81sSIHPjwo/N0FSLYIvpLGkUwm6x0PDMn6V1JzgSxwOjDCzb4qqb/uOneyzr8bGo+nF\nUpUKZZJdhaSpvs/Fya5CUq2f/Pj3UebrYta2XUd74+PRUfdrVb9S3I4Zq6QPjQEkNQVygHx/zszs\nbuB8oAIwVlJLgj8qhUXvbLa0p3xEGa8AfYF1wCeSDjOzX4BOBCds7pJ0S8HCnHPxF68LquMt6YFQ\nUibwX+AxK9A9ldTMzKaY2T0EJ1BaAp8C50qqGO6TOzSeTRDcAPpHlNEU+N3MHiGYR9hbUn1grZm9\nBNwHdNxV7XPObaEYHsmQrDnCCpIms+XymReBBwrZ73JJ3Qh6i9OBj8xsg6T2wARJG4EPgX8SBLTh\nks4ARkaUcRJwuqRNwCJgMLAPcK+kzcAmYOCuaKRzbgtfhqsAM0svYtsoYFT4/JJt7HM3cHeB934G\nIi/Vvyl8/y7grgJFfBI+nHOJ4stwOeecL8PlnCv1UncZLg+EzrmESdE46IHQOZcYuXeWpCIPhM65\nhCls8YlU4IHQOZcw3iN0zpVugjQPhM45l5qR0AOhcy4h/GSJc86RukPjpC+64JwrPRTDfzGVEz2d\n5yGSJkrKltS/sDIieSB0ziVOHJafiUjn2RNoDZwiqXWB3eYCZxOkAYnKh8bOuYRQ/M4a56XzDMpV\nbjrPvLzGZjY73BZTEhfvETrnEibGoXG803lG5T1C51zixNYjjFs6z1h5IHTOJUychsYxpfPcHj40\nds4lSCwD45giZdR0ntvLA6FzLiHildfYzLKB3HSePwHDc9N5SuoLIGkfSfOBE4AnJU0rqkwfGjvn\nEiZed5aY2YcE+Yoi37sl4vl4giFzTDwQOucSxpfhcs6Vbp68yTlX2vmiC845hw+NnXPOe4TOOeeB\n0DlX6qXq0FhmO3WLXqkgaSkwJ4lVqAUsS+Lxk6k0tx2S3/5GZpYZj4IkfUzQnmiWmdlR8ThmrDwQ\nFgOSJkS5Cb3EKs1tB29/ovgtds65Us8DoXOu1PNAWDw8lewKJFFpbjt4+xPC5widc6We9widc6We\nB0LnXKnngdA5V+p5IHTOlXoeCEsASf5zLIWkVL1zt/jxe42LGUkyM5PUDkgHVprZrNz3k12/ZIn4\nXqoCaWa2Mtl12pUif96SegJLzWxCkqtVbHlPopgJ/7EfBbwKHAH8JGmv0hwEIe97OQYYDrwi6XpJ\n5ZNdr10lIgheBtwBrEhujYo3D4TFiAINgGuBPsBk4HdgceQ+SapeUknal+B7ORWYCPQn6DGXWJIO\nAM4EupjZ75IOlHRcaf0d2Bk+NC5Gwl7PUuAjoDtwDnCMmS2R1B/4xswWF1lIyVWJ4C6MHsBhwAlm\ntkZSMzObmdyqxUch0x+/AxOAJyStBPYG/gZqA/9NQhWLLQ+EKS5i7qsasB7YCHQJH/XMbKOkzsB1\nwC9E9A5LskKCwt/AuQSjnBPNbJ6kXsAVkk4kmEstttMHBeYELwXWAe8CbwPHAM8DPwOXAGWTVM1i\nywNhiguDYC/gGoJk1ouB04ApwD2S1gC9gUFm9mPyapo4EX8cegGdgIrAIGA80BRoHA6V/wVcZ2bF\nfv4sIgheTDD8P83MlgAfhw8knQmcApyRrHoWV36vcYqTtD/wBMEv/zHAWWbWWlIdoB9BcrBpZvZN\naTpzLOlI4E6CXuA7wHtmdrmk24HdCBYAHWpmHxfn7yUi6KcRDP9fJgjwc4BeQHNgDMEfyEHAjWY2\nJVn1La48EKagiF/+skB7oCbBkO9m4GQzmy1pTzP7NakVTSJJg4H3gEzgJuAMM/s9YntFM1ubrPrF\nm6Sm4QmRh4CGQBngV4KA/xNwN1DZzFYnsZrFlgfCFCXpcGAfYBbwELAAONjM1knqSnCi5GozW5q8\nWiaepC5AOeBAYA8gCxhoZr9KOhWoZWaPSEozs83JrGu8SGpMcCb8RGAcQdunm9nciOHwsWa2PmmV\nLOb88pkUJKktcDTwuZkNA54EqgINJB0PPAa8XgqDYHOCYeFUYCTBVMGzYRDcl6BnOBWgpARBADOb\nDZwP/AfoamYfA4sknU9wkuwqD4I7x0+WpBhJlYC3gEXA/QBmdmt4adjtBH+8rjGzj4rz3Nf2Cv84\n3Al8H14itFjSBcBNknoALQhOjIxMZj3jKRwVzAV+M7O3JG0EHpW0MZz7bAD0N7OfklvT4s+Hxikk\n/MVeBLQEXgOeNLNHI7anARlmtjFJVUyK8La5dcAzBHOCFwJzzWyzpKbhtopmNrM4/3EoWHdJzxF0\nVgYBc8wsR9JdBJfI9AD+V1zbmmp8aJxkuXcBSNqHYOhzo5lNJThLfKmkgbn7mtnm0hIEI76X1gQ9\n4VbAecBS4CqgPoCZ/W5mf+ReNF1cA0OB6wRPkXS0mZ0DrAFuBJqEu04BngWWFNe2piLvEaaA8Kb5\nKwnuFOgKPGFmD0vaG/gEuMPMHktiFZMi/F7+jyAIfgc8QDAH+CSQTXDt5ILk1TD+wnuHTwPODf8g\nIum/BJfObAQ6AH1KWruTzQNhkkmqCbwE/NvMvgyvjzsfmGxmd0rqAFQ1s1HJrGeihUPet4FjCUYu\nZwDVgCHAb8BzwO1mNi1plYwzSXUJ2ne+mS2WVM7MNoTbegKNgVE+Jxh/frIkwSS1IPirPtrM5pvZ\ncknzgNaSvjazT8N/ELdLWmBmL4SfK7ZzXzuoEsHc3x/hJUPPEgSJmwnmzE4t7t9HIT/Tv4DyBL8f\nH0cEwTZm9lEy6lha+BxhAoXzXgOAF4F/S3ooPEs8GahMcP8wBNeKTQGuktQSiu/cV6wi5gTTAcK7\nIyYD50iqYWZzgNcJhsQnFPfvo8Cc4EWS+gKbgC+B5uFZciSdAgySVD15tS35fGicYOHQ9xbgLII5\nrykEq4WksWXlkHYE1xFeBnxoZp8np7aJEXEnTR/gEKACwQmR7gRzpvWAT4FLCS6hGUhwr22xX2Ai\nYk7wbDObLqk9wW2DTYA/Ce6l7l+SpgBSkfcIE8zMPgWWEQztjgFmEtwzegBBb2dF+LoJwWIKJf42\nujAIHg0MJpj760YwP/gj8HT4/32BC4D5BHeWbEhObXeOpM4Rvb3aBD/r44EFkk4A9iK4dOqG8P89\nPAjuet4jTKDc277CuyD6AG8Q3ET/EDCb4NaptwnODg4n6PVMTVJ1EyIcEmcAjxKsJ1ifYIHVRQQn\nB04wsznhNZQ9CILlBWY2OTk13nHhvePHAl8BG8xspaQ3Cf74VSOYI2wJvFoarxJIJg+ESRD2BF4m\nmBO83MyeDN/PWyhAUp2SMPTblojhcH0zWyipAsG0wKtArzBIzAcmEQTD9ZIOITh5Uqx7yeGtgkMI\nrg4oC+wHjDGzGZIGEPxenAdkF/e50OLCh8ZJYME6cjcCPxCsoJLbW1wb9nwo4UEwLQyCvYBPJTUy\ns3UEc6QLgJbhZUMfAnfn3kdrZl8XxyAoaU9JB0g6TFI1M/sFGAHcC5Qxs2eBXySdRzAPepeZbfIg\nmDh++UzyTAKmAQdLeiN3kYCStFhAQZLKm9n6cHqgC3AfwUmCOQpW4N5McPLo/4CDCYbAY5JY5Z0W\nBvvbCNYPrAy0kNSbYCpgE3CbgiXFphPME5/o1wkmng+NkyicK8wws/8luy67Wnj5xyDgZjNbLelY\noAHBqtIdgIsI5kxHECwyWrM4zgNGUpBt8FaCxSC+Ct8bRHBWuJeZTVWw7H5/gmHyLDPblKz6lmYe\nCF1CSKpBcJF0OsFiomsJVtepQHBd5QKCebGHzWx0suoZL2F7lwF9zWxEbm843HYrwZ0y7QjmCE8g\nuExqXrLqW9p5IHS7VMSZ8tz/X0VwWdB5BMGvgpmtktSEoEc4wMy+T2ad4yUcFt9NsIbg8gK3zH1J\nsI7gREnpZpaT1MqWcj5H6HaZ8OzoGeEyWmnhElKPEMyNPUxwr/B3ChKz3wr8q6QEQQAz+0DSZmCc\npM7hmfAy4fD3L4LvAQ+Cyednjd0uEd5T/RbBNXK5eYXHEwwH/0OwwvT1kjoC3wMXmtk7ubfalRTh\nPcIXAxMkVTezTQqW169LKUm9Whx4j9DFXbiG4MvAP83svYj3/yC4XGh/govIM4A7gOPNbD6UzHuq\nLVhN/GLga0lPEMwPnhdeRuVSgM8RurgLL4352szSwtcVwusEkfQAwYXTZxKcNKloQU6OEi+8bOYt\noIPfNpdafGjs4i4869tL0kxJNcNltMqHm78j+AO82cyWlJYgCGBmI4BqHgRTjwdCt0tEzI2NC5fR\nys2ytgFYJals7l00pYmVoFzLJUmp+0V0iRN5ogDyTqDcDbxvZhtL8l00rnjxOUK3y4XLzL9JkKz+\nGjP7MMlVci4fD4QuIRTk6N3NzN5Odl2cK8gDoUuoUph7xRUDHgidc6WenyxxzpV6Hgidc6WeB0Ln\nXKnngdDlIylH0mRJUyW9LqniTpTVVdKI8HlfSdcXsW81Sf+3A8e4VdLVsb5fYJ/nJfXfjmM1llSi\nk2mVVh4IXUHrzKy9mbUlyKZ3YeRGBbb798bM3jOzu4vYpRrBEv3OJZwHQleUb4A9wp7QT+HKKROB\n3SUdKelbSRPDnmNlCJanl/SzpNHAcbkFSTpb0mPh8zqS3pb0Q/g4kOCOk2Zhb/TecL9rJI2X9KOk\nf0WUdaOkGZI+B1pEa4SkC8JyfpD0ZoFe7hGSvpH0S7goApLSJd0bcex/7OwX6VKbB0JXKEkZQE+C\nZEoQBJyhZtYBWAPcBBxhZh0JbqG7MlxYYQhBzuaDCdbcK8wjwFdm1g7oSJDE6npgZtgbvUbSkcCe\nBInd2wOdJB0iqRNwMkGek+OAfWJozltmtk94vJ8IVsfO1Rg4lCDR+n/DNpwH/Glm+4TlXxCuoO1K\nKF+P0BVUQVJu0qRvgGcIkq7PMbOx4fv7A62BMeE6qmWBbwmSk8/KTbkp6SVgQCHHOIxgGa7c1Zn/\nDJM7RToyfEwKX1cmCIxVgLcj8j+/R3RtJd1OMPyuDHwSsW14eM/zr5J+D9twJLB3xPxh1fDYv8Rw\nLFcMeSB0Ba0zs/aRb4TBbk3kW8BnZnZKgf3aA/G6Ql8E+X2fLHCMy3fgGM8D/czsB0lnA10jthUs\ny8JjX2JmkQETSY2387iumPChsdsRY4GDJO0BIKlimJ/kZ6CJpGbhfqds4/NfAAPDz6ZL2g1YTdDb\ny/UJcG7E3GOWpNrA18CxkipIqkIwDI+mCvCHpDLAaQW2nSApLaxzU2BGeOyB4f5Iai6pUgzHccWU\n9wjddjOzpWHP6lVJ5cK3bzKzXyQNAD6QtAwYDbQtpIjLgKcknQfkAAPN7FtJY8LLUz4K5wlbAd+G\nPdK/gdPDrG/DgMkESdO/iaHKNxMsCDuHYM4zMuDOAL4C6hDkTVkv6WmCucOJCg6+FOgX27fjiiO/\n19g5V+r50Ng5V+p5IHTOlXoeCJ1zpZ4HQudcqeeB0DlX6nkgdM6Veh4InXOl3v8DjRL+Eb+DzU8A\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1cfc384a58>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "matrix = score_model(predicted_l)\n",
    "utils.plot_confusion_matrix(matrix, classes=[\"Agree\",\"Disagree\", \"Discuss\"],\n",
    "                      title='Normalized confusion matrix')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for m in []:\n",
    "    model = m[0]\n",
    "    predicted = eval_model(model)\n",
    "    predicted_l = [i.item() for i in list(predicted)]\n",
    "    matrix = score_model(predicted_l)\n",
    "    utils.plot_confusion_matrix(matrix, classes=[\"Agree\",\"Disagree\", \"Discuss\"],\n",
    "                      title='Normalized confusion matrix')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "true_label = [(2 if x[-1] == \"agree\" else (1 if x[-1] == \"discuss\" else 0)) for x in stances_val.values]\n",
    "[list(x) for x in list(confusion_matrix(true_label,predicted_l))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torch.save(model2.state_dict(), './saved_models/LSTM_1_256.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# train finalized model below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_final = CNN(EMBEDDING_DIM, N_FILTERS, FILTER_SIZES, OUTPUT_DIM, DROPOUT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fts = train_feats+val_feats\n",
    "tr_labels = [str(x[-1]) for x in stances_tr.values]+[str(x[-1]) for x in stances_val.values]\n",
    "\n",
    "def train_final(model, total_batch, optimizer, criterion):\n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    model_final.train()\n",
    "    for i in range(total_batch):\n",
    "        batch_x,batch_y = get_batch(fts, tr_labels,i,batch_size)\n",
    "        inputs = Variable(torch.FloatTensor(batch_x))\n",
    "        labels = Variable(torch.LongTensor(batch_y))\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        predictions = model_final(inputs)\n",
    "        loss = criterion(predictions, labels)\n",
    "        _, predicted = torch.max(predictions.data, 1)\n",
    "        acc = binary_accuracy(predicted, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "    return epoch_loss / total_batch, epoch_acc / total_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batches_train= int(len(fts)/batch_size)\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    train_loss, train_acc = train_final(model_f, batches_train, optimizer, criterion)\n",
    "    print(f'| Epoch: {epoch+1:02} | Train Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}% |')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torch.save(model_final.state_dict(), './CNN_model_softmax_final.pth')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}