training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Namespace(batch_size=64, decay=0.99, dpout_fc=0.0, dpout_model=0.0, enc_lstm_dim=2048, encoder_type='BLSTMEncoder', fc_dim=512, gpu_id=0, lrshrink=5, max_norm=5.0, minlr=1e-05, n_classes=3, n_enc_layers=1, n_epochs=20, nlipath='dataset/SNLI/', nonlinear_fc=0, optimizer='sgd,lr=0.1', outputdir='savedir2/', outputmodelname='model.pickle', pool_type='max', seed=1234)\n",
      "** TRAIN DATA : Found 549367 pairs of train sentences.\n",
      "** DEV DATA : Found 9842 pairs of dev sentences.\n",
      "** TEST DATA : Found 9824 pairs of test sentences.\n",
      "Found 38957(/43479) words with glove vectors\n",
      "Vocab size : 38957\n",
      "NLINet (\n",
      "  (encoder): BLSTMEncoder (\n",
      "    (enc_lstm): LSTM(300, 2048, bidirectional=True)\n",
      "  )\n",
      "  (classifier): Sequential (\n",
      "    (0): Linear (16384 -> 512)\n",
      "    (1): Linear (512 -> 512)\n",
      "    (2): Linear (512 -> 3)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# Copyright (c) 2017-present, Facebook, Inc.\n",
    "# All rights reserved.\n",
    "#\n",
    "# This source code is licensed under the license found in the\n",
    "# LICENSE file in the root directory of this source tree. \n",
    "#\n",
    "\n",
    "import time\n",
    "import argparse\n",
    "import sys, os\n",
    "\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "\n",
    "from data import get_nli, get_batch, build_vocab\n",
    "from mutils import get_optimizer, dotdict\n",
    "from models import NLINet\n",
    "\n",
    "\n",
    "GLOVE_PATH = \"dataset/GloVe/glove.840B.300d.txt\"\n",
    "\n",
    "\n",
    "parser = argparse.ArgumentParser(description='NLI training')\n",
    "# paths\n",
    "parser.add_argument(\"--nlipath\", type=str, default='dataset/SNLI/', help=\"NLI data path (SNLI or MultiNLI)\")\n",
    "parser.add_argument(\"--outputdir\", type=str, default='savedir2/', help=\"Output directory\")\n",
    "parser.add_argument(\"--outputmodelname\", type=str, default='model.pickle')\n",
    "\n",
    "\n",
    "# training\n",
    "parser.add_argument(\"--n_epochs\", type=int, default=20)\n",
    "parser.add_argument(\"--batch_size\", type=int, default=64)\n",
    "parser.add_argument(\"--dpout_model\", type=float, default=0., help=\"encoder dropout\")\n",
    "parser.add_argument(\"--dpout_fc\", type=float, default=0., help=\"classifier dropout\")\n",
    "parser.add_argument(\"--nonlinear_fc\", type=float, default=0, help=\"use nonlinearity in fc\")\n",
    "parser.add_argument(\"--optimizer\", type=str, default=\"sgd,lr=0.1\", help=\"adam or sgd,lr=0.1\")\n",
    "parser.add_argument(\"--lrshrink\", type=float, default=5, help=\"shrink factor for sgd\")\n",
    "parser.add_argument(\"--decay\", type=float, default=0.99, help=\"lr decay\")\n",
    "parser.add_argument(\"--minlr\", type=float, default=1e-5, help=\"minimum lr\")\n",
    "parser.add_argument(\"--max_norm\", type=float, default=5., help=\"max norm (grad clipping)\")\n",
    "\n",
    "#model\n",
    "parser.add_argument(\"--encoder_type\", type=str, default='BLSTMEncoder', help=\"see list of encoders\")\n",
    "parser.add_argument(\"--enc_lstm_dim\", type=int, default=2048, help=\"encoder nhid dimension\")\n",
    "parser.add_argument(\"--n_enc_layers\", type=int, default=1, help=\"encoder num layers\")\n",
    "parser.add_argument(\"--fc_dim\", type=int, default=512, help=\"nhid of fc layers\")\n",
    "parser.add_argument(\"--n_classes\", type=int, default=3, help=\"entailment/neutral/contradiction\")\n",
    "parser.add_argument(\"--pool_type\", type=str, default='max', help=\"max or mean\")\n",
    "\n",
    "# gpu\n",
    "parser.add_argument(\"--gpu_id\", type=int, default=0, help=\"GPU ID\")\n",
    "parser.add_argument(\"--seed\", type=int, default=1234, help=\"seed\")\n",
    "\n",
    "\n",
    "params, _ = parser.parse_known_args(\" \".split())\n",
    "\n",
    "# set gpu device\n",
    "torch.cuda.set_device(params.gpu_id)\n",
    "\n",
    "# print parameters passed, and all parameters\n",
    "# print('\\ntogrep : {0}\\n'.format(sys.argv[1:]))\n",
    "print(params)\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "SEED\n",
    "\"\"\"\n",
    "np.random.seed(params.seed)\n",
    "torch.manual_seed(params.seed)\n",
    "torch.cuda.manual_seed(params.seed)\n",
    "\n",
    "\"\"\"\n",
    "DATA\n",
    "\"\"\"\n",
    "train, valid, test = get_nli(params.nlipath)\n",
    "word_vec = build_vocab(train['s1'] + train['s2'] + valid['s1'] + valid['s2'] + test['s1'] + test['s2'], GLOVE_PATH)\n",
    "\n",
    "for split in ['s1', 's2']:\n",
    "    for data_type in ['train', 'valid', 'test']:\n",
    "        eval(data_type)[split] = np.array([['<s>'] + [word for word in sent.split() if word in word_vec] +\\\n",
    "                                          ['</s>'] for sent in eval(data_type)[split]])        \n",
    "\n",
    "params.word_emb_dim = 300\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "MODEL\n",
    "\"\"\"\n",
    "# model config\n",
    "config_nli_model = {\n",
    "    'n_words'        :  len(word_vec)          ,\n",
    "    'word_emb_dim'   :  params.word_emb_dim   ,\n",
    "    'enc_lstm_dim'   :  params.enc_lstm_dim   ,\n",
    "    'n_enc_layers'   :  params.n_enc_layers   ,\n",
    "    'dpout_model'    :  params.dpout_model    ,\n",
    "    'dpout_fc'       :  params.dpout_fc       ,\n",
    "    'fc_dim'         :  params.fc_dim         ,\n",
    "    'bsize'          :  params.batch_size     ,\n",
    "    'n_classes'      :  params.n_classes      ,\n",
    "    'pool_type'      :  params.pool_type      ,\n",
    "    'nonlinear_fc'   :  params.nonlinear_fc   ,\n",
    "    'encoder_type'   :  params.encoder_type   ,\n",
    "    'use_cuda'       :  True                  ,\n",
    "\n",
    "}\n",
    "\n",
    "# model\n",
    "encoder_types = ['BLSTMEncoder', 'BLSTMprojEncoder', 'BGRUlastEncoder', 'InnerAttentionMILAEncoder',\\\n",
    "                 'InnerAttentionYANGEncoder', 'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder']\n",
    "assert params.encoder_type in encoder_types, \"encoder_type must be in \" + str(encoder_types)\n",
    "nli_net = NLINet(config_nli_model)\n",
    "print(nli_net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'dict' object has no attribute 'head'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-c88a9af15b66>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mword_vec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'dict' object has no attribute 'head'"
     ]
    }
   ],
   "source": [
    "word_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NLINet (\n",
      "  (encoder): BLSTMEncoder (\n",
      "    (enc_lstm): LSTM(300, 2048, bidirectional=True)\n",
      "  )\n",
      "  (classifier): Sequential (\n",
      "    (0): Linear (16384 -> 512)\n",
      "    (1): Linear (512 -> 512)\n",
      "    (2): Linear (512 -> 3)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "\n",
    "nli_net = NLINet(config_nli_model)\n",
    "print(nli_net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# loss\n",
    "weight = torch.FloatTensor(params.n_classes).fill_(1)\n",
    "loss_fn = nn.CrossEntropyLoss(weight=weight)\n",
    "loss_fn.size_average = False\n",
    "\n",
    "# optimizer\n",
    "optim_fn, optim_params = get_optimizer(params.optimizer)\n",
    "optimizer = optim_fn(nli_net.parameters(), **optim_params)\n",
    "\n",
    "# cuda by default\n",
    "nli_net.cuda()\n",
    "loss_fn.cuda()\n",
    "#src_embeddings.cuda()\n",
    "\n",
    "\n",
    "    \n",
    "\"\"\"\n",
    "TRAIN\n",
    "\"\"\"\n",
    "#src_embeddings.volatile = True\n",
    "val_acc_best = -1e10\n",
    "adam_stop = False\n",
    "stop_training = False\n",
    "lr = optim_params['lr'] if 'sgd' in params.optimizer else None\n",
    "#index_pad =word2id['<p>']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def trainepoch(epoch):\n",
    "    print('\\nTRAINING : Epoch ' + str(epoch))\n",
    "    nli_net.train()\n",
    "    all_costs   = []\n",
    "    logs        = []\n",
    "    words_count = 0\n",
    "    \n",
    "    last_time = time.time()\n",
    "    correct = 0.\n",
    "    # shuffle the data\n",
    "    permutation = np.random.permutation(len(train['s1']))\n",
    "\n",
    "    s1 = train['s1'][permutation]\n",
    "    s2 = train['s2'][permutation]\n",
    "    target = train['label'][permutation]\n",
    "    \n",
    "\n",
    "    optimizer.param_groups[0]['lr'] = optimizer.param_groups[0]['lr'] * params.decay if epoch>1\\\n",
    "                                    and 'sgd' in params.optimizer else optimizer.param_groups[0]['lr']\n",
    "    print('Learning rate : {0}'.format(optimizer.param_groups[0]['lr']))\n",
    "\n",
    "    for stidx in range(0, len(s1), params.batch_size):\n",
    "        # prepare batch\n",
    "        s1_batch, s1_len = get_batch(s1[stidx:stidx + params.batch_size], word_vec)\n",
    "        s2_batch, s2_len = get_batch(s2[stidx:stidx + params.batch_size], word_vec)\n",
    "        s1_batch, s2_batch = Variable(s1_batch.cuda()), Variable(s2_batch.cuda())\n",
    "        tgt_batch = Variable(torch.LongTensor(target[stidx:stidx + params.batch_size])).cuda()\n",
    "        k = s1_batch.size(1)  # actual batch size\n",
    "#         print(s1_batch, s1_len) \n",
    "        # model forward\n",
    "        output = nli_net((s1_batch, s1_len), (s2_batch, s2_len))\n",
    "\n",
    "        pred = output.data.max(1)[1]\n",
    "        correct += pred.long().eq(tgt_batch.data.long()).cpu().sum()\n",
    "        assert len(pred) == len(s1[stidx:stidx + params.batch_size])\n",
    "        \n",
    "        # loss\n",
    "        loss = loss_fn(output, tgt_batch)\n",
    "        all_costs.append(loss.data[0])\n",
    "        words_count += (s1_batch.nelement() + s2_batch.nelement()) / params.word_emb_dim\n",
    "\n",
    "        # backward\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        \n",
    "        # gradient clipping (off by default)\n",
    "        shrink_factor = 1\n",
    "        total_norm = 0\n",
    "        \n",
    "        for p in nli_net.parameters():\n",
    "            if p.requires_grad:\n",
    "                p.grad.data.div_(k)  # divide by the actual batch size\n",
    "                total_norm += p.grad.data.norm() ** 2\n",
    "        total_norm = np.sqrt(total_norm)\n",
    "        \n",
    "        if total_norm > params.max_norm:\n",
    "            shrink_factor = params.max_norm / total_norm\n",
    "        current_lr = optimizer.param_groups[0]['lr'] # current lr (no external \"lr\", for adam)\n",
    "        optimizer.param_groups[0]['lr'] = current_lr * shrink_factor # just for update\n",
    "        \n",
    "        # optimizer step\n",
    "        optimizer.step()\n",
    "        optimizer.param_groups[0]['lr'] = current_lr\n",
    "\n",
    "        if len(all_costs) == 100:\n",
    "            logs.append( '{0} ; loss {1} ; sentence/s {2} ; words/s {3} ; accuracy train : {4}'.format(\n",
    "                    stidx, round(np.mean(all_costs),2), int(len(all_costs) * params.batch_size / (time.time() - last_time)),\n",
    "                    int(words_count * 1.0 / (time.time() - last_time)), round(100.*correct/(stidx+k), 2) ))\n",
    "            print(logs[-1])\n",
    "            last_time = time.time()\n",
    "            words_count = 0\n",
    "            all_costs = []\n",
    "    train_acc = round(100 * correct/len(s1), 2)\n",
    "    print('results : epoch {0} ; mean accuracy train : {1}'.format(epoch, train_acc))\n",
    "    return train_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s1 = test['s1'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ list(['<s>', 'This', 'church', 'choir', 'sings', 'to', 'the', 'masses', 'as', 'they', 'sing', 'joyous', 'songs', 'from', 'the', 'book', 'at', 'a', 'church', '.', '</s>']),\n",
       "       list(['<s>', 'This', 'church', 'choir', 'sings', 'to', 'the', 'masses', 'as', 'they', 'sing', 'joyous', 'songs', 'from', 'the', 'book', 'at', 'a', 'church', '.', '</s>']),\n",
       "       list(['<s>', 'This', 'church', 'choir', 'sings', 'to', 'the', 'masses', 'as', 'they', 'sing', 'joyous', 'songs', 'from', 'the', 'book', 'at', 'a', 'church', '.', '</s>']),\n",
       "       ...,\n",
       "       list(['<s>', 'A', 'man', 'in', 'a', 'black', 'leather', 'jacket', 'and', 'a', 'book', 'in', 'his', 'hand', 'speaks', 'in', 'a', 'classroom', '.', '</s>']),\n",
       "       list(['<s>', 'A', 'man', 'in', 'a', 'black', 'leather', 'jacket', 'and', 'a', 'book', 'in', 'his', 'hand', 'speaks', 'in', 'a', 'classroom', '.', '</s>']),\n",
       "       list(['<s>', 'A', 'man', 'in', 'a', 'black', 'leather', 'jacket', 'and', 'a', 'book', 'in', 'his', 'hand', 'speaks', 'in', 'a', 'classroom', '.', '</s>'])], dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(epoch, eval_type='valid', final_eval=False):\n",
    "    nli_net.eval()\n",
    "    correct = 0.\n",
    "    global val_acc_best, lr, stop_training, adam_stop\n",
    "#     print(\"val acc best, lr, stop_training, adam_stop: \", val_acc_best, lr, stop_training, adam_stop)\n",
    "    if eval_type == 'valid':\n",
    "        print('\\nVALIDATION : Epoch {0}'.format(epoch))\n",
    "    \n",
    "    s1    = valid['s1']    if eval_type == 'valid' else test['s1']\n",
    "    s2    = valid['s2']    if eval_type == 'valid' else test['s2']\n",
    "    target = valid['label'] if eval_type == 'valid' else test['label']\n",
    "\n",
    "    for i in range(0, len(s1), params.batch_size):\n",
    "        # prepare batch\n",
    "        s1_batch, s1_len = get_batch(s1[i:i + params.batch_size], word_vec)\n",
    "        s2_batch, s2_len = get_batch(s2[i:i + params.batch_size], word_vec)\n",
    "        s1_batch, s2_batch = Variable(s1_batch.cuda()), Variable(s2_batch.cuda())\n",
    "        tgt_batch = Variable(torch.LongTensor(target[i:i + params.batch_size])).cuda()\n",
    "        k = s1_batch.size(1)  # actual batch size\n",
    "#         print(\"yo, I am here\")\n",
    "        first_arg = (s1_batch, s1_len)\n",
    "        second_arg = (s2_batch, s2_len)\n",
    "        # model forward\n",
    "        output = nli_net(first_arg, second_arg )\n",
    "        \n",
    "        pred = output.data.max(1)[1]\n",
    "        correct += pred.long().eq(tgt_batch.data.long()).cpu().sum()\n",
    "        \n",
    "        \n",
    "    # save model\n",
    "    eval_acc  = round(100 * correct / len(s1),2)\n",
    "    if final_eval:\n",
    "        print('finalgrep : accuracy {0} : {1}'.format(eval_type, eval_acc))\n",
    "    else:\n",
    "        print('togrep : results : epoch {0} ; mean accuracy {1} : {2}'.format(epoch, eval_type, eval_acc))\n",
    "    \n",
    "    if eval_type == 'valid' and epoch <= params.n_epochs:\n",
    "        if eval_acc > val_acc_best:\n",
    "            print('saving model at epoch {0}'.format(epoch))\n",
    "            if not os.path.exists(params.outputdir):\n",
    "                os.makedirs(params.outputdir)\n",
    "            torch.save(nli_net, os.path.join(params.outputdir, params.outputmodelname))\n",
    "            val_acc_best = eval_acc\n",
    "        else:\n",
    "            if 'sgd' in params.optimizer:\n",
    "                optimizer.param_groups[0]['lr'] = optimizer.param_groups[0]['lr'] / params.lrshrink\n",
    "                print('Shrinking lr by : {0}. New lr = {1}'.format(params.lrshrink, optimizer.param_groups[0]['lr']))\n",
    "                if optimizer.param_groups[0]['lr'] < params.minlr:\n",
    "                    stop_training = True\n",
    "            if 'adam' in params.optimizer:\n",
    "                # early stopping (at 2nd decrease in accuracy)\n",
    "                stop_training = adam_stop\n",
    "                adam_stop = True\n",
    "    return eval_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'label': array([1, 2, 0, ..., 1, 2, 0]),\n",
       " 's1': array([ list(['<s>', 'A', 'person', 'on', 'a', 'horse', 'jumps', 'over', 'a', 'broken', 'down', 'airplane', '.', '</s>']),\n",
       "        list(['<s>', 'A', 'person', 'on', 'a', 'horse', 'jumps', 'over', 'a', 'broken', 'down', 'airplane', '.', '</s>']),\n",
       "        list(['<s>', 'A', 'person', 'on', 'a', 'horse', 'jumps', 'over', 'a', 'broken', 'down', 'airplane', '.', '</s>']),\n",
       "        ...,\n",
       "        list(['<s>', 'A', 'man', 'is', 'surfing', 'in', 'a', 'bodysuit', 'in', 'beautiful', 'blue', 'water', '.', '</s>']),\n",
       "        list(['<s>', 'A', 'man', 'is', 'surfing', 'in', 'a', 'bodysuit', 'in', 'beautiful', 'blue', 'water', '.', '</s>']),\n",
       "        list(['<s>', 'A', 'man', 'is', 'surfing', 'in', 'a', 'bodysuit', 'in', 'beautiful', 'blue', 'water', '.', '</s>'])], dtype=object),\n",
       " 's2': array([ list(['<s>', 'A', 'person', 'is', 'training', 'his', 'horse', 'for', 'a', 'competition', '.', '</s>']),\n",
       "        list(['<s>', 'A', 'person', 'is', 'at', 'a', 'diner', ',', 'ordering', 'an', 'omelette', '.', '</s>']),\n",
       "        list(['<s>', 'A', 'person', 'is', 'outdoors', ',', 'on', 'a', 'horse', '.', '</s>']),\n",
       "        ...,\n",
       "        list(['<s>', 'A', 'man', 'in', 'a', 'bodysuit', 'is', 'competing', 'in', 'a', 'surfing', 'competition', '.', '</s>']),\n",
       "        list(['<s>', 'A', 'man', 'in', 'a', 'business', 'suit', 'is', 'heading', 'to', 'a', 'board', 'meeting', '.', '</s>']),\n",
       "        list(['<s>', 'On', 'the', 'beautiful', 'blue', 'water', 'there', 'is', 'a', 'man', 'in', 'a', 'bodysuit', 'surfing', '.', '</s>'])], dtype=object)}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DONT RUN THE CELL BELOW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "TRAINING : Epoch 1\n",
      "Learning rate : 0.1\n",
      "6336 ; loss 70.29 ; sentence/s 115 ; words/s 6686 ; accuracy train : 34.27\n",
      "332736 ; loss 36.46 ; sentence/s 115 ; words/s 6598 ; accuracy train : 68.7\n",
      "339136 ; loss 37.81 ; sentence/s 112 ; words/s 6834 ; accuracy train : 68.84\n",
      "345536 ; loss 36.21 ; sentence/s 115 ; words/s 6617 ; accuracy train : 68.99\n",
      "351936 ; loss 36.33 ; sentence/s 115 ; words/s 6642 ; accuracy train : 69.14\n",
      "358336 ; loss 36.69 ; sentence/s 115 ; words/s 6632 ; accuracy train : 69.29\n",
      "364736 ; loss 36.53 ; sentence/s 115 ; words/s 6622 ; accuracy train : 69.42\n",
      "371136 ; loss 36.49 ; sentence/s 113 ; words/s 6720 ; accuracy train : 69.55\n",
      "377536 ; loss 36.33 ; sentence/s 115 ; words/s 6551 ; accuracy train : 69.68\n",
      "383936 ; loss 36.07 ; sentence/s 115 ; words/s 6626 ; accuracy train : 69.8\n",
      "390336 ; loss 36.08 ; sentence/s 117 ; words/s 6522 ; accuracy train : 69.92\n",
      "396736 ; loss 34.88 ; sentence/s 114 ; words/s 6653 ; accuracy train : 70.05\n",
      "403136 ; loss 36.02 ; sentence/s 115 ; words/s 6655 ; accuracy train : 70.16\n",
      "543936 ; loss 34.56 ; sentence/s 114 ; words/s 6675 ; accuracy train : 72.24\n",
      "results : epoch 1 ; mean accuracy train : 72.31\n",
      "\n",
      "VALIDATION : Epoch 1\n",
      "togrep : results : epoch 1 ; mean accuracy valid : 80.51\n",
      "saving model at epoch 1\n",
      "\n",
      "TRAINING : Epoch 2\n",
      "Learning rate : 0.099\n",
      "6336 ; loss 32.96 ; sentence/s 114 ; words/s 6713 ; accuracy train : 79.34\n",
      "12736 ; loss 32.37 ; sentence/s 114 ; words/s 6700 ; accuracy train : 79.8\n",
      "19136 ; loss 33.25 ; sentence/s 115 ; words/s 6624 ; accuracy train : 79.67\n",
      "25536 ; loss 33.26 ; sentence/s 115 ; words/s 6639 ; accuracy train : 79.52\n",
      "31936 ; loss 33.3 ; sentence/s 115 ; words/s 6687 ; accuracy train : 79.51\n"
     ]
    }
   ],
   "source": [
    "\n",
    "epoch = 1\n",
    "while not stop_training and epoch <= params.n_epochs:\n",
    "    train_acc = trainepoch(epoch)\n",
    "    eval_acc = evaluate(epoch, 'valid')\n",
    "    epoch += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'savedir2/model.pickle'"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.join(params.outputdir, params.outputmodelname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./savedir2/model.pickle'"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'./savedir2/model.pickle'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "TEST : Epoch 2\n",
      "finalgrep : accuracy test : 80.58\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "80.58"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Run best model on test set.\n",
    "del nli_net\n",
    "nli_net = torch.load(os.path.join(params.outputdir, params.outputmodelname))\n",
    "\n",
    "print('\\nTEST : Epoch {0}'.format(epoch))\n",
    "# evaluate(1e6, 'valid', True)\n",
    "evaluate(0, 'test', True)\n",
    "\n",
    "# Save encoder instead of full model\n",
    "# torch.save(nli_net.encoder, os.path.join(params.outputdir, params.outputmodelname))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BLSTMEncoder (\n",
       "  (enc_lstm): LSTM(300, 2048, bidirectional=True)\n",
       ")"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nli_net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# print('\\nTRAINING : Epoch ' + str(epoch))\n",
    "# nli_net.train()\n",
    "# all_costs   = []\n",
    "# logs        = []\n",
    "# words_count = 0\n",
    "\n",
    "# last_time = time.time()\n",
    "# correct = 0.\n",
    "# # shuffle the data\n",
    "# permutation = np.random.permutation(len(train['s1']))\n",
    "\n",
    "# s1 = train['s1'][permutation]\n",
    "# s2 = train['s2'][permutation]\n",
    "# target = train['label'][permutation]\n",
    "\n",
    "\n",
    "# optimizer.param_groups[0]['lr'] = optimizer.param_groups[0]['lr'] * params.decay if epoch>1\\\n",
    "#                                 and 'sgd' in params.optimizer else optimizer.param_groups[0]['lr']\n",
    "# print('Learning rate : {0}'.format(optimizer.param_groups[0]['lr']))\n",
    "\n",
    "# for stidx in range(0, len(s1), params.batch_size):\n",
    "#     # prepare batch\n",
    "#     s1_batch, s1_len = get_batch(s1[stidx:stidx + params.batch_size], word_vec)\n",
    "#     s2_batch, s2_len = get_batch(s2[stidx:stidx + params.batch_size], word_vec)\n",
    "#     s1_batch, s2_batch = Variable(s1_batch.cuda()), Variable(s2_batch.cuda())\n",
    "#     tgt_batch = Variable(torch.LongTensor(target[stidx:stidx + params.batch_size])).cuda()\n",
    "#     k = s1_batch.size(1)  # actual batch size\n",
    "#     print(\"First positional argument: \", s1_batch, \"First positional argument: \", s1_len) \n",
    "#     # model forward\n",
    "#     output = nli_net((s1_batch, s1_len), (s2_batch, s2_len))\n",
    "    \n",
    "#     first_arg = (s1_batch, s1_len)\n",
    "#     second_arg = (s2_batch, s2_len)\n",
    "#     # model forward\n",
    "#     output = nli_net(first_arg, second_arg )\n",
    "\n",
    "#     pred = output.data.max(1)[1]\n",
    "#     correct += pred.long().eq(tgt_batch.data.long()).cpu().sum()\n",
    "#     assert len(pred) == len(s1[stidx:stidx + params.batch_size])\n",
    "\n",
    "#     # loss\n",
    "#     loss = loss_fn(output, tgt_batch)\n",
    "#     all_costs.append(loss.data[0])\n",
    "#     words_count += (s1_batch.nelement() + s2_batch.nelement()) / params.word_emb_dim\n",
    "\n",
    "#     # backward\n",
    "#     optimizer.zero_grad()\n",
    "#     loss.backward()\n",
    "\n",
    "#     # gradient clipping (off by default)\n",
    "#     shrink_factor = 1\n",
    "#     total_norm = 0\n",
    "\n",
    "#     for p in nli_net.parameters():\n",
    "#         if p.requires_grad:\n",
    "#             p.grad.data.div_(k)  # divide by the actual batch size\n",
    "#             total_norm += p.grad.data.norm() ** 2\n",
    "#     total_norm = np.sqrt(total_norm)\n",
    "\n",
    "#     if total_norm > params.max_norm:\n",
    "#         shrink_factor = params.max_norm / total_norm\n",
    "#     current_lr = optimizer.param_groups[0]['lr'] # current lr (no external \"lr\", for adam)\n",
    "#     optimizer.param_groups[0]['lr'] = current_lr * shrink_factor # just for update\n",
    "\n",
    "#     # optimizer step\n",
    "#     optimizer.step()\n",
    "#     optimizer.param_groups[0]['lr'] = current_lr\n",
    "\n",
    "#     if len(all_costs) == 100:\n",
    "#         logs.append( '{0} ; loss {1} ; sentence/s {2} ; words/s {3} ; accuracy train : {4}'.format(\n",
    "#                 stidx, round(np.mean(all_costs),2), int(len(all_costs) * params.batch_size / (time.time() - last_time)),\n",
    "#                 int(words_count * 1.0 / (time.time() - last_time)), round(100.*correct/(stidx+k), 2) ))\n",
    "#         print(logs[-1])\n",
    "#         last_time = time.time()\n",
    "#         words_count = 0\n",
    "#         all_costs = []\n",
    "# train_acc = round(100 * correct/len(s1), 2)\n",
    "# print('results : epoch {0} ; mean accuracy train : {1}'.format(epoch, train_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eval_type = 'valid'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class NLINet(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super(NLINet, self).__init__()\n",
    "\n",
    "        # classifier\n",
    "        self.nonlinear_fc = config['nonlinear_fc']\n",
    "        self.fc_dim = config['fc_dim']\n",
    "        self.n_classes = 3\n",
    "        self.enc_lstm_dim = config['enc_lstm_dim']\n",
    "        self.encoder_type = config['encoder_type']\n",
    "        self.dpout_fc = config['dpout_fc']\n",
    "\n",
    "        self.encoder = eval(self.encoder_type)(config)\n",
    "        self.inputdim = 4*2*self.enc_lstm_dim\n",
    "        self.inputdim = 4*self.inputdim if self.encoder_type in \\\n",
    "                        [\"ConvNetEncoder\", \"InnerAttentionMILAEncoder\"] else self.inputdim\n",
    "        self.inputdim = self.inputdim/2 if self.encoder_type==\"LSTMEncoder\" else self.inputdim\n",
    "        if self.nonlinear_fc:\n",
    "            self.classifier = nn.Sequential(\n",
    "                nn.Dropout(p=self.dpout_fc),\n",
    "                nn.Linear(self.inputdim, self.fc_dim),\n",
    "                nn.Tanh(),\n",
    "                nn.Dropout(p=self.dpout_fc),\n",
    "                nn.Linear(self.fc_dim, self.fc_dim),\n",
    "                nn.Tanh(),\n",
    "                nn.Dropout(p=self.dpout_fc),\n",
    "                nn.Linear(self.fc_dim, self.n_classes),\n",
    "                )\n",
    "        else:\n",
    "            self.classifier = nn.Sequential(\n",
    "                nn.Linear(self.inputdim, self.fc_dim),\n",
    "                nn.Linear(self.fc_dim, self.fc_dim),\n",
    "                nn.Linear(self.fc_dim, self.n_classes)\n",
    "                )\n",
    "\n",
    "    def forward(self, s1, s2):\n",
    "        # s1 : (s1, s1_len)\n",
    "        u = self.encoder(s1)\n",
    "        v = self.encoder(s2)\n",
    "\n",
    "        features = torch.cat((u, v, torch.abs(u-v), u*v), 1)\n",
    "        output = self.classifier(features)\n",
    "        return output\n",
    "\n",
    "    def encode(self, s1):\n",
    "        emb = self.encoder(s1)\n",
    "        return emb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "VALIDATION : Epoch 1\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "forward() takes 2 positional arguments but 3 were given",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-163-eb896590b0f3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     21\u001b[0m     \u001b[0msecond_arg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0ms2_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms2_len\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m     \u001b[0;31m# model forward\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m     \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnli_net\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst_arg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msecond_arg\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    204\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    207\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: forward() takes 2 positional arguments but 3 were given"
     ]
    }
   ],
   "source": [
    "nli_net.eval()\n",
    "correct = 0.\n",
    "global val_acc_best, lr, stop_training, adam_stop\n",
    "#     print(\"val acc best, lr, stop_training, adam_stop: \", val_acc_best, lr, stop_training, adam_stop)\n",
    "if eval_type == 'valid':\n",
    "    print('\\nVALIDATION : Epoch {0}'.format(epoch))\n",
    "\n",
    "s1    = valid['s1']    if eval_type == 'valid' else test['s1']\n",
    "s2    = valid['s2']    if eval_type == 'valid' else test['s2']\n",
    "target = valid['label'] if eval_type == 'valid' else test['label']\n",
    "\n",
    "for i in range(0, len(s1), params.batch_size):\n",
    "    # prepare batch\n",
    "    s1_batch, s1_len = get_batch(s1[i:i + params.batch_size], word_vec)\n",
    "    s2_batch, s2_len = get_batch(s2[i:i + params.batch_size], word_vec)\n",
    "    s1_batch, s2_batch = Variable(s1_batch.cuda()), Variable(s2_batch.cuda())\n",
    "    tgt_batch = Variable(torch.LongTensor(target[i:i + params.batch_size])).cuda()\n",
    "    k = s1_batch.size(1)  # actual batch size\n",
    "#         print(\"yo, I am here\")\n",
    "    first_arg = (s1_batch, s1_len)\n",
    "    second_arg = (s2_batch, s2_len)\n",
    "    # model forward\n",
    "    output = nli_net(first_arg, second_arg )\n",
    "\n",
    "    pred = output.data.max(1)[1]\n",
    "    correct += pred.long().eq(tgt_batch.data.long()).cpu().sum()\n",
    "\n",
    "\n",
    "# save model\n",
    "eval_acc  = round(100 * correct / len(s1),2)\n",
    "if final_eval:\n",
    "    print('finalgrep : accuracy {0} : {1}'.format(eval_type, eval_acc))\n",
    "else:\n",
    "    print('togrep : results : epoch {0} ; mean accuracy {1} : {2}'.format(epoch, eval_type, eval_acc))\n",
    "\n",
    "if eval_type == 'valid' and epoch <= params.n_epochs:\n",
    "    if eval_acc > val_acc_best:\n",
    "        print('saving model at epoch {0}'.format(epoch))\n",
    "        if not os.path.exists(params.outputdir):\n",
    "            os.makedirs(params.outputdir)\n",
    "        torch.save(nli_net, os.path.join(params.outputdir, params.outputmodelname))\n",
    "        val_acc_best = eval_acc\n",
    "    else:\n",
    "        if 'sgd' in params.optimizer:\n",
    "            optimizer.param_groups[0]['lr'] = optimizer.param_groups[0]['lr'] / params.lrshrink\n",
    "            print('Shrinking lr by : {0}. New lr = {1}'.format(params.lrshrink, optimizer.param_groups[0]['lr']))\n",
    "            if optimizer.param_groups[0]['lr'] < params.minlr:\n",
    "                stop_training = True\n",
    "        if 'adam' in params.optimizer:\n",
    "            # early stopping (at 2nd decrease in accuracy)\n",
    "            stop_training = adam_stop\n",
    "            adam_stop = True\n",
    "print(eval_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}