generate_predictions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "import time\n",
    "import argparse\n",
    "import sys, os\n",
    "\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "\n",
    "from data import get_nli, get_batch, build_vocab\n",
    "from mutils import get_optimizer, dotdict\n",
    "from models import NLINet\n",
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "GLOVE_PATH = \"/home/ubuntu/Dropbox/xai/AdditionalStuff/RepEval2017/multiNLI/data/glove.840B.300d.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loadDataset(filename, size=-1):\n",
    "    label_category = {\n",
    "        'neutral': 0,\n",
    "        'entailment': 1,\n",
    "        'contradiction': 2\n",
    "    }\n",
    "    dataset = []\n",
    "    sentence1 = []\n",
    "    sentence2 = []\n",
    "    labels = []\n",
    "    with open(filename, 'r') as f:\n",
    "        i = 0\n",
    "        not_found = 0\n",
    "        for line in f:\n",
    "            row = json.loads(line, 'utf-8')\n",
    "            if size == -1 or i < size:\n",
    "                dataset.append(row)\n",
    "                label = row['gold_label'].strip()\n",
    "                if label in label_category:\n",
    "                    sentence1.append( row['sentence1'].strip() )\n",
    "                    sentence2.append( row['sentence2'].strip() )\n",
    "\n",
    "                    labels.append( label_category[label] )\n",
    "                    i += 1\n",
    "                else:\n",
    "                    not_found += 1\n",
    "            else:\n",
    "                break;\n",
    "        if not_found > 0:\n",
    "            print('Label not recognized %d' % not_found)\n",
    "                \n",
    "    return (dataset, sentence1, sentence2, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label not recognized 185\n",
      "Label not recognized 168\n",
      "Label not recognized 9796\n",
      "Label not recognized 9847\n"
     ]
    }
   ],
   "source": [
    "(train_dataset, train_sentence1, train_sentence2, train_labels) = loadDataset('./multinli_all/multinli_0.9_train.jsonl')\n",
    "(dev_matched_dataset, dev_matched_sentence1, dev_matched_sentence2, dev_matched_labels) = loadDataset('./multinli_all/multinli_0.9_dev_matched.jsonl')\n",
    "(dev_mismatched_dataset, dev_mismatched_sentence1, dev_mismatched_sentence2, dev_mismatched_labels) = loadDataset('./multinli_all/multinli_0.9_dev_mismatched.jsonl')\n",
    "(test_matched_dataset, test_matched_sentence1, test_matched_sentence2, test_matched_labels) = loadDataset('./multinli_all/multinli_0.9_test_matched_unlabeled.jsonl')\n",
    "(test_mismatched_dataset, test_mismatched_sentence1, test_mismatched_sentence2, test_mismatched_labels) = loadDataset('./multinli_all/multinli_0.9_test_mismatched_unlabeled.jsonl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_set = pd.DataFrame(train_dataset)\n",
    "dev_matched_set = pd.DataFrame(dev_matched_dataset)\n",
    "dev_mismatched_set = pd.DataFrame(dev_mismatched_dataset)\n",
    "test_matched_set = pd.DataFrame(test_matched_dataset)\n",
    "test_mismatched_set = pd.DataFrame(test_mismatched_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Get rid of data_points without labels. (Needs to be modified later)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dev_matched_set = dev_matched_set[dev_matched_set['gold_label']!='-']\n",
    "dev_mismatched_set = dev_mismatched_set[dev_mismatched_set['gold_label']!='-']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_sent1 = train_set['sentence1'].apply(TreebankWordTokenizer().tokenize)\n",
    "train_sent2 = train_set['sentence2'].apply(TreebankWordTokenizer().tokenize)\n",
    "dev_m_sent1 = dev_matched_set['sentence1'].apply(TreebankWordTokenizer().tokenize)\n",
    "dev_m_sent2 = dev_matched_set['sentence2'].apply(TreebankWordTokenizer().tokenize)\n",
    "dev_mis_sent1 = dev_mismatched_set['sentence1'].apply(TreebankWordTokenizer().tokenize)\n",
    "dev_mis_sent2 = dev_mismatched_set['sentence2'].apply(TreebankWordTokenizer().tokenize)\n",
    "test_m_sent1 = test_matched_set['sentence1'].apply(TreebankWordTokenizer().tokenize)\n",
    "test_m_sent2 = test_matched_set['sentence2'].apply(TreebankWordTokenizer().tokenize)\n",
    "test_mis_sent1 = test_mismatched_set['sentence1'].apply(TreebankWordTokenizer().tokenize)\n",
    "test_mis_sent2 = test_mismatched_set['sentence2'].apply(TreebankWordTokenizer().tokenize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_word_dict(sentences):\n",
    "    # create vocab of words\n",
    "    word_dict = {}\n",
    "    for sent in sentences:\n",
    "        for word in sent:\n",
    "            if word not in word_dict:\n",
    "                word_dict[word] = ''\n",
    "    word_dict['<s>'] = ''\n",
    "    word_dict['</s>'] = ''\n",
    "    word_dict['<p>'] = ''\n",
    "    return word_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_glove(word_dict, glove_path):\n",
    "    # create word_vec with glove vectors\n",
    "    word_vec = {}\n",
    "    with open(glove_path) as f:\n",
    "        for line in f:\n",
    "            word, vec = line.split(' ', 1)\n",
    "            if word in word_dict:\n",
    "                word_vec[word] = np.array(list(map(float, vec.split())))\n",
    "    print('Found {0}(/{1}) words with glove vectors'.format(len(word_vec), len(word_dict)))\n",
    "    return word_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_vocab(sentences, glove_path):\n",
    "    word_dict = get_word_dict(sentences)\n",
    "    word_vec = get_glove(word_dict, glove_path)\n",
    "    print('Vocab size : {0}'.format(len(word_vec)))\n",
    "    return word_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train1 = list(train_sent1)+list(train_sent2)\n",
    "dev1 = list(dev_m_sent1)+list(dev_m_sent2)+list(dev_mis_sent1)+list(dev_mis_sent2)\n",
    "test1 = list(test_m_sent1)+list(test_m_sent2)+list(test_mis_sent1)+list(test_mis_sent2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 92111(/109562) words with glove vectors\n",
      "Vocab size : 92111\n"
     ]
    }
   ],
   "source": [
    "word_vec = build_vocab(train1+dev1+test1, GLOVE_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_set = {}\n",
    "dev_matched_set = {}\n",
    "dev_mismatched_set = {}\n",
    "test_matched_set = {}\n",
    "test_mismatched_set = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_set['sentence1']  =list(train_sent1   )            \n",
    "train_set['sentence2']  =list(  train_sent2  )\n",
    "dev_matched_set['sentence1']  =list(  dev_m_sent1  )\n",
    "dev_matched_set['sentence2']  =list(  dev_m_sent2  )\n",
    "dev_mismatched_set['sentence1'] =list(  dev_mis_sent1) \n",
    "dev_mismatched_set['sentence2'] =list(  dev_mis_sent2 )\n",
    "test_matched_set['sentence1'] =list(  test_m_sent1 )\n",
    "test_matched_set['sentence2'] =list(  test_m_sent2 )\n",
    "test_mismatched_set['sentence1'] =list(  test_mis_sent1) \n",
    "test_mismatched_set['sentence2']=list(  test_mis_sent2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 92111(/109562) words with glove vectors\n",
      "Vocab size : 92111\n"
     ]
    }
   ],
   "source": [
    "word_vec = build_vocab(train_set['sentence1'] +train_set['sentence2'] +dev_matched_set['sentence1'] + dev_matched_set['sentence2'] +dev_mismatched_set['sentence1'] +dev_mismatched_set['sentence2'] + test_matched_set['sentence1'] + test_matched_set['sentence2'] +test_mismatched_set['sentence1'] +test_mismatched_set['sentence2'], GLOVE_PATH )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for split in ['sentence1', 'sentence2']:\n",
    "    for data_type in ['train_set', 'dev_matched_set', 'dev_mismatched_set', 'test_matched_set','test_mismatched_set']:\n",
    "        eval(data_type)[split] = np.array([['<s>'] + [word for word in sent if word in word_vec] +\\\n",
    "                                          ['</s>'] for sent in eval(data_type)[split]])        \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_set['label'] = np.array(train_labels)\n",
    "dev_matched_set['label'] = np.array(dev_matched_labels)\n",
    "dev_mismatched_set['label'] = np.array(dev_mismatched_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sentence1': array([ list(['<s>', 'That', 'which', 'binds', 'together', 'Chinese', '.', '</s>']),\n",
       "        list(['<s>', 'The', 'actual', 'length', 'of', 'an', 'individual', 'worker', \"'s\", 'H-2A', 'visa', 'varies', 'depending', 'upon', 'the', 'geographic', 'location', 'of', 'the', 'employer', 'and', 'the', 'nature', 'of', 'the', 'farmwork', 'to', 'be', 'performed', '.', '</s>']),\n",
       "        list(['<s>', 'Every', 'man', 'I', 'put', 'down', 'left', 'me', 'empty', '.', '</s>']),\n",
       "        ...,\n",
       "        list(['<s>', 'As', 'capital', 'of', 'a', 'wealthy', 'colony', ',', 'Spanish', 'Town', 'had', 'its', 'fair', 'share', 'of', 'fine', 'buildings', 'that', 'included', 'courthouses', ',', 'administrative', 'offices', ',', 'and', 'official', 'residences', '.', '</s>']),\n",
       "        list(['<s>', 'The', 'H-2A', 'program', ',', 'in', 'effect', ',', 'establishes', 'a', 'ceiling', 'for', 'the', 'terms', 'of', 'work', 'that', 'U.S.', 'workers', 'can', 'demand', 'for', 'similar', 'employment', '.', '</s>']),\n",
       "        list(['<s>', 'Now', ',', 'after', 'Slate', 'careers', 'that', 'span', 'almost', '18', 'months', ',', 'Betsy', 'and', 'Bill', 'are', 'both', 'retiring', '--', 'at', 'ages', '40', 'and', '30', 'respectively', '.', '</s>'])], dtype=object),\n",
       " 'sentence2': array([ list(['<s>', 'This', 'is', 'a', 'shared', 'value', 'among', 'Chinese', 'people', '.', '</s>']),\n",
       "        list(['<s>', 'The', 'location', 'of', 'the', 'employer', 'effects', 'the', 'length', 'of', 'the', 'worker', \"'s\", 'H-2A', 'visa', '.', '</s>']),\n",
       "        list(['<s>', 'I', 'felt', 'empty', 'after', 'every', 'man', 'I', 'put', 'down', '.', '</s>']),\n",
       "        ...,\n",
       "        list(['<s>', 'Spanish', 'Town', 'was', 'dilapidated', 'and', 'crummy', 'while', 'it', 'was', 'serving', 'as', 'capital', '.', '</s>']),\n",
       "        list(['<s>', 'The', 'H-2A', 'program', 'reduces', 'average', 'wages', 'by', 'more', 'than', '10', 'percent', '.', '</s>']),\n",
       "        list(['<s>', 'Betsy', 'is', 'retiring', 'while', 'Bill', 'continues', 'working', '.', '</s>'])], dtype=object)}"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_matched_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "GLOVE_PATH = \"dataset/GloVe/glove.840B.300d.txt\"\n",
    "\n",
    "\n",
    "parser = argparse.ArgumentParser(description='NLI training')\n",
    "# paths\n",
    "parser.add_argument(\"--nlipath\", type=str, default='dataset/MultiNLI/', help=\"NLI data path (SNLI or MultiNLI)\")\n",
    "parser.add_argument(\"--outputdir\", type=str, default='savedir3/', help=\"Output directory\")\n",
    "parser.add_argument(\"--outputmodelname\", type=str, default='model.pickle')\n",
    "\n",
    "\n",
    "# training\n",
    "parser.add_argument(\"--n_epochs\", type=int, default=20)\n",
    "parser.add_argument(\"--batch_size\", type=int, default=64)\n",
    "parser.add_argument(\"--dpout_model\", type=float, default=0., help=\"encoder dropout\")\n",
    "parser.add_argument(\"--dpout_fc\", type=float, default=0., help=\"classifier dropout\")\n",
    "parser.add_argument(\"--nonlinear_fc\", type=float, default=0, help=\"use nonlinearity in fc\")\n",
    "parser.add_argument(\"--optimizer\", type=str, default=\"sgd,lr=0.1\", help=\"adam or sgd,lr=0.1\")\n",
    "parser.add_argument(\"--lrshrink\", type=float, default=5, help=\"shrink factor for sgd\")\n",
    "parser.add_argument(\"--decay\", type=float, default=0.99, help=\"lr decay\")\n",
    "parser.add_argument(\"--minlr\", type=float, default=1e-5, help=\"minimum lr\")\n",
    "parser.add_argument(\"--max_norm\", type=float, default=5., help=\"max norm (grad clipping)\")\n",
    "\n",
    "#model\n",
    "parser.add_argument(\"--encoder_type\", type=str, default='BLSTMEncoder', help=\"see list of encoders\")\n",
    "parser.add_argument(\"--enc_lstm_dim\", type=int, default=2048, help=\"encoder nhid dimension\")\n",
    "parser.add_argument(\"--n_enc_layers\", type=int, default=1, help=\"encoder num layers\")\n",
    "parser.add_argument(\"--fc_dim\", type=int, default=512, help=\"nhid of fc layers\")\n",
    "parser.add_argument(\"--n_classes\", type=int, default=3, help=\"entailment/neutral/contradiction\")\n",
    "parser.add_argument(\"--pool_type\", type=str, default='max', help=\"max or mean\")\n",
    "\n",
    "# gpu\n",
    "parser.add_argument(\"--gpu_id\", type=int, default=0, help=\"GPU ID\")\n",
    "parser.add_argument(\"--seed\", type=int, default=1234, help=\"seed\")\n",
    "\n",
    "\n",
    "params, _ = parser.parse_known_args(\" \".split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(params.seed)\n",
    "torch.manual_seed(params.seed)\n",
    "torch.cuda.manual_seed(params.seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NLINet (\n",
      "  (encoder): BLSTMEncoder (\n",
      "    (enc_lstm): LSTM(300, 2048, bidirectional=True)\n",
      "  )\n",
      "  (classifier): Sequential (\n",
      "    (0): Linear (16384 -> 512)\n",
      "    (1): Linear (512 -> 512)\n",
      "    (2): Linear (512 -> 3)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "params.word_emb_dim = 300\n",
    "\"\"\"\n",
    "MODEL\n",
    "\"\"\"\n",
    "# model config\n",
    "config_nli_model = {\n",
    "    'n_words'        :  len(word_vec)          ,\n",
    "    'word_emb_dim'   :  params.word_emb_dim   ,\n",
    "    'enc_lstm_dim'   :  params.enc_lstm_dim   ,\n",
    "    'n_enc_layers'   :  params.n_enc_layers   ,\n",
    "    'dpout_model'    :  params.dpout_model    ,\n",
    "    'dpout_fc'       :  params.dpout_fc       ,\n",
    "    'fc_dim'         :  params.fc_dim         ,\n",
    "    'bsize'          :  params.batch_size     ,\n",
    "    'n_classes'      :  params.n_classes      ,\n",
    "    'pool_type'      :  params.pool_type      ,\n",
    "    'nonlinear_fc'   :  params.nonlinear_fc   ,\n",
    "    'encoder_type'   :  params.encoder_type   ,\n",
    "    'use_cuda'       :  True                  ,\n",
    "\n",
    "}\n",
    "\n",
    "# model\n",
    "encoder_types = ['BLSTMEncoder', 'BLSTMprojEncoder', 'BGRUlastEncoder', 'InnerAttentionMILAEncoder',\\\n",
    "                 'InnerAttentionYANGEncoder', 'InnerAttentionNAACLEncoder', 'ConvNetEncoder', 'LSTMEncoder']\n",
    "assert params.encoder_type in encoder_types, \"encoder_type must be in \" + str(encoder_types)\n",
    "nli_net = NLINet(config_nli_model)\n",
    "print(nli_net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# loss\n",
    "weight = torch.FloatTensor(params.n_classes).fill_(1)\n",
    "loss_fn = nn.CrossEntropyLoss(weight=weight)\n",
    "loss_fn.size_average = False\n",
    "\n",
    "# optimizer\n",
    "optim_fn, optim_params = get_optimizer(params.optimizer)\n",
    "optimizer = optim_fn(nli_net.parameters(), **optim_params)\n",
    "\n",
    "# cuda by default\n",
    "nli_net.cuda()\n",
    "loss_fn.cuda()\n",
    "#src_embeddings.cuda()\n",
    "\n",
    "\n",
    "    \n",
    "\"\"\"\n",
    "TRAIN\n",
    "\"\"\"\n",
    "#src_embeddings.volatile = True\n",
    "val_acc_best = -1e10\n",
    "adam_stop = False\n",
    "stop_training = False\n",
    "lr = optim_params['lr'] if 'sgd' in params.optimizer else None\n",
    "#index_pad =word2id['<p>']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def trainepoch(epoch):\n",
    "    print('\\nTRAINING : Epoch ' + str(epoch))\n",
    "    nli_net.train()\n",
    "    all_costs   = []\n",
    "    logs        = []\n",
    "    words_count = 0\n",
    "    \n",
    "    last_time = time.time()\n",
    "    correct = 0.\n",
    "    # shuffle the data\n",
    "    permutation = np.random.permutation(len(train_set['sentence1']))\n",
    "\n",
    "    s1 = train_set['sentence1'][permutation]\n",
    "    s2 = train_set['sentence2'][permutation]\n",
    "    target = train_set['label'][permutation]\n",
    "    \n",
    "\n",
    "    optimizer.param_groups[0]['lr'] = optimizer.param_groups[0]['lr'] * params.decay if epoch>1\\\n",
    "                                    and 'sgd' in params.optimizer else optimizer.param_groups[0]['lr']\n",
    "    print('Learning rate : {0}'.format(optimizer.param_groups[0]['lr']))\n",
    "\n",
    "    for stidx in range(0, len(s1), params.batch_size):\n",
    "        # prepare batch\n",
    "        s1_batch, s1_len = get_batch(s1[stidx:stidx + params.batch_size], word_vec)\n",
    "        s2_batch, s2_len = get_batch(s2[stidx:stidx + params.batch_size], word_vec)\n",
    "        s1_batch, s2_batch = Variable(s1_batch.cuda()), Variable(s2_batch.cuda())\n",
    "        tgt_batch = Variable(torch.LongTensor(target[stidx:stidx + params.batch_size])).cuda()\n",
    "        k = s1_batch.size(1)  # actual batch size\n",
    "#         print(s1_batch, s1_len) \n",
    "        # model forward\n",
    "        output = nli_net((s1_batch, s1_len), (s2_batch, s2_len))\n",
    "\n",
    "        pred = output.data.max(1)[1]\n",
    "        correct += pred.long().eq(tgt_batch.data.long()).cpu().sum()\n",
    "        assert len(pred) == len(s1[stidx:stidx + params.batch_size])\n",
    "        \n",
    "        # loss\n",
    "        loss = loss_fn(output, tgt_batch)\n",
    "        all_costs.append(loss.data[0])\n",
    "        words_count += (s1_batch.nelement() + s2_batch.nelement()) / params.word_emb_dim\n",
    "\n",
    "        # backward\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        \n",
    "        # gradient clipping (off by default)\n",
    "        shrink_factor = 1\n",
    "        total_norm = 0\n",
    "        \n",
    "        for p in nli_net.parameters():\n",
    "            if p.requires_grad:\n",
    "                p.grad.data.div_(k)  # divide by the actual batch size\n",
    "                total_norm += p.grad.data.norm() ** 2\n",
    "        total_norm = np.sqrt(total_norm)\n",
    "        \n",
    "        if total_norm > params.max_norm:\n",
    "            shrink_factor = params.max_norm / total_norm\n",
    "        current_lr = optimizer.param_groups[0]['lr'] # current lr (no external \"lr\", for adam)\n",
    "        optimizer.param_groups[0]['lr'] = current_lr * shrink_factor # just for update\n",
    "        \n",
    "        # optimizer step\n",
    "        optimizer.step()\n",
    "        optimizer.param_groups[0]['lr'] = current_lr\n",
    "\n",
    "        if len(all_costs) == 100:\n",
    "            logs.append( '{0} ; loss {1} ; sentence/s {2} ; words/s {3} ; accuracy train : {4}'.format(\n",
    "                    stidx, round(np.mean(all_costs),2), int(len(all_costs) * params.batch_size / (time.time() - last_time)),\n",
    "                    int(words_count * 1.0 / (time.time() - last_time)), round(100.*correct/(stidx+k), 2) ))\n",
    "            print(logs[-1])\n",
    "            last_time = time.time()\n",
    "            words_count = 0\n",
    "            all_costs = []\n",
    "    train_acc = round(100 * correct/len(s1), 2)\n",
    "    print('results : epoch {0} ; mean accuracy train : {1}'.format(epoch, train_acc))\n",
    "    return train_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(epoch, eval_type='valid', matched = True, final_eval=False):\n",
    "    nli_net.eval()\n",
    "    correct = 0.\n",
    "    global val_acc_best, lr, stop_training, adam_stop\n",
    "    \n",
    "    if eval_type == 'valid' and matched==True:\n",
    "        print('\\nVALIDATION, matched : Epoch {0}'.format(epoch))\n",
    "    else:\n",
    "        print('\\nVALIDATION, mismatched : Epoch {0}'.format(epoch))\n",
    "        \n",
    "        \n",
    "    if matched:\n",
    "        data_m_set = dev_matched_set\n",
    "    else:\n",
    "        data_m_set = dev_mismatched_set\n",
    "    \n",
    "    \n",
    "    s1    = data_m_set['sentence1']    \n",
    "    s2    = data_m_set['sentence2']    \n",
    "    target = data_m_set['label'] \n",
    "\n",
    "\n",
    "    for i in range(0, len(s1), params.batch_size):\n",
    "        # prepare batch\n",
    "        s1_batch, s1_len = get_batch(s1[i:i + params.batch_size], word_vec)\n",
    "        s2_batch, s2_len = get_batch(s2[i:i + params.batch_size], word_vec)\n",
    "        s1_batch, s2_batch = Variable(s1_batch.cuda()), Variable(s2_batch.cuda())\n",
    "        tgt_batch = Variable(torch.LongTensor(target[i:i + params.batch_size])).cuda()\n",
    "        k = s1_batch.size(1)  # actual batch size\n",
    "            \n",
    "            \n",
    "        # model forward\n",
    "        output = nli_net((s1_batch, s1_len), (s2_batch, s2_len))\n",
    "        \n",
    "        pred = output.data.max(1)[1]\n",
    "        correct += pred.long().eq(tgt_batch.data.long()).cpu().sum()\n",
    "        \n",
    "        \n",
    "    # save model\n",
    "    eval_acc  = round(100 * correct / len(s1),2)\n",
    "    if final_eval:\n",
    "        print('finalgrep : accuracy {0} : {1}'.format(eval_type, eval_acc))\n",
    "    else:\n",
    "        print('togrep : results : epoch {0} ; mean accuracy {1} : {2}'.format(epoch, eval_type, eval_acc))\n",
    "    \n",
    "    if eval_type == 'valid' and epoch <= params.n_epochs:\n",
    "        if eval_acc > val_acc_best:\n",
    "            print('saving model at epoch {0}'.format(epoch))\n",
    "            if not os.path.exists(params.outputdir):\n",
    "                os.makedirs(params.outputdir)\n",
    "            torch.save(nli_net, os.path.join(params.outputdir, params.outputmodelname))\n",
    "            val_acc_best = eval_acc\n",
    "        else:\n",
    "            if 'sgd' in params.optimizer:\n",
    "                optimizer.param_groups[0]['lr'] = optimizer.param_groups[0]['lr'] / params.lrshrink\n",
    "                print('Shrinking lr by : {0}. New lr = {1}'.format(params.lrshrink, optimizer.param_groups[0]['lr']))\n",
    "                if optimizer.param_groups[0]['lr'] < params.minlr:\n",
    "                    stop_training = True\n",
    "            if 'adam' in params.optimizer:\n",
    "                # early stopping (at 2nd decrease in accuracy)\n",
    "                stop_training = adam_stop\n",
    "                adam_stop = True\n",
    "    return eval_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# epoch = 1\n",
    "# while not stop_training and epoch <= params.n_epochs:\n",
    "#     train_acc = trainepoch(epoch)\n",
    "#     eval_acc = evaluate(epoch, eval_type='valid', matched = True)\n",
    "#     eval_acc = evaluate(epoch, eval_type='valid', matched = False)\n",
    "#     epoch+=1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_matched_set_1 = pd.DataFrame(test_matched_dataset)\n",
    "test_mismatched_set_1 = pd.DataFrame(test_mismatched_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "label_category = {\n",
    "    'neutral': 0,\n",
    "    'entailment': 1,\n",
    "    'contradiction': 2\n",
    "}\n",
    "category_label = {v: k for k, v in label_category.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_csvs(epoch, matched = True):\n",
    "    predictions_list = []\n",
    "    nli_net.eval()\n",
    "    correct = 0.\n",
    "    global val_acc_best, lr, stop_training, adam_stop\n",
    "        \n",
    "    if matched:\n",
    "        data_m_set = test_matched_set\n",
    "        pairId = list(test_matched_set_1['pairID'])\n",
    "    else:\n",
    "        data_m_set = test_mismatched_set\n",
    "        pairId = list(test_mismatched_set_1['pairID'])\n",
    "    \n",
    "    \n",
    "    s1    = data_m_set['sentence1']    \n",
    "    s2    = data_m_set['sentence2']    \n",
    "\n",
    "    params.batch_size=1\n",
    "    for i in range(0, len(s1), params.batch_size):\n",
    "        # prepare batch\n",
    "        s1_batch, s1_len = get_batch(s1[i:i + params.batch_size], word_vec)\n",
    "        s2_batch, s2_len = get_batch(s2[i:i + params.batch_size], word_vec)\n",
    "        s1_batch, s2_batch = Variable(s1_batch.cuda()), Variable(s2_batch.cuda())\n",
    "        k = s1_batch.size(1)  # actual batch size\n",
    "            \n",
    "            \n",
    "        # model forward\n",
    "        output = nli_net((s1_batch, s1_len), (s2_batch, s2_len))\n",
    "        \n",
    "        pred = output.data.max(1)[1].cpu()\n",
    "        predictions_list.append(pred.numpy())\n",
    "    \n",
    "    final_predictions = [category_label[k[0][0]] for k in predictions_list]\n",
    "        \n",
    "    return final_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del nli_net\n",
    "epoch=0\n",
    "nli_net = torch.load(os.path.join(params.outputdir, params.outputmodelname))\n",
    "final_val = generate_csvs(epoch)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_val2  = generate_csvs(epoch, matched=False)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "final_val2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savetxt(\"matched_test.csv\", np.column_stack((list(test_matched_set_1['pairID']),final_val)),delimiter=\",\", fmt=\"%s, %s\",header=' pairId, gold_label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_out = pd.read_csv('matched_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#  pairId</th>\n",
       "      <th>gold_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9847</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9848</td>\n",
       "      <td>entailment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9849</td>\n",
       "      <td>entailment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9850</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9851</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9852</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9853</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>9854</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9855</td>\n",
       "      <td>entailment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9856</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>9857</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>9858</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>9859</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>9860</td>\n",
       "      <td>entailment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>9861</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>9862</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>9863</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>9864</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>9865</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>9866</td>\n",
       "      <td>entailment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>9867</td>\n",
       "      <td>entailment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>9868</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>9869</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>9870</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>9871</td>\n",
       "      <td>entailment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>9872</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>9873</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>9874</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>9875</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>9876</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9766</th>\n",
       "      <td>19613</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9767</th>\n",
       "      <td>19614</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9768</th>\n",
       "      <td>19615</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9769</th>\n",
       "      <td>19616</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9770</th>\n",
       "      <td>19617</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9771</th>\n",
       "      <td>19618</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9772</th>\n",
       "      <td>19619</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9773</th>\n",
       "      <td>19620</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9774</th>\n",
       "      <td>19621</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9775</th>\n",
       "      <td>19622</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9776</th>\n",
       "      <td>19623</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9777</th>\n",
       "      <td>19624</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9778</th>\n",
       "      <td>19625</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9779</th>\n",
       "      <td>19626</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9780</th>\n",
       "      <td>19627</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9781</th>\n",
       "      <td>19628</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9782</th>\n",
       "      <td>19629</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9783</th>\n",
       "      <td>19630</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9784</th>\n",
       "      <td>19631</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9785</th>\n",
       "      <td>19632</td>\n",
       "      <td>entailment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9786</th>\n",
       "      <td>19633</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9787</th>\n",
       "      <td>19634</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9788</th>\n",
       "      <td>19635</td>\n",
       "      <td>entailment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9789</th>\n",
       "      <td>19636</td>\n",
       "      <td>entailment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9790</th>\n",
       "      <td>19637</td>\n",
       "      <td>entailment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9791</th>\n",
       "      <td>19638</td>\n",
       "      <td>entailment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9792</th>\n",
       "      <td>19639</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9793</th>\n",
       "      <td>19640</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9794</th>\n",
       "      <td>19641</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9795</th>\n",
       "      <td>19642</td>\n",
       "      <td>contradiction</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9796 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      #  pairId      gold_label\n",
       "0          9847         neutral\n",
       "1          9848      entailment\n",
       "2          9849      entailment\n",
       "3          9850         neutral\n",
       "4          9851   contradiction\n",
       "5          9852   contradiction\n",
       "6          9853         neutral\n",
       "7          9854         neutral\n",
       "8          9855      entailment\n",
       "9          9856         neutral\n",
       "10         9857         neutral\n",
       "11         9858   contradiction\n",
       "12         9859         neutral\n",
       "13         9860      entailment\n",
       "14         9861         neutral\n",
       "15         9862         neutral\n",
       "16         9863         neutral\n",
       "17         9864         neutral\n",
       "18         9865         neutral\n",
       "19         9866      entailment\n",
       "20         9867      entailment\n",
       "21         9868   contradiction\n",
       "22         9869         neutral\n",
       "23         9870         neutral\n",
       "24         9871      entailment\n",
       "25         9872         neutral\n",
       "26         9873   contradiction\n",
       "27         9874         neutral\n",
       "28         9875   contradiction\n",
       "29         9876         neutral\n",
       "...         ...             ...\n",
       "9766      19613   contradiction\n",
       "9767      19614   contradiction\n",
       "9768      19615   contradiction\n",
       "9769      19616         neutral\n",
       "9770      19617   contradiction\n",
       "9771      19618         neutral\n",
       "9772      19619         neutral\n",
       "9773      19620         neutral\n",
       "9774      19621         neutral\n",
       "9775      19622         neutral\n",
       "9776      19623         neutral\n",
       "9777      19624   contradiction\n",
       "9778      19625   contradiction\n",
       "9779      19626   contradiction\n",
       "9780      19627         neutral\n",
       "9781      19628   contradiction\n",
       "9782      19629         neutral\n",
       "9783      19630         neutral\n",
       "9784      19631         neutral\n",
       "9785      19632      entailment\n",
       "9786      19633   contradiction\n",
       "9787      19634         neutral\n",
       "9788      19635      entailment\n",
       "9789      19636      entailment\n",
       "9790      19637      entailment\n",
       "9791      19638      entailment\n",
       "9792      19639         neutral\n",
       "9793      19640   contradiction\n",
       "9794      19641         neutral\n",
       "9795      19642   contradiction\n",
       "\n",
       "[9796 rows x 2 columns]"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a = [1,2,3,4,5]\n",
    "b = ['a', 'b', 'c', 'd', 'e']\n",
    "np.savetxt(\"f1.csv\", np.column_stack((a,b)),delimiter=\",\", fmt=\"%s, %s\",header=\"val1, val2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "k1 = pd.read_csv(\"f1.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['# val1', ' val2'], dtype='object')"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k1.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python 3.5.3 :: Continuum Analytics, Inc.\r\n"
     ]
    }
   ],
   "source": [
    "!python --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "appdirs==1.4.3\r\n",
      "bleach==2.0.0\r\n",
      "cffi==1.10.0\r\n",
      "decorator==4.0.11\r\n",
      "entrypoints==0.2.2\r\n",
      "html5lib==0.999999999\r\n",
      "ipykernel==4.6.1\r\n",
      "ipython==6.0.0\r\n",
      "ipython-genutils==0.2.0\r\n",
      "ipywidgets==6.0.0\r\n",
      "jedi==0.10.2\r\n",
      "Jinja2==2.9.6\r\n",
      "jsonschema==2.6.0\r\n",
      "jupyter==1.0.0\r\n",
      "jupyter-client==5.0.1\r\n",
      "jupyter-console==5.1.0\r\n",
      "jupyter-core==4.3.0\r\n",
      "MarkupSafe==1.0\r\n",
      "mistune==0.7.4\r\n",
      "nbconvert==5.1.1\r\n",
      "nbformat==4.3.0\r\n",
      "nltk==3.2.4\r\n",
      "notebook==5.0.0\r\n",
      "numpy==1.13.1\r\n",
      "olefile==0.44\r\n",
      "packaging==16.8\r\n",
      "pandas==0.20.2\r\n",
      "pandocfilters==1.4.1\r\n",
      "pexpect==4.2.1\r\n",
      "pickleshare==0.7.4\r\n",
      "Pillow==4.2.1\r\n",
      "prompt-toolkit==1.0.14\r\n",
      "ptyprocess==0.5.1\r\n",
      "pycparser==2.17\r\n",
      "Pygments==2.2.0\r\n",
      "pyparsing==2.2.0\r\n",
      "python-dateutil==2.6.0\r\n",
      "pytz==2017.2\r\n",
      "PyYAML==3.12\r\n",
      "pyzmq==16.0.2\r\n",
      "qtconsole==4.3.0\r\n",
      "requests==2.14.2\r\n",
      "scikit-learn==0.18.2\r\n",
      "scipy==0.19.1\r\n",
      "simplegeneric==0.8.1\r\n",
      "six==1.10.0\r\n",
      "terminado==0.6\r\n",
      "testpath==0.3\r\n",
      "torch==0.1.12.post2\r\n",
      "torchvision==0.1.8\r\n",
      "tornado==4.5.1\r\n",
      "traitlets==4.3.2\r\n",
      "wcwidth==0.1.7\r\n",
      "webencodings==0.5.1\r\n",
      "widgetsnbextension==2.0.0\r\n"
     ]
    }
   ],
   "source": [
    "!pip freeze"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}