notebooks/Tensorflow_RNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# High-level RNN TF Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "from common.params_lstm import *\n",
    "from common.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Force one-gpu\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OS:  linux\n",
      "Python:  3.5.2 |Anaconda custom (64-bit)| (default, Jul  2 2016, 17:53:06) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "Numpy:  1.14.1\n",
      "Tensorflow:  1.4.0\n",
      "GPU:  ['Tesla P100-PCIE-16GB', 'Tesla P100-PCIE-16GB']\n",
      "CUDA Version 8.0.61\n",
      "CuDNN Version  6.0.21\n"
     ]
    }
   ],
   "source": [
    "print(\"OS: \", sys.platform)\n",
    "print(\"Python: \", sys.version)\n",
    "print(\"Numpy: \", np.__version__)\n",
    "print(\"Tensorflow: \", tf.__version__)\n",
    "print(\"GPU: \", get_gpu_name())\n",
    "print(get_cuda_version())\n",
    "print(\"CuDNN Version \", get_cudnn_version())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_symbol(CUDNN=True, \n",
    "                  maxf=MAXFEATURES, edim=EMBEDSIZE, nhid=NUMHIDDEN, batchs=BATCHSIZE):\n",
    "    word_vectors = tf.contrib.layers.embed_sequence(X, vocab_size=maxf, embed_dim=edim)\n",
    "    word_list = tf.unstack(word_vectors, axis=1)\n",
    "    \n",
    "    if not CUDNN:\n",
    "        cell = tf.contrib.rnn.GRUCell(nhid)\n",
    "        outputs, states = tf.contrib.rnn.static_rnn(cell, word_list, dtype=tf.float32)\n",
    "    else:\n",
    "        # Using cuDNN since vanilla RNN\n",
    "        cudnn_cell = tf.contrib.cudnn_rnn.CudnnGRU(num_layers=1, \n",
    "                                                   num_units=nhid, \n",
    "                                                   input_size=edim)\n",
    "        params_size_t = cudnn_cell.params_size()\n",
    "        params = tf.Variable(tf.random_uniform([params_size_t], -0.1, 0.1), validate_shape=False)   \n",
    "        input_h = tf.Variable(tf.zeros([1, batchs, nhid]))\n",
    "        outputs, states = cudnn_cell(input_data=word_list,\n",
    "                                     input_h=input_h,\n",
    "                                     params=params)\n",
    "        logits = tf.layers.dense(outputs[-1], 2, activation=None, name='output')\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_model(m, y, lr=LR, b1=BETA_1, b2=BETA_2, eps=EPS):\n",
    "    # Single-class labels, don't need dense one-hot\n",
    "    # Expects unscaled logits, not output of tf.nn.softmax\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=m, labels=y)\n",
    "    loss = tf.reduce_mean(xentropy)\n",
    "    optimizer = tf.train.AdamOptimizer(lr, b1, b2, eps)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "    return training_op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing train set...\n",
      "Preparing test set...\n",
      "Trimming to 30000 max-features\n",
      "Padding to length 150\n",
      "(25000, 150) (25000, 150) (25000,) (25000,)\n",
      "int32 int32 int32 int32\n",
      "CPU times: user 5.9 s, sys: 417 ms, total: 6.32 s\n",
      "Wall time: 6.32 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Data into format for library\n",
    "x_train, x_test, y_train, y_test = imdb_for_library(seq_len=MAXLEN, max_features=MAXFEATURES)\n",
    "print(x_train.shape, x_test.shape, y_train.shape, y_test.shape)\n",
    "print(x_train.dtype, x_test.dtype, y_train.dtype, y_test.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 737 ms, sys: 76.1 ms, total: 814 ms\n",
      "Wall time: 820 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Place-holders\n",
    "X = tf.placeholder(tf.int32, shape=[None, MAXLEN])\n",
    "y = tf.placeholder(tf.int32, shape=[None])\n",
    "sym = create_symbol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 836 ms, sys: 693 ms, total: 1.53 s\n",
      "Wall time: 1.54 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model = init_model(sym, y)\n",
    "init = tf.global_variables_initializer()\n",
    "sess = tf.Session()\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.84375\n",
      "1 Train accuracy: 0.96875\n",
      "2 Train accuracy: 0.984375\n",
      "CPU times: user 19 s, sys: 2.77 s, total: 21.8 s\n",
      "Wall time: 22.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Main training loop: 22s\n",
    "correct = tf.nn.in_top_k(sym, y, 1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "for j in range(EPOCHS):\n",
    "    for data, label in yield_mb(x_train, y_train, BATCHSIZE, shuffle=True):\n",
    "        sess.run(model, feed_dict={X: data, y: label})\n",
    "    # Log\n",
    "    acc_train = sess.run(accuracy, feed_dict={X: data, y: label})\n",
    "    print(j, \"Train accuracy:\", acc_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8.67 s, sys: 651 ms, total: 9.32 s\n",
      "Wall time: 9.19 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Main evaluation loop: 9.19s\n",
    "n_samples = (y_test.shape[0]//BATCHSIZE)*BATCHSIZE\n",
    "y_guess = np.zeros(n_samples, dtype=np.int)\n",
    "y_truth = y_test[:n_samples]\n",
    "c = 0\n",
    "for data, label in yield_mb(x_test, y_test, BATCHSIZE):\n",
    "    pred = tf.argmax(sym, 1)\n",
    "    output = sess.run(pred, feed_dict={X: data})\n",
    "    y_guess[c*BATCHSIZE:(c+1)*BATCHSIZE] = output\n",
    "    c += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.8598557692307692\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: \", 1.*sum(y_guess == y_truth)/len(y_guess))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}