2. O'Reilly Generate.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Generating captions from Flickr-30K feature embeddings\n",
    "This notebook generates captions from the Flickr-30K dataset. This allows us to analyze the quality of captions generated by the LSTM we trained in the `O'Reilly Training.ipynb` notebook. Unfortunately, we can not see the images associated with the Feature embedding. For that please look at the `O'Reilly Generate from images.ipynb` notebook.\n",
    "\n",
    "*Note:* The `model_path` variable should be the same as the `model_path` variable set in `O'Reilly Training.ipynb`\n",
    "\n",
    "# Make sure to run the 'O'Reilly Training.ipynb' notebook for at least one epoch before running this notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import os\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "\n",
    "import tensorflow.python.platform\n",
    "from keras.preprocessing import sequence\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Downloading Data\n",
    "As mentioned in the README, in order to run this notebook, you will need VGG-16 image embeddings for the Flickr-30K dataset. These image embeddings are available from our [Google Drive](https://drive.google.com/file/d/0B5o40yxdA9PqTnJuWGVkcFlqcG8/view?usp=sharing).\n",
    "\n",
    "Additionally, you will need the corresponding captions for these images (`results_20130124.token`), which can also be downloaded from our [Google Drive](https://drive.google.com/file/d/0B2vTU3h54lTydXFjSVM5T2t4WmM/view?usp=sharing).\n",
    "\n",
    "(**Note**: These are the same downloads from `O'Reilly Training.ipynb`. If you downloaded them while working through that notebook there is no need to download them again)\n",
    "\n",
    "Place all of these downloads in the `./data/` folder.\n",
    "\n",
    "The feature embeddings will be in `./data/feats.npy` and the embeddings' corresponding captions will be saved to `./data/results_20130124.token` ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "model_path = './models/tensorflow'\n",
    "feature_path = './data/feats.npy'\n",
    "annotation_path = './data/results_20130124.token'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "### Set Hyperparameters ###\n",
    "dim_embed = 256\n",
    "dim_hidden = 256\n",
    "dim_in = 4096\n",
    "batch_size = 1\n",
    "learning_rate = 0.001\n",
    "momentum = 0.9\n",
    "n_epochs = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_data(annotation_path, feature_path):\n",
    "     annotations = pd.read_table(annotation_path, sep='\\t', header=None, names=['image', 'caption'])\n",
    "     return np.load(feature_path,'r'), annotations['caption'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "feats, captions = get_data(annotation_path, feature_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(feats.shape)\n",
    "print(captions.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(captions[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class Caption_Generator():\n",
    "    def __init__(self, dim_in, dim_embed, dim_hidden, batch_size, n_lstm_steps, n_words, init_b=None):\n",
    "\n",
    "        self.dim_in = dim_in\n",
    "        self.dim_embed = dim_embed\n",
    "        self.dim_hidden = dim_hidden\n",
    "        self.batch_size = batch_size\n",
    "        self.n_lstm_steps = n_lstm_steps\n",
    "        self.n_words = n_words\n",
    "        \n",
    "        # declare the variables to be used for our word embeddings\n",
    "        with tf.device(\"/cpu:0\"):\n",
    "            self.word_embedding = tf.Variable(tf.random_uniform([self.n_words, self.dim_embed], -0.1, 0.1), name='word_embedding')\n",
    "\n",
    "        self.embedding_bias = tf.Variable(tf.zeros([dim_embed]), name='embedding_bias')\n",
    "        \n",
    "        # declare the LSTM itself\n",
    "        self.lstm = tf.contrib.rnn.BasicLSTMCell(dim_hidden)\n",
    "        \n",
    "        # declare the variables to be used to embed the image feature embedding to the word embedding space\n",
    "        self.img_embedding = tf.Variable(tf.random_uniform([dim_in, dim_hidden], -0.1, 0.1), name='img_embedding')\n",
    "        self.img_embedding_bias = tf.Variable(tf.zeros([dim_hidden]), name='img_embedding_bias')\n",
    "\n",
    "        # declare the variables to go from an LSTM output to a word encoding output\n",
    "        self.word_encoding = tf.Variable(tf.random_uniform([dim_hidden, n_words], -0.1, 0.1), name='word_encoding')\n",
    "        \n",
    "        # optional initialization setter for encoding bias variable \n",
    "        if init_b is not None:\n",
    "            self.word_encoding_bias = tf.Variable(init_b, name='word_encoding_bias')\n",
    "        else:\n",
    "            self.word_encoding_bias = tf.Variable(tf.zeros([n_words]), name='word_encoding_bias')\n",
    "\n",
    "    def build_model(self):\n",
    "        # declaring the placeholders for our extracted image feature vectors, our caption, and our mask\n",
    "        # (describes how long our caption is with an array of 0/1 values of length `maxlen`  \n",
    "        img = tf.placeholder(tf.float32, [self.batch_size, self.dim_in])\n",
    "        caption_placeholder = tf.placeholder(tf.int32, [self.batch_size, self.n_lstm_steps])\n",
    "        mask = tf.placeholder(tf.float32, [self.batch_size, self.n_lstm_steps])\n",
    "        \n",
    "        # getting an initial LSTM embedding from our image_imbedding\n",
    "        image_embedding = tf.matmul(img, self.img_embedding) + self.img_embedding_bias\n",
    "        \n",
    "        # setting initial state of our LSTM\n",
    "        state = self.lstm.zero_state(self.batch_size, dtype=tf.float32)\n",
    "\n",
    "        total_loss = 0.0\n",
    "        with tf.variable_scope(\"RNN\"):\n",
    "            for i in range(self.n_lstm_steps): \n",
    "                if i > 0:\n",
    "                   # if this isn’t the first iteration of our LSTM we need to get the word_embedding corresponding\n",
    "                   # to the (i-1)th word in our caption \n",
    "                    with tf.device(\"/cpu:0\"):\n",
    "                        current_embedding = tf.nn.embedding_lookup(self.word_embedding, caption_placeholder[:,i-1]) + self.embedding_bias\n",
    "                else:\n",
    "                     #if this is the first iteration of our LSTM we utilize the embedded image as our input \n",
    "                    current_embedding = image_embedding\n",
    "                if i > 0: \n",
    "                    # allows us to reuse the LSTM tensor variable on each iteration\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "                out, state = self.lstm(current_embedding, state)\n",
    "\n",
    "                \n",
    "                if i > 0:\n",
    "                    #get the one-hot representation of the next word in our caption \n",
    "                    labels = tf.expand_dims(caption_placeholder[:, i], 1)\n",
    "                    ix_range=tf.range(0, self.batch_size, 1)\n",
    "                    ixs = tf.expand_dims(ix_range, 1)\n",
    "                    concat = tf.concat([ixs, labels],1)\n",
    "                    onehot = tf.sparse_to_dense(\n",
    "                            concat, tf.stack([self.batch_size, self.n_words]), 1.0, 0.0)\n",
    "\n",
    "\n",
    "                    #perform a softmax classification to generate the next word in the caption\n",
    "                    logit = tf.matmul(out, self.word_encoding) + self.word_encoding_bias\n",
    "                    xentropy = tf.nn.softmax_cross_entropy_with_logits(logits=logit, labels=onehot)\n",
    "                    xentropy = xentropy * mask[:,i]\n",
    "\n",
    "                    loss = tf.reduce_sum(xentropy)\n",
    "                    total_loss += loss\n",
    "\n",
    "            total_loss = total_loss / tf.reduce_sum(mask[:,1:])\n",
    "            return total_loss, img,  caption_placeholder, mask\n",
    "\n",
    "\n",
    "    def build_generator(self, maxlen, batchsize=1):\n",
    "        #same setup as `build_model` function \n",
    "        img = tf.placeholder(tf.float32, [self.batch_size, self.dim_in])\n",
    "        image_embedding = tf.matmul(img, self.img_embedding) + self.img_embedding_bias\n",
    "        state = self.lstm.zero_state(batchsize,dtype=tf.float32)\n",
    "\n",
    "        #declare list to hold the words of our generated captions\n",
    "        all_words = []\n",
    "        with tf.variable_scope(\"RNN\"):\n",
    "            # in the first iteration we have no previous word, so we directly pass in the image embedding\n",
    "            # and set the `previous_word` to the embedding of the start token ([0]) for the future iterations\n",
    "            output, state = self.lstm(image_embedding, state)\n",
    "            previous_word = tf.nn.embedding_lookup(self.word_embedding, [0]) + self.embedding_bias\n",
    "\n",
    "            for i in range(maxlen):\n",
    "                tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "                out, state = self.lstm(previous_word, state)\n",
    "\n",
    "\n",
    "                # get a get maximum probability word and it's encoding from the output of the LSTM\n",
    "                logit = tf.matmul(out, self.word_encoding) + self.word_encoding_bias\n",
    "                best_word = tf.argmax(logit, 1)\n",
    "\n",
    "                with tf.device(\"/cpu:0\"):\n",
    "                    # get the embedding of the best_word to use as input to the next iteration of our LSTM \n",
    "                    previous_word = tf.nn.embedding_lookup(self.word_embedding, best_word)\n",
    "\n",
    "                previous_word += self.embedding_bias\n",
    "\n",
    "                all_words.append(best_word)\n",
    "\n",
    "        return img, all_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "if not os.path.exists('data/ixtoword.npy'):\n",
    "    print ('You must run 1. O\\'reilly Training.ipynb first.')\n",
    "else:\n",
    "    ixtoword = np.load('data/ixtoword.npy').tolist()\n",
    "    n_words = len(ixtoword)\n",
    "    maxlen=15\n",
    "    sess = tf.InteractiveSession()\n",
    "    caption_generator = Caption_Generator(dim_in, dim_hidden, dim_embed, batch_size, maxlen+2, n_words)\n",
    "\n",
    "\n",
    "    image, generated_words = caption_generator.build_generator(maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Note:\n",
    "In order for the call to `saver.restore(sess, tf.train.latest_checkpoint(model_path))` to work you must have run `O' Reilly Training.ipynb` for 1 full epoch. This is because the call in `O' Reilly Training.ipynb` to save the graph to `model_path` only occurs after successfully completing one full epoch of training. If you would like to run this notebook to sanity check the code, uncomment `sanity_check=True`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def test(sess,image,generated_words,ixtoword,idx=0): # Naive greedy search\n",
    "\n",
    "    feats, captions = get_data(annotation_path, feature_path)\n",
    "    feat = np.array([feats[idx]])\n",
    "    \n",
    "    saver = tf.train.Saver()\n",
    "    sanity_check= False\n",
    "    # sanity_check=True\n",
    "    if not sanity_check:\n",
    "        saved_path=tf.train.latest_checkpoint(model_path)\n",
    "        saver.restore(sess, saved_path)\n",
    "    else:\n",
    "        tf.global_variables_initializer().run()\n",
    "\n",
    "    generated_word_index= sess.run(generated_words, feed_dict={image:feat})\n",
    "    generated_word_index = np.hstack(generated_word_index)\n",
    "\n",
    "    generated_sentence = [ixtoword[x] for x in generated_word_index]\n",
    "    print(generated_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "test(sess,image,generated_words,ixtoword,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}