RNN-GSN_opendeep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Imports!\n",
    "# standard libraries\n",
    "import logging\n",
    "import math\n",
    "# third party\n",
    "import theano\n",
    "# internal references\n",
    "from opendeep.data import MNIST\n",
    "from opendeep.log import config_root_logger\n",
    "from opendeep.models import Model, RNN, GSN\n",
    "from opendeep.optimization import RMSProp\n",
    "\n",
    "config_root_logger()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Let's define a new model combining the RNN and GSNs.\n",
    "class RNN_GSN(Model):\n",
    "    def __init__(self):\n",
    "        super(RNN_GSN, self).__init__()\n",
    "\n",
    "        gsn_hiddens = 500\n",
    "        gsn_layers = 2\n",
    "\n",
    "        # RNN that takes in images (3D sequences) and outputs gsn hiddens (3D sequence of them)\n",
    "        self.rnn = RNN(\n",
    "            input_size=28 * 28,\n",
    "            hidden_size=100,\n",
    "            # needs to output hidden units for odd layers of GSN\n",
    "            output_size=gsn_hiddens * (math.ceil(gsn_layers/2.)),\n",
    "            layers=1,\n",
    "            activation='tanh',\n",
    "            hidden_activation='relu',\n",
    "            weights_init='uniform', weights_interval='montreal',\n",
    "            r_weights_init='identity'\n",
    "        )\n",
    "\n",
    "        # Create the GSN that will encode the input space\n",
    "        gsn = GSN(\n",
    "            input_size=28 * 28,\n",
    "            hidden_size=gsn_hiddens,\n",
    "            layers=gsn_layers,\n",
    "            walkbacks=4,\n",
    "            visible_activation='sigmoid',\n",
    "            hidden_activation='tanh',\n",
    "            image_height=28,\n",
    "            image_width=28\n",
    "        )\n",
    "        # grab the input arguments\n",
    "        gsn_args = gsn.args.copy()\n",
    "        # grab the parameters it initialized\n",
    "        gsn_params = gsn.get_params()\n",
    "\n",
    "        # Now hook the two up! RNN should output hiddens for GSN into a 3D tensor (1 set for each timestep)\n",
    "        # Therefore, we need to use scan to create the GSN reconstruction for each timestep given the hiddens\n",
    "        def step(hiddens, x):\n",
    "            gsn = GSN(\n",
    "                inputs_hook=(28*28, x),\n",
    "                hiddens_hook=(gsn_hiddens, hiddens),\n",
    "                params_hook=(gsn_params),\n",
    "                **gsn_args\n",
    "            )\n",
    "            # return the reconstruction and cost!\n",
    "            return gsn.get_outputs(), gsn.get_train_cost()\n",
    "\n",
    "        (outputs, costs), scan_updates = theano.scan(\n",
    "            fn=lambda h, x: step(h, x),\n",
    "            sequences=[self.rnn.output, self.rnn.input],\n",
    "            outputs_info=[None, None]\n",
    "        )\n",
    "\n",
    "        self.outputs = outputs\n",
    "\n",
    "        self.updates = dict()\n",
    "        self.updates.update(self.rnn.get_updates())\n",
    "        self.updates.update(scan_updates)\n",
    "\n",
    "        self.cost = costs.sum()\n",
    "        self.params = gsn_params + self.rnn.get_params()\n",
    "\n",
    "    # Model functions necessary for training\n",
    "    def get_inputs(self):\n",
    "        return self.rnn.get_inputs()\n",
    "    def get_params(self):\n",
    "        return self.params\n",
    "    def get_train_cost(self):\n",
    "        return self.cost\n",
    "    def get_updates(self):\n",
    "        return self.updates\n",
    "    def get_outputs(self):\n",
    "        return self.outputs\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Now we can instantiate and train the model!\n",
    "rnn_gsn = RNN_GSN()\n",
    "\n",
    "# data! (needs to be 3d for rnn).\n",
    "mnist = MNIST(sequence_number=1, seq_3d=True, seq_length=30)\n",
    "\n",
    "# optimizer!\n",
    "optimizer = RMSProp(\n",
    "    model=rnn_gsn,\n",
    "    dataset=mnist,\n",
    "    epochs=500,\n",
    "    batch_size=50,\n",
    "    save_freq=10,\n",
    "    stop_patience=30,\n",
    "    stop_threshold=.9995,\n",
    "    learning_rate=1e-6,\n",
    "    decay=.95,\n",
    "    max_scaling=1e5,\n",
    "    grad_clip=5.,\n",
    "    hard_clip=False\n",
    ")\n",
    "# train!\n",
    "optimizer.train()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}