4.1 Convolutional Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convolution Network "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Adapted notebook from Author: Aymeric Damien\n",
    "# Project: https://github.com/aymericdamien/TensorFlow-Examples/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import MINST data\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"tmp/data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "learning_rate = 0.001\n",
    "training_iters = 100000\n",
    "batch_size = 128\n",
    "display_step = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Network Parameters\n",
    "n_input = 784 # MNIST data input (img shape: 28*28)\n",
    "n_classes = 10 # MNIST total classes (0-9 digits)\n",
    "dropout = 0.75 # Dropout, probability to keep units"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# tf Graph input\n",
    "x = tf.placeholder(tf.float32, [None, n_input])\n",
    "y = tf.placeholder(tf.float32, [None, n_classes])\n",
    "keep_prob = tf.placeholder(tf.float32) #dropout (keep probability)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create model\n",
    "def conv2d(img, w, b):\n",
    "    return tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(img, w, strides=[1, 1, 1, 1], \n",
    "                                                  padding='SAME'),b))\n",
    "\n",
    "def max_pool(img, k):\n",
    "    return tf.nn.max_pool(img, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='SAME')\n",
    "\n",
    "def conv_net(_X, _weights, _biases, _dropout):\n",
    "    # Reshape input picture\n",
    "    _X = tf.reshape(_X, shape=[-1, 28, 28, 1])\n",
    "\n",
    "    # Convolution Layer\n",
    "    conv1 = conv2d(_X, _weights['wc1'], _biases['bc1'])\n",
    "    # Max Pooling (down-sampling)\n",
    "    conv1 = max_pool(conv1, k=2)\n",
    "    # Apply Dropout\n",
    "    conv1 = tf.nn.dropout(conv1, _dropout)\n",
    "\n",
    "    # Convolution Layer\n",
    "    conv2 = conv2d(conv1, _weights['wc2'], _biases['bc2'])\n",
    "    # Max Pooling (down-sampling)\n",
    "    conv2 = max_pool(conv2, k=2)\n",
    "    # Apply Dropout\n",
    "    conv2 = tf.nn.dropout(conv2, _dropout)\n",
    "\n",
    "    # Fully connected layer\n",
    "    # Reshape conv2 output to fit dense layer input\n",
    "    dense1 = tf.reshape(conv2, [-1, _weights['wd1'].get_shape().as_list()[0]]) \n",
    "    # Relu activation\n",
    "    dense1 = tf.nn.relu(tf.add(tf.matmul(dense1, _weights['wd1']), _biases['bd1']))\n",
    "    # Apply Dropout\n",
    "    dense1 = tf.nn.dropout(dense1, _dropout) # Apply Dropout\n",
    "\n",
    "    # Output, class prediction\n",
    "    out = tf.add(tf.matmul(dense1, _weights['out']), _biases['out'])\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Store layers weight & bias\n",
    "weights = {\n",
    "    # 5x5 conv, 1 input, 32 outputs\n",
    "    'wc1': tf.Variable(tf.random_normal([5, 5, 1, 32])), \n",
    "    # 5x5 conv, 32 inputs, 64 outputs\n",
    "    'wc2': tf.Variable(tf.random_normal([5, 5, 32, 64])), \n",
    "    # fully connected, 7*7*64 inputs, 1024 outputs\n",
    "    'wd1': tf.Variable(tf.random_normal([7*7*64, 1024])), \n",
    "    # 1024 inputs, 10 outputs (class prediction)\n",
    "    'out': tf.Variable(tf.random_normal([1024, n_classes])) \n",
    "}\n",
    "\n",
    "biases = {\n",
    "    'bc1': tf.Variable(tf.random_normal([32])),\n",
    "    'bc2': tf.Variable(tf.random_normal([64])),\n",
    "    'bd1': tf.Variable(tf.random_normal([1024])),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Construct model\n",
    "pred = conv_net(x, weights, biases, keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Evaluate model\n",
    "correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initializing the variables\n",
    "init = tf.initialize_all_variables()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 2560, Minibatch Loss= 21154.437500, Training Accuracy= 0.30469\n",
      "Iter 5120, Minibatch Loss= 12081.548828, Training Accuracy= 0.50781\n",
      "Iter 7680, Minibatch Loss= 11403.013672, Training Accuracy= 0.53125\n",
      "Iter 10240, Minibatch Loss= 5651.076172, Training Accuracy= 0.70312\n",
      "Iter 12800, Minibatch Loss= 5953.035645, Training Accuracy= 0.67188\n",
      "Iter 15360, Minibatch Loss= 2218.375732, Training Accuracy= 0.80469\n",
      "Iter 17920, Minibatch Loss= 2644.182373, Training Accuracy= 0.78906\n",
      "Iter 20480, Minibatch Loss= 1479.909546, Training Accuracy= 0.87500\n",
      "Iter 23040, Minibatch Loss= 1315.046265, Training Accuracy= 0.91406\n",
      "Iter 25600, Minibatch Loss= 3178.376709, Training Accuracy= 0.77344\n",
      "Iter 28160, Minibatch Loss= 2294.122559, Training Accuracy= 0.81250\n",
      "Iter 30720, Minibatch Loss= 1196.053589, Training Accuracy= 0.84375\n",
      "Iter 33280, Minibatch Loss= 1676.132812, Training Accuracy= 0.85156\n",
      "Iter 35840, Minibatch Loss= 428.335327, Training Accuracy= 0.92969\n",
      "Iter 38400, Minibatch Loss= 340.574097, Training Accuracy= 0.95312\n",
      "Iter 40960, Minibatch Loss= 1397.641113, Training Accuracy= 0.89844\n",
      "Iter 43520, Minibatch Loss= 558.186829, Training Accuracy= 0.92188\n",
      "Iter 46080, Minibatch Loss= 606.679749, Training Accuracy= 0.95312\n",
      "Iter 48640, Minibatch Loss= 1555.515625, Training Accuracy= 0.88281\n",
      "Iter 51200, Minibatch Loss= 384.410248, Training Accuracy= 0.92188\n",
      "Iter 53760, Minibatch Loss= 362.374695, Training Accuracy= 0.95312\n",
      "Iter 56320, Minibatch Loss= 807.723511, Training Accuracy= 0.92969\n",
      "Iter 58880, Minibatch Loss= 544.502686, Training Accuracy= 0.94531\n",
      "Iter 61440, Minibatch Loss= 863.515930, Training Accuracy= 0.92188\n",
      "Iter 64000, Minibatch Loss= 836.524353, Training Accuracy= 0.89844\n",
      "Iter 66560, Minibatch Loss= 656.470886, Training Accuracy= 0.93750\n",
      "Iter 69120, Minibatch Loss= 856.812866, Training Accuracy= 0.89844\n",
      "Iter 71680, Minibatch Loss= 580.534241, Training Accuracy= 0.94531\n",
      "Iter 74240, Minibatch Loss= 708.007568, Training Accuracy= 0.91406\n",
      "Iter 76800, Minibatch Loss= 971.588013, Training Accuracy= 0.91406\n",
      "Iter 79360, Minibatch Loss= 383.844879, Training Accuracy= 0.92969\n",
      "Iter 81920, Minibatch Loss= 909.780396, Training Accuracy= 0.91406\n",
      "Iter 84480, Minibatch Loss= 600.898804, Training Accuracy= 0.95312\n",
      "Iter 87040, Minibatch Loss= 267.563171, Training Accuracy= 0.96094\n",
      "Iter 89600, Minibatch Loss= 772.752686, Training Accuracy= 0.89844\n",
      "Iter 92160, Minibatch Loss= 613.072388, Training Accuracy= 0.91406\n",
      "Iter 94720, Minibatch Loss= 433.241302, Training Accuracy= 0.96094\n",
      "Iter 97280, Minibatch Loss= 684.323486, Training Accuracy= 0.92969\n",
      "Iter 99840, Minibatch Loss= 563.071777, Training Accuracy= 0.92188\n",
      "Optimization Finished!\n",
      "Testing Accuracy: 0.964844\n"
     ]
    }
   ],
   "source": [
    "# Launch the graph\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    step = 1\n",
    "    # Keep training until reach max iterations\n",
    "    while step * batch_size < training_iters:\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        # Fit training using batch data\n",
    "        sess.run(optimizer, feed_dict={x: batch_xs, y: batch_ys, keep_prob: dropout})\n",
    "        if step % display_step == 0:\n",
    "            # Calculate batch accuracy\n",
    "            acc = sess.run(accuracy, feed_dict={x: batch_xs, y: batch_ys, keep_prob: 1.})\n",
    "            # Calculate batch loss\n",
    "            loss = sess.run(cost, feed_dict={x: batch_xs, y: batch_ys, keep_prob: 1.})\n",
    "            print \"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \\\n",
    "                  \"{:.6f}\".format(loss) + \", Training Accuracy= \" + \"{:.5f}\".format(acc)\n",
    "        step += 1\n",
    "    print \"Optimization Finished!\"\n",
    "    # Calculate accuracy for 256 mnist test images\n",
    "    print \"Testing Accuracy:\", sess.run(accuracy, feed_dict={x: mnist.test.images[:256], \n",
    "                                                             y: mnist.test.labels[:256], \n",
    "                                                             keep_prob: 1.})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}