digit_recognizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 397,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import time\n",
    "import numpy as np\n",
    "from scipy import ndimage\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Conv2D, Dropout, Flatten, MaxPooling2D, BatchNormalization, Add, Activation, Cropping2D, Concatenate\n",
    "import keras.backend as K\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the MNIST dataset\n",
    "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x21eda2d59e8>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAALY0lEQVR4nO3dX4hc9RnG8eepfzAYwaQZY4iha2UtlUKjDKGQIhb/EHMTvWgxFyUFYb1QUBSp2It6GUpUelGEWINpsUpBxQihNQRBlCKOkibZxhobNjUmZCfkwqiIjb692GNZ4+7sZM45c6Z5vx9YZvacmZ2XId+c2TmT/BwRAnDu+1bTAwAYDmIHkiB2IAliB5IgdiCJ84f5YMuWLYuxsbFhPiSQytTUlE6cOOG59pWK3fY6Sb+VdJ6k30fE5l63HxsbU6fTKfOQAHpot9vz7hv4Zbzt8yT9TtKtkq6RtNH2NYP+PAD1KvM7+xpJ70fEoYj4XNJzkjZUMxaAqpWJfaWkD2Z9f6TY9jW2J2x3bHe63W6JhwNQRpnY53oT4BufvY2IrRHRjoh2q9Uq8XAAyigT+xFJq2Z9f4Wko+XGAVCXMrG/JWnc9pW2L5R0h6Qd1YwFoGoDn3qLiNO275H0V82cetsWEZOVTQagUqXOs0fETkk7K5oFQI34uCyQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kMRQl2zG6HnwwQd77t+yZUvP/e+9917P/ePj42c9E+rBkR1IgtiBJIgdSILYgSSIHUiC2IEkiB1IgvPsyb377rs999vuuf/ll1/uuf/+++8/65lQj1Kx256SdErSF5JOR0S7iqEAVK+KI/tPIuJEBT8HQI34nR1IomzsIekV22/bnpjrBrYnbHdsd7rdbsmHAzCosrGvjYjrJN0q6W7b1595g4jYGhHtiGi3Wq2SDwdgUKVij4ijxeW0pBclraliKADVGzh22xfbvuSr65JukbS/qsEAVKvMu/HLJb1YnIc9X9KfIuIvlUwFoHIDxx4RhyT9sMJZANSIU29AEsQOJEHsQBLEDiRB7EASxA4kQexAEsQOJEHsQBLEDiRB7EASxA4kQexAEsQOJEHsQBLEDiRB7EASxA4kQexAEsQOJEHsQBIs2ZxcRNS6H6ODIzuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBOfZkyuW3K5tP0bHgkd229tsT9veP2vbUtu7bB8sLpfUOyaAsvp5Gf+0pHVnbHtI0u6IGJe0u/gewAhbMPaIeE3SyTM2b5C0vbi+XdJtFc8FoGKDvkG3PCKOSVJxedl8N7Q9Ybtju9Ptdgd8OABl1f5ufERsjYh2RLRbrVbdDwdgHoPGftz2CkkqLqerGwlAHQaNfYekTcX1TZJeqmYcAHXp59Tbs5L+Jul7to/YvlPSZkk32z4o6ebiewAjbMEP1UTExnl23VjxLABqxMdlgSSIHUiC2IEkiB1IgtiBJPgnrue4Tz75pOf+ycnJUj//0ksvLXV/DA9HdiAJYgeSIHYgCWIHkiB2IAliB5IgdiAJzrOf4z799NOe+w8fPlzq5y9atKjU/TE8HNmBJIgdSILYgSSIHUiC2IEkiB1IgtiBJDjPnlxElNp/0003VTkOasSRHUiC2IEkiB1IgtiBJIgdSILYgSSIHUiC8+zJ2W56BAxJP+uzb7M9bXv/rG2P2P7Q9p7ia329YwIoq5+X8U9LWjfH9scjYnXxtbPasQBUbcHYI+I1SSeHMAuAGpV5g+4e23uLl/lL5ruR7QnbHdudbrdb4uEAlDFo7E9IukrSaknHJD063w0jYmtEtCOi3Wq1Bnw4AGUNFHtEHI+ILyLiS0lPSlpT7VgAqjZQ7LZXzPr2dkn757stgNGw4Hl2289KukHSMttHJP1a0g22V0sKSVOS7qpxRpTwxhtvND0CRsSCsUfExjk2P1XDLABqxMdlgSSIHUiC2IEkiB1IgtiBJPgnrue4yy+/vNT9x8fHe+5fvHhxqZ+P4eHIDiRB7EASxA4kQexAEsQOJEHsQBLEDiTBefZz3KFDh0rd/+qrr+65f9GiRaV+PoaHIzuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBOfZz3FLly7tuT8ieu6fmprquf+zzz7ruf+iiy7quR/Dw5EdSILYgSSIHUiC2IEkiB1IgtiBJIgdSILz7Oe4ycnJnvtt99w/NjbWcz/n0f9/LHhkt73K9qu2D9ietH1vsX2p7V22DxaXS+ofF8Cg+nkZf1rSAxHxfUk/knS37WskPSRpd0SMS9pdfA9gRC0Ye0Qci4h3iuunJB2QtFLSBknbi5ttl3RbXUMCKO+s3qCzPSbpWklvSloeEcekmb8QJF02z30mbHdsd7rdbrlpAQys79htL5b0vKT7IuKjfu8XEVsjoh0R7VarNciMACrQV+y2L9BM6M9ExAvF5uO2VxT7V0iarmdEAFXo5914S3pK0oGIeGzWrh2SNhXXN0l6qfrxAFSln/PsayX9XNI+23uKbQ9L2izpz7bvlPRvST+tZ0QAVVgw9oh4XdJ8n7y4sdpxANSFj8sCSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0k4Iob2YO12OzqdztAeD8im3W6r0+nM+b9Bc2QHkiB2IAliB5IgdiAJYgeSIHYgCWIHkuhnffZVtl+1fcD2pO17i+2P2P7Q9p7ia3394wIYVD/rs5+W9EBEvGP7Eklv295V7Hs8IrbUNx6AqvSzPvsxSceK66dsH5C0su7BAFTrrH5ntz0m6VpJbxab7rG91/Y220vmuc+E7Y7tTrfbLTUsgMH1HbvtxZKel3RfRHwk6QlJV0larZkj/6Nz3S8itkZEOyLarVargpEBDKKv2G1foJnQn4mIFyQpIo5HxBcR8aWkJyWtqW9MAGX18268JT0l6UBEPDZr+4pZN7td0v7qxwNQlX7ejV8r6eeS9tneU2x7WNJG26slhaQpSXfVMiGASvTzbvzrkub697E7qx8HQF34BB2QBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSQx1yWbbXUmHZ21aJunE0AY4O6M626jOJTHboKqc7TsRMef//zbU2L/x4HYnItqNDdDDqM42qnNJzDaoYc3Gy3ggCWIHkmg69q0NP34vozrbqM4lMdughjJbo7+zAxiepo/sAIaE2IEkGond9jrb/7T9vu2HmphhPranbO8rlqHuNDzLNtvTtvfP2rbU9i7bB4vLOdfYa2i2kVjGu8cy440+d00vfz7039ltnyfpPUk3Szoi6S1JGyPiH0MdZB62pyS1I6LxD2DYvl7Sx5L+EBE/KLb9RtLJiNhc/EW5JCJ+OSKzPSLp46aX8S5WK1oxe5lxSbdJ+oUafO56zPUzDeF5a+LIvkbS+xFxKCI+l/ScpA0NzDHyIuI1SSfP2LxB0vbi+nbN/GEZunlmGwkRcSwi3imun5L01TLjjT53PeYaiiZiXynpg1nfH9Forfcekl6x/bbtiaaHmcPyiDgmzfzhkXRZw/OcacFlvIfpjGXGR+a5G2T587KaiH2upaRG6fzf2oi4TtKtku4uXq6iP30t4z0scywzPhIGXf68rCZiPyJp1azvr5B0tIE55hQRR4vLaUkvavSWoj7+1Qq6xeV0w/P8zygt4z3XMuMageeuyeXPm4j9LUnjtq+0faGkOyTtaGCOb7B9cfHGiWxfLOkWjd5S1DskbSqub5L0UoOzfM2oLOM93zLjavi5a3z584gY+pek9Zp5R/5fkn7VxAzzzPVdSX8vviabnk3Ss5p5WfcfzbwiulPStyXtlnSwuFw6QrP9UdI+SXs1E9aKhmb7sWZ+NdwraU/xtb7p567HXEN53vi4LJAEn6ADkiB2IAliB5IgdiAJYgeSIHYgCWIHkvgveI+GWVd9y20AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Show an example image from the training data along with its label\n",
    "index = int(random.random()*len(x_train))\n",
    "print(y_train[index])\n",
    "plt.imshow(x_train[index], cmap='Greys')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28)\n",
      "(10000, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "# 60k training examples, 28 x 28 pixel images\n",
    "print(x_train.shape)\n",
    "# 10k testing examples\n",
    "print(x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reshape, only one layer (grayscale)\n",
    "x_train = x_train.reshape(60000, 28, 28, 1).astype('float32')\n",
    "x_test = x_test.reshape(10000, 28, 28, 1).astype('float32')\n",
    "input_shape = (28, 28, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize\n",
    "x_train /= 255.0\n",
    "x_test /= 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create model - convolution, pooling, flatten, relu, dropout, softmax\n",
    "model = Sequential()\n",
    "model.add(Conv2D(28, kernel_size=(3,3), input_shape = input_shape))\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128, activation=tf.nn.relu))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(10, activation=tf.nn.softmax))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "60000/60000 [==============================] - 11s 176us/step - loss: 0.1975 - accuracy: 0.9414\n",
      "Epoch 2/5\n",
      "60000/60000 [==============================] - 10s 171us/step - loss: 0.0827 - accuracy: 0.9743\n",
      "Epoch 3/5\n",
      "60000/60000 [==============================] - 10s 174us/step - loss: 0.0551 - accuracy: 0.9826\n",
      "Epoch 4/5\n",
      "60000/60000 [==============================] - 10s 165us/step - loss: 0.0413 - accuracy: 0.9870\n",
      "Epoch 5/5\n",
      "60000/60000 [==============================] - 10s 165us/step - loss: 0.0327 - accuracy: 0.9894\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x22f4bffbdd8>"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(optimizer='adam', \n",
    "              loss='sparse_categorical_crossentropy', \n",
    "              metrics=['accuracy'])\n",
    "model.fit(x=x_train, y=y_train, epochs=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 74us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.053561915939213944, 0.9830999970436096]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pretty much the same despite having twice as many neurons in the second to last layer\n",
    "model.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction: 2\n",
      "Actual: 2\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x22f4face860>"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAM0klEQVR4nO3db6xU9Z3H8c9HC0+kGliuhAhZsPHBGpOlZIKbaBrXZuu/B9DEEnjQYDShJhDbpA+qbAI+NJttyWo2GLqSsitrrRYjJma3BIhKTIgjQUWIixraUpE7hGhp/MMq331wD5sL3DlzmTkzZy7f9yuZzMz5zrm/byZ8ODPnNzM/R4QAXP6uqLsBAINB2IEkCDuQBGEHkiDsQBLfGORgs2fPjgULFgxySCCVo0eP6uTJk56o1lPYbd8p6V8kXSnp3yLisbLHL1iwQM1ms5chAZRoNBpta12/jLd9paR/lXSXpBslrbR9Y7d/D0B/9fKefYmk9yPiw4g4I+nXkpZW0xaAqvUS9usk/XHc/WPFtvPYXm27abvZarV6GA5AL3oJ+0QnAS767G1EbI6IRkQ0RkZGehgOQC96CfsxSfPH3Z8n6aPe2gHQL72E/Q1JN9heaHu6pBWSdlTTFoCqdT31FhFf2V4r6b81NvW2JSLerawzAJXqaZ49Il6W9HJFvQDoIz4uCyRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEgP9KWkMXqeFO5cvX15af/7550vrN910U2l97969bWtXX3116b72hL+IjC5xZAeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJJhnvwycPn26bW39+vWl+27fvr20fsUV5ceDQ4cOldZnzZrVtvbss8+W7nvvvfeW1nFpOLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBLMs08BZ8+eLa1v27atbe3xxx+vup3KvPLKK6V15tmr1VPYbR+VdFrS15K+iohGFU0BqF4VR/a/j4iTFfwdAH3Ee3YgiV7DHpJ+Z/tN26sneoDt1babtputVqvH4QB0q9ew3xIRiyXdJWmN7e9c+ICI2BwRjYhojIyM9DgcgG71FPaI+Ki4HpX0gqQlVTQFoHpdh932Vba/ee62pO9JOlhVYwCq1cvZ+DmSXih+2/sbkv4zIv6rkq5wnrJ5dElas2bNgDrBVNZ12CPiQ0l/W2EvAPqIqTcgCcIOJEHYgSQIO5AEYQeS4CuuU8C+ffvqbgGXAY7sQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATfZ58C7rvvvtJ62dLHhw4dqrgbTFUc2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCebZp4BGo1Fa3717d9va8uXLS/d96623SuuffvppaR1TR8cju+0ttkdtHxy3bZbtnbaPFNcz+9smgF5N5mX8ryTdecG2hyXtiogbJO0q7gMYYh3DHhGvSjp1wealkrYWt7dKWlZxXwAq1u0JujkRcVySiutr2z3Q9mrbTdvNVqvV5XAAetX3s/ERsTkiGhHRGBkZ6fdwANroNuwnbM+VpOJ6tLqWAPRDt2HfIWlVcXuVpBeraQdAv3ScZ7f9jKTbJM22fUzSBkmPSfqN7Qck/UHSD/rZJMqVvT3as2dP6b4rVqworT/33HNd9YTh0zHsEbGyTem7FfcCoI/4uCyQBGEHkiDsQBKEHUiCsANJ8BVX1Ob++++vu4VUOLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBLMs1/mPv7449L6rl27+jr+3Llz29auv/76vo6N83FkB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkmGe/zD3xxBOl9VOnLlzGr1ovvfRS29o111zT17FxPo7sQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AE8+yXgdHR0ba1J598coCdXKxsOWkMVscju+0ttkdtHxy37VHbf7J9oLjc3d82AfRqMi/jfyXpzgm2b4yIRcXl5WrbAlC1jmGPiFcl9fczlQD6rpcTdGttv128zJ/Z7kG2V9tu2m62Wq0ehgPQi27DvknStyQtknRc0s/bPTAiNkdEIyIanKwB6tNV2CPiRER8HRFnJf1S0pJq2wJQta7Cbnv87wN/X9LBdo8FMBw6zrPbfkbSbZJm2z4maYOk22wvkhSSjkr6UR97RAdffvll29onn3zS17GXLVtWWp8zZ05fx8fkdQx7RKycYPNTfegFQB/xcVkgCcIOJEHYgSQIO5AEYQeS4CuuU8CRI0dK6+vXrx9QJxebMWNGaX3atGkD6gSdcGQHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSSYZx8C7733Xml97dq1pfXdu3dX2c4l2b59e2n91ltvbVubP39+T2PffvvtpfXp06f39PcvNxzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJ5tkrcObMmdL6008/XVp/6KGHSuuff/75Jfc0KJ999llp/cEHH+zb2K+//npp/eabb+7b2FMRR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJ59gps3LixtL5u3boBdZJLp++zv/baa21rixcvrrqdodfxyG57vu09tg/bftf2j4vts2zvtH2kuJ7Z/3YBdGsyL+O/kvTTiPgbSX8naY3tGyU9LGlXRNwgaVdxH8CQ6hj2iDgeEfuL26clHZZ0naSlkrYWD9sqaVm/mgTQu0s6QWd7gaRvS9onaU5EHJfG/kOQdG2bfVbbbtputlqt3roF0LVJh932DEm/lfSTiPjzZPeLiM0R0YiIxsjISDc9AqjApMJue5rGgr4tIs79nOgJ23OL+lxJo/1pEUAVOk692bakpyQdjohfjCvtkLRK0mPF9Yt96XAK6PQ1T/THF198UVrfsWNH21rGqbfJzLPfIumHkt6xfaDYtk5jIf+N7Qck/UHSD/rTIoAqdAx7ROyV5Dbl71bbDoB+4eOyQBKEHUiCsANJEHYgCcIOJMFXXCvwyCOPlNanTZtWWt+wYUNP48+bN69tbdOmTaX73nHHHaX1nTt3ltY/+OCD0nov9u/fX1pfuHBhaf2ee+6psp0pjyM7kARhB5Ig7EAShB1IgrADSRB2IAnCDiThiBjYYI1GI5rN5sDGA7JpNBpqNpsTfkuVIzuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4k0THstufb3mP7sO13bf+42P6o7T/ZPlBc7u5/uwC6NZlFIr6S9NOI2G/7m5LetH1u5YCNEfHP/WsPQFUmsz77cUnHi9unbR+WdF2/GwNQrUt6z257gaRvS9pXbFpr+23bW2zPbLPPattN281Wq9VTswC6N+mw254h6beSfhIRf5a0SdK3JC3S2JH/5xPtFxGbI6IREY2RkZEKWgbQjUmF3fY0jQV9W0Rsl6SIOBERX0fEWUm/lLSkf20C6NVkzsZb0lOSDkfEL8ZtnzvuYd+XdLD69gBUZTJn42+R9ENJ79g+UGxbJ2ml7UWSQtJRST/qS4cAKjGZs/F7JU30O9QvV98OgH7hE3RAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkHBGDG8xuSfr9uE2zJZ0cWAOXZlh7G9a+JHrrVpW9/XVETPj7bwMN+0WD282IaNTWQIlh7W1Y+5LorVuD6o2X8UAShB1Iou6wb655/DLD2tuw9iXRW7cG0lut79kBDE7dR3YAA0LYgSRqCbvtO22/Z/t92w/X0UM7to/afqdYhrpZcy9bbI/aPjhu2yzbO20fKa4nXGOvpt6GYhnvkmXGa33u6l7+fODv2W1fKel/JP2DpGOS3pC0MiIODbSRNmwfldSIiNo/gGH7O5L+IunfI+KmYts/SToVEY8V/1HOjIifDUlvj0r6S93LeBerFc0dv8y4pGWS7lONz11JX8s1gOetjiP7EknvR8SHEXFG0q8lLa2hj6EXEa9KOnXB5qWStha3t2rsH8vAteltKETE8YjYX9w+LencMuO1PnclfQ1EHWG/TtIfx90/puFa7z0k/c72m7ZX193MBOZExHFp7B+PpGtr7udCHZfxHqQLlhkfmueum+XPe1VH2CdaSmqY5v9uiYjFku6StKZ4uYrJmdQy3oMywTLjQ6Hb5c97VUfYj0maP+7+PEkf1dDHhCLio+J6VNILGr6lqE+cW0G3uB6tuZ//N0zLeE+0zLiG4Lmrc/nzOsL+hqQbbC+0PV3SCkk7aujjIravKk6cyPZVkr6n4VuKeoekVcXtVZJerLGX8wzLMt7tlhlXzc9d7cufR8TAL5Lu1tgZ+Q8k/WMdPbTp63pJbxWXd+vuTdIzGntZ978ae0X0gKS/krRL0pHietYQ9fYfkt6R9LbGgjW3pt5u1dhbw7clHSgud9f93JX0NZDnjY/LAknwCTogCcIOJEHYgSQIO5AEYQeSIOxAEoQdSOL/AOcV29h/UoUZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Show an example image from the training data along with its label\n",
    "# and its predicted label from the model\n",
    "index = int(random.random()*len(x_test))\n",
    "pred = model.predict(x_test[index].reshape(1, 28, 28, 1))\n",
    "print('Prediction: {}'.format(pred.argmax()))\n",
    "print('Actual: {}'.format(y_test[index]))\n",
    "plt.imshow(x_test[index].reshape(28, 28), cmap='Greys')\n",
    "# Looks like a 1 to me..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "60000/60000 [==============================] - 19s 315us/step - loss: 0.0252 - accuracy: 0.9912\n",
      "Epoch 1/1\n",
      "60000/60000 [==============================] - 10s 160us/step - loss: 0.0234 - accuracy: 0.9922\n"
     ]
    }
   ],
   "source": [
    "# Verify CUDA installation and compare GPU training speed to CPU\n",
    "with tf.device('/CPU:0'):\n",
    "    model.compile(optimizer='adam', \n",
    "              loss='sparse_categorical_crossentropy', \n",
    "              metrics=['accuracy'])\n",
    "    model.fit(x=x_train, y=y_train, epochs=1)\n",
    "with tf.device('/GPU:0'):\n",
    "    model.compile(optimizer='adam', \n",
    "              loss='sparse_categorical_crossentropy', \n",
    "              metrics=['accuracy'])\n",
    "    model.fit(x=x_train, y=y_train, epochs=1)\n",
    "# I don't have a great GPU, but it's better than nothing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Try some different architectures\n",
    "model2 = Sequential()\n",
    "model2.add(Conv2D(32, kernel_size=(3,3), input_shape=input_shape, padding='same', activation=tf.nn.relu))\n",
    "model2.add(Conv2D(64, kernel_size=(3,3), padding='same', activation=tf.nn.relu))\n",
    "model2.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model2.add(Dropout(0.2))\n",
    "model2.add(Flatten())\n",
    "model2.add(Dense(64, activation=tf.nn.relu))\n",
    "model2.add(Dropout(0.5))\n",
    "model2.add(Dense(10, activation=tf.nn.softmax))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "60000/60000 [==============================] - 9s 146us/step - loss: 0.5017 - accuracy: 0.8469\n",
      "Epoch 2/5\n",
      "60000/60000 [==============================] - 8s 136us/step - loss: 0.1613 - accuracy: 0.9537\n",
      "Epoch 3/5\n",
      "60000/60000 [==============================] - 8s 136us/step - loss: 0.1194 - accuracy: 0.9651\n",
      "Epoch 4/5\n",
      "60000/60000 [==============================] - 8s 136us/step - loss: 0.0991 - accuracy: 0.9695\n",
      "Epoch 5/5\n",
      "60000/60000 [==============================] - 8s 136us/step - loss: 0.0839 - accuracy: 0.9757\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x22f544eeef0>"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model2.compile(optimizer='adam', \n",
    "               loss='sparse_categorical_crossentropy',\n",
    "               metrics=['accuracy'])\n",
    "model2.fit(x=x_train, y=y_train, epochs=5, batch_size=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 142us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.03603210315018077, 0.9883999824523926]"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Barely better\n",
    "model2.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "model3 = Sequential()\n",
    "model3.add(Dropout(0.1))\n",
    "model3.add(Conv2D(32, kernel_size=(3,3), input_shape=input_shape, padding='same', activation=tf.nn.relu))\n",
    "model3.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model3.add(Dropout(0.2))\n",
    "model3.add(Conv2D(32, kernel_size=(3,3), padding='same', activation=tf.nn.relu))\n",
    "model3.add(Flatten())\n",
    "model3.add(Dropout(0.2))\n",
    "model3.add(Dense(64, activation=tf.nn.relu))\n",
    "model3.add(Dropout(0.25))\n",
    "model3.add(Dense(10, activation=tf.nn.softmax))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "60000/60000 [==============================] - 4s 67us/step - loss: 0.0795 - accuracy: 0.9755\n",
      "Epoch 2/10\n",
      "60000/60000 [==============================] - 4s 63us/step - loss: 0.0707 - accuracy: 0.9783\n",
      "Epoch 3/10\n",
      "60000/60000 [==============================] - 4s 63us/step - loss: 0.0639 - accuracy: 0.9800\n",
      "Epoch 4/10\n",
      "60000/60000 [==============================] - 4s 63us/step - loss: 0.0578 - accuracy: 0.9819\n",
      "Epoch 5/10\n",
      "60000/60000 [==============================] - 4s 63us/step - loss: 0.0532 - accuracy: 0.9829\n",
      "Epoch 6/10\n",
      "60000/60000 [==============================] - 4s 63us/step - loss: 0.0469 - accuracy: 0.9853\n",
      "Epoch 7/10\n",
      "60000/60000 [==============================] - 4s 63us/step - loss: 0.0440 - accuracy: 0.9864\n",
      "Epoch 8/10\n",
      "60000/60000 [==============================] - 4s 63us/step - loss: 0.0413 - accuracy: 0.9868\n",
      "Epoch 9/10\n",
      "60000/60000 [==============================] - 4s 63us/step - loss: 0.0390 - accuracy: 0.9871\n",
      "Epoch 10/10\n",
      "60000/60000 [==============================] - 4s 63us/step - loss: 0.0383 - accuracy: 0.9872\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x22f558f57b8>"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model3.compile(optimizer='adam',\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "model3.fit(x=x_train, y=y_train, epochs=10, batch_size=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 106us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.028460302224976475, 0.9907000064849854]"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#This one was much faster, and slightly better.\n",
    "model3.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ResNetUnit(x, filters=64, size=3, activation='relu', padding='same'):\n",
    "    y = Conv2D(filters, size, activation=activation, padding=padding)(x)\n",
    "    y = BatchNormalization()(y)\n",
    "    y = Conv2D(filters, size, activation=None, padding=padding)(y)\n",
    "    y = BatchNormalization()(y)\n",
    "    if padding=='same':\n",
    "        y = Add()([x, y])\n",
    "    else:\n",
    "        x = Cropping2D(size-1)(x)\n",
    "        y = Add()([x, y])\n",
    "    y = Activation(activation)(y)\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=input_shape)\n",
    "x = Conv2D(32, 3, activation='relu', input_shape=input_shape)(inputs)\n",
    "x = Conv2D(64, 3, activation='relu')(x)\n",
    "x = MaxPooling2D(2)(x)\n",
    "x = ResNetUnit(x)\n",
    "x = ResNetUnit(x)\n",
    "x = Conv2D(64, 3, activation='relu')(x)\n",
    "x = MaxPooling2D(2)(x)\n",
    "x = Flatten()(x)\n",
    "x = Dense(256, activation='relu')(x)\n",
    "x = Dropout(0.2)(x)\n",
    "outputs = Dense(10, activation='softmax')(x)\n",
    "\n",
    "model4 = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "60000/60000 [==============================] - 20s 327us/step - loss: 0.2691 - accuracy: 0.9251\n",
      "Epoch 2/10\n",
      "60000/60000 [==============================] - 19s 309us/step - loss: 0.0457 - accuracy: 0.9862\n",
      "Epoch 3/10\n",
      "60000/60000 [==============================] - 19s 309us/step - loss: 0.0304 - accuracy: 0.9906\n",
      "Epoch 4/10\n",
      "60000/60000 [==============================] - 19s 310us/step - loss: 0.0251 - accuracy: 0.9922\n",
      "Epoch 5/10\n",
      "60000/60000 [==============================] - 19s 309us/step - loss: 0.0204 - accuracy: 0.9937\n",
      "Epoch 6/10\n",
      "60000/60000 [==============================] - 19s 310us/step - loss: 0.0157 - accuracy: 0.9949\n",
      "Epoch 7/10\n",
      "60000/60000 [==============================] - 19s 310us/step - loss: 0.0118 - accuracy: 0.9964\n",
      "Epoch 8/10\n",
      "60000/60000 [==============================] - 19s 309us/step - loss: 0.0102 - accuracy: 0.9967\n",
      "Epoch 9/10\n",
      "60000/60000 [==============================] - 19s 310us/step - loss: 0.0088 - accuracy: 0.9972\n",
      "Epoch 10/10\n",
      "60000/60000 [==============================] - 19s 309us/step - loss: 0.0087 - accuracy: 0.9972\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x2d275bddef0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model4.compile(optimizer='adam',\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "model4.fit(x=x_train, y=y_train, batch_size=512, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 2s 207us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.024909546506872084, 0.9932000041007996]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model4.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2d32a51e898>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAOHUlEQVR4nO3df4xU9bnH8c+DUpEfiSArEsC7pWrQkLhtJnCTvQEvzW3EfxCMN+WPRqPJNgaTaghWagwmJGbjvbVe9Qbd3pLitZcfSWs00VSUNNEmpmFcEfEias3awhJYJKYQDRV47h97vFlx53uWmTNzZnner2QzM+eZs+dhwmfPzHzPOV9zdwG48E0ouwEArUHYgSAIOxAEYQeCIOxAEBe3cmMzZ870zs7OVm4SCGVgYEDHjh2z0WoNhd3MbpL0H5IukvRf7t6ben5nZ6eq1WojmwSQUKlUatbqfhtvZhdJ+k9JyyVdL2m1mV1f7+8D0FyNfGZfJOkjd//Y3f8uaZukFcW0BaBojYR9jqS/jnh8MFv2NWbWY2ZVM6sODQ01sDkAjWgk7KN9CfCNY2/dvc/dK+5e6ejoaGBzABrRSNgPSpo34vFcSYONtQOgWRoJ+25J15jZt83sW5J+KOnFYtoCULS6h97c/bSZ3SPpFQ0PvW129/cK6wxAoRoaZ3f3lyW9XFAvAJqIw2WBIAg7EARhB4Ig7EAQhB0IgrADQbT0fHa03qZNm5L1J554Ill/5ZVXkvWrrrrqvHtCOdizA0EQdiAIwg4EQdiBIAg7EARhB4Jg6O0CcODAgZq13t7kBX81derUZP2dd95J1hl6Gz/YswNBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIyzjwOnTp1K1pcuXVqzdt999yXXXbduXV09Yfxhzw4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQTDOPg5s3749Wb/00ktr1tasWZNcd8IE/t5H0VDYzWxA0glJZySddvdKEU0BKF4Re/Z/dvdjBfweAE3EezggiEbD7pJ2mtlbZtYz2hPMrMfMqmZWHRoaanBzAOrVaNi73f17kpZLWmNmS859grv3uXvF3SsdHR0Nbg5AvRoKu7sPZrdHJT0vaVERTQEoXt1hN7MpZjbtq/uSfiBpX1GNAShWI9/Gz5L0vJl99Xv+x91/X0hX+Jq8c87Xr19fs5Z3XXjEUXfY3f1jSTcU2AuAJmLoDQiCsANBEHYgCMIOBEHYgSA4xbUN5F0q+osvvkjWu7q6imwHFyj27EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBOPsbaC/v7+h9Rct4pohyMeeHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCYJy9DTz22GPJempKZkmaPHlyke3gAsWeHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCYJy9Bdw9WR8cHEzWb7311iLbaRsHDhxI1p977rmGfv/06dNr1pYtW5Zc94Yb0hMUZ1OVjyu5e3Yz22xmR81s34hlM8zsVTP7MLut/aoCaAtjeRv/a0k3nbPsAUm73P0aSbuyxwDaWG7Y3f11ScfPWbxC0pbs/hZJtxTcF4CC1fsF3Sx3PyxJ2e0VtZ5oZj1mVjWz6tDQUJ2bA9Copn8b7+597l5x90pHR0ezNweghnrDfsTMZktSdnu0uJYANEO9YX9R0u3Z/dslvVBMOwCaJXec3cy2SrpR0kwzOyhpg6ReSTvM7C5Jf5F0WzObHO9OnjyZrL/55pvJem9vb5HtFOrMmTPJ+qOPPlqz9uCDDybXnT9/frJ++eWXJ+vXXnttzdrGjRuT677xxhvJ+sKFC5P1dpQbdndfXaP0/YJ7AdBEHC4LBEHYgSAIOxAEYQeCIOxAEJziOg7kDTE1U97puevWrUvWH3/88Zq1F15IH56xfPnyZP3ii+v/77t79+5kfdWqVcn63r17k/VJkyadd0/Nxp4dCIKwA0EQdiAIwg4EQdiBIAg7EARhB4JgnL0FGr0cV96pns20YcOGZH3Hjh3J+r59+2rWrrvuuuS6zbxcc1dXV7L++eefJ+tffvllss44O4DSEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIyzt8Dx4+dOldc+8i5z/eyzzybrr732WrK+YMGC8+6pFSZOnJis543D553P3t3dfd49NRt7diAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgnH2Fsgb083z2WefJetXXnll3b8779rthw4dStavvvrqurc9np04caLsFs5b7p7dzDab2VEz2zdi2cNmdsjM9mQ/Nze3TQCNGsvb+F9LummU5b9w967s5+Vi2wJQtNywu/vrktr3eE8AY9LIF3T3mNne7G3+9FpPMrMeM6uaWbXRa7EBqF+9Yd8k6TuSuiQdlvTzWk909z53r7h7paOjo87NAWhUXWF39yPufsbdz0r6paRFxbYFoGh1hd3MZo94uFJS7esFA2gLuePsZrZV0o2SZprZQUkbJN1oZl2SXNKApB83scdxL++c7rlz5ybrzzzzTLKed233lMWLFyfrp0+fTtbff//9ZH3hwoXn3VMrnD17Nln/9NNPk/XLLrusyHZaIjfs7r56lMW/akIvAJqIw2WBIAg7EARhB4Ig7EAQhB0IglNcW+CSSy5J1vNOE+3r60vWH3rooZq1CRPSf8+nTZuWrOetnzc01662b9+erH/yySfJet6lptsRe3YgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIJx9jawcePGZH3p0qV1r593+uusWbOS9fvvvz9ZX7FiRbK+du3amrXJkycn182zZMmSZH1wcLBm7c4770yu+/bbbyfrkyZNStbbEXt2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCcfY20N3dnaz39PQk64888kjN2pQpU5Lr3n333cl63jEAq1atStZTl2R29+S6p06dStbzzknv7++vWfvggw+S686bNy9ZH4/YswNBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIyzjwNPPvlkst7Z2Vmztn79+uS6Tz/9dLJ+xx13JOvz589P1lO2bt2arL/00kvJ+m233ZasP/XUUzVrc+bMSa57Icrds5vZPDP7g5ntN7P3zOwn2fIZZvaqmX2Y3U5vfrsA6jWWt/GnJa119+sk/aOkNWZ2vaQHJO1y92sk7coeA2hTuWF398Pu3p/dPyFpv6Q5klZI2pI9bYukW5rVJIDGndcXdGbWKem7kv4kaZa7H5aG/yBIuqLGOj1mVjWz6tDQUGPdAqjbmMNuZlMl/VbSve7+t7Gu5+597l5x90pHR0c9PQIowJjCbmYTNRz037j777LFR8xsdlafLeloc1oEUATLO83QzEzDn8mPu/u9I5b/m6RP3b3XzB6QNMPdk9cdrlQqXq1WC2gbYzUwMJCsb9u2LVnfuXNnsr579+5kfeXKlTVrixcvTq67bNmyZH3BggXJ+vB/3VgqlYqq1eqo//CxjLN3S/qRpHfNbE+27GeSeiXtMLO7JP1FUnrQE0CpcsPu7n+UVOtP5PeLbQdAs3C4LBAEYQeCIOxAEIQdCIKwA0HkjrMXiXF2oLlS4+zs2YEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IIjcsJvZPDP7g5ntN7P3zOwn2fKHzeyQme3Jfm5ufrsA6jWW+dlPS1rr7v1mNk3SW2b2alb7hbv/e/PaA1CUsczPfljS4ez+CTPbL2lOsxsDUKzz+sxuZp2SvivpT9mie8xsr5ltNrPpNdbpMbOqmVWHhoYaahZA/cYcdjObKum3ku51979J2iTpO5K6NLzn//lo67l7n7tX3L3S0dFRQMsA6jGmsJvZRA0H/Tfu/jtJcvcj7n7G3c9K+qWkRc1rE0CjxvJtvEn6laT97v7YiOWzRzxtpaR9xbcHoChj+Ta+W9KPJL1rZnuyZT+TtNrMuiS5pAFJP25KhwAKMZZv4/8oabT5nl8uvh0AzcIRdEAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSDM3Vu3MbMhSZ+MWDRT0rGWNXB+2rW3du1Lord6FdnbP7j7qNd/a2nYv7Fxs6q7V0prIKFde2vXviR6q1ereuNtPBAEYQeCKDvsfSVvP6Vde2vXviR6q1dLeiv1MzuA1il7zw6gRQg7EEQpYTezm8zsgJl9ZGYPlNFDLWY2YGbvZtNQV0vuZbOZHTWzfSOWzTCzV83sw+x21Dn2SuqtLabxTkwzXuprV/b05y3/zG5mF0n6QNK/SDooabek1e7+vy1tpAYzG5BUcffSD8AwsyWSTkp61t0XZsselXTc3XuzP5TT3f2nbdLbw5JOlj2NdzZb0eyR04xLukXSHSrxtUv09a9qwetWxp59kaSP3P1jd/+7pG2SVpTQR9tz99clHT9n8QpJW7L7WzT8n6XlavTWFtz9sLv3Z/dPSPpqmvFSX7tEXy1RRtjnSPrriMcH1V7zvbuknWb2lpn1lN3MKGa5+2Fp+D+PpCtK7udcudN4t9I504y3zWtXz/TnjSoj7KNNJdVO43/d7v49ScslrcnermJsxjSNd6uMMs14W6h3+vNGlRH2g5LmjXg8V9JgCX2Myt0Hs9ujkp5X+01FfeSrGXSz26Ml9/P/2mka79GmGVcbvHZlTn9eRth3S7rGzL5tZt+S9ENJL5bQxzeY2ZTsixOZ2RRJP1D7TUX9oqTbs/u3S3qhxF6+pl2m8a41zbhKfu1Kn/7c3Vv+I+lmDX8j/2dJD5bRQ42+5kt6J/t5r+zeJG3V8Nu6LzX8juguSZdL2iXpw+x2Rhv19t+S3pW0V8PBml1Sb/+k4Y+GeyXtyX5uLvu1S/TVkteNw2WBIDiCDgiCsANBEHYgCMIOBEHYgSAIOxAEYQeC+D9hzzjMXc/FGgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(x_test[100].reshape(28,28), cmap='Greys')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "vals = []\n",
    "for i in range(10000):\n",
    "    vals.append([y_test[i], model4.predict(x_test[i].reshape(1,28,28,1)).argmax()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "mismatch = []\n",
    "for i in range(10000):\n",
    "    if vals[i][0] != vals[i][1]:\n",
    "        mismatch.append([i, vals[i][0], vals[i][1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[115, 4, 9],\n",
       " [340, 5, 3],\n",
       " [449, 3, 5],\n",
       " [582, 8, 3],\n",
       " [684, 7, 2],\n",
       " [740, 4, 9],\n",
       " [846, 7, 9],\n",
       " [947, 8, 9],\n",
       " [1014, 6, 5],\n",
       " [1039, 7, 1],\n",
       " [1242, 4, 9],\n",
       " [1260, 7, 1],\n",
       " [1393, 5, 3],\n",
       " [1459, 2, 3],\n",
       " [1500, 7, 3],\n",
       " [1522, 7, 9],\n",
       " [1737, 5, 3],\n",
       " [1878, 8, 3],\n",
       " [1901, 9, 4],\n",
       " [2070, 7, 9],\n",
       " [2130, 4, 9],\n",
       " [2182, 1, 3],\n",
       " [2293, 9, 4],\n",
       " [2447, 4, 9],\n",
       " [2454, 6, 5],\n",
       " [2597, 5, 3],\n",
       " [2654, 6, 1],\n",
       " [2771, 4, 9],\n",
       " [2896, 8, 0],\n",
       " [2939, 9, 5],\n",
       " [2953, 3, 5],\n",
       " [2959, 2, 3],\n",
       " [2995, 6, 5],\n",
       " [3073, 1, 2],\n",
       " [3225, 7, 9],\n",
       " [3422, 6, 0],\n",
       " [3441, 7, 2],\n",
       " [3520, 6, 4],\n",
       " [3534, 4, 8],\n",
       " [3767, 7, 2],\n",
       " [3906, 1, 3],\n",
       " [3941, 4, 6],\n",
       " [4199, 7, 9],\n",
       " [4382, 4, 9],\n",
       " [4536, 6, 5],\n",
       " [4571, 6, 8],\n",
       " [4783, 4, 9],\n",
       " [4860, 4, 9],\n",
       " [4911, 4, 9],\n",
       " [5165, 0, 6],\n",
       " [5201, 4, 9],\n",
       " [5936, 4, 9],\n",
       " [5937, 5, 3],\n",
       " [5981, 5, 9],\n",
       " [5997, 5, 9],\n",
       " [6571, 9, 5],\n",
       " [6576, 7, 1],\n",
       " [6597, 0, 3],\n",
       " [6651, 0, 6],\n",
       " [6783, 1, 6],\n",
       " [7902, 7, 9],\n",
       " [8061, 4, 9],\n",
       " [8316, 7, 2],\n",
       " [8527, 4, 9],\n",
       " [9505, 7, 2],\n",
       " [9664, 2, 7],\n",
       " [9729, 5, 6],\n",
       " [9792, 4, 9]]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mismatch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2d336aa5198>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAANZ0lEQVR4nO3dYahc9ZnH8d8vNhVi+yKaGxNs3HRLBGV103BJRJeglBTNm6RgN41QIt5wK6g0pC9Wq1DBF8bFNomwFNJ4aVa6SqENRom7Ea1I30Sv4iaxoasbsm1q9F4R1JIX1fTZF/dYrvHOf25mzsyZ5Pl+YJiZ88y552HIL2fm/M+cvyNCAM5/c5puAEB/EHYgCcIOJEHYgSQIO5DEF/q5sQULFsTSpUv7uUkglePHj+u9997zTLWuwm77Jkk7JV0gaXdEbCu9funSpRofH+9mkwAKhoeHW9Y6/hhv+wJJ/ybpZklXSdpo+6pO/x6A3urmO/tKSW9FxLGI+IukJyWtq6ctAHXrJuyXSfrjtOcnqmWfYXvU9rjt8cnJyS42B6Ab3YR9poMAnzv3NiJ2RcRwRAwPDQ11sTkA3egm7CckLZn2/CuS3u6uHQC90k3YX5G0zPZXbX9R0nck7aunLQB163joLSI+sX2XpP/S1NDbWES8UVtnAGrV1Th7ROyXtL+mXgD0EKfLAkkQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5BEX6dsxrlnzpzy/mDz5s3FesTnJgn6m2uuuaa47t13312s4+ywZweSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJBhnR5HtYn1sbKxYL42zt/vbK1euLNZXrVpVrOOzugq77eOSPpJ0WtInETFcR1MA6lfHnv3GiHivhr8DoIf4zg4k0W3YQ9IB26/aHp3pBbZHbY/bHp+cnOxycwA61W3Yr4+IFZJulnSn7dVnviAidkXEcEQMDw0Ndbk5AJ3qKuwR8XZ1PyFpr6Ty4VMAjek47LYvsv3lTx9L+qakI3U1BqBe3RyNv1TS3mqs9AuS/iMi/rOWrjAwnn766a7WHxkZaVmbmJgornvvvfcW6y+88EJHPWXVcdgj4pikf6yxFwA9xNAbkARhB5Ig7EAShB1IgrADSfATVxStXbu2q/VLQ2/btm3r6m/j7LBnB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkGGdHY0qXmZakNWvW9KmTHNizA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjLOjp0pTOrebsvmWW26pu53U2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs6Onbr/99pa1hx56qLjusmXL6m4ntbZ7dttjtidsH5m27GLbz9l+s7qf39s2AXRrNh/jfy7ppjOW3SPp+YhYJun56jmAAdY27BHxkqT3z1i8TtKe6vEeSetr7gtAzTo9QHdpRJyUpOp+YasX2h61PW57fHJyssPNAehWz4/GR8SuiBiOiOGhoaFebw5AC52G/V3biyWpup+oryUAvdBp2PdJ2lQ93iTpqXraAdArbcfZbT8h6QZJC2yfkPQjSdsk/dL2iKQ/SPp2L5tEc06dOlWs79y5s1gv/Z59xYoVHfWEzrQNe0RsbFH6Rs29AOghTpcFkiDsQBKEHUiCsANJEHYgCX7iep774IMPivV58+YV6/v27SvW77///o7//v79+4vrol7s2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZz3NXXnllsf7kk08W61u3bi3W2027vGPHjpY1rlzUX+zZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtnPAwcPHmxZe+edd4rrthtHb7d+u8tBj4yMFOvoH/bsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AE4+wDYGJiolhvd+320lh5u9+b33bbbcV6u3H00pTMkrR79+6Wtc2bNxfXRb3a7tltj9mesH1k2rIHbP/J9uvVbW1v2wTQrdl8jP+5pJtmWL49IpZXN6b2AAZc27BHxEuS3u9DLwB6qJsDdHfZPlR9zJ/f6kW2R22P2x6fnJzsYnMAutFp2H8q6WuSlks6KenHrV4YEbsiYjgihrnAINCcjsIeEe9GxOmI+Kukn0laWW9bAOrWUdhtL5729FuSjrR6LYDB0Hac3fYTkm6QtMD2CUk/knSD7eWSQtJxSd/rYY/nvHZzpN93333F+mOPPVasr1u3rmXt8OHDxXU3bNjQ8d+WpKuvvrpYHx0dbVm74ooriuuuXr26WMfZaRv2iNg4w+Lyvz4AA4fTZYEkCDuQBGEHkiDsQBKEHUiCn7j2waZNm4r1Z555plhftGhRsf7oo4+2rF1yySXFdefNm1esf/zxx8X6HXfcUayvX7++Ze3BBx8srnvhhRcW66tWrSrW8Vns2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZa3Dq1Klivd2loNv9jHTv3r1n3VNd5s6d29X6S5YsaVnbtWtXcd3FixcX6y+//HLH286IPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4ew1efPHFYr3dtMm33nprjd2cPw4cOFCsHzt2rFhnnP2z2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs9eg3bXZFy5cWKw//PDDxfrll19erJ+r108/ePBgsX7ttdcW63PmlPdVp0+fPuuezmdt9+y2l9j+je2jtt+w/f1q+cW2n7P9ZnU/v/ftAujUbD7GfyLpBxFxpaRrJd1p+ypJ90h6PiKWSXq+eg5gQLUNe0ScjIjXqscfSToq6TJJ6yTtqV62R1LreX4ANO6sDtDZXirp65IOSro0Ik5KU/8hSJrxi6ntUdvjtscnJye76xZAx2YddttfkvQrSVsi4sPZrhcRuyJiOCKGh4aGOukRQA1mFXbbczUV9F9ExK+rxe/aXlzVF0ua6E2LAOrQdujNU7/PfEzS0Yj4ybTSPkmbJG2r7p/qSYfngHZDXzt27CjWH3nkkWJ9zZo1xfr27dtb1kZGRorr9tru3btb1rZu3Vpct93Q2s6dOzvqKavZjLNfL+m7kg7bfr1a9kNNhfyXtkck/UHSt3vTIoA6tA17RPxWUqurL3yj3nYA9AqnywJJEHYgCcIOJEHYgSQIO5AEP3Htgw0bNhTrN954Y7F+3XXXFesbN248657qsmXLlmL90KFDLWuLFi0qrvv4448X6+fqT3ubwp4dSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnH0AtLvU9LPPPtunTj5vYqJ8TZKxsbFivfRb+3bnB8ybN69Yx9lhzw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSTDOfg5YtmxZY9tudw7Ahx/OenIgNIw9O5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4k0TbstpfY/o3to7bfsP39avkDtv9k+/Xqtrb37QLo1GxOqvlE0g8i4jXbX5b0qu3nqtr2iHikd+0BqMts5mc/Kelk9fgj20clXdbrxgDU66y+s9teKunrkg5Wi+6yfcj2mO35LdYZtT1ue3xycrKrZgF0btZht/0lSb+StCUiPpT0U0lfk7RcU3v+H8+0XkTsiojhiBgeGhqqoWUAnZhV2G3P1VTQfxERv5akiHg3Ik5HxF8l/UzSyt61CaBbszkab0mPSToaET+ZtnzxtJd9S9KR+tsDUJfZHI2/XtJ3JR22/Xq17IeSNtpeLikkHZf0vZ50CKAWszka/1tJnqG0v/52APQKZ9ABSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeScET0b2P2pKT/m7ZogaT3+tbA2RnU3ga1L4neOlVnb38XETNe/62vYf/cxu3xiBhurIGCQe1tUPuS6K1T/eqNj/FAEoQdSKLpsO9qePslg9rboPYl0Vun+tJbo9/ZAfRP03t2AH1C2IEkGgm77Zts/972W7bvaaKHVmwft324moZ6vOFexmxP2D4ybdnFtp+z/WZ1P+Mcew31NhDTeBemGW/0vWt6+vO+f2e3fYGk/5G0RtIJSa9I2hgRv+trIy3YPi5pOCIaPwHD9mpJf5b07xHxD9Wyf5X0fkRsq/6jnB8R/zIgvT0g6c9NT+NdzVa0ePo045LWS7pNDb53hb7+WX1435rYs6+U9FZEHIuIv0h6UtK6BvoYeBHxkqT3z1i8TtKe6vEeTf1j6bsWvQ2EiDgZEa9Vjz+S9Ok0442+d4W++qKJsF8m6Y/Tnp/QYM33HpIO2H7V9mjTzczg0og4KU3945G0sOF+ztR2Gu9+OmOa8YF57zqZ/rxbTYR9pqmkBmn87/qIWCHpZkl3Vh9XMTuzmsa7X2aYZnwgdDr9ebeaCPsJSUumPf+KpLcb6GNGEfF2dT8haa8Gbyrqdz+dQbe6n2i4n78ZpGm8Z5pmXAPw3jU5/XkTYX9F0jLbX7X9RUnfkbSvgT4+x/ZF1YET2b5I0jc1eFNR75O0qXq8SdJTDfbyGYMyjXeracbV8HvX+PTnEdH3m6S1mjoi/7+S7muihxZ9/b2k/65ubzTdm6QnNPWx7mNNfSIakXSJpOclvVndXzxAvT0u6bCkQ5oK1uKGevsnTX01PCTp9eq2tun3rtBXX943TpcFkuAMOiAJwg4kQdiBJAg7kARhB5Ig7EAShB1I4v8BroDx8CcEMbkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# wtf is this? \n",
    "plt.imshow(x_test[582].reshape(28,28), cmap='Greys')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2d33a0e5978>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAOQElEQVR4nO3df4jVdb7H8dc7dwVTIVtHk3ZovEvkxtJ1l0EWWpau25UpDFuwUEJMJIsydmKtlQ1apQi53HVJqI3ZsrWbtWztRhJ1ryJiGSSNNaVe6datyZ0dcUb6YxOqdfK9f8zXy2jz/Zzj+X7Pjzvv5wOGc873fb7n++bgy+8538/5fj/m7gIw8V3Q7AYANAZhB4Ig7EAQhB0IgrADQXyjkRubOXOmd3R0NHKTQCj9/f06ceKEjVcrFHYz65L0iKRJkp5w902p53d0dKi3t7fIJgEkdHZ25tZq/hhvZpMkPSrpOklXSlpuZlfW+noA6qvId/YFkj5094/c/e+S/iBpSTltAShbkbBfKukvYx4PZMvOYmZrzKzXzHqHh4cLbA5AEUXCPt5BgK/99tbde9y9090729raCmwOQBFFwj4gqX3M429LGizWDoB6KRL2tyRdbmZzzWyypGWSdpTTFoCy1Tz05u4jZrZW0n9pdOhtq7sfLq0zAKUqNM7u7q9IeqWkXgDUET+XBYIg7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgiIZeShoo06lTp5L1o0eP5tZef/315LpTpkxJ1lNXcZWkWbNmJevTp09P1uuBPTsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBME4O1rWyMhIsv74448n693d3bk1969NXnQWs3FnPa7aJZdckqwvWrQot/bUU08V2nYe9uxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EATj7Giavr6+ZH39+vXJ+q5du2re9qpVq5L1SuPsixcvTtbb29vPu6d6KxR2M+uX9JmkrySNuHv6jH4ATVPGnv1f3P1ECa8DoI74zg4EUTTsLmmnmR0wszXjPcHM1phZr5n1Dg8PF9wcgFoVDfvV7v4DSddJusvMfnzuE9y9x9073b2zra2t4OYA1KpQ2N19MLsdkvSipAVlNAWgfDWH3cymmtn0M/clLZJ0qKzGAJSryNH42ZJezMYjvyHpWXf/z1K6woQxODiYW+vq6kquW+kYz8KFC5P17du359YqXdd9Iqo57O7+kaR/LrEXAHXE0BsQBGEHgiDsQBCEHQiCsANBcIorCjlxIn0O1COPPJJbO3nyZHLdRx99NFm/7bbbkvVJkyYl69GwZweCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIBhnR9KpU6eS9fvvvz9Z7+npya2tXr06ue4dd9yRrOP8sGcHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAYZ0fSPffck6w/8cQTyfq6detyaxs3bqypJ9SGPTsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBME4+wQ3MjKSrPf19SXrzz77bKHtr1q1Krd24YUXFnptnJ+Ke3Yz22pmQ2Z2aMyyi81sl5l9kN3OqG+bAIqq5mP87yV1nbNsvaTd7n65pN3ZYwAtrGLY3f01SZ+es3iJpG3Z/W2Sbiy5LwAlq/UA3Wx3PyZJ2e2svCea2Roz6zWz3uHh4Ro3B6Couh+Nd/ced+909862trZ6bw5AjlrDftzM5khSdjtUXksA6qHWsO+QtDK7v1LSS+W0A6BeKo6zm9lzkq6RNNPMBiT9StImSX80s9WSjkq6qZ5NIi11bfd9+/Yl17322mvLbucsCxcuzK2ZWXLdpUuXJusPP/xwsj59+vRkPZqKYXf35Tmln5TcC4A64ueyQBCEHQiCsANBEHYgCMIOBMEprhPA2rVrc2uVLvVcb0ND+b+3qjT09thjjyXr7777brL+6quv5tamTp2aXHciYs8OBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0Ewzt4CUqeoStLevXuT9Z6entxapbHsShYtWpSsL1u2LFm/9dZbc2uDg4OFXrvS6bsvvPBCbm3lypW5tYmKPTsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBME4ewN8+eWXyXpqnFySuru7k/XJkyfn1pYsWZJcd8OGDcn6vHnzkvULLqh9f1FphqArrrgiWX/jjTeS9f7+/vNtaUJjzw4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQTDOXqXPP/88t/bMM88k133ooYeS9YGBgZp6OuP222/PrW3ZsqXQa9dTpfP4t27dWuj1V6xYUWj9iabint3MtprZkJkdGrNsg5n91cz6sr/r69smgKKq+Rj/e0ld4yz/jbvPz/5eKbctAGWrGHZ3f03Spw3oBUAdFTlAt9bM3ss+5s/Ie5KZrTGzXjPrHR4eLrA5AEXUGvbfSvqOpPmSjkn6dd4T3b3H3TvdvbPSiQ8A6qemsLv7cXf/yt1PS/qdpAXltgWgbDWF3czmjHn4U0mH8p4LoDVUHGc3s+ckXSNpppkNSPqVpGvMbL4kl9QvKX+gd4LYtGlTbq3SOHpRV111VbK+efPmum6/Xp5++ulC6y9cuDBZv+yyywq9/kRTMezuvnycxU/WoRcAdcTPZYEgCDsQBGEHgiDsQBCEHQiCU1yr9MUXX9S87gMPPJCs79y5M1l/8803k/Xnn38+t7Z8+XiDKY2zf//+3Nq9995b6LW3b9+erE+aNKnQ60807NmBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjG2Uvg7sn64sWLk/U777wzWZ89e3ayvm7dutzaDTfckFx32rRpyXqlyz2vXbs2WU9NR33RRRcl1z148GCyPmvWrGQdZ2PPDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBMM5epU8++SS3ZmbJdVPndEvSvHnzkvVbbrklWd+zZ09u7f3330+um5qKWpK6u7uT9XfeeSdZnzt3bm5t7969yXXb29uTdZwf9uxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EATj7FXasmVLbu3jjz9Ornv33XcX2vbp06eT9cHBwdzaggULCm27kptvvjlZf/DBB3NrjKM3VsU9u5m1m9keMztiZofN7GfZ8ovNbJeZfZDdzqh/uwBqVc3H+BFJP3f370r6oaS7zOxKSesl7Xb3yyXtzh4DaFEVw+7ux9z97ez+Z5KOSLpU0hJJ27KnbZN0Y72aBFDceR2gM7MOSd+XtF/SbHc/Jo3+hyBp3AuCmdkaM+s1s97h4eFi3QKoWdVhN7Npkv4kqdvd/1bteu7e4+6d7t7Z1tZWS48ASlBV2M3smxoN+nZ3/3O2+LiZzcnqcyQN1adFAGWoOPRmo+dvPinpiLtvHlPaIWmlpE3Z7Ut16bBFpC5bvHHjxuS6lS4VXXRoroiOjo5kff369HHXpUuXJuszZjBI0yqqGWe/WtIKSQfNrC9b9kuNhvyPZrZa0lFJN9WnRQBlqBh2d98nKe/qDD8ptx0A9cLPZYEgCDsQBGEHgiDsQBCEHQiCU1xL0NXVlawfPnw4WX/55ZeT9QMHDpx3T2dMmTIlWb/vvvsKrY//P9izA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQjLM3QKWx6ptuSp8dXKkOVIM9OxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRRMexm1m5me8zsiJkdNrOfZcs3mNlfzawv+7u+/u0CqFU1F68YkfRzd3/bzKZLOmBmu7Lab9z93+vXHoCyVDM/+zFJx7L7n5nZEUmX1rsxAOU6r+/sZtYh6fuS9meL1prZe2a21cxm5Kyzxsx6zax3eHi4ULMAald12M1smqQ/Sep2979J+q2k70iar9E9/6/HW8/de9y9090729raSmgZQC2qCruZfVOjQd/u7n+WJHc/7u5fuftpSb+TtKB+bQIoqpqj8SbpSUlH3H3zmOVzxjztp5IOld8egLJUczT+akkrJB00s75s2S8lLTez+ZJcUr+k2+vSIYBSVHM0fp8kG6f0SvntAKgXfkEHBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0Iwty9cRszG5b0yZhFMyWdaFgD56dVe2vVviR6q1WZvV3m7uNe/62hYf/axs163b2zaQ0ktGpvrdqXRG+1alRvfIwHgiDsQBDNDntPk7ef0qq9tWpfEr3VqiG9NfU7O4DGafaeHUCDEHYgiKaE3cy6zOx9M/vQzNY3o4c8ZtZvZgezaah7m9zLVjMbMrNDY5ZdbGa7zOyD7HbcOfaa1FtLTOOdmGa8qe9ds6c/b/h3djObJOl/JP2rpAFJb0la7u7/3dBGcphZv6ROd2/6DzDM7MeSTkp62t2/ly37N0mfuvum7D/KGe7+ixbpbYOkk82exjubrWjO2GnGJd0o6VY18b1L9HWzGvC+NWPPvkDSh+7+kbv/XdIfJC1pQh8tz91fk/TpOYuXSNqW3d+m0X8sDZfTW0tw92Pu/nZ2/zNJZ6YZb+p7l+irIZoR9ksl/WXM4wG11nzvLmmnmR0wszXNbmYcs939mDT6j0fSrCb3c66K03g30jnTjLfMe1fL9OdFNSPs400l1Urjf1e7+w8kXSfpruzjKqpT1TTejTLONOMtodbpz4tqRtgHJLWPefxtSYNN6GNc7j6Y3Q5JelGtNxX18TMz6Ga3Q03u5/+00jTe400zrhZ475o5/Xkzwv6WpMvNbK6ZTZa0TNKOJvTxNWY2NTtwIjObKmmRWm8q6h2SVmb3V0p6qYm9nKVVpvHOm2ZcTX7vmj79ubs3/E/S9Ro9Iv+/ku5vRg85ff2TpHezv8PN7k3Scxr9WHdKo5+IVkv6lqTdkj7Ibi9uod7+Q9JBSe9pNFhzmtTbjzT61fA9SX3Z3/XNfu8SfTXkfePnskAQ/IIOCIKwA0EQdiAIwg4EQdiBIAg7EARhB4L4B9r8Nrdx3sARAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# I swear this is a 6.\n",
    "plt.imshow(x_test[9729].reshape(28,28), cmap='Greys')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=input_shape)\n",
    "x = Conv2D(32, 3, activation='relu', input_shape=input_shape)(inputs)\n",
    "x = Dropout(0.1)(x)\n",
    "x = Conv2D(64, 3, activation='relu')(x)\n",
    "x = Dropout(0.1)(x)\n",
    "x = MaxPooling2D(2)(x)\n",
    "x = ResNetUnit(x)\n",
    "x = ResNetUnit(x)\n",
    "x = Conv2D(64, 3, activation='relu')(x)\n",
    "x = Dropout(0.1)(x)\n",
    "x = MaxPooling2D(2)(x)\n",
    "x = Flatten()(x)\n",
    "x = Dense(256, activation='relu')(x)\n",
    "x = Dropout(0.2)(x)\n",
    "outputs = Dense(10, activation='softmax')(x)\n",
    "\n",
    "model5 = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "60000/60000 [==============================] - 26s 442us/step - loss: 0.7608 - accuracy: 0.7875\n",
      "Epoch 2/10\n",
      "60000/60000 [==============================] - 24s 403us/step - loss: 0.0853 - accuracy: 0.9728\n",
      "Epoch 3/10\n",
      "60000/60000 [==============================] - 24s 405us/step - loss: 0.0566 - accuracy: 0.9827\n",
      "Epoch 4/10\n",
      "60000/60000 [==============================] - 24s 405us/step - loss: 0.0422 - accuracy: 0.9868\n",
      "Epoch 5/10\n",
      "60000/60000 [==============================] - 24s 406us/step - loss: 0.0337 - accuracy: 0.9898\n",
      "Epoch 6/10\n",
      "60000/60000 [==============================] - 24s 405us/step - loss: 0.0282 - accuracy: 0.9911\n",
      "Epoch 7/10\n",
      "60000/60000 [==============================] - 24s 405us/step - loss: 0.0242 - accuracy: 0.9922\n",
      "Epoch 8/10\n",
      "60000/60000 [==============================] - 24s 404us/step - loss: 0.0218 - accuracy: 0.9932\n",
      "Epoch 9/10\n",
      "60000/60000 [==============================] - 24s 404us/step - loss: 0.0202 - accuracy: 0.9934\n",
      "Epoch 10/10\n",
      "60000/60000 [==============================] - 24s 404us/step - loss: 0.0172 - accuracy: 0.9948\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x2d3465c9c18>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model5.compile(optimizer='adam',\n",
    "               loss='sparse_categorical_crossentropy',\n",
    "               metrics=['accuracy'])\n",
    "model5.fit(x_train, y_train, batch_size=1000, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 2s 249us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.020051686681550926, 0.9939000010490417]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model5.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ~20 layers deep?\n",
    "inputs = keras.Input(shape=input_shape)\n",
    "x = Conv2D(32, 3, activation='relu', input_shape=input_shape)(inputs)\n",
    "x = Conv2D(64, 3, activation='relu')(x)\n",
    "x = MaxPooling2D(2)(x)\n",
    "for _ in range(8):\n",
    "    x = ResNetUnit(x)\n",
    "x = Conv2D(64, 3, activation='relu')(x)\n",
    "x = MaxPooling2D(2)(x)\n",
    "x = Flatten()(x)\n",
    "x = Dense(256, activation='relu')(x)\n",
    "x = Dropout(0.5)(x)\n",
    "outputs = Dense(10, activation='softmax')(x)\n",
    "\n",
    "model6 = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "60000/60000 [==============================] - 54s 899us/step - loss: 0.9550 - accuracy: 0.8152\n",
      "Epoch 2/30\n",
      "60000/60000 [==============================] - 51s 850us/step - loss: 0.0664 - accuracy: 0.9792\n",
      "Epoch 3/30\n",
      "60000/60000 [==============================] - 51s 850us/step - loss: 0.0427 - accuracy: 0.9868\n",
      "Epoch 4/30\n",
      "60000/60000 [==============================] - 51s 851us/step - loss: 0.0314 - accuracy: 0.9903\n",
      "Epoch 5/30\n",
      "60000/60000 [==============================] - 51s 852us/step - loss: 0.0225 - accuracy: 0.9931\n",
      "Epoch 6/30\n",
      "60000/60000 [==============================] - 51s 852us/step - loss: 0.0191 - accuracy: 0.9941\n",
      "Epoch 7/30\n",
      "60000/60000 [==============================] - 51s 851us/step - loss: 0.0153 - accuracy: 0.9949\n",
      "Epoch 8/30\n",
      "60000/60000 [==============================] - 51s 851us/step - loss: 0.0129 - accuracy: 0.9960\n",
      "Epoch 9/30\n",
      "60000/60000 [==============================] - 51s 851us/step - loss: 0.0145 - accuracy: 0.9952\n",
      "Epoch 10/30\n",
      "60000/60000 [==============================] - 51s 852us/step - loss: 0.0097 - accuracy: 0.9972\n",
      "Epoch 11/30\n",
      "60000/60000 [==============================] - 51s 853us/step - loss: 0.0078 - accuracy: 0.9975\n",
      "Epoch 12/30\n",
      "60000/60000 [==============================] - 51s 852us/step - loss: 0.0078 - accuracy: 0.9973\n",
      "Epoch 13/30\n",
      "60000/60000 [==============================] - 51s 853us/step - loss: 0.0087 - accuracy: 0.9973\n",
      "Epoch 14/30\n",
      "60000/60000 [==============================] - 51s 852us/step - loss: 0.0066 - accuracy: 0.9977\n",
      "Epoch 15/30\n",
      "60000/60000 [==============================] - 51s 856us/step - loss: 0.0074 - accuracy: 0.9976\n",
      "Epoch 16/30\n",
      "60000/60000 [==============================] - 51s 855us/step - loss: 0.0066 - accuracy: 0.9979\n",
      "Epoch 17/30\n",
      "60000/60000 [==============================] - 52s 861us/step - loss: 0.0068 - accuracy: 0.9976\n",
      "Epoch 18/30\n",
      "60000/60000 [==============================] - 52s 862us/step - loss: 0.0055 - accuracy: 0.9982\n",
      "Epoch 19/30\n",
      "60000/60000 [==============================] - 52s 863us/step - loss: 0.0081 - accuracy: 0.9973\n",
      "Epoch 20/30\n",
      "60000/60000 [==============================] - 52s 863us/step - loss: 0.0079 - accuracy: 0.9977\n",
      "Epoch 21/30\n",
      "60000/60000 [==============================] - 52s 866us/step - loss: 0.0057 - accuracy: 0.9981\n",
      "Epoch 22/30\n",
      "60000/60000 [==============================] - 52s 864us/step - loss: 0.0058 - accuracy: 0.9982\n",
      "Epoch 23/30\n",
      "60000/60000 [==============================] - 52s 865us/step - loss: 0.0062 - accuracy: 0.9979\n",
      "Epoch 24/30\n",
      "60000/60000 [==============================] - 52s 866us/step - loss: 0.0045 - accuracy: 0.9986\n",
      "Epoch 25/30\n",
      "60000/60000 [==============================] - 52s 865us/step - loss: 0.0075 - accuracy: 0.9975\n",
      "Epoch 26/30\n",
      "60000/60000 [==============================] - 52s 866us/step - loss: 0.0051 - accuracy: 0.9981\n",
      "Epoch 27/30\n",
      "60000/60000 [==============================] - 52s 868us/step - loss: 0.0032 - accuracy: 0.9990\n",
      "Epoch 28/30\n",
      "60000/60000 [==============================] - 52s 867us/step - loss: 0.0049 - accuracy: 0.9983\n",
      "Epoch 29/30\n",
      "60000/60000 [==============================] - 52s 867us/step - loss: 0.0043 - accuracy: 0.9987\n",
      "Epoch 30/30\n",
      "60000/60000 [==============================] - 52s 865us/step - loss: 0.0056 - accuracy: 0.9982\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x21f9d8d2c88>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model6.compile(optimizer='adam',\n",
    "               loss='sparse_categorical_crossentropy',\n",
    "               metrics=['accuracy'])\n",
    "model6.fit(x_train, y_train, batch_size=1000, epochs=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 5s 499us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.026167441274324993, 0.992900013923645]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model6.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create new data to improve training\n",
    "# Assume image is 28 x 28 x 1 as it is in this dataset...\n",
    "# Only integral shifts\n",
    "def translate(image, dx, dy):\n",
    "    res = np.copy(image)\n",
    "    for i in range(28):\n",
    "        for j in range(28):\n",
    "            if i-dx < 0 or i-dx >= 28 or j-dy < 0 or j-dy >= 28:\n",
    "                res[i,j,0] = 0.0\n",
    "            else:\n",
    "                res[i,j,0] = image[i-dx,j-dy,0]\n",
    "    return image\n",
    "\n",
    "def rotate(image, theta):\n",
    "    return ndimage.rotate(image, theta, reshape=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create augmented training set\n",
    "extra_train_x = []\n",
    "extra_train_y = []\n",
    "# Copy over\n",
    "for image in x_train:\n",
    "    extra_train_x.append(image)\n",
    "for label in y_train:\n",
    "    extra_train_y.append(label)\n",
    "# Up to 300k total images, shifted +/- 1 pixel and rotated +/-30 degrees\n",
    "for _ in range(4):\n",
    "    for (image, label) in zip(x_train, y_train):\n",
    "        trans_image = translate(rotate(image, -30+60*random.random()), -1+int(3*random.random()), -1+int(3*random.random()))\n",
    "        extra_train_x.append(trans_image)\n",
    "        extra_train_y.append(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "extra_train_x = np.asarray(extra_train_x).astype('float32')\n",
    "extra_train_y = np.asarray(extra_train_y).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert image values so there aren't any negatives...\n",
    "fastconv = Sequential()\n",
    "fastconv.add(Activation('relu'))\n",
    "extra_train_x = fastconv.predict(extra_train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x21f9dc80e48>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAPyUlEQVR4nO3db4xUZZbH8d+xUUQcVOyWJWhgdlSyBiNOyj8BY9xMdqImihOddXwxcaOR8V9gEhPXiFFeklVnMi90klaJuBkxMzCKJqhDyKgxGrVUVnGJiIjK2EoTEocRRGjOvujrpsW+z9PUrX9wvp+kU9V16nadVPrXt6tO3fuYuwvA4e+ITjcAoD0IOxAEYQeCIOxAEIQdCGJcOx+st7fXZ8yY0c6HBELZsmWLtm/fbqPVKoXdzC6W9DtJPZIedvclqfvPmDFD9Xq9ykOiywwNDSXrPT09beoEklSr1UprDf8bb2Y9kh6QdImkMyRdY2ZnNPrzALRWldfs50ra5O6b3f0bSU9ImtectgA0W5WwT5P06Yjvtxa3fYeZzTezupnVBwcHKzwcgCqqhH20NwG+99lbd+9395q71/r6+io8HIAqqoR9q6RTRnx/sqTPqrUDoFWqhP0NSaeZ2Q/N7ChJv5D0dHPaAtBsDY/e3H2fmd0q6XkNj96Wuvt7TesMXSF3VGSV0dqHH36YrOde9k2aNClZT/VuNuoo+rBWac7u7qslrW5SLwBaiI/LAkEQdiAIwg4EQdiBIAg7EARhB4Jo6/HsaI3UPDk3Jz/iiPTf+9w8emBgIFlfuHBhaW3Tpk3JbZ944olkPTdnjzhLT2HPDgRB2IEgCDsQBGEHgiDsQBCEHQiC0dshIDc+S42YcuOnvXv3JuvjxqV/Re66665kfcWKFaW1m266KbktZzZqLvbsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEc/bD3Keffpqs33vvvcn67Nmzk/WdO3cm64sWLSqt3XzzzcltjzvuuGQdB4c9OxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EwZy9Dfbv35+s5445z9U3b95cWsvN0V988cVk/ZhjjknWFyxYkKyfeeaZpTXm6O1VKexmtkXSTklDkva5e60ZTQFovmbs2f/V3bc34ecAaCFeswNBVA27S/qLmb1pZvNHu4OZzTezupnVBwcHKz4cgEZVDftcd/+xpEsk3WJmFx54B3fvd/eau9c4gSDQOZXC7u6fFZfbJD0p6dxmNAWg+RoOu5lNNLMffHtd0k8lrW9WYwCaq8q78VMkPVnMgMdJetzdn2tKV10od+72lNyyyDmpObok3X333aW1F154Iblt7rzxc+fOTdYvuOCCZL3K84bmajjs7r5Z0llN7AVACzF6A4Ig7EAQhB0IgrADQRB2IAgOcR2j3GGmKXv27EnW77vvvmQ9d5jqvn37SmvTp09Pbrtx48Zk/U9/+lOyftlllyXrqeetylLUOHjs2YEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCObshSoz361btya3ffzxx5P13Cz7xBNPTNZ7e3tLa3PmzElue/vttyfru3btStZ37NiRrE+ePLm0xhy9vdizA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQYebsVY+d/uqrr0prjz76aHLb3Jw99bOl/Kmozz///NJa7lj5np6eZJ1jzg8f7NmBIAg7EARhB4Ig7EAQhB0IgrADQRB2IIgwc/bcPHjTpk3J+j333FNaW716dXLb8ePHJ+tHH310sp6bs/f19ZXWqs7Rh4aGkvVx49K/Qvv37y+tVV3KGgcn+2yb2VIz22Zm60fcNtnM1pjZB8XlCa1tE0BVY/nT+qikiw+47Q5Ja939NElri+8BdLFs2N39JUkHnntonqRlxfVlkq5ocl8AmqzRF01T3H1AkorLk8ruaGbzzaxuZvXBwcEGHw5AVS1/h8Td+9295u611BtJAFqr0bB/YWZTJam43Na8lgC0QqNhf1rStcX1ayWtak47AFolO2c3s+WSLpLUa2ZbJd0jaYmkP5rZ9ZI+kfTzVjbZDKl5ryStW7cuWV+xYkVp7fTTT09uO2/evGT99ddfT9bff//9ZH337t2ltY8//ji5bW799ipzdKnaLL3qsfTM+L8rG3Z3v6ak9JMm9wKgheL9eQOCIuxAEIQdCIKwA0EQdiAIy403mqlWq3m9Xm/b4x2M3AjpueeeK61t3Lgxue2VV16ZrE+cODFZX7x4cbL+zDPPlNamTJmS3Pb+++9P1nPLRb/99tvJ+pFHHllamzVrVnLb3POS+0Rm7tDhlEP1FNq1Wk31en3U5tizA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQYebsrTxcMne65dSseSw++uijZP3ll18urS1fvjy5bW5OPmnSpGQ9dwjsli1bSmvTpk1LbnvyyScn6yedVHo2NEnSnDlzSmvXXXddcttjjz02We9WzNkBEHYgCsIOBEHYgSAIOxAEYQeCIOxAEGHm7J3UytMt5+zduzdZf/XVV5P1tWvXJut79uxJ1lNLfk2YMCG57cqVK5P13OcXUvXnn38+ue2pp56arOc+W5FbKrtVmLMDIOxAFIQdCIKwA0EQdiAIwg4EQdiBILKruKK6Vi8PnJrj52bRF154YaV67jMEqXruWPjjjz8+WV+6dGmyftRRR5XWVq1aldz2tttuS9Y7NUevIvtbaGZLzWybma0fcdtiM/ubma0rvi5tbZsAqhrLLudRSRePcvtv3X128bW6uW0BaLZs2N39JUk72tALgBaq8mLyVjN7p/g3/4SyO5nZfDOrm1k99TlpAK3VaNh/L+lHkmZLGpBUujqgu/e7e83da7mF+AC0TkNhd/cv3H3I3fdLekjSuc1tC0CzNRR2M5s64tufSVpfdl8A3SE7Zzez5ZIuktRrZlsl3SPpIjObLcklbZH0qxb2iIxWz/GrSPW2Y0f6fd/du3cn67n117/55pvSWm9vb3LbnENx/fZs2N39mlFufqQFvQBooe7dJQBoKsIOBEHYgSAIOxAEYQeC4BBXVJIb+6VONd3f35/cds2aNQ3/bEm64oorSmtXX311ctucbhyt5bBnB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgmLMjKXcoZ67+1FNPldYefvjh5La7du1K1qdPn56sL1iwoLSWOzz2UDyENYc9OxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EwZwdSbl5cm5Jr5UrV5bWcss9jx8/Plm/4YYbkvWZM2cm6ymH4hw9hz07EARhB4Ig7EAQhB0IgrADQRB2IAjCDgTBnD243HHbe/fuTdYfeuihZP2VV14preXOOX/55Zcn61XP/R5Nds9uZqeY2V/NbIOZvWdmC4vbJ5vZGjP7oLg8ofXtAmjUWP6N3yfpNnf/F0nnS7rFzM6QdIekte5+mqS1xfcAulQ27O4+4O5vFdd3StogaZqkeZKWFXdbJql8rR0AHXdQb9CZ2QxJZ0t6TdIUdx+Qhv8gSDqpZJv5ZlY3s3ruc9QAWmfMYTezYyWtlPRrd//7WLdz9353r7l7ra+vr5EeATTBmMJuZkdqOOh/cPc/Fzd/YWZTi/pUSdta0yKAZsiO3mz4WL9HJG1w99+MKD0t6VpJS4rLVS3pEC2VO5Rzw4YNyfqzzz6brI8bV/4rNmXKlOS2N954Y7I+ceLEZD01VjwcD2HNGcucfa6kX0p618zWFbfdqeGQ/9HMrpf0iaSft6ZFAM2QDbu7vyyp7M/gT5rbDoBW4eOyQBCEHQiCsANBEHYgCMIOBMEhroeB1CmZc4eRVj2E9ZNPPknWU49/1VVXJbedNWtWsp4TcZaewp4dCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Jgzn4YSM2y9+3bl9w2taSyJK1evTpZHxoaStYnTJhQWsudKjondxps5uzfxZ4dCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Jgzn4ISB2vLqXn7P39/cltH3jggWT966+/TtbPOuusZH3hwoWltZkzZya3zWGOfnDYswNBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEGNZn/0USY9J+idJ+yX1u/vvzGyxpBskDRZ3vdPd0wc/oyG5c7/v3LmztPbaa68lt92+fXuynjt3+6JFi5L1s88+u7SWOx49hzn7wRnLh2r2SbrN3d8ysx9IetPM1hS137r7fa1rD0CzjGV99gFJA8X1nWa2QdK0VjcGoLkO6jW7mc2QdLakb/83vNXM3jGzpWZ2Qsk2882sbmb1wcHB0e4CoA3GHHYzO1bSSkm/dve/S/q9pB9Jmq3hPf/9o23n7v3uXnP3Wl9fXxNaBtCIMYXdzI7UcND/4O5/liR3/8Ldh9x9v6SHJJ3bujYBVJUNuw2/5fmIpA3u/psRt08dcbefSVrf/PYANMtY3o2fK+mXkt41s3XFbXdKusbMZktySVsk/aolHUJffvllsp46jPSxxx5Lbnveeecl60uWLEnWzznnnGQd3WMs78a/LGm0gSYzdeAQwifogCAIOxAEYQeCIOxAEIQdCIKwA0FwKulDwOeff95w/cEHH0xuO3/+/GS9p6cnWc9JHcbKIartxZ4dCIKwA0EQdiAIwg4EQdiBIAg7EARhB4KwqqfzPagHMxuU9PGIm3olpc9l3Dnd2lu39iXRW6Oa2dt0dx/1/G9tDfv3Htys7u61jjWQ0K29dWtfEr01ql298W88EARhB4LodNj7O/z4Kd3aW7f2JdFbo9rSW0dfswNon07v2QG0CWEHguhI2M3sYjN738w2mdkdneihjJltMbN3zWydmdU73MtSM9tmZutH3DbZzNaY2QfF5ahr7HWot8Vm9rfiuVtnZpd2qLdTzOyvZrbBzN4zs4XF7R197hJ9teV5a/trdjPrkbRR0r9J2irpDUnXuPv/trWREma2RVLN3Tv+AQwzu1DSPyQ95u6zitv+S9IOd19S/KE8wd3/s0t6WyzpH51exrtYrWjqyGXGJV0h6T/Uwecu0de/qw3PWyf27OdK2uTum939G0lPSJrXgT66nru/JGnHATfPk7SsuL5Mw78sbVfSW1dw9wF3f6u4vlPSt8uMd/S5S/TVFp0I+zRJn474fqu6a713l/QXM3vTzNLnbOqMKe4+IA3/8kg6qcP9HCi7jHc7HbDMeNc8d40sf15VJ8I+2onHumn+N9fdfyzpEkm3FP+uYmzGtIx3u4yyzHhXaHT586o6Efatkk4Z8f3Jkj7rQB+jcvfPisttkp5U9y1F/cW3K+gWl9s63M//66ZlvEdbZlxd8Nx1cvnzToT9DUmnmdkPzewoSb+Q9HQH+vgeM5tYvHEiM5so6afqvqWon5Z0bXH9WkmrOtjLd3TLMt5ly4yrw89dx5c/d/e2f0m6VMPvyH8oaVEneijp658l/U/x9V6ne5O0XMP/1u3V8H9E10s6UdJaSR8Ul5O7qLf/lvSupHc0HKypHertAg2/NHxH0rri69JOP3eJvtryvPFxWSAIPkEHBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0H8HxyR+xryMIiSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(extra_train_x[2*60000+0].reshape(28,28), cmap='Greys')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "300000/300000 [==============================] - 258s 861us/step - loss: 0.2514 - accuracy: 0.9365\n",
      "Epoch 2/100\n",
      "300000/300000 [==============================] - 256s 854us/step - loss: 0.0388 - accuracy: 0.9886\n",
      "Epoch 3/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0271 - accuracy: 0.9919\n",
      "Epoch 4/100\n",
      "300000/300000 [==============================] - 257s 856us/step - loss: 0.0213 - accuracy: 0.9936\n",
      "Epoch 5/100\n",
      "300000/300000 [==============================] - 256s 854us/step - loss: 0.0179 - accuracy: 0.9945\n",
      "Epoch 6/100\n",
      "300000/300000 [==============================] - 256s 855us/step - loss: 0.0160 - accuracy: 0.9951\n",
      "Epoch 7/100\n",
      "300000/300000 [==============================] - 257s 855us/step - loss: 0.0162 - accuracy: 0.9950\n",
      "Epoch 8/100\n",
      "300000/300000 [==============================] - 270s 901us/step - loss: 0.0133 - accuracy: 0.9960\n",
      "Epoch 9/100\n",
      "300000/300000 [==============================] - 272s 906us/step - loss: 0.0135 - accuracy: 0.9960\n",
      "Epoch 10/100\n",
      "300000/300000 [==============================] - 271s 905us/step - loss: 0.0117 - accuracy: 0.9963\n",
      "Epoch 11/100\n",
      "300000/300000 [==============================] - 272s 906us/step - loss: 0.0104 - accuracy: 0.9968\n",
      "Epoch 12/100\n",
      "300000/300000 [==============================] - 272s 906us/step - loss: 0.0099 - accuracy: 0.9969\n",
      "Epoch 13/100\n",
      "300000/300000 [==============================] - 272s 905us/step - loss: 0.0115 - accuracy: 0.9965\n",
      "Epoch 14/100\n",
      "300000/300000 [==============================] - 266s 887us/step - loss: 0.0096 - accuracy: 0.9970\n",
      "Epoch 15/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0093 - accuracy: 0.9971\n",
      "Epoch 16/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0092 - accuracy: 0.9972\n",
      "Epoch 17/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0076 - accuracy: 0.9976\n",
      "Epoch 18/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0077 - accuracy: 0.9977\n",
      "Epoch 19/100\n",
      "300000/300000 [==============================] - 256s 854us/step - loss: 0.0074 - accuracy: 0.9977\n",
      "Epoch 20/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0082 - accuracy: 0.9974\n",
      "Epoch 21/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0068 - accuracy: 0.9978\n",
      "Epoch 22/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0077 - accuracy: 0.9978\n",
      "Epoch 23/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0051 - accuracy: 0.9984\n",
      "Epoch 24/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0062 - accuracy: 0.9981\n",
      "Epoch 25/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0063 - accuracy: 0.9980\n",
      "Epoch 26/100\n",
      "300000/300000 [==============================] - 258s 858us/step - loss: 0.0052 - accuracy: 0.9984\n",
      "Epoch 27/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0055 - accuracy: 0.9984\n",
      "Epoch 28/100\n",
      "300000/300000 [==============================] - 257s 856us/step - loss: 0.0049 - accuracy: 0.9985\n",
      "Epoch 29/100\n",
      "300000/300000 [==============================] - 256s 854us/step - loss: 0.0042 - accuracy: 0.9987\n",
      "Epoch 30/100\n",
      "300000/300000 [==============================] - 256s 853us/step - loss: 0.0056 - accuracy: 0.9983\n",
      "Epoch 31/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0044 - accuracy: 0.9987\n",
      "Epoch 32/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0038 - accuracy: 0.9988\n",
      "Epoch 33/100\n",
      "300000/300000 [==============================] - 256s 853us/step - loss: 0.0047 - accuracy: 0.9986\n",
      "Epoch 34/100\n",
      "300000/300000 [==============================] - 255s 851us/step - loss: 0.0045 - accuracy: 0.9987\n",
      "Epoch 35/100\n",
      "300000/300000 [==============================] - 257s 856us/step - loss: 0.0026 - accuracy: 0.9991\n",
      "Epoch 36/100\n",
      "300000/300000 [==============================] - 257s 856us/step - loss: 0.0038 - accuracy: 0.9989\n",
      "Epoch 37/100\n",
      "300000/300000 [==============================] - 256s 852us/step - loss: 0.0040 - accuracy: 0.9988\n",
      "Epoch 38/100\n",
      "300000/300000 [==============================] - 257s 856us/step - loss: 0.0040 - accuracy: 0.9989\n",
      "Epoch 39/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0040 - accuracy: 0.9988\n",
      "Epoch 40/100\n",
      "300000/300000 [==============================] - 257s 856us/step - loss: 0.0033 - accuracy: 0.9990\n",
      "Epoch 41/100\n",
      "300000/300000 [==============================] - 257s 855us/step - loss: 0.0035 - accuracy: 0.9990\n",
      "Epoch 42/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0030 - accuracy: 0.9991\n",
      "Epoch 43/100\n",
      "300000/300000 [==============================] - 256s 855us/step - loss: 0.0027 - accuracy: 0.9992\n",
      "Epoch 44/100\n",
      "300000/300000 [==============================] - 257s 856us/step - loss: 0.0032 - accuracy: 0.9991\n",
      "Epoch 45/100\n",
      "300000/300000 [==============================] - 258s 859us/step - loss: 0.0018 - accuracy: 0.9995\n",
      "Epoch 46/100\n",
      "300000/300000 [==============================] - 266s 888us/step - loss: 0.0027 - accuracy: 0.9992\n",
      "Epoch 47/100\n",
      "300000/300000 [==============================] - 272s 905us/step - loss: 0.0029 - accuracy: 0.9992\n",
      "Epoch 48/100\n",
      "300000/300000 [==============================] - 272s 906us/step - loss: 0.0029 - accuracy: 0.9992\n",
      "Epoch 49/100\n",
      "300000/300000 [==============================] - 271s 905us/step - loss: 0.0026 - accuracy: 0.9993\n",
      "Epoch 50/100\n",
      "300000/300000 [==============================] - 272s 905us/step - loss: 0.0026 - accuracy: 0.9993\n",
      "Epoch 51/100\n",
      "300000/300000 [==============================] - 271s 905us/step - loss: 0.0024 - accuracy: 0.9993\n",
      "Epoch 52/100\n",
      "300000/300000 [==============================] - 270s 901us/step - loss: 0.0025 - accuracy: 0.9992\n",
      "Epoch 53/100\n",
      "300000/300000 [==============================] - 257s 856us/step - loss: 0.0026 - accuracy: 0.9992\n",
      "Epoch 54/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0019 - accuracy: 0.9995\n",
      "Epoch 55/100\n",
      "300000/300000 [==============================] - 256s 855us/step - loss: 0.0031 - accuracy: 0.9991\n",
      "Epoch 56/100\n",
      "300000/300000 [==============================] - 256s 855us/step - loss: 0.0023 - accuracy: 0.9993\n",
      "Epoch 57/100\n",
      "300000/300000 [==============================] - 257s 856us/step - loss: 0.0026 - accuracy: 0.9993\n",
      "Epoch 58/100\n",
      "300000/300000 [==============================] - 256s 853us/step - loss: 0.0014 - accuracy: 0.9996\n",
      "Epoch 59/100\n",
      "300000/300000 [==============================] - 256s 855us/step - loss: 0.0033 - accuracy: 0.9991\n",
      "Epoch 60/100\n",
      "300000/300000 [==============================] - 257s 856us/step - loss: 0.0021 - accuracy: 0.9994\n",
      "Epoch 61/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0018 - accuracy: 0.9995\n",
      "Epoch 62/100\n",
      "300000/300000 [==============================] - 257s 855us/step - loss: 0.0013 - accuracy: 0.9997\n",
      "Epoch 63/100\n",
      "300000/300000 [==============================] - 256s 855us/step - loss: 0.0019 - accuracy: 0.9994\n",
      "Epoch 64/100\n",
      "300000/300000 [==============================] - 257s 856us/step - loss: 0.0022 - accuracy: 0.9994\n",
      "Epoch 65/100\n",
      "300000/300000 [==============================] - 256s 854us/step - loss: 0.0018 - accuracy: 0.9995\n",
      "Epoch 66/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0017 - accuracy: 0.9995\n",
      "Epoch 67/100\n",
      "300000/300000 [==============================] - 257s 856us/step - loss: 0.0018 - accuracy: 0.9995\n",
      "Epoch 68/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0022 - accuracy: 0.9994\n",
      "Epoch 69/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0021 - accuracy: 0.9994\n",
      "Epoch 70/100\n",
      "300000/300000 [==============================] - 256s 855us/step - loss: 0.0013 - accuracy: 0.9996\n",
      "Epoch 71/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0017 - accuracy: 0.9995\n",
      "Epoch 72/100\n",
      "300000/300000 [==============================] - 257s 855us/step - loss: 0.0012 - accuracy: 0.9996\n",
      "Epoch 73/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0022 - accuracy: 0.9994\n",
      "Epoch 74/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0019 - accuracy: 0.9995\n",
      "Epoch 75/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0016 - accuracy: 0.9995\n",
      "Epoch 76/100\n",
      "300000/300000 [==============================] - 258s 859us/step - loss: 0.0016 - accuracy: 0.9995\n",
      "Epoch 77/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0014 - accuracy: 0.9996\n",
      "Epoch 78/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0031 - accuracy: 0.9992\n",
      "Epoch 79/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0020 - accuracy: 0.9994\n",
      "Epoch 80/100\n",
      "300000/300000 [==============================] - 258s 859us/step - loss: 9.1947e-04 - accuracy: 0.9997\n",
      "Epoch 81/100\n",
      "300000/300000 [==============================] - 258s 860us/step - loss: 4.8038e-04 - accuracy: 0.9999\n",
      "Epoch 82/100\n",
      "300000/300000 [==============================] - 258s 860us/step - loss: 0.0017 - accuracy: 0.9995\n",
      "Epoch 83/100\n",
      "300000/300000 [==============================] - 258s 860us/step - loss: 0.0021 - accuracy: 0.9995\n",
      "Epoch 84/100\n",
      "300000/300000 [==============================] - 258s 859us/step - loss: 0.0015 - accuracy: 0.9996\n",
      "Epoch 85/100\n",
      "300000/300000 [==============================] - 257s 856us/step - loss: 9.9274e-04 - accuracy: 0.9997\n",
      "Epoch 86/100\n",
      "300000/300000 [==============================] - 258s 859us/step - loss: 0.0023 - accuracy: 0.9994\n",
      "Epoch 87/100\n",
      "300000/300000 [==============================] - 258s 860us/step - loss: 0.0015 - accuracy: 0.9996\n",
      "Epoch 88/100\n",
      "300000/300000 [==============================] - 259s 862us/step - loss: 0.0015 - accuracy: 0.9996\n",
      "Epoch 89/100\n",
      "300000/300000 [==============================] - 258s 861us/step - loss: 0.0016 - accuracy: 0.9996\n",
      "Epoch 90/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0011 - accuracy: 0.9997\n",
      "Epoch 91/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 0.0024 - accuracy: 0.9993\n",
      "Epoch 92/100\n",
      "300000/300000 [==============================] - 257s 855us/step - loss: 0.0015 - accuracy: 0.9996\n",
      "Epoch 93/100\n",
      "300000/300000 [==============================] - 257s 857us/step - loss: 8.0358e-04 - accuracy: 0.9998\n",
      "Epoch 94/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 8.9849e-04 - accuracy: 0.9998\n",
      "Epoch 95/100\n",
      "300000/300000 [==============================] - 258s 860us/step - loss: 0.0013 - accuracy: 0.9997\n",
      "Epoch 96/100\n",
      "300000/300000 [==============================] - 257s 858us/step - loss: 0.0014 - accuracy: 0.9996\n",
      "Epoch 97/100\n",
      "300000/300000 [==============================] - 258s 859us/step - loss: 0.0012 - accuracy: 0.9997\n",
      "Epoch 98/100\n",
      "300000/300000 [==============================] - 258s 860us/step - loss: 0.0016 - accuracy: 0.9996\n",
      "Epoch 99/100\n",
      "300000/300000 [==============================] - 258s 861us/step - loss: 0.0018 - accuracy: 0.9995\n",
      "Epoch 100/100\n",
      "300000/300000 [==============================] - 258s 859us/step - loss: 0.0012 - accuracy: 0.9997\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x220f39d5c18>"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model6.compile(optimizer='adam',\n",
    "               loss='sparse_categorical_crossentropy',\n",
    "               metrics=['accuracy'])\n",
    "model6.fit(extra_train_x, extra_train_y, batch_size=1000, epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 5s 501us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.05039574176039329, 0.9943000078201294]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model6.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "model7a = Sequential()\n",
    "model7a.add(Conv2D(32, 3, activation='relu', padding='same', input_shape=(28,28,1)))\n",
    "model7a.add(Conv2D(32, 3, activation='relu', padding='same', input_shape=(26,26,1)))\n",
    "model7a.add(Conv2D(32, 3, activation='relu', padding='same', input_shape=(24,24,1)))\n",
    "model7a.add(Conv2D(32, 3, activation='relu', padding='same', input_shape=(22,22,1)))\n",
    "model7a.add(Conv2D(32, 3, activation='relu', padding='same', input_shape=(20,20,1)))\n",
    "model7a.add(Conv2D(32, 3, activation='relu', padding='same', input_shape=(18,18,1)))\n",
    "model7a.add(Conv2D(32, 3, activation='relu', padding='same', input_shape=(16,16,1)))\n",
    "model7a.add(Conv2D(32, 3, activation='relu', padding='same', input_shape=(14,14,1)))\n",
    "model7a.add(Conv2D(32, 3, activation='relu', padding='same', input_shape=(12,12,1)))\n",
    "model7a.add(Conv2D(32, 3, activation='relu', padding='same', input_shape=(10,10,1)))\n",
    "model7a.add(Conv2D(32, 3, activation='relu', padding='same', input_shape=(8,8,1)))\n",
    "model7a.add(Conv2D(32, 3, activation='relu', padding='same', input_shape=(6,6,1)))\n",
    "model7a.add(Conv2D(32, 3, activation='relu', padding='same', input_shape=(4,4,1)))\n",
    "model7a.add(Flatten())\n",
    "model7a.add(Dense(256, activation='relu'))\n",
    "model7a.add(Dropout(0.5))\n",
    "model7a.add(Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "60000/60000 [==============================] - 41s 686us/step - loss: 0.2221 - accuracy: 0.9319\n",
      "Epoch 2/10\n",
      "60000/60000 [==============================] - 40s 671us/step - loss: 0.0675 - accuracy: 0.9807\n",
      "Epoch 3/10\n",
      "60000/60000 [==============================] - 40s 671us/step - loss: 0.0468 - accuracy: 0.9858\n",
      "Epoch 4/10\n",
      "60000/60000 [==============================] - 40s 669us/step - loss: 0.0375 - accuracy: 0.9890\n",
      "Epoch 5/10\n",
      "60000/60000 [==============================] - 40s 669us/step - loss: 0.0313 - accuracy: 0.9907\n",
      "Epoch 6/10\n",
      "60000/60000 [==============================] - 40s 669us/step - loss: 0.0282 - accuracy: 0.9913\n",
      "Epoch 7/10\n",
      "60000/60000 [==============================] - 40s 667us/step - loss: 0.0225 - accuracy: 0.9933\n",
      "Epoch 8/10\n",
      "60000/60000 [==============================] - 40s 667us/step - loss: 0.0190 - accuracy: 0.9939\n",
      "Epoch 9/10\n",
      "60000/60000 [==============================] - 40s 668us/step - loss: 0.0213 - accuracy: 0.9933\n",
      "Epoch 10/10\n",
      "60000/60000 [==============================] - 40s 669us/step - loss: 0.0175 - accuracy: 0.9946\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x220119f87f0>"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model7a.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
    "model7a.fit(x_train, y_train, batch_size=200, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 3s 327us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.025978392983453523, 0.991599977016449]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model7a.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=input_shape)\n",
    "x = ResNetUnit(inputs, 64, 3, padding='valid')\n",
    "x = MaxPooling2D(2)(x)\n",
    "x = ResNetUnit(x, 64, 3, padding='valid')\n",
    "y = ResNetUnit(inputs, 32, 7, padding='valid')\n",
    "y = MaxPooling2D(2)(y)\n",
    "z = ResNetUnit(inputs, 16, 11, padding='valid')\n",
    "#print(x.shape)\n",
    "#print(y.shape)\n",
    "#print(z.shape)\n",
    "w = Concatenate()([x, y, z])\n",
    "w = Flatten()(w)\n",
    "w = Dense(512, activation='relu')(w)\n",
    "w = Dropout(0.5)(w)\n",
    "outputs = Dense(10, activation='softmax')(w)\n",
    "model8 = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "60000/60000 [==============================] - 39s 646us/step - loss: 0.7570 - accuracy: 0.8684\n",
      "Epoch 2/10\n",
      "60000/60000 [==============================] - 36s 597us/step - loss: 0.0975 - accuracy: 0.9724\n",
      "Epoch 3/10\n",
      "60000/60000 [==============================] - 36s 597us/step - loss: 0.0658 - accuracy: 0.9810\n",
      "Epoch 4/10\n",
      "60000/60000 [==============================] - 36s 597us/step - loss: 0.0520 - accuracy: 0.9843\n",
      "Epoch 5/10\n",
      "60000/60000 [==============================] - 36s 595us/step - loss: 0.0419 - accuracy: 0.9879\n",
      "Epoch 6/10\n",
      "60000/60000 [==============================] - 36s 596us/step - loss: 0.0362 - accuracy: 0.9895\n",
      "Epoch 7/10\n",
      "60000/60000 [==============================] - 36s 595us/step - loss: 0.0344 - accuracy: 0.9900\n",
      "Epoch 8/10\n",
      "60000/60000 [==============================] - 36s 594us/step - loss: 0.0308 - accuracy: 0.9908\n",
      "Epoch 9/10\n",
      "60000/60000 [==============================] - 36s 596us/step - loss: 0.0282 - accuracy: 0.9915\n",
      "Epoch 10/10\n",
      "60000/60000 [==============================] - 36s 596us/step - loss: 0.0232 - accuracy: 0.9928\n",
      "10000/10000 [==============================] - 4s 386us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.05592482680416167, 0.9847999811172485]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model8.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
    "model8.fit(x_train, y_train, batch_size=500, epochs=10)\n",
    "model8.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(None, 4, 4, 64) (None, 4, 4, 64) (None, 4, 4, 64)\n"
     ]
    }
   ],
   "source": [
    "inputs = keras.Input(shape=(28,28,1))\n",
    "x = ResNetUnit(inputs, 64, 3, padding='valid') # 24\n",
    "x = ResNetUnit(x, 64, 3, padding='valid') # 20\n",
    "x = ResNetUnit(x, 64, 3, padding='valid') # 16\n",
    "x = ResNetUnit(x, 64, 3, padding='valid') # 12\n",
    "x = ResNetUnit(x, 64, 3, padding='valid') # 8\n",
    "x = ResNetUnit(x, 64, 3, padding='valid') # 4\n",
    "y = ResNetUnit(inputs, 64, 7, padding='valid') # 16\n",
    "y = ResNetUnit(y, 64, 7, padding='valid') # 4\n",
    "z = ResNetUnit(inputs, 64, 11, padding='valid') # 8\n",
    "z = ResNetUnit(z, 64, 3, padding='valid') # 4\n",
    "print(x.shape, y.shape, z.shape)\n",
    "w = Concatenate()([x, y, z])\n",
    "w = Flatten()(w)\n",
    "w = Dense(512, activation='relu')(w)\n",
    "w = Dropout(0.5)(w)\n",
    "outputs = Dense(10, activation='softmax')(w)\n",
    "model9 = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "60000/60000 [==============================] - 104s 2ms/step - loss: 0.1386 - accuracy: 0.9617\n",
      "Epoch 2/50\n",
      "60000/60000 [==============================] - 100s 2ms/step - loss: 0.0651 - accuracy: 0.9815\n",
      "Epoch 3/50\n",
      "60000/60000 [==============================] - 102s 2ms/step - loss: 0.0485 - accuracy: 0.9863\n",
      "Epoch 4/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0387 - accuracy: 0.9891\n",
      "Epoch 5/50\n",
      "60000/60000 [==============================] - 104s 2ms/step - loss: 0.0313 - accuracy: 0.9911\n",
      "Epoch 6/50\n",
      "60000/60000 [==============================] - 104s 2ms/step - loss: 0.0315 - accuracy: 0.9911\n",
      "Epoch 7/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0260 - accuracy: 0.9922\n",
      "Epoch 8/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0273 - accuracy: 0.9921\n",
      "Epoch 9/50\n",
      "60000/60000 [==============================] - 104s 2ms/step - loss: 0.0226 - accuracy: 0.9933\n",
      "Epoch 10/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0179 - accuracy: 0.9945\n",
      "Epoch 11/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0157 - accuracy: 0.9952\n",
      "Epoch 12/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0184 - accuracy: 0.9945\n",
      "Epoch 13/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0176 - accuracy: 0.9948\n",
      "Epoch 14/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0155 - accuracy: 0.9955\n",
      "Epoch 15/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0121 - accuracy: 0.9964\n",
      "Epoch 16/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0153 - accuracy: 0.9957\n",
      "Epoch 17/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0162 - accuracy: 0.9954\n",
      "Epoch 18/50\n",
      "60000/60000 [==============================] - 107s 2ms/step - loss: 0.0132 - accuracy: 0.9966\n",
      "Epoch 19/50\n",
      "60000/60000 [==============================] - 109s 2ms/step - loss: 0.0118 - accuracy: 0.9964\n",
      "Epoch 20/50\n",
      "60000/60000 [==============================] - 109s 2ms/step - loss: 0.0104 - accuracy: 0.9972\n",
      "Epoch 21/50\n",
      "60000/60000 [==============================] - 110s 2ms/step - loss: 0.0098 - accuracy: 0.9976\n",
      "Epoch 22/50\n",
      "60000/60000 [==============================] - 110s 2ms/step - loss: 0.0106 - accuracy: 0.9971\n",
      "Epoch 23/50\n",
      "60000/60000 [==============================] - 111s 2ms/step - loss: 0.0113 - accuracy: 0.9968\n",
      "Epoch 24/50\n",
      "60000/60000 [==============================] - 111s 2ms/step - loss: 0.0085 - accuracy: 0.9974\n",
      "Epoch 25/50\n",
      "60000/60000 [==============================] - 110s 2ms/step - loss: 0.0072 - accuracy: 0.9981\n",
      "Epoch 26/50\n",
      "60000/60000 [==============================] - 110s 2ms/step - loss: 0.0073 - accuracy: 0.9981\n",
      "Epoch 27/50\n",
      "60000/60000 [==============================] - 110s 2ms/step - loss: 0.0080 - accuracy: 0.9979\n",
      "Epoch 28/50\n",
      "60000/60000 [==============================] - 111s 2ms/step - loss: 0.0106 - accuracy: 0.9972\n",
      "Epoch 29/50\n",
      "60000/60000 [==============================] - 111s 2ms/step - loss: 0.0079 - accuracy: 0.9976\n",
      "Epoch 30/50\n",
      "60000/60000 [==============================] - 105s 2ms/step - loss: 0.0068 - accuracy: 0.9983\n",
      "Epoch 31/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0062 - accuracy: 0.9982\n",
      "Epoch 32/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0155 - accuracy: 0.9967\n",
      "Epoch 33/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0070 - accuracy: 0.9978\n",
      "Epoch 34/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0040 - accuracy: 0.9987\n",
      "Epoch 35/50\n",
      "60000/60000 [==============================] - 104s 2ms/step - loss: 0.0051 - accuracy: 0.9987\n",
      "Epoch 36/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0082 - accuracy: 0.9981\n",
      "Epoch 37/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0055 - accuracy: 0.9986\n",
      "Epoch 38/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0050 - accuracy: 0.9985\n",
      "Epoch 39/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0067 - accuracy: 0.9985\n",
      "Epoch 40/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0076 - accuracy: 0.9984\n",
      "Epoch 41/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0050 - accuracy: 0.9986\n",
      "Epoch 42/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0076 - accuracy: 0.9983\n",
      "Epoch 43/50\n",
      "60000/60000 [==============================] - 104s 2ms/step - loss: 0.0044 - accuracy: 0.9988\n",
      "Epoch 44/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0029 - accuracy: 0.9992\n",
      "Epoch 45/50\n",
      "60000/60000 [==============================] - 104s 2ms/step - loss: 0.0047 - accuracy: 0.9988\n",
      "Epoch 46/50\n",
      "60000/60000 [==============================] - 104s 2ms/step - loss: 0.0063 - accuracy: 0.9984\n",
      "Epoch 47/50\n",
      "60000/60000 [==============================] - 104s 2ms/step - loss: 0.0039 - accuracy: 0.9990\n",
      "Epoch 48/50\n",
      "60000/60000 [==============================] - 104s 2ms/step - loss: 0.0038 - accuracy: 0.9990\n",
      "Epoch 49/50\n",
      "60000/60000 [==============================] - 104s 2ms/step - loss: 0.0058 - accuracy: 0.9988\n",
      "Epoch 50/50\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.0041 - accuracy: 0.9989\n",
      "10000/10000 [==============================] - 9s 863us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.05033699312680069, 0.9932000041007996]"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model9.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
    "model9.fit(x_train, y_train, batch_size=200, epochs=50)\n",
    "model9.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
      "170500096/170498071 [==============================] - 345s 2us/step\n"
     ]
    }
   ],
   "source": [
    "#99.48% is still my best. Oh well.\n",
    "# Just curious what happens on data well outside the domain of applicability\n",
    "(train_x_c10, train_y_c10), (test_x_c10, test_y_c10) = tf.keras.datasets.cifar10.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9059\n",
      "[0]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x22116b732b0>"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD5CAYAAADhukOtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAdn0lEQVR4nO2de2xd15Xev3VfJEVSpCjqZUm2bFl27CR+ZDSKW08yTtxkPO4UiqedQdwiMIo0GhQJ0ADTh5MWkxToH5miSZA/igyU2h2nyDhJJwlsdNyZuK6nnqSOY9m1ZTvyQ9bDelAi9aAokiJ5H6t/3KuO7O5vkebjUpn9/QCCl3vdfc4++5x1zuX+7lrL3B1CiL/5FJZ7AEKI9iBnFyIT5OxCZIKcXYhMkLMLkQlydiEyobSQzmZ2F4BvACgC+E/u/pXo/f0V8yu6LGmLBMB0j5hGsMFG0O90cP+bLhaT7TafAc4KP4DopPU10kdXrvGjLgTjL6QPed5ESu98ReBaLTCSk9Oo8AOrOZ8r9/mdbI+OjpjmMx+jFxqYnGkkBzlvZzezIoD/COBjAI4CeNbMHnX3X7A+V3QZvv23K0lbNfDOChllNBnTM9w2GXT8Nnqp7WB/d7K9aMEHpODaiG9wdWobDHb3d6fOJ9s3jEzSPpUy397KPj7KuvOBMKeu8sMKb8L1YLJOn+aTbKX0wU1t4uf5ZHU6GAffV3QjaJCbcNPG9sUPukGunm/9dIz2WcjH+B0A9rv7AXefAfBdADsXsD0hxBKyEGffCODIJX8fbbUJIS5DFuLsqc8s/99nCzPbZWZ7zGzP2Rl9NVeI5WIhzn4UwOZL/t4E4Pg73+Tuu919u7tvX1VZkpUsIcQcWIizPwtgm5ldbWYVAJ8E8OjiDEsIsdjMezXe3Wtm9jkAf4Gm9Pagu78SdjKgxFbWa/ypXyykP/5Hi+CB0oRKYOsq8I3adHop+cLZcdqnFAySHRcA1EtcGhrtCFafK+ltVoJ9VQLtrRLMRy1amSZr65HM58HqfjXoVy4FUln1QrK9VkurQgDQCPTGYijL8TmOnqp1cmwWysfpLUYy8IJ0dnd/DMBjC9mGEKI96Bt0QmSCnF2ITJCzC5EJcnYhMkHOLkQmLGg1fj4wNSH6uk0h0msIxaBLZxD40RN88ae/Ky3J1CsraJ9aFPgR6IMXgsiPYpHbeovpHUbBa+VgfkuFQKeMAmHIGY2i3lhwBwCUSnyM3SuCSMVa+sidRVcB6K5wWc4C6S0i6sYku+DSoUE3hUAq1ZNdiEyQswuRCXJ2ITJBzi5EJsjZhciEtq7GG4AiWfm1YPm8QIIqLFhFLga3sUqwsntHYYLadsxMveudzQTL4FOd3DgRrJ8XgnXaTWSFPxpHNFdRYIUFS+tGOgbxPainU6cBiANogmlEo5zu2N3H01J1dHbyDQbJs6JAGA/SrjVIvyjNVZ1sr1g8RfvoyS5EJsjZhcgEObsQmSBnFyIT5OxCZIKcXYhMaHsgDIsIiAIkmIxjUZKuIOcay4MHANcZl7WKSNcZCiWXMPIjyEEX5K6rBffoDrLJk8F8RFFIFgVWzCOoJZLyooAnI6W3AKBRCkJGiMzaubKHdqmXO6itVq3yfnVeh6pe52Nkl7FHkTBGJMBggvVkFyIT5OxCZIKcXYhMkLMLkQlydiEyQc4uRCYsSHozs0MAzqOZLqvm7tvjDkCRRLdFEpXNo/xTMdCTwnx3UXkfFrEXyB0W7M2DAygF0VXlYIyslFAh2lcUERfYPNLRmDLEe4T6azGIVCw0ghJb5BIvlnkiwqrxg65Wp6mtVo8i4oIINiKxxapt+pijPouhs3/E3XlcnRDiskAf44XIhIU6uwP4sZk9Z2a7FmNAQoilYaEf42939+NmthbA42b2qrs/dekbWjeBXQCwoWuBexNCzJsFPdnd/Xjr9zCAHwHYkXjPbnff7u7bVwV1xYUQS8u8nd3Mus2s9+JrAB8H8PJiDUwIsbgs5GP8OgA/aslOJQB/4u5/HnUw8MSBUdJDKtcF5YI8kKcskGpQ4jamXkXSGy94BZjzKKlAqQnlK/O0bMSSdgJAoDQhHH8YwZZuD6pawYJ9RSaM82MrkAC2yckZvrngvNSnuK0YaZjB+GtEewslYqKJRtfivJ3d3Q8AuHm+/YUQ7UXSmxCZIGcXIhPk7EJkgpxdiEyQswuRCW1NOOkO1BtpmaQRyWFEtggTFEbJKEMdJ5J/yP6CGmWNaIzBvTaSXYI8lTTKLoq+KxYC6SoYSBSp2CB6XiEKywrmoxFIh4Uq79fdmT6A6U6+vbOnxvk4GhVqq3TwRJWN8LlKpDeWVDLYlwXRjXqyC5EJcnYhMkHOLkQmyNmFyAQ5uxCZ0N7yTxavFgbd3qUBKAZli4wE1gA83x0AFFguvGilO7A1gtXWRjAOi1a0a+ltRgEShSiHXnC+okAYKobMM8CnWObHXAny03Vu3pZsb9x2D+3z1pOPUFvh/BlqM/BAmOh67OhKu2EhuE7LpXQOPQvkEz3ZhcgEObsQmSBnFyIT5OxCZIKcXYhMkLMLkQltld7MgBLJ8RbmVaOBGmFECCUOoIlsTFrheclQr1JTKQjWaXgn32aQoK5K5qQR5evjewrLCc03ZRwjDF4q8nJNpf4eanvqtt9Ltj86tIn2+dgVx6ht45tclhud4kEyrFwTAHR1p891sSPYXiDNMvRkFyIT5OxCZIKcXYhMkLMLkQlydiEyQc4uRCbMKr2Z2YMAfgvAsLu/r9U2AOB7ALYAOATgd9397Kx7c6BBctB5oJVRNSxKkBZElPGQLCDS7Hw6XTKo2nMF7VPd+B6+PTIXANA1dojaOsePU1uZzIk7lwDj8lVB2ahIwoxyChI8GEclqBt1/s7fprZHzgwk2wfe+DPa50O/zse+uvtaavurl4aobWySy7Pl3hXJ9r6VK2mf8YmJtCGSnLnp//HHAO56R9v9AJ5w920Anmj9LYS4jJnV2Vv11t8ZxLsTwEOt1w8B+MQij0sIscjM93/2de4+BACt32sXb0hCiKVgyRfozGyXme0xsz1npufzJUohxGIwX2c/aWYbAKD1e5i90d13u/t2d98+0BEtBAkhlpL5OvujAO5rvb4PAI8OEEJcFsxFensYwB0ABs3sKIAvAfgKgO+b2acBvAXgd+a6QybzeBDJ5UQqK4SJHoPkf/WgRE4wI1M77ku2V+75F7TPa7V+atsflBkqjZ2gtv4DP6O22998OtneMfQ431eNj6NgfELSRYuaOEuWWAui7+p8i6W1XPL674O/QW0df/ZEsv36gVHaZ+vN26mt+y0+V4Ovj1Db2VNT1Fb3dCmnteu4pNt1Jn19lIr8up/V2d39XmK6c7a+QojLB32DTohMkLMLkQlydiEyQc4uRCbI2YXIhPbWegPAAr2ixIas3lhUly26jVmZR3Ida/BIoxfrfcn23p//hPb5+eHz1Ha+j0srHkkotUFqe+7KtAy1afUNtM9HR/6K2gZG9lBbceYCtVlhOtk+g3SEFwAUg5O2/9Z04kgAePrpfdTWP30k2V7t20z7rBzkyT5n9vNrrjbO5+PMKI86HO9JX487uvl57iymZcpSibu0nuxCZIKcXYhMkLMLkQlydiEyQc4uRCbI2YXIhLZLb6wKmIdFxdIRVFHNtlKFb69U4f0eO72R2l7b80qyvXL4HO2z/9UD1Na57mpqazhPUDjYzWuAdfanpa3hDVfRPn955x9Q2+aZd2Yk+2t6Dz9PbYMHn0m2d468SPuUbvwQte0+xmuzFd96mNr6K5PJ9puu4Jd+5RxJ5gjg9dcPUtupUS69HTvPpdTJw+lcrX+/xq/TQrEr2c5kakBPdiGyQc4uRCbI2YXIBDm7EJkgZxciE9q7Gm8Ai++ISgkVST4zC3LQlRvpUk0AcHrgDmrr/nv/htquOJkuu3Tg9Zdon+JKvpp99lB6dR8AZqb4+E/VuO1QV3qlfu3AXtrn+PBpapt+z3XUtuL9O6ntxb/1mWT76nFeIgldZWqaePItaqsbD645Pj6WbF/Pu+DcK3yujp3g4+/uC8o1vZVWBQCgXk0rL9MzXBWYvpDeXr3Og7z0ZBciE+TsQmSCnF2ITJCzC5EJcnYhMkHOLkQmzKX804MAfgvAsLu/r9X2ZQCfAXCx3s0X3f2x2bblDaBWTUsDdef3nVJHOqjlwhgvFzQ2yWWQ/Xf/E2prFLisNXo2LbtMjpykfWpVXvane5DnGKs0uKxYd37clXpaxhmZ4dLP6T3/i9reeJkHrvT19lBb77p0QFH3Jh7807uW5+S79eY11Fa9iZ/PQ//zz5PtJ57lefceGd5Pbf09PEBpdSUIrgmk4IHB9cl2C87zqaG0DFyr8v3M5cn+xwDuSrR/3d1vaf3M6uhCiOVlVmd396cA8G+GCCF+KVjI/+yfM7O9Zvagma1atBEJIZaE+Tr7NwFsBXALgCEAX2VvNLNdZrbHzPacnYkSVAghlpJ5Obu7n3T3urs3AHwLwI7gvbvdfbu7b18VZIgRQiwt83J2M9twyZ/3AHh5cYYjhFgq5iK9PQzgDgCDZnYUwJcA3GFmt6CZUO4QAF6b5xK8AUxNpp/utTr/iF8mMtTEBI+SOrztY9T2avcGanv9uaepbfjN9D2tei6dQwwAKoiikPgxR3fhhvFPSKxsVKHQz/sU+GVQsw5qOzkR2PanI+nK+4/RPp0FnsOtBC4pVYh0BQAf+PDdyfa909fQPhjnMmXfKn4+Oyo8N+CNv5IuhwUANpVe/z49foj2OTuSnt96jct1szq7u9+baH5gtn5CiMsLfYNOiEyQswuRCXJ2ITJBzi5EJsjZhciEtiacrDccY5Mk6g28PE6jkJaougpcBjkywMsF/fQvHqG288d4uSZjEWzBd4UKRX4/LRaCUj1BAs7OYJsFIst5IMlYgc+9F/klMoNuamsgLYu699I+MC69VRpc3txy9gVqc5I70lbyiL31tWFqGzg3Sm2rPZ3cEgA6vUptpRXpUk5TSLcDwKnhU8n2GkleCejJLkQ2yNmFyAQ5uxCZIGcXIhPk7EJkgpxdiExoq/RWaxjOTKZlHg8iwGZIwNBYJ4962zTN5ZgPj49QGwJlqEoi2KqR9uaBrBXda4NNXuAqDg7WBsgweERWtco3ODnOo7WiMaKcLqYWRfqV/Ry3FXjizutXpJMvAkD5zV8k22++gstag+v5RfA//jeX5bob/Hrs2noltXXekE60efQwT2R6biwtU6rWmxBCzi5ELsjZhcgEObsQmSBnFyIT2roaP1MHjpxLrxaa8ftOGekgjkKBryLf3nmE2m68aoLapqd5wIiT1ecCMwCYqvMx1qb5MU/xYeBkja8kHzudDvA4O8FXs4tBnrlb3vteaiv3plf+AaDc1ZfeV4nvq6PMV+o7KjzAY0PlV/k4OtJqiG1Ijw8Azhb5avzqbXwcBya5DV38GvlgYV+yfdOx12if/mvSrvvwC3w/erILkQlydiEyQc4uRCbI2YXIBDm7EJkgZxciE+ZS/mkzgG8DWA+gAWC3u3/DzAYAfA/AFjRLQP2uu/NEYQCmGo5XJ4i80uCyRd9AWuLp6F1D+wzV04EYAHDtGr6v4REuUdVJiapaIBt6gctkFgSFNIL4k1ogD56bTh9blC/uk5/9ArVdfcvN1Fad4fM4SsYxPc0PbGqS2+rT49R2sMpLQ1VJmaSTT/yMb28TDxoCy0MIYOJcOi8cAJwbOkFttXXp9n/8Hi4PgvhRd2e6lBQwtyd7DcDvu/sNAG4D8FkzuxHA/QCecPdtAJ5o/S2EuEyZ1dndfcjdn2+9Pg9gH4CNAHYCeKj1tocAfGKpBimEWDjv6n92M9sC4FYAzwBY5+5DQPOGAGDtYg9OCLF4zPnrsmbWA+AHAD7v7mMWlA1+R79dAHYBQI+WA4VYNubkfmZWRtPRv+PuP2w1nzSzDS37BgDJFB7uvtvdt7v79k45uxDLxqzuZ81H+AMA9rn71y4xPQrgvtbr+wDwMitCiGVnLh/jbwfwKQAvmdnFxG5fBPAVAN83s08DeAvA78y2oboVcKaQljVWrOYy2vUf/+1k+3U33Ej7rKhwGae7/1lqu7L/DWo7+Go6ku7CJI+iK5b4FJeCW+2KCu/XM8Mlu6uKaell9Souk00+/Z+p7c2f8X1ZY5LaeiwtUfXUuXTVO8NtU+d5froL53nZpSsqaVmub4xfH+8f4ydmZTfPKVgOSnbN9PH8dINr0/npVq3iUYU1EiFYKPLxzers7v4T8NSCd87WXwhxeaD/ooXIBDm7EJkgZxciE+TsQmSCnF2ITGhrwsmGA1Mk/2JXzyDtN7lyY7L91Ar+Dd0zQdmltybS2wOAzV1PUdvaLSRZ5sFjtM/QMJfl6kHUW6nIy/hYnR/br/eMJtt9mkdkTbzM5caxOo8A27CSy0nFRvq4x6b5JXfyDJfeVq7ZQG3r+nhkYaOR3ubpQZ74shREFTa/X5bmqsF0sk8AGPfguVpMR2iOT/HjqpMh8qtGT3YhskHOLkQmyNmFyAQ5uxCZIGcXIhPk7EJkQlult+6K41evSmsG9cIB2q9r/2PJ9lqVJ0MscXUKKHDjgekL1Lbiyq3J9nXXcjnm4JF0HS8AODvBJZ7xSS5DjVa5ZDdTS5/Sg6M86u3IZCe1/aN7P0JtjcM/pbajp9JzPHSOJ4e8YccHqe2jd95EbS899Ti1nTqaHkdXFxep1vVzyWvb9TwJZE8X77fiAq/5Vyal5boHeNJUn+hPtheDqDc92YXIBDm7EJkgZxciE+TsQmSCnF2ITGjravz69V34l//qfUlbw3n+rkYxvWptpddon2KBH1ohyAtnzlfjX3pyf7J9hOTVA4DxKj+u4Um+Gr+izFd2izW+snt8Im2rlXn5py984V5q29rLV/F/fIQHfgxNpANh1q/jedU+spXPY2OMlzXq6SP1kwAcJkFKpyf4c66zl18fG25K54sDgEJwDfdd4NcViulz4938GkA5rQqYVuOFEHJ2ITJBzi5EJsjZhcgEObsQmSBnFyITZpXezGwzgG8DWI9miqvd7v4NM/sygM8AGGm99Yvuno5YubitUgmV1auSNi/zYAwjakJcSZYfWqCQoB7kXDtxIF1maLyDB6acHOcy2VBQZujuHXw+xs5xeeXoaDrAY30PD/xY/Sovh3VuM5fKrlzL5bxjh9JzUq+P0z6vfofn/+u+ictQ00HU0zUD6Vxz587xcQx28n1Vg1xy5ejZadzmJBdhfHmT4Kug01x09hqA33f3582sF8BzZnYxzOjr7v4f5rANIcQyM5dab0MAhlqvz5vZPgA8PasQ4rLkXf3PbmZbANwK4JlW0+fMbK+ZPWhm6c/nQojLgjk7u5n1APgBgM+7+xiAbwLYCuAWNJ/8XyX9dpnZHjPbMzLKExcIIZaWOTm7NTPj/wDAd9z9hwDg7ifdve7uDQDfArAj1dfdd7v7dnffvqafL34JIZaWWZ3dmkveDwDY5+5fu6T90hId9wB4efGHJ4RYLOayGn87gE8BeMnMXmi1fRHAvWZ2CwAHcAjA782+KQNAnu41Ll95I31PahT5vcqMbw+BzQpcurjuzvS65N7nTtI+I6M8l1z/Gi7xdHdyOenwCR6JdoHoiismeZ+9T/L79HUf4p/Game5nLdpZXr8o+e5FDnK5CQAW4JyTYdO8hJb3kjncfu1nbfTPtfewOXGjgqfR0wHxZciGY35RPAo9gIzLkB6c/efkC2EmroQ4vJC36ATIhPk7EJkgpxdiEyQswuRCXJ2ITKhrQkn4Q6nyRIDSYNEDBVItBAAeBD9YyV+jytUuG3rrenEhsNneDLBkz87T20f2sSP+fiLPMHi8AhPVDl1IX3cx8r8VP/KFh5hV3QuAY6N8W9EvnIsfZ5fm+Tz+w9v5uMYPsHneOWmdBJTALj+N3cm269ew6+PqQM8CvDo6SFqKwXXVanC558FfHb2BSWj3r3ypie7ELkgZxciE+TsQmSCnF2ITJCzC5EJcnYhMqHN0lsDqE2mTWEEW3qYjUBeaxQDW51HJ00EEs/EWDqCbSToY5NcPqkOc+nw9BkeLVcJyoadJ8ko35jhMl+Pcwmt/AaX+TyoY3eUKI4HL/C5P96ZvjYAYONGHhG3fef7qW3D+65NttcOv0j7TNX5MU86H7+N83Pm1UDu7UjPf9cFfl4qlh5jrcqvNz3ZhcgEObsQmSBnFyIT5OxCZIKcXYhMkLMLkQntld7MgDKRUIJEjyim+xQKPCKrECSVHDuertkGACf28qimscn0Nlc0+L7u+mAftTWKvN/Ae3up7dwEn6trSN2zQhdPbsnTNQITZ7mU09PF5bByb9rWqHFZqzTJNcXOKX7Ozhw5QG2rjx9Ntp8fGUm2A8CJY6eobZLIrwBghSDJaWdQI47Uj2tUucxXXkFcN5Cj9WQXIhPk7EJkgpxdiEyQswuRCXJ2ITJh1tV4M+sE8BSAjtb7/9Tdv2RmVwP4LoABAM8D+JR7EFHR3BiMraCH5ZrSq5IeJNyKbL2b+6nt+msHqa1RYjnSgntmNQiOCPoVi1xpQIGv0tJuxlfOUQyCXYKyXFG+M2MBI8aPa3qGj6NxIchRWF5JTcX+8WT7Kpb4DUBv32ZqmxkfpbbaNL/8p6f5PFaJQhGktENHR3ryg8X4OT3ZpwF81N1vRrM8811mdhuAPwTwdXffBuAsgE/PYVtCiGViVmf3Jhdvj+XWjwP4KIA/bbU/BOATSzJCIcSiMNf67MVWBddhAI8DeBPAqLtf/Gx1FEC6xKkQ4rJgTs7u7nV3vwXAJgA7ANyQeluqr5ntMrM9ZrZnZDT+l14IsXS8q9V4dx8F8JcAbgPQb3+dQmYTgOOkz2533+7u29f081rfQoilZVZnN7M1Ztbfet0F4O8A2AfgSQD/oPW2+wA8slSDFEIsnLkEwmwA8JCZFdG8OXzf3f+bmf0CwHfN7N8B+D8AHph9Uw7yaR8W5IwDCe6IdAYjwTMAgEJwjzP+6aNQ6ki2OwlkAADQcldAkZS1ao6Dm1DnY2yQ/GkebS8IKLJIywkCgLyR7tdAkO8OwTg6g0s1ONVeTQfQWIGfl1I3Py+dnTxACQ0uHUZzRauYedBnKi1FFoNre1Znd/e9AG5NtB9A8/93IcQvAfoGnRCZIGcXIhPk7EJkgpxdiEyQswuRCebR8v5i78xsBMDh1p+DAHiyr/ahcbwdjePt/LKN4yp3X5MytNXZ37Zjsz3uvn1Zdq5xaBwZjkMf44XIBDm7EJmwnM6+exn3fSkax9vRON7O35hxLNv/7EKI9qKP8UJkwrI4u5ndZWavmdl+M7t/OcbQGschM3vJzF4wsz1t3O+DZjZsZi9f0jZgZo+b2Rut36uWaRxfNrNjrTl5wczubsM4NpvZk2a2z8xeMbN/1mpv65wE42jrnJhZp5n93MxebI3j37barzazZ1rz8T2zIEQzhbu39QdAEc20VtcAqAB4EcCN7R5HayyHAAwuw34/DOADAF6+pO3fA7i/9fp+AH+4TOP4MoB/3ub52ADgA63XvQBeB3Bju+ckGEdb5wTNAOee1usygGfQTBjzfQCfbLX/EYB/+m62uxxP9h0A9rv7AW+mnv4ugJ3LMI5lw92fAnDmHc070UzcCbQpgScZR9tx9yF3f771+jyayVE2os1zEoyjrXiTRU/yuhzOvhHAkUv+Xs5klQ7gx2b2nJntWqYxXGSduw8BzYsOwNplHMvnzGxv62P+kv87cSlmtgXN/AnPYBnn5B3jANo8J0uR5HU5nD2VwmS5JIHb3f0DAH4TwGfN7MPLNI7LiW8C2IpmjYAhAF9t147NrAfADwB83t15jeb2j6Ptc+ILSPLKWA5nPwrg0pIbNFnlUuPux1u/hwH8CMubeeekmW0AgNbv4eUYhLufbF1oDQDfQpvmxMzKaDrYd9z9h63mts9JahzLNSetfb/rJK+M5XD2ZwFsa60sVgB8EsCj7R6EmXWbWe/F1wA+DuDluNeS8iiaiTuBZUzgedG5WtyDNsyJNRPdPQBgn7t/7RJTW+eEjaPdc7JkSV7btcL4jtXGu9Fc6XwTwL9epjFcg6YS8CKAV9o5DgAPo/lxsIrmJ51PA1gN4AkAb7R+DyzTOP4LgJcA7EXT2Ta0YRy/huZH0r0AXmj93N3uOQnG0dY5AXATmklc96J5Y/mDS67ZnwPYD+C/Auh4N9vVN+iEyAR9g06ITJCzC5EJcnYhMkHOLkQmyNmFyAQ5uxCZIGcXIhPk7EJkwv8FyDN980SAliUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 9059\n",
    "ind = int(random.random()*50000)\n",
    "print(ind)\n",
    "print(train_y_c10[ind])\n",
    "plt.imshow(train_x_c10[ind])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x22116d45710>"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAVFklEQVR4nO3da4zV9ZkH8O8zXAZB7vfhJpZBBjYurANuZF2rYrUaI4Z0U140bGJ2+qImbdIXa9wX5Y2J2WzbNHHTSBcski5NYxVNMPWCJl6iyHARUFAQhwoM14HhPjDw7Is5NCPO//uM5z/nkv19PwmZmfPM7/x/53/Ow//MeX4Xc3eIyP9/NZXugIiUh5JdJBFKdpFEKNlFEqFkF0lE/3IebMSIEV5XV1fOQ/5NVHW4evUqjdfUFP//opnlikd9Z/H+/flTHN33lStXaDyP6HFHor7369ev6GNfvnw517Er5fDhw2hvb+/xweVKdjN7AMBvAPQD8D/u/jT7/bq6Ojz//PNFH4+9cKOTf+nSJRq/cOECjQ8aNCgzxl5UQJxwUfvOzk4aZ49t7NixRbcFgPb2dhrPk7DR447uOzovw4YNy4xFz8nhw4dpPDpvec5LdGFh993U1JR9v8V2yMz6AfhvAN8HMBvAUjObXez9iUhp5fmbfQGAve6+z90vAfgjgEf6plsi0tfyJPskAF91+/lA4bavMbMmM2s2s+aTJ0/mOJyI5JEn2Xv6w+Ebfzi7+wp3b3T3xpEjR+Y4nIjkkSfZDwCY0u3nyQAO5euOiJRKnmTfBKDezKab2UAAPwTwSt90S0T6WtGlN3fvNLPHAbyGrtLbKnf/JE9nolo3K69FZZjovgcOHEjjrN4c1aIvXrxI43mxMlL0uPOccyDfGIHovEUlqDxjBKI6eltbG42fO3eOxiPsvEWPm8VZHuSqs7v7qwBezXMfIlIeGi4rkgglu0gilOwiiVCyiyRCyS6SCCW7SCLKOp/d3WltNM+c86jOnnf+MauLRrXmvPXiCDt+VMvOO88/z1TOvMeOHht7TUSvl+PHj9N4R0cHjdfW1tI4GxsxYMAA2padF3ZOdWUXSYSSXSQRSnaRRCjZRRKhZBdJhJJdJBFlLb0BvGwQlVryiEpEead6lqotkH+paSYqQeUt3ZVyCe6o73mOHZXWosfNViMG8i1zzUpz7DHryi6SCCW7SCKU7CKJULKLJELJLpIIJbtIIpTsIoko+xTXPFsA51l+N+9S03lq2aWsk/fm/pm8YxvyTFPNu2Vz1H7IkCGZsWga6fDhw0t2bIDvAhvdd7QDbRZd2UUSoWQXSYSSXSQRSnaRRCjZRRKhZBdJhJJdJBFlrbObGa0hlrLWHcmzHHQp5+ED+evRTCW3bC7l4wJ4PTqarx5tyVxXV0fjbL46AJw/f57Gi8XOd65kN7MWAGcAXAHQ6e6Nee5PREqnL67sd7s7X1FfRCpOf7OLJCJvsjuA181ss5k19fQLZtZkZs1m1nzy5MmchxORYuV9G7/Q3Q+Z2TgAb5jZbnd/p/svuPsKACsAoKGhoXSfsIkIlevK7u6HCl+PAngJwIK+6JSI9L2ik93MhpjZ0GvfA/gegJ191TER6Vt53saPB/BSoVbaH8D/uvtf8nQmz9zoaJ58VE8uZa281Nsis3pydOxSx1nf846buHz5Mo2z18T+/ftp2zfffJPGFy9eTOMjR46kcSZ6PZw9e7aotkUnu7vvA/D3xbYXkfJS6U0kEUp2kUQo2UUSoWQXSYSSXSQRZV9KOk+Ji5Vq8k7VzFMGiqYz5p0mWuqpoHmOXcq+RectWh68vb09M/baa6/RtlOmTKHxffv20fitt95K42yp6eicnj59uqi2urKLJELJLpIIJbtIIpTsIolQsoskQskukgglu0giylpnB0pXK887XbK2tpbG2bTCvXv30rbRdtJjxoyh8Wj74DzbIuc9b9FzFj12JupbtHXxiy++mBkbO3YsbTtv3jwaX7duHY2zWjgALFy4MDM2dOhQ2ratrS0zxs6ZruwiiVCyiyRCyS6SCCW7SCKU7CKJULKLJELJLpKIqqqz5xHNKY/mPkdLBz/zzDOZsajeO3XqVBofN24cjc+aNYvG2dzpCRMm0LZR3/M+X2w55+jY0XMaLQfd2tqaGVu2bBlte+TIERrfsGEDjUfz2Vkdf/z48bRtsWMXdGUXSYSSXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFEVFWdPc+2ytH2vVFtMqr5PvTQQ5mxmTNn0rYnTpyg8Wi76YEDB9L4Z599lhk7dOgQbTt58mQanzRpEo1HfSvlVtjvvfcejd9///2ZsWj9guPHj9N4dF4XLFhA4yNGjCgqBgATJ07MjA0YMCAzFl7ZzWyVmR01s53dbhtlZm+Y2Z7C1+I3oxaRsujN2/jfA3jgutueALDB3esBbCj8LCJVLEx2d38HwPXr4DwCYHXh+9UAFvdxv0SkjxX7Ad14d28FgMLXzMHdZtZkZs1m1nzq1KkiDycieZX803h3X+Huje7eGH3wICKlU2yyHzGziQBQ+Hq077okIqVQbLK/AuDaHMFlAF7um+6ISKmEdXYzWwvguwDGmNkBAL8A8DSAP5nZYwD+CuAHfdGZqCZ76dKlomIAMHr0aBq/++67abyhoSEzNmfOHNo2mhsdre3O1qwHgKNHs99YtbS00LZsbXUgrqPX19fTODs3dXV1tO1HH31E49OnT6fxGTNmZMai18uXX35J49HrKZrPPmrUqMwY27sdAKZNm5YZY+MHwmR396UZoXujtiJSPTRcViQRSnaRRCjZRRKhZBdJhJJdJBFlneLq7uF0ToZNU42mHK5atYrGo6mcbOnfaHveaCnpqLwVLanc0dGRGWtvb6dto8e9detWGt+4cSONr1+/PjMWbVUdndclS5bQOJtOfeONN9K2586do/G77rqLxll5DADY0PHoOYleD1l0ZRdJhJJdJBFKdpFEKNlFEqFkF0mEkl0kEUp2kUSUfSlpNp1z2LBhtO3gwYMzY2vWrKFto+mzY8eOpXFW+2TL9wJAW9v1S/h9XbTMdTQFltVdoxr+nXfeSePz58+n8TNnztD4V199lRnbu3cvbRuNnfjwww9pnI0BePTRR2nbRYsW0ThbzhmIpyVfuHAhMxa9HtjriY1j0ZVdJBFKdpFEKNlFEqFkF0mEkl0kEUp2kUQo2UUSUdY6e79+/WgtndXRAb41cVTvjerJ0dK/0RiAUmLzsgFeW82zDXZv4tFzNnv27KJiQLyVdYRtlb1u3TradtCgQTQ+fPhwGo+2fGZrELz99tu0LTvnbJ68ruwiiVCyiyRCyS6SCCW7SCKU7CKJULKLJELJLpKIstbZa2pqaP0yqie3trZmxpYuzdpstku0dns0J531rbOzk7bNW8uO6s2sb/3786c477HziObpR6+HaN73yJEjM2N33HEHbctq9EC8TgDbkhngfY8eFxtTwtY2CK/sZrbKzI6a2c5uty03s4Nmtq3w78HofkSksnrzNv73AB7o4fZfu/vcwr9X+7ZbItLXwmR393cA8HWVRKTq5fmA7nEz2154m5/5x5GZNZlZs5k1R2uxiUjpFJvsvwXwHQBzAbQC+GXWL7r7CndvdPfG6EMLESmdopLd3Y+4+xV3vwrgdwAW9G23RKSvFZXsZtZ9Hd1HAezM+l0RqQ5hnd3M1gL4LoAxZnYAwC8AfNfM5gJwAC0Aftybg9XU1GDo0KGZ8UuXLtH2N9xwQ2Ysml8czXfPI6oXR6J6cjQGgNXSozp5dN/RPuZR3y9fvkzjTHReo8fG1mZvaGigbdl69wDQ0tJC4/fddx+Ns8e2Z88e2nbTpk2ZMXZOwmR3955Gq6yM2olIddFwWZFEKNlFEqFkF0mEkl0kEUp2kURU1VLSURnn3LlzmbFdu3bRtqxs15tjs1JJVL6Klltm0xKBeDrlmDFjMmNDhgyhbaPHvXXrVhqPhkCz5ywqy0Wl2IsXLxbdPmobldaivu3YsYPGWUmTnTOAl9fYOdWVXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFEKNlFElHWOru70xphVF9k9eyDBw/StpMmTaLxqNbN5F1uOarDR8tB19bWFn3s5cuX0/jrr79O49HWxmx8QzT2gS0FDcRjCNh5qauro22XLFlC41988QWNR2MjRo8enRmbPn06bbt///7MGHu+dGUXSYSSXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFElLXO3t7ejvXr12fGN2/eTNt3dHRkxqItdqP4vHnzaJzNw4/mZUfbIrMlj4G4js62jD558iRtGy15vGjRIhqPthdm8+WjsQ1RHT7aYYg9Z9F8dNYW4GsIAMCpU6donI3NOHbsGG3LxpRoPruIKNlFUqFkF0mEkl0kEUp2kUQo2UUSoWQXSURZ6+ydnZ203h1tq8zWbo+2Fo7myu/evZvGZ8+enRmbOnUqbRuJ5sNH9Wj22KIa/fz583Md+/Tp0zTO6vBR30aMGEHj0diJ9vb2ovoFAMePH6fxaAxANLaC1cqjrarZuAo2riG8spvZFDN728x2mdknZvbTwu2jzOwNM9tT+MpXGhCRiurN2/hOAD939wYA/wjgJ2Y2G8ATADa4ez2ADYWfRaRKhcnu7q3uvqXw/RkAuwBMAvAIgNWFX1sNYHGpOiki+X2rD+jM7CYA8wBsBDDe3VuBrv8QAIzLaNNkZs1m1hz93SwipdPrZDezGwH8GcDP3J1/KtONu69w90Z3b4wWCBSR0ulVspvZAHQl+h/c/cXCzUfMbGIhPhHA0dJ0UUT6Qlh6s646wEoAu9z9V91CrwBYBuDpwteXo/saPHgw5s6dmxmPlt9lJaqofBUt1/z555/TONuCt76+nraN3tFEZZxoS2hWHhs6dChtG53ztWvX0visWbNonC2ZHPVt48aNNP7CCy/Q+MMPP5wZu/nmm2nbaNpxNA11y5YtNM7+pI1eL6ykyPKgN3X2hQB+BGCHmW0r3PYkupL8T2b2GIC/AvhBL+5LRCokTHZ3fw9AVpX/3r7tjoiUiobLiiRCyS6SCCW7SCKU7CKJULKLJKKsU1yPHTuGZ599NjPOpu5F8Wg552ja4OLFfGg/m7IY1cGjOnr0uKMtm9n9R/f91FNP0Xi0JPLMmTNpnI1viGrV27dvp/FoS2c2TfWWW26hbaNad3Re3n//fRpnr8doGWv2nLIlsnVlF0mEkl0kEUp2kUQo2UUSoWQXSYSSXSQRSnaRRJS1zt7R0YE9e/ZkxqPldxm2hC4Q16rnzJlD42y76GgZ6ygezdtmc8IBoKGhITN2/vx52va2226j8Wie/4EDB2icLbMdPSdNTU00/umnn9I4qzm3tbXRtsOHD6fxaIm1aFwHE72Wi6Uru0gilOwiiVCyiyRCyS6SCCW7SCKU7CKJULKLJKKsdXYzw6BBgzLj0Zx0VoePapPRuvLRGuT33HNPZixagzyq6UY128mTJ9M4q+my8w0A997LFwiO5uqPG9fjrl9/c/bs2cxYNGc8es6iGv+0adMyY9GWzc899xyNf/DBBzQejW9gjz1PnZ3liK7sIolQsoskQskukgglu0gilOwiiVCyiyRCyS6SiN7szz4FwPMAJgC4CmCFu//GzJYD+DcA1xb/ftLdX2X3VVNTE+6TzrC6a1Sjj+qeb731Fo2zevL8+fNp22iv76j9qFGjaJztsR7Vqg8ePEjj0bzsaI3z8ePHZ8Zqa2tp2927d9P4TTfdRONs7YQ1a9bQti0tLTQerQMwYcIEGmdz7VkM4K8ntkZAbwbVdAL4ubtvMbOhADab2RuF2K/d/b96cR8iUmG92Z+9FUBr4fszZrYLwKRSd0xE+ta3+pvdzG4CMA/AtXWUHjez7Wa2ysx63IvHzJrMrNnMmqO32iJSOr1OdjO7EcCfAfzM3U8D+C2A7wCYi64r/y97aufuK9y90d0bo3HWIlI6vUp2MxuArkT/g7u/CADufsTdr7j7VQC/A7CgdN0UkbzCZLeuj2NXAtjl7r/qdvvEbr/2KICdfd89Eekrvfk0fiGAHwHYYWbbCrc9CWCpmc0F4ABaAPw4b2eiMhHbqjZalnjGjBk0Hm3/y6ZLRiWkMWPG0Hi0bDErrQF8Gmu/fv1o26i0xpaC7s39s3MT/VkXTR0+evQoja9cuTIz9u6779K2s2bNovGo79F5Zc95dN9sWXP2WunNp/HvAeip57SmLiLVRSPoRBKhZBdJhJJdJBFKdpFEKNlFEqFkF0lEWZeSdndcvHgxMx7V2dkyufX19bTt7bffTuPRVE1WT/74449p2+hxnT59msajvrHxB2wp596IxghE03dPnDiRGYvGD0SPO1omm9Xpp0+fTttGcTbuAoinJbPXRHROoymwWXRlF0mEkl0kEUp2kUQo2UUSoWQXSYSSXSQRSnaRRFie7WG/9cHMjgHY3+2mMQCOl60D30619q1a+wWob8Xqy75Nc/exPQXKmuzfOLhZs7s3VqwDRLX2rVr7BahvxSpX3/Q2XiQRSnaRRFQ62VdU+PhMtfatWvsFqG/FKkvfKvo3u4iUT6Wv7CJSJkp2kURUJNnN7AEz+8zM9prZE5XoQxYzazGzHWa2zcyaK9yXVWZ21Mx2drttlJm9YWZ7Cl/5gvfl7dtyMztYOHfbzOzBCvVtipm9bWa7zOwTM/tp4faKnjvSr7Kct7L/zW5m/QB8DuA+AAcAbAKw1N0/LWtHMphZC4BGd6/4AAwz+2cAZwE87+5/V7jtPwG0ufvThf8oR7r7v1dJ35YDOFvpbbwLuxVN7L7NOIDFAP4VFTx3pF//gjKct0pc2RcA2Ovu+9z9EoA/AnikAv2oeu7+DoC2625+BMDqwver0fViKbuMvlUFd2919y2F788AuLbNeEXPHelXWVQi2ScB+KrbzwdQXfu9O4DXzWyzmTVVujM9GO/urUDXiwfAuAr353rhNt7ldN0241Vz7orZ/jyvSiR7T1tJVVP9b6G7/wOA7wP4SeHtqvROr7bxLpcethmvCsVuf55XJZL9AIAp3X6eDOBQBfrRI3c/VPh6FMBLqL6tqI9c20G38JXvblhG1bSNd0/bjKMKzl0ltz+vRLJvAlBvZtPNbCCAHwJ4pQL9+AYzG1L44ARmNgTA91B9W1G/AmBZ4ftlAF6uYF++plq28c7aZhwVPncV3/7c3cv+D8CD6PpE/gsA/1GJPmT062YAHxf+fVLpvgFYi663dZfR9Y7oMQCjAWwAsKfwdVQV9W0NgB0AtqMrsSZWqG//hK4/DbcD2Fb492Clzx3pV1nOm4bLiiRCI+hEEqFkF0mEkl0kEUp2kUQo2UUSoWQXSYSSXSQR/wcrk9019yiszwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "croppedR = train_x_c10[ind][2:30,2:30,0]\n",
    "croppedG = train_x_c10[ind][2:30,2:30,1]\n",
    "croppedB = train_x_c10[ind][2:30,2:30,2]\n",
    "plt.imshow(croppedG, cmap='Greys')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]]\n",
      "[[0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]]\n",
      "[[1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "# Pretty much always classifies with 1 in the predicted class and 0 everywhere else.\n",
    "# This is an interesting example in that all three color channels have different predictions.\n",
    "print(np.round(model9.predict(croppedR.reshape(1,28,28,1)), 2))\n",
    "print(np.round(model9.predict(croppedG.reshape(1,28,28,1)), 2))\n",
    "print(np.round(model9.predict(croppedB.reshape(1,28,28,1)), 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's try adding a new non-digit class.\n",
    "train_x_cc = np.asarray([train_x_c10[i][2:30,2:30,1] for i in range(50000)]).reshape(50000, 28, 28, 1).astype('float32')\n",
    "train_x_cc /= 255.0\n",
    "train_y_cc = np.asarray([10 for i in range(50000)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(110000, 28, 28, 1)\n",
      "(110000,)\n"
     ]
    }
   ],
   "source": [
    "train_aug_x = np.concatenate((x_train, train_x_cc))\n",
    "print(train_aug_x.shape)\n",
    "train_aug_y = np.concatenate((y_train, train_y_cc))\n",
    "print(train_aug_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(28,28,1))\n",
    "x = Conv2D(32, 3, activation='relu', input_shape=input_shape)(inputs)\n",
    "x = Conv2D(64, 3, activation='relu')(x)\n",
    "x = MaxPooling2D(2)(x)\n",
    "x = ResNetUnit(x)\n",
    "x = ResNetUnit(x)\n",
    "x = ResNetUnit(x)\n",
    "x = Flatten()(x)\n",
    "x = Dense(256, activation='relu')(x)\n",
    "x = Dropout(0.5)(x)\n",
    "outputs = Dense(11, activation='softmax')(x)\n",
    "model10 = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "110000/110000 [==============================] - 50s 458us/step - loss: 0.2913 - accuracy: 0.9264\n",
      "Epoch 2/10\n",
      "110000/110000 [==============================] - 49s 446us/step - loss: 0.0645 - accuracy: 0.9820\n",
      "Epoch 3/10\n",
      "110000/110000 [==============================] - 49s 445us/step - loss: 0.0410 - accuracy: 0.9885\n",
      "Epoch 4/10\n",
      "110000/110000 [==============================] - 49s 445us/step - loss: 0.0310 - accuracy: 0.9915\n",
      "Epoch 5/10\n",
      "110000/110000 [==============================] - 49s 444us/step - loss: 0.0253 - accuracy: 0.9931\n",
      "Epoch 6/10\n",
      "110000/110000 [==============================] - 49s 444us/step - loss: 0.0227 - accuracy: 0.9937\n",
      "Epoch 7/10\n",
      "110000/110000 [==============================] - 49s 445us/step - loss: 0.0196 - accuracy: 0.9945\n",
      "Epoch 8/10\n",
      "110000/110000 [==============================] - 49s 445us/step - loss: 0.0193 - accuracy: 0.9946\n",
      "Epoch 9/10\n",
      "110000/110000 [==============================] - 49s 444us/step - loss: 0.0289 - accuracy: 0.9925\n",
      "Epoch 10/10\n",
      "110000/110000 [==============================] - 49s 444us/step - loss: 0.0169 - accuracy: 0.9953\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x22100d10e10>"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model10.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
    "model10.fit(train_aug_x, train_aug_y, batch_size=200, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_x_cc = np.asarray([test_x_c10[i][2:30,2:30,1] for i in range(10000)]).reshape(10000, 28, 28, 1).astype('float32')\n",
    "test_x_cc /= 255.0\n",
    "test_y_cc = np.asarray([10 for i in range(10000)])\n",
    "test_aug_x = np.concatenate((x_test, test_x_cc))\n",
    "test_aug_y = np.concatenate((y_test, test_y_cc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20000/20000 [==============================] - 5s 258us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.015903293651354396, 0.9961000084877014]"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 99.61%, not bad!\n",
    "model10.evaluate(test_aug_x, test_aug_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x221282eac50>]"
      ]
     },
     "execution_count": 354,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD4CAYAAAATpHZ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAeOklEQVR4nO3df5BdZZ3n8fcnnQ6BSUiCHUMKgmEwjiLrgtuFUlTNsMXIBmqKONYoZHeYzBQrzq7sqkNtGXUFi7FKZnZHa8ZlZcNIgaIo6w8MVhykkClKRaBBBoEsS4hBMmTzA7gd9N7Yt7u/+8c9HS99z03uzT3n3O5zP6+qrr73uc895zm3O988/TzP+T6KCMzMrBwW9LsBZmaWHQd1M7MScVA3MysRB3UzsxJxUDczK5GF/W6AWRmMjIzE2rVr+90MK6lHH330QESs7KSug7pZBtauXcvY2Fi/m2ElJen5Tut6+MXMrEQc1M3MSsRB3cysRBzUzcxKxEHdzKxEHNRtoEi6RdI+SU+2eV2S/k7SDklPSHp70W0064WDug2aW4H1R3j9YmBd8nUV8IUC2mSWGQd1GygR8QDw8hGqbAC+FA0/AZZLWl1M62xQ3Xj/Du7bvjeTYzmom73WKcALTc93J2UtJF0laUzS2P79+wtpnJXTTf/4HD/ccSCTYzmom72WUspSd5KJiC0RMRoRoytXdnQHt1mLyalpXv31JMuOH87keA7qZq+1G1jT9PxU4MU+tcUGwMFDkwAsd1A3y8VW4E+SVTDvBMYjYk+/G2XlValOALD8hEWZHM8JvWygSLoDuAAYkbQbuA4YBoiIm4BtwCXADqAK/Fl/WmqDolKrA7DshGx66g7qNlAiYuNRXg/ggwU1x4zxaiOoe/jFzKwEKrVsh18c1M3M+qjinrqZWXnMBPUTHdTNzOa/8VqdExcvZGhB2i0S3XNQNzPro0p1IrPxdHBQNzPrq0qtzvKMljOCg7qZWV9VqvXMUgSAg7qZWV8drDmom5mVhodfzMxKYno6GhOlx3ui1Mxs3vvlxCTTgXvqZmZlMJP3xWPqZmYlcDhFgNepm5nNf79J5uWeupnZvJd1Mi9wUDcz65usN8gAB3Uzs74ZT7ayy3KitKedjyStB/4WGAL+PiJuOFL9kZGRWLt27TGfr7EpTX9JnWdSK7K93bSrrHbt2sWBAwf8Qdi8UanWOWHREMctHMrsmMcc1CUNATcC76KxA/sjkrZGxNPt3rN27VoeeuihYz2lg/oRZBHU8/qPoajP4R3veEch5zHLSqVWz3Q8HXobfjkX2BEROyNiAvgasCGbZpmZld94rZ7Z5hgzegnqpwAvND3fnZS9hqSrJI1JGtu/f38PpzMzK5fxarZ5X6C3MfW0v9Vb/s6OiC3AFoDR0dG019MPPkfHiKenp1PL09rb7hp6HY7o5lzdnD+tvN1x08rbHXfBgta+QzdtMCurSm2C3x5Zkukxe+mp7wbWND0/FXixt+aYmQ2OSg499V6C+iPAOkmnS1oEXA5szaZZZmblFhFUavVM16hDD8MvETEp6WrgHhpLGm+JiKcya5mZWYkdqk8zMTmdadpd6HGdekRsA7Zl1BYzs4GRR94X8B2lZmZ9kUfeF+ixp96tiOh4dcNcXQUxNTWVWj401PkdYZ2uVMli5Uk3n2OvdbNY7dPNtZnNZzNBPesxdffUzcz6YHxm+CXjMXUHdTOzPvjNBhnuqZuZzXszaXcd1M16IGm9pGck7ZC0OeX10yTdL+mnkp6QdEk/2mnlV6nWGR4Sxw9nl6ERCp4oldQyEVb0JFivE3HtJkS7uXW/19v8u9Hr59tNGoduzpWWOqBb3V5bh5lF/ytwZ0R8QdKZNJbsru25sWazjNfqLDt+Ueb/9t1Tt0HSSWbRAE5MHi/DqS8sJ+O1icyHXsBB3QZLJ5lFPwX8saTdNHrp/6ndwZyB1HpRqWafSx0c1G2wdJJZdCNwa0ScClwCfFlS6r+TiNgSEaMRMbpy5cqMm2pll0cyL3BQt8HSSWbRK4E7ASLiQWAxMFJI62ygzIypZ81B3QZJJ5lFfwFcCCDpLTSCusdWLHOVaj5j6oWufoHW1SPtNp1oV96rXleDZLFqI02Rq4C6WdGS196nWbSh28+sXWZRSdcDYxGxFbgGuFnSR2gMzfxpOE+BZWxicppfTUzlMqZeeFA366e0zKIRcW3T46eB84tulw2W8ZxuPAIPv5iZFW4m78uyEzymbmY27+WVdhcc1M3MCpdXMi/ow5j67DmnLHKG56XXScK88ojnlVKgSHlNwJrNBzPJvJbNtYlSSbuAV4EpYDIiRrNolJlZmR2eKM1hnXoWPfV/HREHMjiOmdlAGK9OIMHSxdkPlnhM3cysYJVanWXHD7NgQfZDiL0G9QC+L+lRSVelVXDSIzOz18ormRf0HtTPj4i3AxcDH5T0u7MrOOmRmdlrVWr1XNaoQ49j6hHxYvJ9n6Rv08hX/UA3x5iamuqlCXNGN2kNOl21kcXqjrS0BlmkYMhrZVCRG4iY9ct4dYLlOQX1Y+6pS/otSUtnHgMXAU9m1TAzs7Kq1PJJuwu99dRXAd9OelELga9GxD9k0iozsxLLc0z9mIN6ROwE/mWGbTEzK72p6eDgofzG1L2k0cysQK8eqhORT94X6EOagNmTdL3eSt+tH/3oRy1ln//851PrrlmzpqVs6dKlqXU3btzYUnbSSSel1h0ZyX4jnaxyjvfy/ix+PkWfz6xoeeZ9AffUzcwKlWfeF3BQNzMrVJ4bZICDuplZoSrVZIOMHJJ5gYO6mVmh3FM3MyuRmYnSvMbUC139EhEdr37pdWVDu/e///3vbyl79tlnU+um3WK/aFH6n0w33HBDS9mKFStS65533nmp5bMtXNj5j2doaCi1fHJysqWsWq12fIy0FUAAH/3oR1vKTjvttNS6aT9jr1yxQVWp1lly3EKGh/LpU7unbmZWoEptIrdeOjiom5kVaryaX94XcFA3MytUnsm8wEHdzKxQlepELnuTzig8TUCn8ppcu+uuu1rKHn744dS6Z511VkvZ9u3bU+uOjY21lN19992pdb///e+3lJ1xxhktZT//+c9T35/2OQwPp//PnzZRunz58tS6e/fubSlLmywGeOMb39hS9pGPfCS1bppuJsh7TXVgNpeM1+osc0/dzGz+i4hc0+6Cg7qZWWF+NTHF5HR49YuZWRnkfTcpOKibmRUm77wv4KBuA0bSeknPSNohaXObOu+T9LSkpyR9teg2WnmN55xLHTpY/SLpFuAPgH0RcVZSdhLwdWAtsAt4X0S80skJZ69uyCtNQDtvfvObW8rWrVvX8fvPPvvs1PLLLruspez6669Prfv888+3lKWtJtm5c2fH7Wq3+iXt8z355JNT6771rW9tKatUKql13/SmN3XctrxWtHR7DElDwI3Au4DdwCOStkbE00111gEfA86PiFckvb7nhpolKnNk+OVWYP2sss3AfRGxDrgveW42150L7IiInRExAXwN2DCrzvuBG2c6KRGxr+A2Wokd3vWon8MvEfEA8PKs4g3Abcnj24B3Z9wuszycArzQ9Hx3UtbsTcCbJP1I0k8kze7QHCbpKkljksb279+fQ3OtbCq1xph6v3vqaVZFxB6A5HvbP1Gbf/EPHDhwjKczy0TamN7sMZyFwDrgAmAj8PeSUu/WiogtETEaEaMrV67MtKFWTuPVOsctXMDi4fSsqlnIfaK0+Rc/jw2XzbqwG2jOJXwq8GJKne9ERD0ifg48QyPIm/WsknMyLzj2NAF7Ja2OiD2SVgMdjzvOntxqN9nVLj94J8c80nHTJu26mZTtZnLuuOOOSy1Pm6xNux0/rV477W7nn5qaail78MEHU+u+/PLsUbb2ud8vvPDCjtuWpt1n3s3newyT6Y8A6ySdDvwzcDnwb2fVuYtGD/1WSSM0hmM6n7E2O4JKLd+8L3DsPfWtwKbk8SbgO9k0xyw/ETEJXA3cA2wH7oyIpyRdL+nSpNo9wEuSngbuB/5LRLzUnxZb2VSq+eZ9gc6WNN5BY3xxRNJu4DrgBuBOSVcCvwDem2cjzbISEduAbbPKrm16HMBfJF9mmRqv1TntpBNyPcdRg3pEbGzzUm9/f5uZDZhKtc6/OCXfnrrvKDUzK0ilNpH7RKmDuplZAQ7VpzhUn2b5CflOlBa+ScZc3PCgm5UY09PTHR+j3YqUXm+bb3fcNNVqtaVs48b0EbW0a/v0pz+dWrfdyp40vf7M5+LvjFm3DiYpAvJMuwvuqZuZFaKIvC/goG5mVogi8r6Ag7qZWSFmcqm7p25mVgKVgsbUC58ona3dJGXapF271AHdTKR1k1Kgm3Ollbdrb1654tN8+ctfbilrl1Hwda97XUvZG97whtS63VxDrxPD7c5V5Odo1qsiNsgA99TNzApRqU0wtEAsOS7fvrSDuplZASrVOsuPH879L0wHdTOzAlRq+SfzAgd1M7NCjFfruU+SwhyYKC36bsG083WTi7ybu0SLnOBrt0n1xz72sZaydhO4P/jBD1rKVq1alVo37XPM689KT4haGVRqE6xc0vmd2MfKPXUzswKM1+q5530BB3Uzs0JUChp+cVA3M8vZ5NQ0rx6azH2NOjiom5nl7uChSQCWu6duZjb//SbvS/5j6p3sUXoL8AfAvog4Kyn7FPB+YOZ+848nez9mppvb+dNWR7TLe97pudodt5tc5kX67ne/m1qedm2XXXZZat0zzjij4/N1s9qnG9383GeXO++6zVWH877MkeGXW4H1KeWfi4izk69MA7qZWZkczvsyF4ZfIuIB4OXcW2JmVlKVWnHDL72MJVwt6QlJt0ha0a6SpKskjUkaO3DgQA+nMzObnypzqafexheAM4CzgT3A37SrGBFbImI0IkZHRkaO8XRmZvPXTFA/ca6mCYiIvTOPJd0MpM/SdaCbTZ/bTYT1OnmZxURpWkqBXq+t3bkmJydbyu6+++7UusPDrb9En/jEJ1LrpqUP6CbVQTfX2w1PgNp8N16rs3TxQoYW5J/y4piioaTVTU//EHgym+aYmZVPpTpRyI1H0NmSxjuAC4ARSbuB64ALJJ0NBLAL+ECObTQzm9fGa/XcN5yecdSgHhEbU4q/mENbzMxKqVKrF9ZTn5t30piZlUhRudTBQd3MLHdF9tT7vklGNzq9XRy6u2V9Lqyu6Ga1z+23395S9uCDD6bWfe9739tSdvrpp6fW7fXW/3btzSJlg9l8NT0djYnSgsbU3VO3gSJpvaRnJO2QtPkI9f5IUkgaLbJ9Vj6/nJhkOvCYulnWJA0BNwIXA2cCGyWdmVJvKfCfgYeKbaGV0UzeF4+pm2XvXGBHROyMiAnga8CGlHp/Cfw1cKjIxlk5HU4RUEDeF3BQt8FyCvBC0/PdSdlhks4B1kTEUe+Sbs5rtH///qNVtwH1m2ReAz5RmtcO8mmTdmm3x7fTTUqBbiYO0+r+9Kc/TX3/Nddc01LWLq/Odddd11LWzfW20831dlO3lzZ08DuTVuFwQyQtAD4H/Gkn54+ILcAWgNHRUc/wWqqKh1/McrMbWNP0/FTgxabnS4GzgH+UtAt4J7DVk6XWi5kNMorI0AgO6jZYHgHWSTpd0iLgcmDrzIsRMR4RIxGxNiLWAj8BLo2Isf4018pgPNnKrogMjeCgbgMkIiaBq4F7gO3AnRHxlKTrJV3a39ZZWVWqdY4fHmLxcO/Dnp2Ys2PqZnlItl7cNqvs2jZ1LyiiTVZu4wXeTQruqZuZ5apSKy7vC5Skp5626qHdrelpm1m0q9vrKpFubo8/dKh1SfSmTZtS6/76179uKXvPe96TWrddSoBOdZOaIQt5rXoy65fxqnvqZmalUakVl/cFHNTNzHJVcU/dzKwcIqIxpu6gbmY2/x2qTzMxOV3o8Esne5SuAb4EnAxMA1si4m8lnQR8HVhLY5/S90XEK0c73uwJtiwmxtImJNMmRDtt05GO0W7yNO06FixI/z8z7Xxpec937dqV+v63ve1tLWWf/OQnOz5XFpz33Ozois77Ap311CeBayLiLTRum/5gkq50M3BfRKwD7kuem5lZoui8L9BBUI+IPRHxWPL4VRp34p1CI2XpbUm124B359VIM7P56HDa3bkU1JtJWgucQ2PzgFURsQcagR94fZv3OD2pmQ2k8WT4ZU5OlEpaAnwT+HBEHOz0fRGxJSJGI2J05cqVx9JGM7N5qegNMqDDoC5pmEZA/0pEfCsp3itpdfL6amBfPk00M5ufxgtOuwudrX4R8EVge0R8tumlrcAm4Ibk+3c6OWGnq126WV2RVreb97drU9pKlyw2mHjppZdayh5++OGWsoUL0388N910U0vZihUrem5Xr5+ZV8SYvValVmd4SJywqJgMjdBZ7pfzgSuAn0l6PCn7OI1gfqekK4FfAK1r8szMBlilWmfZ8YsKzWl01KAeET8kfRswgAuzbY6ZWXmM1yYKXaMOvqPUzCw3lWq90PF0cFA3M8tN0cm8oOB86pJaJhq7yTneblyqm2OkaXc7f7vyTh08mL7y86KLLmopS0tJcPPNN6e+Py1NQBa6GffzpKjZ0Y3X6rxl9YmFntM9dTOznFSqHlM3MyuFiclpfjUxVWjeF3BQNzPLxeEbj9xTNzOb/w7nfXFP3cxs/utH3hcoePULtK6waLfCpJsVLd2sxEg7Xxa3/qe14fbbb0+t+9xzz7WUpa08Oe+881Lfn9fdaV7RYpadfuR9AffUzcxy8ZueuoO6mdm8VzncUy92+MVB3cwsB+PVCSRYurjYUW4HdTOzHFRqdZYdP8yCBcVlaIQ+TJTO1m7SL21Cs9d0AO2O2043OcN37tzZUvaZz3ym84YVqCwTorOvoyzXZeXQj2Re4J66mVkuKrU6ywpezggO6jZgJK2X9IykHZI2p7z+F5KelvSEpPskvaEf7bT5b7w64Z66WZ4kDQE3AhcDZwIbJZ05q9pPgdGIeBvwDeCvi22llcXMmHrRHNRtkJwL7IiInRExAXwN2NBcISLuj4hq8vQnwKkFt9FKoh+51KGDoC5pjaT7JW2X9JSkDyXln5L0z5IeT74uyb+5Zj05BXih6fnupKydK4HvtXtR0lWSxiSN7d+/P6MmWhlMTQcHD/VnorST1S+TwDUR8ZikpcCjku5NXvtcRPz3bk44e4VCu9UvaeXtVq6kbTDRzXHbSVtN0e79P/7xj1vK2m2SkXaMM8+cPQoAixcvPloTD5tvKz/afY5p15HhtaWdNPXgkv4YGAV+r93BImILsAVgdHR0fv0ALFevHqoTQV8mSjvZeHoPsCd5/Kqk7Ry5d2M2V+0G1jQ9PxV4cXYlSb8PfAL4vYj4dUFtsxLpV94X6HJMXdJa4BzgoaTo6mSVwC2SVmTcNrOsPQKsk3S6pEXA5cDW5gqSzgH+F3BpROzrQxutBPqV9wW6COqSlgDfBD4cEQeBLwBnAGfT6Mn/TZv3edzR5oSImASuBu4BtgN3RsRTkq6XdGlS7b8BS4D/ncwVbW1zOLO2Kn3aIAM6vKNU0jCNgP6ViPgWQETsbXr9ZuC7ae/1uKPNJRGxDdg2q+zapse/X3ijrHQq1ZkNMubgmLoaM1pfBLZHxGebylcn4+0Afwg8mU8TX9OW1PKFC1svI23y9EjH6FS7SbtuJvjS8qTfddddLWXdTJR2M/E4F3TzOXb6M8srz7xZt/q1lR101lM/H7gC+Jmkx5Oyj9O4ceNsGqsHdgEfyKWFZmbzzMyYej9uPupk9csPSV8Kti2lzMxs4FWqdZYct5DhoeLv7/QdpWZmGavUJvrSSwcHdTOzzI1X+5P3BRzUzcwyV6n1J+8L9GGTjNkrFNqtguhmg4q09AHdrITIYtXEFVdc0VFZN+bCypVe0yocqbzX43q1i81VleoEv3Py0r6c2z11M7OMjdfqfVmjDg7qZmaZioi+pd0FB3Uzs0xVJ6aYnI6+JPMCB3Uzs0z1M+8LFDxR+uijjx4YGhp6Pnk6Ahwo8vwF8XX1j/cTtb7rZ94XKDioR8TKmceSxiJitMjzF8HXZTbYxvuYdhc8/GJmlql+D784qJuZZejwBhkDuKRxSx/PnSdfl9kAq9QaY+oD11NPNs8oHV+X2WAbr9ZZtHABi4eH+nJ+D7+YmWWoUq33bY06OKibmWWqUpvo29AL9CGoS1ov6RlJOyRtLvr8WZJ0i6R9kp5sKjtJ0r2Snk2+r+hnG4+FpDWS7pe0XdJTkj6UlM/7azPLW6On3p9JUig4qEsaAm4ELgbOpLEl3plFtiFjtwLrZ5VtBu6LiHXAfcnz+WYSuCYi3gK8E/hg8nMqw7WZ5Wq8VmfZAPXUzwV2RMTOiJgAvgZsKLgNmYmIB4CXZxVvAG5LHt8GvLvQRmUgIvZExGPJ41eB7cAplODazPI2XhusMfVTgBeanu9OyspkVUTsgUZwBF7f5/b0RNJa4BzgIUp2bWZ56GeGRig+qKftatD/nSAslaQlwDeBD0fEwX63x2yuO1SfolafYvkJAzKmTqNnvqbp+anAiwW3IW97Ja0GSL7v63N7jomkYRoB/SsR8a2kuBTXZpaXg0mKgH7tTwrFB/VHgHWSTpe0CLgc2FpwG/K2FdiUPN4EfKePbTkmauwT90Vge0R8tumleX9tZnnqd94XKD5L46Skq4F7gCHgloh4qsg2ZEnSHcAFwIik3cB1wA3AnZKuBH4BvLd/LTxm5wNXAD+T9HhS9nHKcW1muel33hfow8bTEbEN2Fb0efMQERvbvHRhoQ3JWET8kPT5D5jn12aWp5lc6oM0UWpmVlqVARxTNzMrrZkNMgbp5iMzs9Kq1CYYWiCWHlf4yPZhDuo2UI6We0jScZK+nrz+UHLzlVlHKtU6y44fprGArD8c1G1gdJh76ErglYh4I/A54K+KbaXNZ5U+pwiAPqx+Meujw7mHACTN5B56uqnOBuBTyeNvAP9DkiKi6zufH33+FTZ/84neWmzzyouVGutWLe1rGxzUbZCk5R56R7s6yX0V48DrgAOzDybpKuAqgNNOO63lZCcsGmLdqiWZNNzmh3WrlnDxWav72gYHdRskneQe6jg/UbLF3xaA0dHRljpvWX0i//Pf/atu22jWE4+p2yDpJPfQ4TqSFgLLaE2vbDZnOajbIOkk91Bzfps/An5wLOPpZv3i4RcbGO1yD0m6HhiLiK00Epl9WdIOGj30y/vXYrPuOajbQEnLPRQR1zY9PoQTldk85uEXM7MScVA3MysRB3UzsxJxUDczKxF5tZZZ7yTtB55PeWmElLtR+8RtaTVX2gFHbssbImJlJwdxUDfLkaSxiBjtdzvAbZnL7YDs2uLhFzOzEnFQNzMrEQd1s3xt6XcDmrgtreZKOyCjtnhM3cysRNxTNzMrEQd1M7MScVA3y8Bc2dBa0hpJ90vaLukpSR9KqXOBpHFJjydf16YdK6P27JL0s+Q8YymvS9LfJZ/LE5LenkMbfqfpWh+XdFDSh2fVye0zkXSLpH2SnmwqO0nSvZKeTb6vaPPeTUmdZyVtSqvTIiL85S9/9fBFI43vc8BvA4uAfwLOnFXnPwI3JY8vB76eU1tWA29PHi8F/m9KWy4AvlvQZ7MLGDnC65cA36Ox49Q7gYcK+Fn9Pxo38xTymQC/C7wdeLKp7K+BzcnjzcBfpbzvJGBn8n1F8njF0c7nnrpZ7w5vaB0RE8DMhtbNNgC3JY+/AVwoKW3rvJ5ExJ6IeCx5/Cqwnca+q3PVBuBL0fATYLmkPDf5vBB4LiLS7v7NRUQ8QOvuWc2/D7cB7055678B7o2IlyPiFeBeYP3Rzuegbta7tA2tZwfS12xoDcxsaJ2bZIjnHOChlJfPk/RPkr4n6a05NiOA70t6NNmoe7ZOPrssXQ7c0ea1oj4TgFURsQca/xEDr0+pc0yfjTfJMOtdphtaZ0HSEuCbwIcj4uCslx+jMfzwS0mXAHcB63JqyvkR8aKk1wP3Svo/Sc/1cFNT3pPL55JsYXgp8LGUl4v8TDp1TJ+Ne+pmvZtTG1pLGqYR0L8SEd+a/XpEHIyIXyaPtwHDkkbyaEtEvJh83wd8m8ZQVbNOPrusXAw8FhF7U9pZ2GeS2DszzJR835dS55g+Gwd1s97NmQ2tk3H6LwLbI+KzbeqcPDOeL+lcGnHgpRza8luSls48Bi4CnpxVbSvwJ8kqmHcC4zPDEjnYSJuhl6I+kybNvw+bgO+k1LkHuEjSimR1zEVJ2RF5+MWsRzG3NrQ+H7gC+Jmkx5OyjwOnJW29icZ/Kv9B0iRQAy7P4z8YYBXw7SRWLgS+GhH/IOnPm9qyjcYKmB1AFfizHNqBpBOAdwEfaCprbkdun4mkO2isrhmRtBu4DrgBuFPSlcAvSPbFlTQK/HlE/PuIeFnSX9LoNABcHxFH/evOaQLMzErEwy9mZiXioG5mViIO6mZmJeKgbmZWIg7qZmYl4qBuZlYiDupmZiXy/wHGPufKPRPiYgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Still doesn't do well on noisy digits.\n",
    "testimg = 11*test_aug_x[0] + 1*test_aug_x[10000]\n",
    "testimg = testimg/np.max(testimg)\n",
    "pred = np.round(model10.predict(testimg.reshape(1,28,28,1)), 2).reshape(11)\n",
    "f, (ax1, ax2) = plt.subplots(1,2)\n",
    "ax1.imshow(testimg.reshape(28,28), cmap='Greys')\n",
    "ax2.plot(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 2s 248us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.03172433795391189, 0.9922000169754028]"
      ]
     },
     "execution_count": 307,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How does it do on just the digits? 99.22%\n",
    "model10.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cross-validation\n",
    "x_combined = np.concatenate((x_train, x_test))\n",
    "y_combined = np.concatenate((y_train, y_test))\n",
    "num_folds = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I feel like there should be a way to do this in the library to begin with.\n",
    "# Turns out this is broken anyway.\n",
    "def reset_weights(model):\n",
    "    for layer in model.layers: \n",
    "        if isinstance(layer, tf.keras.Model):\n",
    "            reset_weights(layer)\n",
    "        continue\n",
    "    for k, initializer in layer.__dict__.items():\n",
    "        if \"initializer\" not in k:\n",
    "            continue\n",
    "        var = getattr(layer, k.replace(\"_initializer\", \"\"))\n",
    "        var.assign(initializer(var.shape, var.dtype))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "metadata": {},
   "outputs": [],
   "source": [
    "def crossvalidate(model, x_data, y_data, folds, batch_size=200, epochs=20, seed=0):\n",
    "    # Assume model hasn't been trained yet, because weight resetting doesn't work.\n",
    "    np.random.seed(seed)\n",
    "    model.save_weights('initial.h5')\n",
    "    size = len(x_data)\n",
    "    losses = []\n",
    "    accuracies = []\n",
    "    avgloss = 0\n",
    "    avgacc = 0\n",
    "    for i in range(folds):\n",
    "        # reset_weights(model)\n",
    "        model.load_weights('initial.h5')\n",
    "        model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
    "        start_index = int(size*i/folds)\n",
    "        end_index = int(size*(i+1)/folds)\n",
    "        print('Fold {}: {} - {}'.format(i, start_index, end_index))\n",
    "        xtest = x_data[start_index:end_index]\n",
    "        ytest = y_data[start_index:end_index]\n",
    "        xtrain = np.concatenate((x_data[0:start_index], x_data[end_index:size]))\n",
    "        ytrain = np.concatenate((y_data[0:start_index], y_data[end_index:size]))\n",
    "        model.fit(xtrain, ytrain, batch_size=batch_size, epochs=epochs)\n",
    "        scores = model.evaluate(xtest, ytest)\n",
    "        losses.append(scores[0])\n",
    "        accuracies.append(scores[1])\n",
    "        print(scores[1])\n",
    "        avgloss += scores[0]\n",
    "        avgacc += scores[1]\n",
    "    avgloss /= folds\n",
    "    avgacc /= folds\n",
    "    sstdev = 0\n",
    "    for i in range(folds):\n",
    "        sstdev += (accuracies[i] - avgacc)**2\n",
    "    sstdev = (sstdev/(folds-1))**0.5\n",
    "    print('Average accuracy with {}-fold cross-validation: {:.5f} \\u00b1 {:.5f}'.format(folds, avgacc, sstdev))\n",
    "    return (avgacc, losses, accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0: 0 - 7000\n",
      "Epoch 1/5\n",
      "63000/63000 [==============================] - 28s 450us/step - loss: 0.0590 - accuracy: 0.9821\n",
      "Epoch 2/5\n",
      "63000/63000 [==============================] - 27s 429us/step - loss: 0.0327 - accuracy: 0.9901\n",
      "Epoch 3/5\n",
      "63000/63000 [==============================] - 27s 429us/step - loss: 0.0253 - accuracy: 0.9922\n",
      "Epoch 4/5\n",
      "63000/63000 [==============================] - 27s 429us/step - loss: 0.0200 - accuracy: 0.9939\n",
      "Epoch 5/5\n",
      "63000/63000 [==============================] - 27s 429us/step - loss: 0.0166 - accuracy: 0.9946\n",
      "7000/7000 [==============================] - 2s 238us/step\n",
      "0.9932857155799866\n",
      "Fold 1: 7000 - 14000\n",
      "Epoch 1/5\n",
      "63000/63000 [==============================] - 28s 449us/step - loss: 0.0569 - accuracy: 0.9823\n",
      "Epoch 2/5\n",
      "63000/63000 [==============================] - 27s 428us/step - loss: 0.0342 - accuracy: 0.9899\n",
      "Epoch 3/5\n",
      "63000/63000 [==============================] - 27s 428us/step - loss: 0.0243 - accuracy: 0.9925\n",
      "Epoch 4/5\n",
      "63000/63000 [==============================] - 27s 428us/step - loss: 0.0205 - accuracy: 0.9934\n",
      "Epoch 5/5\n",
      "63000/63000 [==============================] - 27s 427us/step - loss: 0.0191 - accuracy: 0.9936\n",
      "7000/7000 [==============================] - 2s 247us/step\n",
      "0.9918571710586548\n",
      "Fold 2: 14000 - 21000\n",
      "Epoch 1/5\n",
      "63000/63000 [==============================] - 28s 445us/step - loss: 0.0596 - accuracy: 0.9819\n",
      "Epoch 2/5\n",
      "63000/63000 [==============================] - 27s 426us/step - loss: 0.0347 - accuracy: 0.9892\n",
      "Epoch 3/5\n",
      "63000/63000 [==============================] - 27s 426us/step - loss: 0.0242 - accuracy: 0.9925\n",
      "Epoch 4/5\n",
      "63000/63000 [==============================] - 27s 426us/step - loss: 0.0203 - accuracy: 0.9934\n",
      "Epoch 5/5\n",
      "63000/63000 [==============================] - 27s 426us/step - loss: 0.0164 - accuracy: 0.9947\n",
      "7000/7000 [==============================] - 2s 245us/step\n",
      "0.9917142987251282\n",
      "Fold 3: 21000 - 28000\n",
      "Epoch 1/5\n",
      "63000/63000 [==============================] - 28s 445us/step - loss: 0.0599 - accuracy: 0.9822\n",
      "Epoch 2/5\n",
      "63000/63000 [==============================] - 27s 427us/step - loss: 0.0341 - accuracy: 0.9897\n",
      "Epoch 3/5\n",
      "63000/63000 [==============================] - 27s 427us/step - loss: 0.0252 - accuracy: 0.9922\n",
      "Epoch 4/5\n",
      "63000/63000 [==============================] - 27s 429us/step - loss: 0.0189 - accuracy: 0.9939\n",
      "Epoch 5/5\n",
      "63000/63000 [==============================] - 27s 432us/step - loss: 0.0173 - accuracy: 0.9946\n",
      "7000/7000 [==============================] - 2s 246us/step\n",
      "0.9912857413291931\n",
      "Fold 4: 28000 - 35000\n",
      "Epoch 1/5\n",
      "63000/63000 [==============================] - 29s 467us/step - loss: 0.0617 - accuracy: 0.9822\n",
      "Epoch 2/5\n",
      "63000/63000 [==============================] - 29s 454us/step - loss: 0.0350 - accuracy: 0.9897\n",
      "Epoch 3/5\n",
      "63000/63000 [==============================] - 29s 454us/step - loss: 0.0248 - accuracy: 0.9924\n",
      "Epoch 4/5\n",
      "63000/63000 [==============================] - 29s 455us/step - loss: 0.0208 - accuracy: 0.9934\n",
      "Epoch 5/5\n",
      "63000/63000 [==============================] - 30s 471us/step - loss: 0.0154 - accuracy: 0.9950\n",
      "7000/7000 [==============================] - 2s 267us/step\n",
      "0.9901428818702698\n",
      "Fold 5: 35000 - 42000\n",
      "Epoch 1/5\n",
      "63000/63000 [==============================] - 30s 473us/step - loss: 0.0588 - accuracy: 0.9817\n",
      "Epoch 2/5\n",
      "63000/63000 [==============================] - 28s 452us/step - loss: 0.0322 - accuracy: 0.9897\n",
      "Epoch 3/5\n",
      "63000/63000 [==============================] - 29s 460us/step - loss: 0.0247 - accuracy: 0.9919\n",
      "Epoch 4/5\n",
      "63000/63000 [==============================] - 30s 483us/step - loss: 0.0181 - accuracy: 0.9945\n",
      "Epoch 5/5\n",
      "63000/63000 [==============================] - 29s 468us/step - loss: 0.0164 - accuracy: 0.9948\n",
      "7000/7000 [==============================] - 2s 292us/step\n",
      "0.9902856945991516\n",
      "Fold 6: 42000 - 49000\n",
      "Epoch 1/5\n",
      "63000/63000 [==============================] - 31s 493us/step - loss: 0.0597 - accuracy: 0.9815\n",
      "Epoch 2/5\n",
      "63000/63000 [==============================] - 30s 470us/step - loss: 0.0334 - accuracy: 0.9900\n",
      "Epoch 3/5\n",
      "63000/63000 [==============================] - 30s 469us/step - loss: 0.0240 - accuracy: 0.9927\n",
      "Epoch 4/5\n",
      "63000/63000 [==============================] - 30s 471us/step - loss: 0.0197 - accuracy: 0.9940\n",
      "Epoch 5/5\n",
      "63000/63000 [==============================] - 30s 473us/step - loss: 0.0160 - accuracy: 0.9951\n",
      "7000/7000 [==============================] - 2s 306us/step\n",
      "0.9892857074737549\n",
      "Fold 7: 49000 - 56000\n",
      "Epoch 1/5\n",
      "63000/63000 [==============================] - 31s 500us/step - loss: 0.0594 - accuracy: 0.9816\n",
      "Epoch 2/5\n",
      "63000/63000 [==============================] - 29s 468us/step - loss: 0.0330 - accuracy: 0.9896\n",
      "Epoch 3/5\n",
      "63000/63000 [==============================] - 30s 472us/step - loss: 0.0240 - accuracy: 0.9925\n",
      "Epoch 4/5\n",
      "63000/63000 [==============================] - 30s 476us/step - loss: 0.0199 - accuracy: 0.9937\n",
      "Epoch 5/5\n",
      "63000/63000 [==============================] - 30s 474us/step - loss: 0.0158 - accuracy: 0.9948\n",
      "7000/7000 [==============================] - ETA:  - 2s 297us/step\n",
      "0.9911428689956665\n",
      "Fold 8: 56000 - 63000\n",
      "Epoch 1/5\n",
      "63000/63000 [==============================] - 31s 499us/step - loss: 0.0596 - accuracy: 0.9815\n",
      "Epoch 2/5\n",
      "63000/63000 [==============================] - 30s 469us/step - loss: 0.0329 - accuracy: 0.9897\n",
      "Epoch 3/5\n",
      "63000/63000 [==============================] - 30s 472us/step - loss: 0.0245 - accuracy: 0.9920\n",
      "Epoch 4/5\n",
      "63000/63000 [==============================] - 30s 472us/step - loss: 0.0204 - accuracy: 0.9937\n",
      "Epoch 5/5\n",
      "63000/63000 [==============================] - 30s 471us/step - loss: 0.0176 - accuracy: 0.9944s -\n",
      "7000/7000 [==============================] - 2s 296us/step\n",
      "0.9884285926818848\n",
      "Fold 9: 63000 - 70000\n",
      "Epoch 1/5\n",
      "63000/63000 [==============================] - 31s 498us/step - loss: 0.0589 - accuracy: 0.9815\n",
      "Epoch 2/5\n",
      "63000/63000 [==============================] - 30s 472us/step - loss: 0.0350 - accuracy: 0.9889\n",
      "Epoch 3/5\n",
      "63000/63000 [==============================] - 31s 496us/step - loss: 0.0260 - accuracy: 0.9919\n",
      "Epoch 4/5\n",
      "63000/63000 [==============================] - 30s 479us/step - loss: 0.0195 - accuracy: 0.9940\n",
      "Epoch 5/5\n",
      "63000/63000 [==============================] - 30s 474us/step - loss: 0.0183 - accuracy: 0.9943\n",
      "7000/7000 [==============================] - 2s 297us/step\n",
      "0.9944285750389099\n",
      "Average accuracy with 10-fold cross-validation: 0.99119 ± 0.00179\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.99118572473526,\n",
       " [0.022284390567602324,\n",
       "  0.028721844047723738,\n",
       "  0.026113871110586582,\n",
       "  0.032248963425518015,\n",
       "  0.03622376112113541,\n",
       "  0.032728482961521616,\n",
       "  0.029867032854069425,\n",
       "  0.030832724159566817,\n",
       "  0.04267810324100355,\n",
       "  0.0172787805354206],\n",
       " [0.9932857155799866,\n",
       "  0.9918571710586548,\n",
       "  0.9917142987251282,\n",
       "  0.9912857413291931,\n",
       "  0.9901428818702698,\n",
       "  0.9902856945991516,\n",
       "  0.9892857074737549,\n",
       "  0.9911428689956665,\n",
       "  0.9884285926818848,\n",
       "  0.9944285750389099])"
      ]
     },
     "execution_count": 403,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "crossvalidate(model5, x_combined, y_combined, num_folds, batch_size=200, epochs=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0: 0 - 7000\n",
      "Epoch 1/25\n",
      "63000/63000 [==============================] - 71s 1ms/step - loss: 0.3300 - accuracy: 0.9169\n",
      "Epoch 2/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0591 - accuracy: 0.9829\n",
      "Epoch 3/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0445 - accuracy: 0.9867\n",
      "Epoch 4/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0376 - accuracy: 0.9888\n",
      "Epoch 5/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0340 - accuracy: 0.9901\n",
      "Epoch 6/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0290 - accuracy: 0.9915\n",
      "Epoch 7/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0277 - accuracy: 0.9917\n",
      "Epoch 8/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0255 - accuracy: 0.9925\n",
      "Epoch 9/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0225 - accuracy: 0.9932\n",
      "Epoch 10/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0226 - accuracy: 0.9930\n",
      "Epoch 11/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0232 - accuracy: 0.9929\n",
      "Epoch 12/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0204 - accuracy: 0.9939\n",
      "Epoch 13/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0191 - accuracy: 0.9943\n",
      "Epoch 14/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0196 - accuracy: 0.9944\n",
      "Epoch 15/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0183 - accuracy: 0.9944\n",
      "Epoch 16/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0165 - accuracy: 0.9950\n",
      "Epoch 17/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0150 - accuracy: 0.9953\n",
      "Epoch 18/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0137 - accuracy: 0.9958\n",
      "Epoch 19/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0140 - accuracy: 0.9958\n",
      "Epoch 20/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0170 - accuracy: 0.9949\n",
      "Epoch 21/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0145 - accuracy: 0.9956\n",
      "Epoch 22/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0137 - accuracy: 0.9961\n",
      "Epoch 23/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0138 - accuracy: 0.9955\n",
      "Epoch 24/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0111 - accuracy: 0.9966\n",
      "Epoch 25/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0092 - accuracy: 0.9973\n",
      "7000/7000 [==============================] - 4s 624us/step\n",
      "0.9937142729759216\n",
      "Fold 1: 7000 - 14000\n",
      "Epoch 1/25\n",
      "63000/63000 [==============================] - 72s 1ms/step - loss: 0.2822 - accuracy: 0.9288\n",
      "Epoch 2/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0605 - accuracy: 0.9821\n",
      "Epoch 3/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0464 - accuracy: 0.9858\n",
      "Epoch 4/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0364 - accuracy: 0.9898\n",
      "Epoch 5/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0297 - accuracy: 0.9913\n",
      "Epoch 6/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0290 - accuracy: 0.9911\n",
      "Epoch 7/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0277 - accuracy: 0.9916\n",
      "Epoch 8/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0240 - accuracy: 0.9930\n",
      "Epoch 9/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0226 - accuracy: 0.9934\n",
      "Epoch 10/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0205 - accuracy: 0.9936\n",
      "Epoch 11/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0186 - accuracy: 0.9943\n",
      "Epoch 12/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0212 - accuracy: 0.9937\n",
      "Epoch 13/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0169 - accuracy: 0.9946\n",
      "Epoch 14/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0168 - accuracy: 0.9950\n",
      "Epoch 15/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0163 - accuracy: 0.9953\n",
      "Epoch 16/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0151 - accuracy: 0.9953\n",
      "Epoch 17/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0153 - accuracy: 0.9953\n",
      "Epoch 18/25\n",
      "63000/63000 [==============================] - 67s 1ms/step - loss: 0.0149 - accuracy: 0.9953\n",
      "Epoch 19/25\n",
      "63000/63000 [==============================] - 67s 1ms/step - loss: 0.0132 - accuracy: 0.9958\n",
      "Epoch 20/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0129 - accuracy: 0.9960\n",
      "Epoch 21/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0115 - accuracy: 0.9964\n",
      "Epoch 22/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0129 - accuracy: 0.9958\n",
      "Epoch 23/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0139 - accuracy: 0.9963\n",
      "Epoch 24/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0118 - accuracy: 0.9964\n",
      "Epoch 25/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0091 - accuracy: 0.9971\n",
      "7000/7000 [==============================] - 4s 612us/step\n",
      "0.9928571581840515\n",
      "Fold 2: 14000 - 21000\n",
      "Epoch 1/25\n",
      "63000/63000 [==============================] - 70s 1ms/step - loss: 0.3166 - accuracy: 0.9230\n",
      "Epoch 2/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0619 - accuracy: 0.9819\n",
      "Epoch 3/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0437 - accuracy: 0.9873\n",
      "Epoch 4/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0355 - accuracy: 0.9893\n",
      "Epoch 5/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0325 - accuracy: 0.9901\n",
      "Epoch 6/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0280 - accuracy: 0.9917\n",
      "Epoch 7/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0271 - accuracy: 0.9920\n",
      "Epoch 8/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0249 - accuracy: 0.9927\n",
      "Epoch 9/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0205 - accuracy: 0.9940\n",
      "Epoch 10/25\n",
      "63000/63000 [==============================] - 61s 975us/step - loss: 0.0246 - accuracy: 0.9930s - loss: 0.0246 - ac\n",
      "Epoch 11/25\n",
      "63000/63000 [==============================] - 60s 951us/step - loss: 0.0201 - accuracy: 0.9937\n",
      "Epoch 12/25\n",
      "63000/63000 [==============================] - 60s 951us/step - loss: 0.0221 - accuracy: 0.9931\n",
      "Epoch 13/25\n",
      "63000/63000 [==============================] - 60s 954us/step - loss: 0.0194 - accuracy: 0.9941\n",
      "Epoch 14/25\n",
      "63000/63000 [==============================] - 60s 955us/step - loss: 0.0165 - accuracy: 0.9953\n",
      "Epoch 15/25\n",
      "63000/63000 [==============================] - 60s 960us/step - loss: 0.0189 - accuracy: 0.9946\n",
      "Epoch 16/25\n",
      "63000/63000 [==============================] - 61s 960us/step - loss: 0.0169 - accuracy: 0.9947\n",
      "Epoch 17/25\n",
      "63000/63000 [==============================] - 61s 971us/step - loss: 0.0163 - accuracy: 0.9951\n",
      "Epoch 18/25\n",
      "63000/63000 [==============================] - 61s 973us/step - loss: 0.0126 - accuracy: 0.9960\n",
      "Epoch 19/25\n",
      "63000/63000 [==============================] - 61s 971us/step - loss: 0.0149 - accuracy: 0.9954\n",
      "Epoch 20/25\n",
      "63000/63000 [==============================] - 62s 976us/step - loss: 0.0147 - accuracy: 0.9956\n",
      "Epoch 21/25\n",
      "63000/63000 [==============================] - 62s 978us/step - loss: 0.0144 - accuracy: 0.9956\n",
      "Epoch 22/25\n",
      "63000/63000 [==============================] - 62s 978us/step - loss: 0.0116 - accuracy: 0.9965\n",
      "Epoch 23/25\n",
      "63000/63000 [==============================] - 62s 977us/step - loss: 0.0088 - accuracy: 0.9972\n",
      "Epoch 24/25\n",
      "63000/63000 [==============================] - 62s 978us/step - loss: 0.0126 - accuracy: 0.9964\n",
      "Epoch 25/25\n",
      "63000/63000 [==============================] - 62s 979us/step - loss: 0.0102 - accuracy: 0.9970\n",
      "7000/7000 [==============================] - 4s 542us/step\n",
      "0.9898571372032166\n",
      "Fold 3: 21000 - 28000\n",
      "Epoch 1/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.2877 - accuracy: 0.9258\n",
      "Epoch 2/25\n",
      "63000/63000 [==============================] - 60s 954us/step - loss: 0.0600 - accuracy: 0.9829\n",
      "Epoch 3/25\n",
      "63000/63000 [==============================] - 60s 953us/step - loss: 0.0455 - accuracy: 0.9867\n",
      "Epoch 4/25\n",
      "63000/63000 [==============================] - 60s 953us/step - loss: 0.0347 - accuracy: 0.9898\n",
      "Epoch 5/25\n",
      "63000/63000 [==============================] - 60s 958us/step - loss: 0.0294 - accuracy: 0.9913\n",
      "Epoch 6/25\n",
      "63000/63000 [==============================] - 61s 966us/step - loss: 0.0268 - accuracy: 0.9920\n",
      "Epoch 7/25\n",
      "63000/63000 [==============================] - 61s 965us/step - loss: 0.0256 - accuracy: 0.9925\n",
      "Epoch 8/25\n",
      "63000/63000 [==============================] - 61s 970us/step - loss: 0.0256 - accuracy: 0.9924\n",
      "Epoch 9/25\n",
      "63000/63000 [==============================] - 61s 970us/step - loss: 0.0210 - accuracy: 0.9936\n",
      "Epoch 10/25\n",
      "63000/63000 [==============================] - 62s 976us/step - loss: 0.0229 - accuracy: 0.9931\n",
      "Epoch 11/25\n",
      "63000/63000 [==============================] - 61s 974us/step - loss: 0.0213 - accuracy: 0.9935\n",
      "Epoch 12/25\n",
      "63000/63000 [==============================] - 62s 978us/step - loss: 0.0193 - accuracy: 0.9943\n",
      "Epoch 13/25\n",
      "63000/63000 [==============================] - 62s 984us/step - loss: 0.0177 - accuracy: 0.9945\n",
      "Epoch 14/25\n",
      "63000/63000 [==============================] - 62s 984us/step - loss: 0.0178 - accuracy: 0.9945\n",
      "Epoch 15/25\n",
      "63000/63000 [==============================] - 62s 988us/step - loss: 0.0183 - accuracy: 0.9951\n",
      "Epoch 16/25\n",
      "63000/63000 [==============================] - 62s 989us/step - loss: 0.0144 - accuracy: 0.9956\n",
      "Epoch 17/25\n",
      "63000/63000 [==============================] - 62s 983us/step - loss: 0.0164 - accuracy: 0.9951\n",
      "Epoch 18/25\n",
      "63000/63000 [==============================] - 62s 986us/step - loss: 0.0147 - accuracy: 0.9957\n",
      "Epoch 19/25\n",
      "63000/63000 [==============================] - 62s 983us/step - loss: 0.0131 - accuracy: 0.9963\n",
      "Epoch 20/25\n",
      "63000/63000 [==============================] - 62s 983us/step - loss: 0.0118 - accuracy: 0.9964\n",
      "Epoch 21/25\n",
      "63000/63000 [==============================] - 63s 1000us/step - loss: 0.0128 - accuracy: 0.9961\n",
      "Epoch 22/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0120 - accuracy: 0.9964\n",
      "Epoch 23/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0132 - accuracy: 0.9962\n",
      "Epoch 24/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0085 - accuracy: 0.9975\n",
      "Epoch 25/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0109 - accuracy: 0.9968\n",
      "7000/7000 [==============================] - 4s 602us/step\n",
      "0.9907143115997314\n",
      "Fold 4: 28000 - 35000\n",
      "Epoch 1/25\n",
      "63000/63000 [==============================] - 69s 1ms/step - loss: 0.3030 - accuracy: 0.9213\n",
      "Epoch 2/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0583 - accuracy: 0.9826\n",
      "Epoch 3/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0438 - accuracy: 0.9874\n",
      "Epoch 4/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0359 - accuracy: 0.9899\n",
      "Epoch 5/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0301 - accuracy: 0.9916\n",
      "Epoch 6/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0297 - accuracy: 0.9913\n",
      "Epoch 7/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0277 - accuracy: 0.9915\n",
      "Epoch 8/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0223 - accuracy: 0.9932\n",
      "Epoch 9/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0237 - accuracy: 0.9933\n",
      "Epoch 10/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0212 - accuracy: 0.9936\n",
      "Epoch 11/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0207 - accuracy: 0.9938\n",
      "Epoch 12/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0191 - accuracy: 0.9941\n",
      "Epoch 13/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0205 - accuracy: 0.9940\n",
      "Epoch 14/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0178 - accuracy: 0.9945\n",
      "Epoch 15/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0202 - accuracy: 0.9941\n",
      "Epoch 16/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0147 - accuracy: 0.9957\n",
      "Epoch 17/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0151 - accuracy: 0.9955\n",
      "Epoch 18/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0164 - accuracy: 0.9948\n",
      "Epoch 19/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0156 - accuracy: 0.9952\n",
      "Epoch 20/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0148 - accuracy: 0.9956\n",
      "Epoch 21/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0131 - accuracy: 0.9963\n",
      "Epoch 22/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0101 - accuracy: 0.9969\n",
      "Epoch 23/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0113 - accuracy: 0.9963\n",
      "Epoch 24/25\n",
      "63000/63000 [==============================] - 63s 993us/step - loss: 0.0134 - accuracy: 0.9959\n",
      "Epoch 25/25\n",
      "63000/63000 [==============================] - 63s 994us/step - loss: 0.0107 - accuracy: 0.9967\n",
      "7000/7000 [==============================] - 4s 513us/step\n",
      "0.9927142858505249\n",
      "Fold 5: 35000 - 42000\n",
      "Epoch 1/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.2869 - accuracy: 0.9277\n",
      "Epoch 2/25\n",
      "63000/63000 [==============================] - 60s 958us/step - loss: 0.0552 - accuracy: 0.9839\n",
      "Epoch 3/25\n",
      "63000/63000 [==============================] - 61s 960us/step - loss: 0.0423 - accuracy: 0.9879\n",
      "Epoch 4/25\n",
      "63000/63000 [==============================] - 61s 962us/step - loss: 0.0322 - accuracy: 0.9904\n",
      "Epoch 5/25\n",
      "63000/63000 [==============================] - 61s 966us/step - loss: 0.0302 - accuracy: 0.9916\n",
      "Epoch 6/25\n",
      "63000/63000 [==============================] - 61s 969us/step - loss: 0.0278 - accuracy: 0.9918\n",
      "Epoch 7/25\n",
      "63000/63000 [==============================] - 62s 980us/step - loss: 0.0237 - accuracy: 0.9926\n",
      "Epoch 8/25\n",
      "63000/63000 [==============================] - 62s 977us/step - loss: 0.0244 - accuracy: 0.9931\n",
      "Epoch 9/25\n",
      "63000/63000 [==============================] - 61s 974us/step - loss: 0.0234 - accuracy: 0.9926\n",
      "Epoch 10/25\n",
      "63000/63000 [==============================] - 61s 974us/step - loss: 0.0233 - accuracy: 0.9931\n",
      "Epoch 11/25\n",
      "63000/63000 [==============================] - 61s 974us/step - loss: 0.0192 - accuracy: 0.9942\n",
      "Epoch 12/25\n",
      "63000/63000 [==============================] - 62s 982us/step - loss: 0.0193 - accuracy: 0.9944\n",
      "Epoch 13/25\n",
      "63000/63000 [==============================] - 62s 980us/step - loss: 0.0181 - accuracy: 0.9945\n",
      "Epoch 14/25\n",
      "63000/63000 [==============================] - 62s 984us/step - loss: 0.0202 - accuracy: 0.9937\n",
      "Epoch 15/25\n",
      "63000/63000 [==============================] - 62s 985us/step - loss: 0.0169 - accuracy: 0.9950\n",
      "Epoch 16/25\n",
      "63000/63000 [==============================] - 62s 987us/step - loss: 0.0151 - accuracy: 0.9956\n",
      "Epoch 17/25\n",
      "63000/63000 [==============================] - 62s 988us/step - loss: 0.0138 - accuracy: 0.9958\n",
      "Epoch 18/25\n",
      "63000/63000 [==============================] - 62s 990us/step - loss: 0.0156 - accuracy: 0.9952\n",
      "Epoch 19/25\n",
      "63000/63000 [==============================] - 62s 988us/step - loss: 0.0153 - accuracy: 0.9956\n",
      "Epoch 20/25\n",
      "63000/63000 [==============================] - 62s 985us/step - loss: 0.0117 - accuracy: 0.9965\n",
      "Epoch 21/25\n",
      "63000/63000 [==============================] - 63s 995us/step - loss: 0.0139 - accuracy: 0.9958\n",
      "Epoch 22/25\n",
      "63000/63000 [==============================] - 63s 995us/step - loss: 0.0141 - accuracy: 0.9958\n",
      "Epoch 23/25\n",
      "63000/63000 [==============================] - 63s 995us/step - loss: 0.0104 - accuracy: 0.9969\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 24/25\n",
      "63000/63000 [==============================] - 63s 993us/step - loss: 0.0122 - accuracy: 0.9964\n",
      "Epoch 25/25\n",
      "63000/63000 [==============================] - 62s 987us/step - loss: 0.0129 - accuracy: 0.9962\n",
      "7000/7000 [==============================] - 4s 516us/step\n",
      "0.9868571162223816\n",
      "Fold 6: 42000 - 49000\n",
      "Epoch 1/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.2814 - accuracy: 0.9279\n",
      "Epoch 2/25\n",
      "63000/63000 [==============================] - 60s 954us/step - loss: 0.0553 - accuracy: 0.9834\n",
      "Epoch 3/25\n",
      "63000/63000 [==============================] - 60s 960us/step - loss: 0.0434 - accuracy: 0.9871\n",
      "Epoch 4/25\n",
      "63000/63000 [==============================] - 61s 968us/step - loss: 0.0389 - accuracy: 0.9890\n",
      "Epoch 5/25\n",
      "63000/63000 [==============================] - 61s 975us/step - loss: 0.0311 - accuracy: 0.9912\n",
      "Epoch 6/25\n",
      "63000/63000 [==============================] - 61s 972us/step - loss: 0.0278 - accuracy: 0.9918\n",
      "Epoch 7/25\n",
      "63000/63000 [==============================] - 61s 972us/step - loss: 0.0243 - accuracy: 0.9930\n",
      "Epoch 8/25\n",
      "63000/63000 [==============================] - 61s 974us/step - loss: 0.0246 - accuracy: 0.9925\n",
      "Epoch 9/25\n",
      "63000/63000 [==============================] - 62s 985us/step - loss: 0.0236 - accuracy: 0.9926\n",
      "Epoch 10/25\n",
      "63000/63000 [==============================] - 62s 990us/step - loss: 0.0194 - accuracy: 0.9944\n",
      "Epoch 11/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0200 - accuracy: 0.9940\n",
      "Epoch 12/25\n",
      "63000/63000 [==============================] - 67s 1ms/step - loss: 0.0190 - accuracy: 0.9944\n",
      "Epoch 13/25\n",
      "63000/63000 [==============================] - 67s 1ms/step - loss: 0.0179 - accuracy: 0.9949\n",
      "Epoch 14/25\n",
      "63000/63000 [==============================] - 67s 1ms/step - loss: 0.0155 - accuracy: 0.9954\n",
      "Epoch 15/25\n",
      "63000/63000 [==============================] - 67s 1ms/step - loss: 0.0154 - accuracy: 0.9956\n",
      "Epoch 16/25\n",
      "63000/63000 [==============================] - 67s 1ms/step - loss: 0.0157 - accuracy: 0.9950\n",
      "Epoch 17/25\n",
      "63000/63000 [==============================] - 68s 1ms/step - loss: 0.0149 - accuracy: 0.9956\n",
      "Epoch 18/25\n",
      "63000/63000 [==============================] - 67s 1ms/step - loss: 0.0139 - accuracy: 0.9957\n",
      "Epoch 19/25\n",
      "63000/63000 [==============================] - 62s 978us/step - loss: 0.0146 - accuracy: 0.9956\n",
      "Epoch 20/25\n",
      "63000/63000 [==============================] - 62s 977us/step - loss: 0.0098 - accuracy: 0.9970\n",
      "Epoch 21/25\n",
      "63000/63000 [==============================] - 62s 990us/step - loss: 0.0123 - accuracy: 0.9962\n",
      "Epoch 22/25\n",
      "63000/63000 [==============================] - 62s 982us/step - loss: 0.0114 - accuracy: 0.9964\n",
      "Epoch 23/25\n",
      "63000/63000 [==============================] - 62s 991us/step - loss: 0.0105 - accuracy: 0.9967\n",
      "Epoch 24/25\n",
      "63000/63000 [==============================] - 62s 989us/step - loss: 0.0104 - accuracy: 0.9969\n",
      "Epoch 25/25\n",
      "63000/63000 [==============================] - 63s 995us/step - loss: 0.0097 - accuracy: 0.9970\n",
      "7000/7000 [==============================] - 4s 515us/step\n",
      "0.99314284324646\n",
      "Fold 7: 49000 - 56000\n",
      "Epoch 1/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.2792 - accuracy: 0.9281\n",
      "Epoch 2/25\n",
      "63000/63000 [==============================] - 60s 956us/step - loss: 0.0549 - accuracy: 0.9849\n",
      "Epoch 3/25\n",
      "63000/63000 [==============================] - 61s 964us/step - loss: 0.0444 - accuracy: 0.9869\n",
      "Epoch 4/25\n",
      "63000/63000 [==============================] - 61s 968us/step - loss: 0.0348 - accuracy: 0.9900\n",
      "Epoch 5/25\n",
      "63000/63000 [==============================] - 61s 971us/step - loss: 0.0318 - accuracy: 0.9908\n",
      "Epoch 6/25\n",
      "63000/63000 [==============================] - 62s 981us/step - loss: 0.0285 - accuracy: 0.9918\n",
      "Epoch 7/25\n",
      "63000/63000 [==============================] - 62s 984us/step - loss: 0.0278 - accuracy: 0.9919\n",
      "Epoch 8/25\n",
      "63000/63000 [==============================] - 62s 989us/step - loss: 0.0240 - accuracy: 0.9926\n",
      "Epoch 9/25\n",
      "63000/63000 [==============================] - 62s 983us/step - loss: 0.0244 - accuracy: 0.9927\n",
      "Epoch 10/25\n",
      "63000/63000 [==============================] - 62s 984us/step - loss: 0.0223 - accuracy: 0.9932\n",
      "Epoch 11/25\n",
      "63000/63000 [==============================] - 62s 984us/step - loss: 0.0198 - accuracy: 0.9940\n",
      "Epoch 12/25\n",
      "63000/63000 [==============================] - 62s 984us/step - loss: 0.0163 - accuracy: 0.9953\n",
      "Epoch 13/25\n",
      "63000/63000 [==============================] - 63s 993us/step - loss: 0.0179 - accuracy: 0.9946\n",
      "Epoch 14/25\n",
      "63000/63000 [==============================] - 62s 991us/step - loss: 0.0202 - accuracy: 0.9943\n",
      "Epoch 15/25\n",
      "63000/63000 [==============================] - 62s 984us/step - loss: 0.0176 - accuracy: 0.9949\n",
      "Epoch 16/25\n",
      "63000/63000 [==============================] - 62s 981us/step - loss: 0.0170 - accuracy: 0.9948\n",
      "Epoch 17/25\n",
      "63000/63000 [==============================] - 62s 976us/step - loss: 0.0137 - accuracy: 0.9961\n",
      "Epoch 18/25\n",
      "63000/63000 [==============================] - 61s 976us/step - loss: 0.0141 - accuracy: 0.9960\n",
      "Epoch 19/25\n",
      "63000/63000 [==============================] - 61s 971us/step - loss: 0.0158 - accuracy: 0.9953\n",
      "Epoch 20/25\n",
      "63000/63000 [==============================] - 61s 965us/step - loss: 0.0151 - accuracy: 0.9957\n",
      "Epoch 21/25\n",
      "63000/63000 [==============================] - 61s 964us/step - loss: 0.0110 - accuracy: 0.9967\n",
      "Epoch 22/25\n",
      "63000/63000 [==============================] - 61s 964us/step - loss: 0.0108 - accuracy: 0.9964\n",
      "Epoch 23/25\n",
      "63000/63000 [==============================] - 61s 964us/step - loss: 0.0120 - accuracy: 0.9963\n",
      "Epoch 24/25\n",
      "63000/63000 [==============================] - 61s 964us/step - loss: 0.0110 - accuracy: 0.9965\n",
      "Epoch 25/25\n",
      "63000/63000 [==============================] - 61s 964us/step - loss: 0.0119 - accuracy: 0.9966\n",
      "7000/7000 [==============================] - 4s 536us/step\n",
      "0.991428554058075\n",
      "Fold 8: 56000 - 63000\n",
      "Epoch 1/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.2691 - accuracy: 0.9297\n",
      "Epoch 2/25\n",
      "63000/63000 [==============================] - 60s 960us/step - loss: 0.0578 - accuracy: 0.9827\n",
      "Epoch 3/25\n",
      "63000/63000 [==============================] - 61s 971us/step - loss: 0.0441 - accuracy: 0.9867\n",
      "Epoch 4/25\n",
      "63000/63000 [==============================] - 61s 971us/step - loss: 0.0363 - accuracy: 0.9896\n",
      "Epoch 5/25\n",
      "63000/63000 [==============================] - 61s 963us/step - loss: 0.0321 - accuracy: 0.9908\n",
      "Epoch 6/25\n",
      "63000/63000 [==============================] - 61s 963us/step - loss: 0.0294 - accuracy: 0.9909\n",
      "Epoch 7/25\n",
      "63000/63000 [==============================] - 61s 963us/step - loss: 0.0268 - accuracy: 0.9921\n",
      "Epoch 8/25\n",
      "63000/63000 [==============================] - 61s 963us/step - loss: 0.0247 - accuracy: 0.9925\n",
      "Epoch 9/25\n",
      "63000/63000 [==============================] - 61s 963us/step - loss: 0.0256 - accuracy: 0.9924\n",
      "Epoch 10/25\n",
      "63000/63000 [==============================] - 61s 967us/step - loss: 0.0193 - accuracy: 0.9943\n",
      "Epoch 11/25\n",
      "63000/63000 [==============================] - 61s 970us/step - loss: 0.0221 - accuracy: 0.9933\n",
      "Epoch 12/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0191 - accuracy: 0.9942\n",
      "Epoch 13/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0205 - accuracy: 0.9939\n",
      "Epoch 14/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0192 - accuracy: 0.9944\n",
      "Epoch 15/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0176 - accuracy: 0.9948\n",
      "Epoch 16/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0157 - accuracy: 0.9950\n",
      "Epoch 17/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0139 - accuracy: 0.9956\n",
      "Epoch 18/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0137 - accuracy: 0.9959\n",
      "Epoch 19/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0141 - accuracy: 0.9956\n",
      "Epoch 20/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0151 - accuracy: 0.9952\n",
      "Epoch 21/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0115 - accuracy: 0.9963\n",
      "Epoch 22/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0125 - accuracy: 0.9963\n",
      "Epoch 23/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0131 - accuracy: 0.9959\n",
      "Epoch 24/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0110 - accuracy: 0.9965\n",
      "Epoch 25/25\n",
      "63000/63000 [==============================] - 66s 1ms/step - loss: 0.0102 - accuracy: 0.9967\n",
      "7000/7000 [==============================] - 4s 601us/step\n",
      "0.9922857284545898\n",
      "Fold 9: 63000 - 70000\n",
      "Epoch 1/25\n",
      "63000/63000 [==============================] - 69s 1ms/step - loss: 0.3006 - accuracy: 0.9244\n",
      "Epoch 2/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0592 - accuracy: 0.9828\n",
      "Epoch 3/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0489 - accuracy: 0.9864\n",
      "Epoch 4/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0390 - accuracy: 0.9889\n",
      "Epoch 5/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0327 - accuracy: 0.9905\n",
      "Epoch 6/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0310 - accuracy: 0.9913\n",
      "Epoch 7/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0262 - accuracy: 0.9919\n",
      "Epoch 8/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0260 - accuracy: 0.9918\n",
      "Epoch 9/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0243 - accuracy: 0.9929\n",
      "Epoch 10/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0242 - accuracy: 0.9923\n",
      "Epoch 11/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0230 - accuracy: 0.9930\n",
      "Epoch 12/25\n",
      "63000/63000 [==============================] - 65s 1ms/step - loss: 0.0235 - accuracy: 0.9930\n",
      "Epoch 13/25\n",
      "63000/63000 [==============================] - 64s 1ms/step - loss: 0.0180 - accuracy: 0.9945\n",
      "Epoch 14/25\n",
      "63000/63000 [==============================] - 62s 980us/step - loss: 0.0202 - accuracy: 0.9939\n",
      "Epoch 15/25\n",
      "63000/63000 [==============================] - 62s 980us/step - loss: 0.0169 - accuracy: 0.9944\n",
      "Epoch 16/25\n",
      "63000/63000 [==============================] - 62s 983us/step - loss: 0.0186 - accuracy: 0.9944\n",
      "Epoch 17/25\n",
      "63000/63000 [==============================] - 62s 983us/step - loss: 0.0168 - accuracy: 0.9952\n",
      "Epoch 18/25\n",
      "63000/63000 [==============================] - 62s 987us/step - loss: 0.0184 - accuracy: 0.9943\n",
      "Epoch 19/25\n",
      "63000/63000 [==============================] - 62s 982us/step - loss: 0.0148 - accuracy: 0.9955\n",
      "Epoch 20/25\n",
      "63000/63000 [==============================] - 62s 985us/step - loss: 0.0133 - accuracy: 0.9961\n",
      "Epoch 21/25\n",
      "63000/63000 [==============================] - 62s 986us/step - loss: 0.0136 - accuracy: 0.9960\n",
      "Epoch 22/25\n",
      "63000/63000 [==============================] - 62s 983us/step - loss: 0.0134 - accuracy: 0.9960\n",
      "Epoch 23/25\n",
      "63000/63000 [==============================] - 62s 979us/step - loss: 0.0147 - accuracy: 0.9954\n",
      "Epoch 24/25\n",
      "63000/63000 [==============================] - 62s 979us/step - loss: 0.0136 - accuracy: 0.9960\n",
      "Epoch 25/25\n",
      "63000/63000 [==============================] - 62s 979us/step - loss: 0.0109 - accuracy: 0.9966\n",
      "7000/7000 [==============================] - 4s 552us/step\n",
      "0.9945714473724365\n",
      "Average accuracy with 10-fold cross-validation: 0.99181 ± 0.00223\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9918142855167389,\n",
       " [0.026611555021946936,\n",
       "  0.03853112976948855,\n",
       "  0.04031716093335594,\n",
       "  0.04902952241230378,\n",
       "  0.03486207676862117,\n",
       "  0.058693780943393774,\n",
       "  0.03936152319094143,\n",
       "  0.0413308753897864,\n",
       "  0.03311518121353633,\n",
       "  0.02597183254967016],\n",
       " [0.9937142729759216,\n",
       "  0.9928571581840515,\n",
       "  0.9898571372032166,\n",
       "  0.9907143115997314,\n",
       "  0.9927142858505249,\n",
       "  0.9868571162223816,\n",
       "  0.99314284324646,\n",
       "  0.991428554058075,\n",
       "  0.9922857284545898,\n",
       "  0.9945714473724365])"
      ]
     },
     "execution_count": 405,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "crossvalidate(model6, x_combined, y_combined, num_folds, batch_size=200, epochs=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 10-fold cross-validation results on model6\n",
    "# 99.45% maximum accuracy\n",
    "# 99.18% +/- 0.22% mean/stdev accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}