chapter02_mathematical-building-blocks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "This is a companion notebook for the book [Deep Learning with Python, Second Edition](https://www.manning.com/books/deep-learning-with-python-second-edition?a_aid=keras&a_bid=76564dff). For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.\n",
    "\n",
    "**If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.**\n",
    "\n",
    "This notebook was generated for TensorFlow 2.6."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "# The mathematical building blocks of neural networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## A first look at a neural network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Loading the MNIST dataset in Keras**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.datasets import mnist\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60000"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, ..., 5, 6, 8], dtype=uint8)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 28, 28)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([7, 2, 1, ..., 4, 5, 6], dtype=uint8)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**The network architecture**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "model = keras.Sequential([\n",
    "    layers.Dense(512, activation=\"relu\"),\n",
    "    layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**The compilation step**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"rmsprop\",\n",
    "              loss=\"sparse_categorical_crossentropy\",\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Preparing the image data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "test_images = test_images.reshape((10000, 28 * 28))\n",
    "test_images = test_images.astype(\"float32\") / 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 784)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_images.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**\"Fitting\" the model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "469/469 [==============================] - 2s 3ms/step - loss: 0.2594 - accuracy: 0.9247\n",
      "Epoch 2/5\n",
      "469/469 [==============================] - 1s 3ms/step - loss: 0.1040 - accuracy: 0.9686\n",
      "Epoch 3/5\n",
      "469/469 [==============================] - 1s 3ms/step - loss: 0.0678 - accuracy: 0.9800\n",
      "Epoch 4/5\n",
      "469/469 [==============================] - 1s 3ms/step - loss: 0.0492 - accuracy: 0.9848\n",
      "Epoch 5/5\n",
      "469/469 [==============================] - 1s 3ms/step - loss: 0.0363 - accuracy: 0.9886\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f917cd1a828>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_images, train_labels, epochs=5, batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Using the model to make batch predictions**\n",
    "\n",
    "result set becomes (10, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4.59112415e-10, 6.26975295e-12, 3.94135725e-07, 2.77792751e-05,\n",
       "       1.10363324e-12, 1.42758685e-08, 1.18216672e-15, 9.99971509e-01,\n",
       "       8.23558644e-09, 3.39408729e-07], dtype=float32)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digits = test_images[0:10]\n",
    "predictions = model.predict(test_digits)\n",
    "predictions[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 10)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4.59112415e-10, 6.26975295e-12, 3.94135725e-07, 2.77792751e-05,\n",
       "       1.10363324e-12, 1.42758685e-08, 1.18216672e-15, 9.99971509e-01,\n",
       "       8.23558644e-09, 3.39408729e-07], dtype=float32)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions[0] # sample 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions[0].argmax() # finds you the index of the max probability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9999715"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions[0][7] # finds you the probability at the index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_labels[0] # compares with actual label for sample 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Evaluating the model on new data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 0s 842us/step - loss: 0.0712 - accuracy: 0.9787\n",
      "test_acc: 0.9786999821662903\n"
     ]
    }
   ],
   "source": [
    "test_loss, test_acc = model.evaluate(test_images, test_labels)\n",
    "print(f\"test_acc: {test_acc}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Data representations for neural networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Scalars (rank-0 tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(12)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "x = np.array(12)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.ndim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Vectors (rank-1 tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([12,  3,  6, 14,  7])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([12, 3, 6, 14, 7])\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5,)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.ndim\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Matrices (rank-2 tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 5)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([[5, 78, 2, 34, 0],\n",
    "              [6, 79, 3, 35, 1],\n",
    "              [7, 80, 4, 36, 2]])\n",
    "x.ndim\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Rank-3 and higher-rank tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 3, 5)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([[[5, 78, 2, 34, 0],\n",
    "               [6, 79, 3, 35, 1],\n",
    "               [7, 80, 4, 36, 2]],\n",
    "              [[5, 78, 2, 34, 0],\n",
    "               [6, 79, 3, 35, 1],\n",
    "               [7, 80, 4, 36, 2]],\n",
    "              [[5, 78, 2, 34, 0],\n",
    "               [6, 79, 3, 35, 1],\n",
    "               [7, 80, 4, 36, 2]]])\n",
    "x.ndim\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Key attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.datasets import mnist\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_images.ndim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('uint8')"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_images.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Displaying the fourth digit**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAANpElEQVR4nO3db6xU9Z3H8c9HtxpDS4TlSpCSvbXyhKwpbSaySbGyaRbUaLAmEokSTIj0ASY2qXENakqMGt0sbWpcmtBVSrUrmrQKD0yRJY3YJ4TRsAqarmggFdF70ZhSo7LY7z64h+aKd35zmf/l+34lNzNzvnPmfDP64cyc35nzc0QIwJnvrH43AKA3CDuQBGEHkiDsQBKEHUji73q5sRkzZsTw8HAvNwmkcvDgQR09etQT1doKu+0rJP1U0tmS/jMiHiw9f3h4WPV6vZ1NAiio1WoNay1/jLd9tqT/kHSlpHmSltue1+rrAeiudr6zXyrpQES8FRHHJW2RtLQzbQHotHbCPlvSH8c9frta9jm2V9uu266Pjo62sTkA7ej60fiI2BgRtYioDQ0NdXtzABpoJ+yHJc0Z9/ir1TIAA6idsO+RNNf212yfI+kGSds60xaATmt56C0iTti+VdJ2jQ29PRYR+zvWGYCOamucPSKek/Rch3oB0EWcLgskQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1Ioq0pm20flHRM0meSTkRErRNNAei8tsJe+eeIONqB1wHQRXyMB5JoN+wh6XnbL9lePdETbK+2XbddHx0dbXNzAFrVbtgXRsS3JF0paY3t75z6hIjYGBG1iKgNDQ21uTkArWor7BFxuLodkfSMpEs70RSAzms57Lan2P7KyfuSFkva16nGAHRWO0fjZ0p6xvbJ1/mviPhtR7oC0HEthz0i3pL0jQ72AqCLGHoDkiDsQBKEHUiCsANJEHYgiU78EAYDbPfu3cX6448/Xqzv2rWrWN+3r/VTK9avX1+sX3jhhcX6iy++WKyvWLGiYW3BggXFdc9E7NmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2c8ATz31VMPabbfdVly32aXCIqJYX7RoUbF+9Gjja5HefvvtxXWbadZbadtbtmxpa9t/i9izA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjLMPgBMnThTre/bsKdZvueWWhrWPPvqouO7ll19erN9zzz3F+sKFC4v1Tz/9tGFt2bJlxXW3b99erDdTqzGp8Hjs2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZB8ATTzxRrK9atarl1168eHGxXvotvCRNnTq15W03e/12x9HnzJlTrK9cubKt1z/TNN2z237M9ojtfeOWTbe9w/Yb1e207rYJoF2T+Rj/C0lXnLLsTkk7I2KupJ3VYwADrGnYI2KXpA9OWbxU0ubq/mZJ13a2LQCd1uoBupkRcaS6/66kmY2eaHu17brterPrnQHonraPxsfYVf8aXvkvIjZGRC0iakNDQ+1uDkCLWg37e7ZnSVJ1O9K5lgB0Q6th3ybp5LjGSklbO9MOgG5pOs5u+0lJiyTNsP22pB9JelDS07ZXSTokqfzD5OTuvvvuYv2BBx4o1m0X62vWrGlYu++++4rrtjuO3sz999/ftdd++OGHi3W+Nn5e07BHxPIGpe92uBcAXcTpskAShB1IgrADSRB2IAnCDiTBT1w74N577y3Wmw2tnXvuucX6kiVLivWHHnqoYe28884rrtvMJ598Uqw///zzxfqhQ4ca1ppNudzsMtZLly4t1vF57NmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2Sfpww8/bFjbsGFDcd1mP1FtNo7+7LPPFuvtOHDgQLF+4403Fuv1er3lbV9//fXF+h133NHya+OL2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs0/S8ePHG9bandaq2SWRR0bKc3Bs2rSpYW3r1vIl/ffv31+sHzt2rFhvdg7BWWc13p/cdNNNxXWnTJlSrOP0sGcHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQYZ5+kc845p2HtggsuKK7bbJx8eHi4WG82lt2O2bNnF+vNpnR+5513ivUZM2Y0rF1zzTXFddFZTffsth+zPWJ737hl62wftr23+ruqu20CaNdkPsb/QtIVEyz/SUTMr/6e62xbADqtadgjYpekD3rQC4AuaucA3a22X6k+5k9r9CTbq23XbdfbPYccQOtaDfvPJH1d0nxJRyStb/TEiNgYEbWIqA0NDbW4OQDtainsEfFeRHwWEX+R9HNJl3a2LQCd1lLYbc8a9/B7kvY1ei6AwdB0nN32k5IWSZph+21JP5K0yPZ8SSHpoKTvd6/FwXD++ec3rDW7rvvVV19drL///vvF+sUXX1ysl+Ypv/nmm4vrTp8+vVi/4YYbivVm4+zN1kfvNA17RCyfYPGjXegFQBdxuiyQBGEHkiDsQBKEHUiCsANJ8BPXDliwYEGxPsinCe/atatYf+GFF4r1Zj+/veiii067J3QHe3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9uQ+/vjjYr3ZOHqzOj9xHRzs2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZk1uyZEm/W0CPsGcHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQYZ09u+/bt/W4BPdJ0z257ju3f2X7N9n7bt1XLp9veYfuN6nZa99sF0KrJfIw/IemHETFP0j9JWmN7nqQ7Je2MiLmSdlaPAQyopmGPiCMR8XJ1/5ik1yXNlrRU0ubqaZslXdulHgF0wGkdoLM9LOmbknZLmhkRR6rSu5JmNlhnte267fogz3kGnOkmHXbbX5b0a0k/iIg/ja9FREiKidaLiI0RUYuI2tDQUFvNAmjdpMJu+0saC/qvIuI31eL3bM+q6rMkjXSnRQCd0HTozWPXCn5U0usR8eNxpW2SVkp6sLrd2pUO0VVvvvlmv1tAj0xmnP3bklZIetX23mrZWo2F/GnbqyQdkrSsKx0C6IimYY+I30tqNBPAdzvbDoBu4XRZIAnCDiRB2IEkCDuQBGEHkuAnrslddtllxfrYyZE4E7BnB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkGGdP7pJLLinW586dW6w3+z18qc6Vi3qLPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4O4rWrl1brK9atarl9R955JHiuvPmzSvWcXrYswNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEpOZn32OpF9KmikpJG2MiJ/aXifpFkmj1VPXRsRz3WoU/XHdddcV61u2bCnWd+zY0bC2bt264rqbNm0q1qdMmVKs4/Mmc1LNCUk/jIiXbX9F0ku2T/4X/ElE/Hv32gPQKZOZn/2IpCPV/WO2X5c0u9uNAeis0/rObntY0jcl7a4W3Wr7FduP2Z7WYJ3Vtuu266OjoxM9BUAPTDrstr8s6deSfhARf5L0M0lflzRfY3v+9ROtFxEbI6IWETWuOQb0z6TCbvtLGgv6ryLiN5IUEe9FxGcR8RdJP5d0affaBNCupmG3bUmPSno9In48bvmscU/7nqR9nW8PQKdM5mj8tyWtkPSq7b3VsrWSltuer7HhuIOSvt+F/tBnU6dOLdaffvrpYv2uu+5qWNuwYUNx3WZDc/wE9vRM5mj87yV5ghJj6sDfEM6gA5Ig7EAShB1IgrADSRB2IAnCDiThiOjZxmq1WtTr9Z5tD8imVqupXq9PNFTOnh3IgrADSRB2IAnCDiRB2IEkCDuQBGEHkujpOLvtUUmHxi2aIelozxo4PYPa26D2JdFbqzrZ2z9ExITXf+tp2L+wcbseEbW+NVAwqL0Nal8SvbWqV73xMR5IgrADSfQ77Bv7vP2SQe1tUPuS6K1VPemtr9/ZAfROv/fsAHqEsANJ9CXstq+w/QfbB2zf2Y8eGrF90Partvfa7uuP76s59EZs7xu3bLrtHbbfqG4nnGOvT72ts324eu/22r6qT73Nsf0726/Z3m/7tmp5X9+7Ql89ed96/p3d9tmS/lfSv0h6W9IeScsj4rWeNtKA7YOSahHR9xMwbH9H0p8l/TIi/rFa9m+SPoiIB6t/KKdFxL8OSG/rJP2539N4V7MVzRo/zbikayXdrD6+d4W+lqkH71s/9uyXSjoQEW9FxHFJWyQt7UMfAy8idkn64JTFSyVtru5v1tj/LD3XoLeBEBFHIuLl6v4xSSenGe/re1foqyf6EfbZkv447vHbGqz53kPS87Zfsr26381MYGZEHKnuvytpZj+bmUDTabx76ZRpxgfmvWtl+vN2cYDuixZGxLckXSlpTfVxdSDF2HewQRo7ndQ03r0ywTTjf9XP967V6c/b1Y+wH5Y0Z9zjr1bLBkJEHK5uRyQ9o8Gbivq9kzPoVrcjfe7nrwZpGu+JphnXALx3/Zz+vB9h3yNpru2v2T5H0g2StvWhjy+wPaU6cCLbUyQt1uBNRb1N0srq/kpJW/vYy+cMyjTejaYZV5/fu75Pfx4RPf+TdJXGjsi/KemufvTQoK+LJP1P9be/371JelJjH+v+T2PHNlZJ+ntJOyW9Iem/JU0foN4el/SqpFc0FqxZfeptocY+or8iaW/1d1W/37tCXz153zhdFkiCA3RAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kMT/A38cJNEbCe0NAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "digit = train_images[4]\n",
    "plt.imshow(digit, cmap=plt.cm.binary)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels[4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Manipulating tensors in NumPy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90, 28, 28)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_slice = train_images[10:100]\n",
    "my_slice.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90, 28, 28)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_slice = train_images[10:100, :, :]\n",
    "my_slice.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90, 28, 28)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_slice = train_images[10:100, 0:28, 0:28]\n",
    "my_slice.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "my_slice = train_images[:, 14:, 14:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "my_slice = train_images[:, 7:-7, 7:-7]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### The notion of data batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "batch = train_images[:128]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "batch = train_images[128:256]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "n = 3\n",
    "batch = train_images[128 * n:128 * (n + 1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "tags": []
   },
   "source": [
    "### Real-world examples of data tensors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Vector data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Timeseries data or sequence data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Image data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "tags": []
   },
   "source": [
    "### Video data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## The gears of neural networks: tensor operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Element-wise operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_relu(x):\n",
    "    assert len(x.shape) == 2\n",
    "    x = x.copy()\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(x.shape[1]):\n",
    "            x[i, j] = max(x[i, j], 0)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_add(x, y):\n",
    "    assert len(x.shape) == 2\n",
    "    assert x.shape == y.shape\n",
    "    x = x.copy()\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(x.shape[1]):\n",
    "            x[i, j] += y[i, j]\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Took: 0.01 s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "x = np.random.random((20, 100))\n",
    "y = np.random.random((20, 100))\n",
    "\n",
    "t0 = time.time()\n",
    "for _ in range(1000):\n",
    "    z = x + y\n",
    "    z = np.maximum(z, 0.)\n",
    "print(\"Took: {0:.2f} s\".format(time.time() - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Took: 1.94 s\n"
     ]
    }
   ],
   "source": [
    "t0 = time.time()\n",
    "for _ in range(1000):\n",
    "    z = naive_add(x, y)\n",
    "    z = naive_relu(z)\n",
    "print(\"Took: {0:.2f} s\".format(time.time() - t0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Broadcasting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "X = np.random.random((32, 10))\n",
    "y = np.random.random((10,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "y = np.expand_dims(y, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "Y = np.concatenate([y] * 32, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_add_matrix_and_vector(x, y):\n",
    "    assert len(x.shape) == 2\n",
    "    assert len(y.shape) == 1\n",
    "    assert x.shape[1] == y.shape[0]\n",
    "    x = x.copy()\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(x.shape[1]):\n",
    "            x[i, j] += y[j]\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "x = np.random.random((64, 3, 32, 10))\n",
    "y = np.random.random((32, 10))\n",
    "z = np.maximum(x, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Tensor product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "x = np.random.random((32,))\n",
    "y = np.random.random((32,))\n",
    "z = np.dot(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7.819117439719728"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_vector_dot(x, y):\n",
    "    assert len(x.shape) == 1\n",
    "    assert len(y.shape) == 1\n",
    "    assert x.shape[0] == y.shape[0]\n",
    "    z = 0.\n",
    "    for i in range(x.shape[0]):\n",
    "        z += x[i] * y[i]\n",
    "    return z # scalar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_matrix_vector_dot(x, y):\n",
    "    assert len(x.shape) == 2\n",
    "    assert len(y.shape) == 1\n",
    "    assert x.shape[1] == y.shape[0]\n",
    "    z = np.zeros(x.shape[0])\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(x.shape[1]):\n",
    "            z[i] += x[i, j] * y[j]\n",
    "    return z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([73.90335158, 76.02916846, 78.15498534])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([[5, 78, 2, 34, 0],\n",
    "              [6, 79, 3, 35, 1],\n",
    "              [7, 80, 4, 36, 2]])\n",
    "y = np.random.random((5,))\n",
    "naive_matrix_vector_dot(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_matrix_vector_dot(x, y):\n",
    "    z = np.zeros(x.shape[0])\n",
    "    for i in range(x.shape[0]):\n",
    "        z[i] = naive_vector_dot(x[i, :], y)\n",
    "    return z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_matrix_dot(x, y):\n",
    "    assert len(x.shape) == 2\n",
    "    assert len(y.shape) == 2\n",
    "    assert x.shape[1] == y.shape[0]\n",
    "    z = np.zeros((x.shape[0], y.shape[1]))\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(y.shape[1]):\n",
    "            row_x = x[i, :]\n",
    "            column_y = y[:, j]\n",
    "            z[i, j] = naive_vector_dot(row_x, column_y)\n",
    "    return z"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Tensor reshaping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "train_images = train_images.reshape((60000, 28 * 28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "x = np.array([[0., 1.],\n",
    "             [2., 3.],\n",
    "             [4., 5.]])\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "x = x.reshape((6, 1))\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "x = np.zeros((300, 20))\n",
    "x = np.transpose(x)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Geometric interpretation of tensor operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### A geometric interpretation of deep learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## The engine of neural networks: gradient-based optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### What's a derivative?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Derivative of a tensor operation: the gradient"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Stochastic gradient descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Chaining derivatives: The Backpropagation algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### The chain rule"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Automatic differentiation with computation graphs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### The gradient tape in TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "x = tf.Variable(0.)\n",
    "with tf.GradientTape() as tape:\n",
    "    y = 2 * x + 3\n",
    "grad_of_y_wrt_x = tape.gradient(y, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "x = tf.Variable(tf.random.uniform((2, 2)))\n",
    "with tf.GradientTape() as tape:\n",
    "    y = 2 * x + 3\n",
    "grad_of_y_wrt_x = tape.gradient(y, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "W = tf.Variable(tf.random.uniform((2, 2)))\n",
    "b = tf.Variable(tf.zeros((2,)))\n",
    "x = tf.random.uniform((2, 2))\n",
    "with tf.GradientTape() as tape:\n",
    "    y = tf.matmul(x, W) + b\n",
    "grad_of_y_wrt_W_and_b = tape.gradient(y, [W, b])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Looking back at our first example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "test_images = test_images.reshape((10000, 28 * 28))\n",
    "test_images = test_images.astype(\"float32\") / 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model = keras.Sequential([\n",
    "    layers.Dense(512, activation=\"relu\"),\n",
    "    layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"rmsprop\",\n",
    "              loss=\"sparse_categorical_crossentropy\",\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model.fit(train_images, train_labels, epochs=5, batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Reimplementing our first example from scratch in TensorFlow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### A simple Dense class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "class NaiveDense:\n",
    "    def __init__(self, input_size, output_size, activation):\n",
    "        self.activation = activation\n",
    "\n",
    "        w_shape = (input_size, output_size)\n",
    "        w_initial_value = tf.random.uniform(w_shape, minval=0, maxval=1e-1)\n",
    "        self.W = tf.Variable(w_initial_value)\n",
    "\n",
    "        b_shape = (output_size,)\n",
    "        b_initial_value = tf.zeros(b_shape)\n",
    "        self.b = tf.Variable(b_initial_value)\n",
    "\n",
    "    def __call__(self, inputs):\n",
    "        return self.activation(tf.matmul(inputs, self.W) + self.b)\n",
    "\n",
    "    @property\n",
    "    def weights(self):\n",
    "        return [self.W, self.b]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### A simple Sequential class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "class NaiveSequential:\n",
    "    def __init__(self, layers):\n",
    "        self.layers = layers\n",
    "\n",
    "    def __call__(self, inputs):\n",
    "        x = inputs\n",
    "        for layer in self.layers:\n",
    "           x = layer(x)\n",
    "        return x\n",
    "\n",
    "    @property\n",
    "    def weights(self):\n",
    "       weights = []\n",
    "       for layer in self.layers:\n",
    "           weights += layer.weights\n",
    "       return weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model = NaiveSequential([\n",
    "    NaiveDense(input_size=28 * 28, output_size=512, activation=tf.nn.relu),\n",
    "    NaiveDense(input_size=512, output_size=10, activation=tf.nn.softmax)\n",
    "])\n",
    "assert len(model.weights) == 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### A batch generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "class BatchGenerator:\n",
    "    def __init__(self, images, labels, batch_size=128):\n",
    "        assert len(images) == len(labels)\n",
    "        self.index = 0\n",
    "        self.images = images\n",
    "        self.labels = labels\n",
    "        self.batch_size = batch_size\n",
    "        self.num_batches = math.ceil(len(images) / batch_size)\n",
    "\n",
    "    def next(self):\n",
    "        images = self.images[self.index : self.index + self.batch_size]\n",
    "        labels = self.labels[self.index : self.index + self.batch_size]\n",
    "        self.index += self.batch_size\n",
    "        return images, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Running one training step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def one_training_step(model, images_batch, labels_batch):\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions = model(images_batch)\n",
    "        per_sample_losses = tf.keras.losses.sparse_categorical_crossentropy(\n",
    "            labels_batch, predictions)\n",
    "        average_loss = tf.reduce_mean(per_sample_losses)\n",
    "    gradients = tape.gradient(average_loss, model.weights)\n",
    "    update_weights(gradients, model.weights)\n",
    "    return average_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "learning_rate = 1e-3\n",
    "\n",
    "def update_weights(gradients, weights):\n",
    "    for g, w in zip(gradients, weights):\n",
    "        w.assign_sub(g * learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras import optimizers\n",
    "\n",
    "optimizer = optimizers.SGD(learning_rate=1e-3)\n",
    "\n",
    "def update_weights(gradients, weights):\n",
    "    optimizer.apply_gradients(zip(gradients, weights))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### The full training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def fit(model, images, labels, epochs, batch_size=128):\n",
    "    for epoch_counter in range(epochs):\n",
    "        print(f\"Epoch {epoch_counter}\")\n",
    "        batch_generator = BatchGenerator(images, labels)\n",
    "        for batch_counter in range(batch_generator.num_batches):\n",
    "            images_batch, labels_batch = batch_generator.next()\n",
    "            loss = one_training_step(model, images_batch, labels_batch)\n",
    "            if batch_counter % 100 == 0:\n",
    "                print(f\"loss at batch {batch_counter}: {loss:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.datasets import mnist\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
    "\n",
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "test_images = test_images.reshape((10000, 28 * 28))\n",
    "test_images = test_images.astype(\"float32\") / 255\n",
    "\n",
    "fit(model, train_images, train_labels, epochs=10, batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Evaluating the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "predictions = model(test_images)\n",
    "predictions = predictions.numpy()\n",
    "predicted_labels = np.argmax(predictions, axis=1)\n",
    "matches = predicted_labels == test_labels\n",
    "print(f\"accuracy: {matches.mean():.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Summary"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "chapter02_mathematical-building-blocks.i",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "deeplearning",
   "language": "python",
   "name": "deeplearning"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}