2.8_implementing-backpropagation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implementing Backpropagation\n",
    "\n",
    "This is an exercise from Udacity Nanodegree **nd101-ent** lesson 2.8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings; warnings.simplefilter('ignore')\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "admissions = pd.read_csv('data/binary.csv')\n",
    "\n",
    "# Make dummy variables for rank\n",
    "data = pd.concat([admissions, pd.get_dummies(admissions['rank'], prefix='rank')], axis=1)\n",
    "data = data.drop('rank', axis=1)\n",
    "\n",
    "# Standarize features\n",
    "for field in ['gre', 'gpa']:\n",
    "    mean, std = data[field].mean(), data[field].std()\n",
    "    data.loc[:,field] = (data[field]-mean)/std\n",
    "    \n",
    "# Split off random 10% of the data for testing\n",
    "np.random.seed(21)\n",
    "sample = np.random.choice(data.index, size=int(len(data)*0.9), replace=False)\n",
    "data, test_data = data.ix[sample], data.drop(sample)\n",
    "\n",
    "# Split into features and targets\n",
    "features, targets = data.drop('admit', axis=1), data['admit']\n",
    "features_test, targets_test = test_data.drop('admit', axis=1), test_data['admit']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have 6 features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gre</th>\n",
       "      <th>gpa</th>\n",
       "      <th>rank_1</th>\n",
       "      <th>rank_2</th>\n",
       "      <th>rank_3</th>\n",
       "      <th>rank_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>106</td>\n",
       "      <td>0.972155</td>\n",
       "      <td>0.446965</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.972155</td>\n",
       "      <td>1.392922</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>61</td>\n",
       "      <td>-0.239793</td>\n",
       "      <td>-0.183673</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>224</td>\n",
       "      <td>1.837832</td>\n",
       "      <td>-1.287291</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>-0.586063</td>\n",
       "      <td>-1.287291</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>1.664697</td>\n",
       "      <td>-0.446439</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>346</td>\n",
       "      <td>0.106478</td>\n",
       "      <td>-0.104844</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>271</td>\n",
       "      <td>-0.412928</td>\n",
       "      <td>-0.157397</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>338</td>\n",
       "      <td>-0.412928</td>\n",
       "      <td>-0.498993</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>240</td>\n",
       "      <td>-1.278605</td>\n",
       "      <td>-1.655163</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>360 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          gre       gpa  rank_1  rank_2  rank_3  rank_4\n",
       "106  0.972155  0.446965       1       0       0       0\n",
       "9    0.972155  1.392922       0       1       0       0\n",
       "61  -0.239793 -0.183673       0       0       0       1\n",
       "224  1.837832 -1.287291       0       1       0       0\n",
       "37  -0.586063 -1.287291       0       0       1       0\n",
       "..        ...       ...     ...     ...     ...     ...\n",
       "28   1.664697 -0.446439       0       1       0       0\n",
       "346  0.106478 -0.104844       0       1       0       0\n",
       "271 -0.412928 -0.157397       0       0       1       0\n",
       "338 -0.412928 -0.498993       1       0       0       0\n",
       "240 -1.278605 -1.655163       0       1       0       0\n",
       "\n",
       "[360 rows x 6 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Architecture\n",
    "\n",
    "![model](images/2-8_nn.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implementation of Backpropagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(21)\n",
    "\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "\n",
    "# ----- Hyperparameters -----------------------------\n",
    "n_hidden= 2  # number of hidden units\n",
    "epochs = 900\n",
    "learnrate = 0.005\n",
    "\n",
    "n_records, n_features = features.shape\n",
    "last_loss = None\n",
    "\n",
    "# Initialize weights\n",
    "weights_input_hidden = np.random.normal(scale=1 / n_features ** .5, size=(n_features, n_hidden))\n",
    "weights_hidden_output = np.random.normal(scale=1 / n_features ** .5, size=n_hidden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss:  0.25135784720177\n",
      "Train loss:  0.25001697258522415\n",
      "Train loss:  0.248717824809504\n",
      "Train loss:  0.24745956973223776\n",
      "Train loss:  0.2462413545575023\n",
      "Train loss:  0.24506231119263724\n",
      "Train loss:  0.24392155942779803\n",
      "Train loss:  0.24281820993418365\n",
      "Train loss:  0.24175136707873493\n",
      "Train loss:  0.2407201315547637\n",
      "Prediction accuracy: 0.725\n"
     ]
    }
   ],
   "source": [
    "for e in range(epochs):\n",
    "    del_w_input_hidden = np.zeros(weights_input_hidden.shape)\n",
    "    del_w_hidden_output = np.zeros(weights_hidden_output.shape)\n",
    "    for x, y in zip(features.values, targets):\n",
    "        \n",
    "        ## forward pass ##\n",
    "        hidden_input = np.dot(x, weights_input_hidden)\n",
    "        hidden_output = sigmoid(hidden_input)\n",
    "        output = sigmoid(np.dot(hidden_output, weights_hidden_output))\n",
    "\n",
    "        ## backward pass ##\n",
    "        error = y - output\n",
    "        output_error_term = error * output * (1 - output)\n",
    "\n",
    "        ## propagate errors to hidden layer\n",
    "\n",
    "        # the hidden layer's contribution to the error:\n",
    "        hidden_error = output_error_term * hidden_output\n",
    "        \n",
    "        # the error term for the hidden layer:\n",
    "        hidden_error_term = np.dot(output_error_term, weights_input_hidden) * hidden_output * (1 - hidden_output)\n",
    "        \n",
    "        # update the change in weights\n",
    "        del_w_hidden_output += output_error_term * hidden_output\n",
    "        del_w_input_hidden += hidden_error_term * x[:,None]\n",
    "\n",
    "    # update weights  (don't forget to division by n_records or number of samples)\n",
    "    weights_input_hidden += learnrate * del_w_input_hidden / n_records\n",
    "    weights_hidden_output += learnrate * del_w_hidden_output / n_records\n",
    "\n",
    "    # Printing out the mean square error on the training set\n",
    "    if e % (epochs / 10) == 0:\n",
    "        hidden_output = sigmoid(np.dot(x, weights_input_hidden))\n",
    "        out = sigmoid(np.dot(hidden_output,\n",
    "                             weights_hidden_output))\n",
    "        loss = np.mean((out - targets) ** 2)\n",
    "\n",
    "        if last_loss and last_loss < loss:\n",
    "            print(\"Train loss: \", loss, \"  WARNING - Loss Increasing\")\n",
    "        else:\n",
    "            print(\"Train loss: \", loss)\n",
    "        last_loss = loss\n",
    "\n",
    "# Calculate accuracy on test data\n",
    "hidden = sigmoid(np.dot(features_test, weights_input_hidden))\n",
    "out = sigmoid(np.dot(hidden, weights_hidden_output))\n",
    "predictions = out > 0.5\n",
    "accuracy = np.mean(predictions == targets_test)\n",
    "print(\"Prediction accuracy: {:.3f}\".format(accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Notes\n",
    "\n",
    "There are two things to mention:\n",
    "\n",
    "1. The calculation of the weights change of the hidden layer `del_w_input_hidden += hidden_error_term * x[:,None]`, we do an **element wise** multiplication of a matrix `hidden_error_term` with the vector `x`, hence must transform the vector to a a **2D-vector**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6, 1)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[:, None].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.00070294, -0.0014713 ],\n",
       "       [ 0.01803146, -0.02124858],\n",
       "       [-0.        ,  0.        ],\n",
       "       [ 0.00209494,  0.00233943],\n",
       "       [-0.        ,  0.        ],\n",
       "       [ 0.        , -0.        ]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hidden_error_term * x[:,None]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. We sum the weights change $\\Delta w$ for each record (feature) from the data set. We therefore must **divide by the number of records** before updating the weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.03660627 -0.07227949]\n",
      " [ 0.32192302 -0.2694654 ]\n",
      " [-0.04282539  0.11350785]\n",
      " [ 0.10634632  0.1189796 ]\n",
      " [-0.81061487  0.00931435]\n",
      " [ 0.27525864 -0.56672298]]\n",
      "360\n"
     ]
    }
   ],
   "source": [
    "print(del_w_input_hidden)\n",
    "print(n_records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.01684089e-04, -2.00776351e-04],\n",
       "       [ 8.94230613e-04, -7.48514989e-04],\n",
       "       [-1.18959405e-04,  3.15299591e-04],\n",
       "       [ 2.95406454e-04,  3.30498894e-04],\n",
       "       [-2.25170798e-03,  2.58731962e-05],\n",
       "       [ 7.64607326e-04, -1.57423050e-03]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del_w_input_hidden / n_records"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}