gradient_descent.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DATA1030 Gradient Descent Interactive Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load libraries and packages\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.datasets import make_regression\n",
    "import plotly\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.graph_objs as go\n",
    "import ipywidgets\n",
    "from ipywidgets import interact"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Regression Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "coefficent: 28.777533858634875\n",
      "added noise: [  9.18115839 -30.87739085  24.90429735  -4.96205858  32.94232532\n",
      "  14.46054785 -66.23260778 -19.67600162 -11.0521372  -25.26260034\n",
      " -59.28576902 -31.67310564 -31.65215819  32.62708851  50.21137962\n",
      "  -3.53647763 -26.92913658  46.7446537   42.07586066  25.94555749\n",
      " -21.90565736  15.2623224  -15.19948048  -7.70915828  16.77198455\n",
      "   1.21480753  25.92441258  -7.17626442 -17.60484091  -9.27837201]\n",
      "y: [  3.08130585 -48.2639338   25.93592732   1.87178372  45.4428258\n",
      "   6.88380644 -72.44927625 -16.20740827 -12.15967691 -25.0337406\n",
      " -66.76530644 -35.83422144 -27.17695128  53.04737598  52.96856006\n",
      "  10.13384942 -20.19197583  36.16208808  58.99412194  38.42855536\n",
      " -32.81240623   6.77988325 -15.82439642 -20.62478531  18.61129032\n",
      "   9.78715027  31.80735422  -4.11214063 -11.04200012 -15.08634424]\n",
      "         x1          y\n",
      "0  0.319039   3.081306\n",
      "1 -1.072969 -48.263934\n",
      "2  0.865408  25.935927\n",
      "3 -0.172428   1.871784\n",
      "4  1.144724  45.442826\n"
     ]
    }
   ],
   "source": [
    "# fix the seed so code is reproducable\n",
    "np.random.seed(1)\n",
    "\n",
    "# generate n_samples points\n",
    "n_samples = 30\n",
    "\n",
    "# generate data\n",
    "X, y, coef = make_regression(n_samples = n_samples, n_features = 1, coef = True, noise= 10, bias=0)\n",
    "print(\"coefficent:\", coef) # the coefficients of the underlying linear model, the bias is 0.\n",
    "print(\"added noise:\", np.dot(X,coef)[:,0]) # noise is added to the label\n",
    "print(\"y:\", y)\n",
    "df = pd.DataFrame()\n",
    "df['x1'] = X[:,0]\n",
    "df['y'] = y\n",
    "print(df.head())\n",
    "\n",
    "# set intercept and slope ranges to explore\n",
    "n_vals = 101\n",
    "theta0 = np.linspace(-100,100,n_vals) # the intercept values to explore\n",
    "theta1 = np.linspace(-100,100,n_vals) # the slope values to explore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction and Cost Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(X,theta):\n",
    "    if len(np.shape(theta)) != 2:\n",
    "        theta = np.array(theta)[np.newaxis,:] # just a numpy trick to make the dot product work\n",
    "    y_pred = theta[0,0] + X.dot(theta[0,1:]) # intercept + theta_i*x_i    \n",
    "    return y_pred\n",
    "\n",
    "def cost_function(X,y_true,theta):\n",
    "    \"\"\"\n",
    "    Take in a numpy array X,y_true, theta and generate the cost function\n",
    "    of using theta as parameter in a linear regression model\n",
    "    \"\"\"\n",
    "    m = len(y)\n",
    "    theta = np.array(theta)[np.newaxis,:] # just a numpy trick to make the dot product work in predict\n",
    "    y_pred = predict(X,theta)\n",
    "    cost = (1/m) * np.sum(np.square(y_true-y_pred)) # this is MSE\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate Cost Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min(cost): 71.43643291686587\n",
      "best intercept: 2.0\n",
      "best slope: 32.0\n"
     ]
    }
   ],
   "source": [
    "# set cost function's value for each theta\n",
    "cost = np.zeros([len(theta0),len(theta1)]) \n",
    "\n",
    "# loop through all intercept-slope combinations and calculate the cost function\n",
    "for i in range(n_vals):\n",
    "    for j in range(n_vals):\n",
    "        theta = [theta0[i],theta1[j]]\n",
    "        cost[i,j] = cost_function(df['x1'].values[:,np.newaxis],df['y'],theta)\n",
    "        \n",
    "# print grid search best thetas  \n",
    "print('min(cost):',np.min(cost))\n",
    "min_coords = np.unravel_index(cost.argmin(),np.shape(cost))\n",
    "print('best intercept:',theta0[min_coords[0]])\n",
    "print('best slope:',theta1[min_coords[1]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Descent Function\n",
    "\n",
    "(For illustration purposes, slower than sklearn implementation.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent(X,y_true,theta,learning_rate=0.01,iterations=100):\n",
    "    '''\n",
    "    X    = Matrix of X with added bias units\n",
    "    y    = Vector of Y\n",
    "    theta=Vector of thetas np.random.randn(j,1)\n",
    "    learning_rate \n",
    "    iterations = no of iterations\n",
    "    \n",
    "    Returns the final theta vector and array of cost history over no of iterations\n",
    "    '''\n",
    "    m = len(y_true)\n",
    "    theta = np.array(theta)[np.newaxis,:]\n",
    "\n",
    "    cost_history = np.zeros(iterations)\n",
    "    theta_history = np.zeros([iterations,np.shape(theta)[1]])\n",
    "    for it in range(iterations):\n",
    "        \n",
    "        y_pred = predict(X,theta)\n",
    "        delta_theta = np.zeros(np.shape(theta)) # the step we take \n",
    "        # the derivative of the cost function with respect to the intercept\n",
    "        delta_theta[0,0] = (1/m) * sum(y_pred - y_true) *learning_rate\n",
    "        # the derivative of the cost function with respect to the slopes * learning_rate\n",
    "        delta_theta[0,1:] = (1/m)*learning_rate*( X.T.dot((y_pred - y_true)))\n",
    "        theta = theta - delta_theta # update theta so we move down the gradient\n",
    "        theta_history[it] = theta[0]\n",
    "        cost_history[it] = cost_function(X,y_true,theta[0])\n",
    "        \n",
    "    return theta[0], cost_history, theta_history"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function to generate gradient descent path data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_grad_desc_data(X,y,theta,learning_rates,iterations=100):\n",
    "    \"\"\"Returns a dictionary of theta history, a dictionary of cost history, and final theta\n",
    "    for given input data, vector of initial thetas, and vector of learning rates\"\"\"\n",
    "    \n",
    "    # create dictionary for theta histories\n",
    "    theta_hist_dict = {}\n",
    "    \n",
    "    # create dictionary for cost histories\n",
    "    cost_hist_dict = {}\n",
    "    \n",
    "    # create dictionary for final theta\n",
    "    theta_dict = {}\n",
    "    \n",
    "    # create df of input data\n",
    "    df = pd.DataFrame()\n",
    "    df['x1'] = X[:,0]\n",
    "    df['y'] = y\n",
    "    \n",
    "    # calculate theta and cost history for each learning rate\n",
    "    for lr in learning_rates:\n",
    "        # calculate cost and theta histories\n",
    "        theta, cost_history, theta_hist = gradient_descent(df['x1'].values[:,np.newaxis],df['y'],[100,-100],lr,iterations)\n",
    "        \n",
    "        # add theta history to dict \n",
    "        theta_hist_dict[lr] = theta_hist\n",
    "        \n",
    "        # add cost history to dict \n",
    "        cost_hist_dict[lr] = cost_history\n",
    "        \n",
    "        # add theta to dict \n",
    "        theta_dict[lr] = theta\n",
    "        \n",
    "    return theta_hist_dict, cost_hist_dict, theta_dict\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set theta range\n",
    "theta = [100,-100]\n",
    "# set learning rates\n",
    "learning_rates = np.arange(0.1,2.1, 0.1).round(2) # round to remove floating point rounding error\n",
    "# generate gradient descent data\n",
    "theta_hist_dict, cost_hist_dict, theta_dict = generate_grad_desc_data(X,y,theta,learning_rates,iterations=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interactive Gradient Descent Plots"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Double Plot - Cost History and Gradient Descent Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e228f85d82f4116ab2cbcb08bb04a8a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(SelectionSlider(description='learning rate', options=(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7,…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0935c7e77d2d4d07b8e86a580faf9587",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FigureWidget({\n",
       "    'data': [{'colorbar': {'title': {'text': 'cost function'}},\n",
       "              'type': 'contour'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Double plot code\n",
    "\n",
    "# set plot range \n",
    "x_min, x_max = np.min(theta0), np.max(theta0)\n",
    "y_min, y_max = np.min(theta1), np.max(theta1)\n",
    "\n",
    "# create figure widget 1 \n",
    "fig = go.FigureWidget( make_subplots(rows=1, cols=2, \n",
    "                                     shared_xaxes=False, \n",
    "                                     subplot_titles=(\"Cost History\", \"Gradient Descent Path\"),\n",
    "                                     horizontal_spacing = 0.1))\n",
    "\n",
    "# add axis labels\n",
    "fig['layout']['xaxis']['title']='nr. of iterations'\n",
    "fig['layout']['xaxis2']['title']='intercept'\n",
    "fig['layout']['yaxis']['title']='cost function'\n",
    "fig['layout']['yaxis2']['title']='slope'\n",
    "\n",
    "\n",
    "# add contour plot\n",
    "fig.add_contour(row=1, col=2)\n",
    "# set plot size\n",
    "fig.update_layout(\n",
    "    autosize=False,\n",
    "    width=1000,\n",
    "    height=500)\n",
    "# add first scatter plot \n",
    "fig.add_scatter(row=1, col=2)\n",
    "\n",
    "# add second scatter plot (path end point)\n",
    "fig.add_scatter(row=1, col=2)\n",
    "\n",
    "# add third scatter plot (cost history)\n",
    "fig.add_scatter(row=1, col=1)\n",
    "\n",
    "# create slider for learning rate\n",
    "slider = ipywidgets.SelectionSlider(options=learning_rates,description=\"learning rate\", orientation='horizontal')\n",
    "\n",
    "@interact(x=slider)\n",
    "# create interactive plot with slider that updates learning rate\n",
    "def update(x=learning_rates[0]):\n",
    "    with fig.batch_update():\n",
    "        # set contour plot colorbar title\n",
    "        fig.data[0].colorbar={\"title\": 'cost function'}\n",
    "        \n",
    "        # set contour plot data and axes\n",
    "        fig.data[0].x=theta0\n",
    "        fig.data[0].y=theta1\n",
    "        fig.data[0].z=cost.T\n",
    "        \n",
    "        # set grad descent path scatterplot point color and shape\n",
    "        fig.data[1].marker.color=\"red\"\n",
    "        fig.data[1].marker.size=8\n",
    "        fig.data[1].marker.symbol=1\n",
    "        fig.data[1].marker.line.width=1\n",
    "        fig.data[1].showlegend= False\n",
    "        # set grad desc path data\n",
    "        fig.data[1].x=theta_hist_dict[x][::10,0]\n",
    "        fig.data[1].y=theta_hist_dict[x][::10,1]\n",
    "        \n",
    "        # set grad descent path end point point color and shape\n",
    "        fig.data[2].marker.color=\"white\"\n",
    "        fig.data[2].marker.size=8\n",
    "        fig.data[2].marker.symbol=1\n",
    "        fig.data[2].marker.line.width=1\n",
    "        fig.data[2].showlegend= False\n",
    "        fig.data[2].mode='lines+markers+text'\n",
    "        fig.data[2].text='end'\n",
    "        fig.data[2].textposition='top right'\n",
    "        fig.data[2].textfont=dict(color='white')\n",
    "        \n",
    "        # set path end point data\n",
    "        fig.data[2].x=np.array(theta_hist_dict[x][::10,0][-1])\n",
    "        fig.data[2].y=np.array(theta_hist_dict[x][::10,1][-1])\n",
    "        \n",
    "        # cost history line plot\n",
    "         # set cost history scatterplot point color and shape\n",
    "        fig.data[3].marker.color=\"red\"\n",
    "        fig.data[3].marker.size=8\n",
    "        fig.data[3].marker.symbol=1\n",
    "        fig.data[3].marker.line.width=1\n",
    "        # set cost history data\n",
    "        fig.data[3].y=cost_hist_dict[x]\n",
    "        fig.data[3].showlegend= False # hide legend\n",
    "        \n",
    "        # generate title\n",
    "        title=\"learning rate = \" + str(x)\n",
    "        \n",
    "        # update title\n",
    "        fig.update_layout(\n",
    "            title={\n",
    "                'text': title,\n",
    "                'y':0.9,\n",
    "                'x':0.41,\n",
    "                'xanchor': 'center',\n",
    "                'yanchor': 'top'},\n",
    "            font=dict(\n",
    "                family=\"arial, monospace\",\n",
    "                size=16)\n",
    "        )\n",
    "        # update y-axis label distance\n",
    "        fig.update_yaxes(\n",
    "        title_standoff = 0)\n",
    "        \n",
    "\n",
    "fig"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient Descent Path Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "016391b28fd9424a800102301a8c4a32",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(SelectionSlider(description='learning rate', options=(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7,…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ad4b6dc19e5498888097ad5198283e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FigureWidget({\n",
       "    'data': [{'colorbar': {'title': {'text': 'cost function'}},\n",
       "              'type': 'contour'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# set plot range \n",
    "x_min, x_max = np.min(theta0), np.max(theta0)\n",
    "y_min, y_max = np.min(theta1), np.max(theta1)\n",
    "\n",
    "# create figure widget 1 \n",
    "fig = go.FigureWidget()\n",
    "# add contour plot\n",
    "fig.add_contour()\n",
    "# set plot size\n",
    "fig.update_layout(\n",
    "    autosize=False,\n",
    "    width=800,\n",
    "    height=640)\n",
    "# add scatter plot\n",
    "fig.add_scatter()\n",
    "\n",
    "# create slider for learning rate\n",
    "slider = ipywidgets.SelectionSlider(options=learning_rates,description=\"learning rate\", orientation='horizontal')\n",
    "\n",
    "@interact(x=slider)\n",
    "# create interactive plot with slider that updates learning rate\n",
    "def update(x=learning_rates[0]):\n",
    "    with fig.batch_update():\n",
    "        # set contour colorscale\n",
    "        #fig.data[0].colorscale = \"RdBu\" # set red blue colorscale\n",
    "       # fig.data[0].contours=dict(\n",
    "        #    size=0.05,\n",
    "         #   start=0,\n",
    "          #  end=1)\n",
    "        fig.data[0].colorbar={\"title\": 'cost function'}\n",
    "        # set contourplot data and axes \n",
    "        fig.data[0].z=cost.T\n",
    "        fig.data[0].x=theta0\n",
    "        fig.data[0].y=theta1\n",
    "\n",
    "\n",
    "        # set scatterplot point color and shape\n",
    "        fig.data[1].marker.color=\"red\"\n",
    "        #fig.data[1].marker.colorscale=[[0, 'rgb(255,0,0)'], [1, 'rgb(0,0,255)']]\n",
    "        fig.data[1].marker.size=8\n",
    "        fig.data[1].marker.symbol=1\n",
    "        fig.data[1].marker.line.width=1\n",
    "        # set grad desc path data\n",
    "        fig.data[1].x=theta_hist_dict[x][::10,0]\n",
    "        fig.data[1].y=theta_hist_dict[x][::10,1]\n",
    "\n",
    "        # generate title\n",
    "        title=\"learning rate = \" + str(x)\n",
    "        \n",
    "        # axes labels\n",
    "        fig.update_layout(\n",
    "            title={\n",
    "                'text': title,\n",
    "                'y':0.9,\n",
    "                'x':0.41,\n",
    "                'xanchor': 'center',\n",
    "                'yanchor': 'top'},\n",
    "            xaxis_title=\"intercept\",\n",
    "            yaxis_title=\"slope\",\n",
    "            font=dict(\n",
    "                family=\"arial, monospace\",\n",
    "                size=16)\n",
    "        )\n",
    "\n",
    "fig"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cost History Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f14eb34c729f48b5aa5fad897e3a74bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(SelectionSlider(description='learning rate', options=(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7,…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a59d3a6fb8f6470b99a80bfb405ac3e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FigureWidget({\n",
       "    'data': [{'marker': {'color': 'red', 'line': {'width': 1}, 'size': 8, 'symbol': 1},\n",
       "       …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# create figure widget\n",
    "fig2 = go.FigureWidget()\n",
    "\n",
    "# set plot size\n",
    "fig2.update_layout(\n",
    "    autosize=False,\n",
    "    width=800,\n",
    "    height=640)\n",
    "# add scatter plot\n",
    "fig2.add_scatter()\n",
    "\n",
    "# create slider for learning rate\n",
    "slider = ipywidgets.SelectionSlider(options=learning_rates,description=\"learning rate\", orientation='horizontal')\n",
    "\n",
    "@interact(x=slider)\n",
    "# create interactive plot with slider that updates learning rate\n",
    "def update(x=learning_rates[0]):\n",
    "    with fig2.batch_update():\n",
    "\n",
    "        # set scatterplot point color and shape\n",
    "        fig2.data[0].marker.color=\"red\"\n",
    "        #fig.data[1].marker.colorscale=[[0, 'rgb(255,0,0)'], [1, 'rgb(0,0,255)']]\n",
    "        fig2.data[0].marker.size=8\n",
    "        fig2.data[0].marker.symbol=1\n",
    "        fig2.data[0].marker.line.width=1\n",
    "        # set grad desc path data\n",
    "        fig2.data[0].y=cost_hist_dict[x]\n",
    "        \n",
    "        # generate title\n",
    "        title=\"cost function history, learning rate = \" + str(x)\n",
    "        \n",
    "        # axes labels\n",
    "        fig2.update_layout(\n",
    "            title={\n",
    "                'text': title,\n",
    "                'y':0.9,\n",
    "                'x':0.41,\n",
    "                'xanchor': 'center',\n",
    "                'yanchor': 'top'},\n",
    "            xaxis_title=\"nr. of iterations\",\n",
    "            yaxis_title=\"cost function\",\n",
    "            font=dict(\n",
    "                family=\"arial, monospace\",\n",
    "                size=16)\n",
    "        )\n",
    "\n",
    "fig2"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}