msan621_ml1/ML-Lecture7.1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Machine Learning 7.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From last class ...."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/Users/tlee010/Desktop/github_repos/fastai/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from fastai.imports import *\n",
    "from fastai.torch_imports import *\n",
    "from fastai.io import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load the MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = './'\n",
    "URL='http://deeplearning.net/data/mnist/' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-3.1638146e-07, 0.99999934)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FILENAME='mnist.pkl.gz'\n",
    "\n",
    "def load_mnist(filename):\n",
    "    return pickle.load(gzip.open(filename, 'rb'), encoding='latin-1')\n",
    "\n",
    "\n",
    "get_data(URL+FILENAME, path+FILENAME)\n",
    "((x, y), (x_valid, y_valid), _) = load_mnist(path+FILENAME)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Normalize the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-8.0261424e-09, 1.0)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean = x.mean()\n",
    "std = x.std()\n",
    "x=(x-mean)/std\n",
    "x.mean(), x.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.0058506131, 0.99243379)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_valid = (x_valid-mean)/std\n",
    "x_valid.mean(), x_valid.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 28, 28)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_imgs = np.reshape(x_valid, (-1,28,28)); x_imgs.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Start our simple NN model (From last lecture)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.metrics import *\n",
    "from fastai.model import *\n",
    "from fastai.dataset import *\n",
    "from fastai.core import *\n",
    "\n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Master Model: The basic NN model  w/pytorch libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = nn.Sequential(\n",
    "    nn.Linear(28*28, 10),\n",
    "    nn.LogSoftmax()\n",
    ")#.cuda() #<--- signals to run on the GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "md = ImageClassifierData.from_arrays(path, (x,y), (x_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 784)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define our loss and optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss=nn.NLLLoss()\n",
    "metrics=[accuracy]\n",
    "opt=optim.Adam(net.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def binary_loss(y,p):\n",
    "    return np.mean(-(y * np.log(p)+(1-y)*np.log(1-p)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.164252033486018"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acts_sample = np.array([1, 0, 0, 1])\n",
    "preds_sample = np.array([0.9, .1, .2, .8])\n",
    "binary_loss(acts_sample, preds_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e50a7d2b72914171ae53af5a967f884b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       0.29864  0.27374  0.92566]                         \n",
      "\n"
     ]
    }
   ],
   "source": [
    "fit(net, md, epochs=1, crit=loss, opt=opt, metrics=metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now let's rebuild this model several times \n",
    "\n",
    "#### (We will manually replace the prebuilt libraries with manual functions)\n",
    "\n",
    "```python\n",
    "net = nn.Sequential(\n",
    "    nn.Linear(28*28, 10),\n",
    "    nn.LogSoftmax()\n",
    ")\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Last week we remade the above model from scratch:\n",
    "\n",
    "**`(nn.module)`** <-- we are ***extending*** a pytorch class. we are borrowing all the methods from the standard module and will add some additional methods to it.\n",
    "\n",
    "**`super().__init__()`** <-- as a result, we have to create or instantiate the standard module first\n",
    "\n",
    "**`self.l1_w = get_weights(28*28, 10) `** is essentially the `ax` part of `y=ax + b`\n",
    "\n",
    "**`self.l1_b = get_weights(10) `** is essentially the `b` part of `y=ax + b`\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rewrite #1 Model - linear / softmax from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weights(*dims): return nn.Parameter(torch.randn(*dims)/dims[0])\n",
    "\n",
    "class LogReg(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.l1_w = get_weights(28*28, 10)  # Layer 1 weights\n",
    "        self.l1_b = get_weights(10)         # Layer 1 bias\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = torch.matmul(x, self.l1_w) + self.l1_b  # Linear Layer\n",
    "        x = torch.log(torch.exp(x)/(1 + torch.exp(x).sum(dim=0)))        # Non-linear (LogSoftmax) Layer\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Equivalent functions\n",
    "\n",
    "\n",
    "#### Original Torch\n",
    "```python\n",
    "net = nn.Sequential(\n",
    "    nn.Linear(28*28, 10), <-- (1)\n",
    "    nn.LogSoftmax()       <-- (2)\n",
    ")\n",
    "```\n",
    "#### Basic Python Version\n",
    "The linear portion of our original model is replicated here:\n",
    "```python\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.l1_w = get_weights(28*28, 10)  <--(1 equivalent)\n",
    "        self.l1_b = get_weights(10)         <--(1 equivalent)\n",
    "    \n",
    "    def forward(self, x):                   \n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = torch.matmul(x, self.l1_w) + self.l1_b                <--(1 equivalent)\n",
    "        x = torch.log(torch.exp(x)/(1 + torch.exp(x).sum(dim=0))) <--(2 equivalent)\n",
    "        return x\n",
    "    \n",
    "```\n",
    "\n",
    "#### Pytorch special method: forward\n",
    "\n",
    "**`forward`** - special hook in the pytorch library, how we are implementing each layer \n",
    "\n",
    "http://pytorch.org/tutorials/beginner/former_torchies/nn_tutorial.html#forward-and-backward-function-hooks\n",
    "\n",
    "#### Softmax activation:\n",
    "\n",
    "- We want probabilities at the end, values should be between 0 and 1\n",
    "- categorical predictions, where we only want to predict **1** of the class (not simultaneous classes, such as a cat and dog in the same picture aka. multilabel which would be sigmoid activation)\n",
    "- all the scores should add up to 1\n",
    "- Side note: Torch uses `torch.log` for numerical stability"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's Run our Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "net2 = LogReg()#.cuda()\n",
    "opt=optim.Adam(net2.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02921b5d3cb9424f9b3cd3468cfc1901",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       2.44347  2.40075  0.90953]                        \n",
      "\n"
     ]
    }
   ],
   "source": [
    "fit(net2, md, epochs=1, crit=loss, opt=opt, metrics=metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using our custom object manually\n",
    "#### first: get a mini batch of the data\n",
    "\n",
    "Will pull from the image generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "dl = iter(md.trn_dl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### since we called it, it will provide the next minibatch set\n",
    "\n",
    "Then we will save the data into the local variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "xmb, ymb = next(dl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Returns a tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "-0.4245 -0.4245 -0.4245  ...  -0.4245 -0.4245 -0.4245\n",
       "-0.4245 -0.4245 -0.4245  ...  -0.4245 -0.4245 -0.4245\n",
       "-0.4245 -0.4245 -0.4245  ...  -0.4245 -0.4245 -0.4245\n",
       "          ...             ⋱             ...          \n",
       "-0.4245 -0.4245 -0.4245  ...  -0.4245 -0.4245 -0.4245\n",
       "-0.4245 -0.4245 -0.4245  ...  -0.4245 -0.4245 -0.4245\n",
       "-0.4245 -0.4245 -0.4245  ...  -0.4245 -0.4245 -0.4245\n",
       "[torch.FloatTensor of size 64x784]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xmb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**`Variable`** - this is how we can get the differentiation. We are tracking operations on this variable, because later we will be using pytorch to do the differentiation. Note the different output of \n",
    "\n",
    "```python\n",
    "    Variable containing:\n",
    "    -0.4245 -0.4245 -0.4245 \n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make a wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "-0.4245 -0.4245 -0.4245  ...  -0.4245 -0.4245 -0.4245\n",
       "-0.4245 -0.4245 -0.4245  ...  -0.4245 -0.4245 -0.4245\n",
       "-0.4245 -0.4245 -0.4245  ...  -0.4245 -0.4245 -0.4245\n",
       "          ...             ⋱             ...          \n",
       "-0.4245 -0.4245 -0.4245  ...  -0.4245 -0.4245 -0.4245\n",
       "-0.4245 -0.4245 -0.4245  ...  -0.4245 -0.4245 -0.4245\n",
       "-0.4245 -0.4245 -0.4245  ...  -0.4245 -0.4245 -0.4245\n",
       "[torch.FloatTensor of size 64x784]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vxmb = Variable(xmb)\n",
    "vxmb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Syntax - treating our model like a function \n",
    "\n",
    "(which is calling the forward command that we implemented)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "\n",
       "Columns 0 to 7 \n",
       " -1.4970 -18.9267  -8.6922  -7.2008 -10.0914  -5.1697 -11.1822 -15.2741\n",
       " -9.0697 -11.9168  -6.6778  -6.3863  -1.6509  -8.4909  -7.1499  -7.1995\n",
       " -5.9551  -7.1476  -1.1607  -6.7816  -7.3475  -7.2832  -6.5625 -10.3792\n",
       " -4.7502 -11.2491  -6.9513  -4.1343  -7.2965  -7.5893 -11.0756  -2.1567\n",
       " -6.7426 -11.6235  -7.4818  -6.0834  -5.0911  -6.3521 -10.0404  -5.7750\n",
       " -1.8195 -10.2395  -6.6171  -7.6511  -7.9903  -4.1114  -2.8303  -9.8166\n",
       " -6.5060  -5.1258  -7.2731  -6.8241  -2.8162  -4.9901  -6.4188  -6.3444\n",
       " -3.8293 -10.2828  -6.5348  -2.0728 -11.5522  -2.4806  -6.7743 -15.9268\n",
       " -6.6867  -2.1434  -5.3255  -5.6275  -7.2797  -6.0414  -5.5714  -6.7775\n",
       " -6.4043  -2.1118  -5.7165  -5.4894  -7.3070  -5.9303  -5.9022  -5.7852\n",
       "\n",
       "Columns 8 to 9 \n",
       " -9.1331 -11.8261\n",
       " -4.7619  -5.0652\n",
       " -5.0495  -9.8351\n",
       " -8.4450  -5.4082\n",
       " -4.8309  -2.2167\n",
       " -7.8708 -11.7794\n",
       " -4.1470  -6.3982\n",
       " -6.4812 -12.6123\n",
       " -5.8198  -6.1695\n",
       " -5.2201  -5.4992\n",
       "[torch.FloatTensor of size 10x10]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = net2(vxmb)\n",
    "preds[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's look at our predictions for this minibatch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 0\n",
       " 4\n",
       " 2\n",
       " 7\n",
       " 9\n",
       " 0\n",
       " 4\n",
       " 3\n",
       " 1\n",
       " 1\n",
       " 4\n",
       " 0\n",
       " 6\n",
       " 2\n",
       " 1\n",
       " 8\n",
       " 3\n",
       " 8\n",
       " 5\n",
       " 3\n",
       " 9\n",
       " 3\n",
       " 6\n",
       " 3\n",
       " 0\n",
       " 2\n",
       " 8\n",
       " 5\n",
       " 2\n",
       " 5\n",
       " 6\n",
       " 3\n",
       " 9\n",
       " 8\n",
       " 9\n",
       " 7\n",
       " 7\n",
       " 1\n",
       " 6\n",
       " 8\n",
       " 7\n",
       " 9\n",
       " 7\n",
       " 8\n",
       " 1\n",
       " 9\n",
       " 8\n",
       " 4\n",
       " 7\n",
       " 1\n",
       " 2\n",
       " 7\n",
       " 9\n",
       " 9\n",
       " 6\n",
       " 1\n",
       " 3\n",
       " 7\n",
       " 6\n",
       " 6\n",
       " 4\n",
       " 9\n",
       " 1\n",
       " 1\n",
       "[torch.LongTensor of size 64]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds.max(1)[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now we can stack, to show that we are combining these different parts together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Example 1\n",
    "\n",
    "net = nn.Sequential(\n",
    "    nn.Linear(28*28, 100),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(100, 10),    \n",
    "    nn.LogSoftmax()\n",
    ")#.cuda() #<--- signals to run on the GPU\n",
    "\n",
    "## Example 2\n",
    "\n",
    "net = nn.Sequential(\n",
    "    nn.Linear(28*28, 100),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(100, 100),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(100, 100),\n",
    "    nn.ReLU(),    \n",
    "    nn.Linear(100, 100),\n",
    "    nn.ReLU(),    \n",
    "    nn.Linear(100, 10),    \n",
    "    nn.LogSoftmax()\n",
    ")#.cuda() #<--- signals to run on the GPU\n",
    "\n",
    "\n",
    "net = nn.Sequential(\n",
    "    nn.Linear(28*28, 100),\n",
    "    nn.Sigmoid(),\n",
    "    nn.Linear(100, 100),\n",
    "    nn.Sigmoid(),\n",
    "    nn.Linear(100, 100),\n",
    "    nn.Sigmoid(), \n",
    "    nn.Linear(100, 100),\n",
    "    nn.ReLU(),    \n",
    "    nn.Linear(100, 10),    \n",
    "    nn.LogSoftmax()\n",
    ")#.cuda() #<--- signals to run on the GPU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Activation is the value calculated in a layer\n",
    "\n",
    "- `Relu` - will tend to drop out negative values\n",
    "- `softmax` - will force many of the activations towards to zero.\n",
    "- `sigmoid` - will limit the values between `0` to `1`\n",
    "- `tanh` - very similar to sigmoid, but between `-1` and `1`\n",
    "- `LeakRelu` - will minimize negative values to a small small decimal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss=nn.NLLLoss()\n",
    "metrics=[accuracy]\n",
    "opt=optim.Adam(net.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc41294c4dfe4090bacf8de2cc5d750e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       0.21213  0.19673  0.94984]                        \n",
      "\n"
     ]
    }
   ],
   "source": [
    "fit(net, md, epochs=1, crit=loss, opt=opt, metrics=metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rewrite #2 Model - matrix multiplication from scratch\n",
    "\n",
    "### Review of python matrix methods and approaches\n",
    "\n",
    "<img src='https://cdn.lynda.com/video/161431-100-635594039429618029_338x600_thumb.jpg' />\n",
    "\n",
    "It's important to learn these concepts to reduce the number of loops and make efficient coding. SIMD - single instruction, multiple data. This is maximizing the CPU calculation. Also there's multiple cores, multiple threads. If you can distribute all these calculations, you take advantage of your computers full potential.\n",
    "\n",
    "GPUs are special processors can run even more things in parallel!\n",
    "\n",
    "<img src='https://image.slidesharecdn.com/tensordecomposition-170301235239/95/a-brief-survey-of-tensors-3-638.jpg?cb=1488412458' style='width:400px'/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = T([10, 6, -4])\n",
    "b = T([2, 8, 7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 12\n",
       " 14\n",
       "  3\n",
       "[torch.LongTensor of size 3]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 0\n",
       " 1\n",
       " 1\n",
       "[torch.ByteTensor of size 3]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a < b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Broadcast: Comparing Arrays against single values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 10\n",
       "  6\n",
       " -4\n",
       "[torch.LongTensor of size 3]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Broadcasting boolean comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1\n",
       " 1\n",
       " 0\n",
       "[torch.ByteTensor of size 3]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a > 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Broadcasting addition of a constant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 11\n",
       "  7\n",
       " -3\n",
       "[torch.LongTensor of size 3]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a + 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Broadcasting multiplication against a constant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1  2  3\n",
       " 4  5  6\n",
       " 7  8  9\n",
       "[torch.LongTensor of size 3x3]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = T([[1, 2, 3], [4,5,6], [7,8,9]]); m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "  2   4   6\n",
       "  8  10  12\n",
       " 14  16  18\n",
       "[torch.LongTensor of size 3x3]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m*2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Broad Casting vector addition against a matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 10\n",
       " 20\n",
       " 30\n",
       "[torch.LongTensor of size 3]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = T([10,20,30]); c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 1  2  3\n",
      " 4  5  6\n",
      " 7  8  9\n",
      "[torch.LongTensor of size 3x3]\n",
      "\n",
      "\n",
      " 10\n",
      " 20\n",
      " 30\n",
      "[torch.LongTensor of size 3]\n",
      "\n",
      "\n",
      " 11  22  33\n",
      " 14  25  36\n",
      " 17  28  39\n",
      "[torch.LongTensor of size 3x3]\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(m), print(c), print(m + c)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Python Numpy version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10, 20, 30])"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = np.array([[1, 2, 3], [4,5,6], [7,8,9]]); m\n",
    "c = np.array([10,20,30]); c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The numpy `expand_dims` method lets us convert the 1-dimensional array `c` into a 2-dimensional array (although one of those dimensions has value 1). This will allow us to use matrix to matrix operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[11, 22, 33],\n",
       "       [14, 25, 36],\n",
       "       [17, 28, 39]])"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m + np.expand_dims(c,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[11, 12, 13],\n",
       "       [24, 25, 26],\n",
       "       [37, 38, 39]])"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m + np.expand_dims(c,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[11, 22, 33],\n",
       "       [14, 25, 36],\n",
       "       [17, 28, 39]])"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m + c[None,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[11, 12, 13],\n",
       "       [24, 25, 26],\n",
       "       [37, 38, 39]])"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m + c[:,None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[10, 20, 30],\n",
       "       [10, 20, 30],\n",
       "       [10, 20, 30]])"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.broadcast_to(c, (3,3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Broadcasting Rules\n",
    "\n",
    "When operating on two arrays, Numpy/PyTorch compares their shapes element-wise. It starts with the **trailing dimensions**, and works its way forward. Two dimensions are **compatible** when\n",
    "\n",
    "- they are equal, or\n",
    "- one of them is 1\n",
    "\n",
    "Arrays do not need to have the same number of dimensions. For example, if you have a $256 \\times 256 \\times 3$ array of RGB values, and you want to scale each color in the image by a different value, you can multiply the image by a one-dimensional array with 3 values. Lining up the sizes of the trailing axes of these arrays according to the broadcast rules, shows that they are compatible:\n",
    "\n",
    "    Image  (3d array): 256 x 256 x 3\n",
    "    Scale  (1d array):             3\n",
    "    Result (3d array): 256 x 256 x 3\n",
    "\n",
    "The [numpy documentation](https://docs.scipy.org/doc/numpy-1.13.0/user/basics.broadcasting.html#general-broadcasting-rules) includes several examples of what dimensions can and can not be broadcast together.\n",
    "\n",
    "```python\n",
    "A      (2d array):  5 x 4\n",
    "B      (1d array):      1\n",
    "Result (2d array):  5 x 4\n",
    "\n",
    "A      (2d array):  5 x 4\n",
    "B      (1d array):      4\n",
    "Result (2d array):  5 x 4\n",
    "\n",
    "A      (3d array):  15 x 3 x 5\n",
    "B      (3d array):  15 x 1 x 5\n",
    "Result (3d array):  15 x 3 x 5\n",
    "\n",
    "A      (3d array):  15 x 3 x 5\n",
    "B      (2d array):       3 x 5\n",
    "Result (3d array):  15 x 3 x 5\n",
    "\n",
    "A      (3d array):  15 x 3 x 5\n",
    "B      (2d array):       3 x 1\n",
    "Result (3d array):  15 x 3 x 5\n",
    "```\n",
    "\n",
    "#### Some bad examples\n",
    "\n",
    "```python\n",
    "A      (1d array):  3\n",
    "B      (1d array):  4 # trailing dimensions do not match\n",
    "\n",
    "A      (2d array):      2 x 1\n",
    "B      (3d array):  8 x 4 x 3 # second from last dimensions mismatched\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10, 20, 30])"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[10, 20, 30]])"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c[None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[10],\n",
       "       [20],\n",
       "       [30]])"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c[:,None]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Note that since these are now considered 2 dim Matrix, these are broadcast operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[100, 200, 300],\n",
       "       [200, 400, 600],\n",
       "       [300, 600, 900]])"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c[None] * c[:,None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[100, 200, 300],\n",
       "       [200, 400, 600],\n",
       "       [300, 600, 900]])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c[:,None] * c[None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[0],\n",
       "        [1],\n",
       "        [2],\n",
       "        [3],\n",
       "        [4]]), array([[0, 1, 2, 3, 4]])]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xg, yg = np.ogrid[0:5,0:5]\n",
    "np.ogrid[0:5,0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1, 2, 3, 4],\n",
       "       [1, 2, 3, 4, 5],\n",
       "       [2, 3, 4, 5, 6],\n",
       "       [3, 4, 5, 6, 7],\n",
       "       [4, 5, 6, 7, 8]])"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xg+yg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[1, 2, 3],\n",
       "        [4, 5, 6],\n",
       "        [7, 8, 9]]), array([10, 20, 30]))"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m,c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([140, 320, 500])"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m @ c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 140\n",
       " 320\n",
       " 500\n",
       "[torch.LongTensor of size 3]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T(m)@ T(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 10,  40,  90],\n",
       "       [ 40, 100, 180],\n",
       "       [ 70, 160, 270]])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m * c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([140, 320, 500])"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(m * c).sum(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Matrix Matrix Product\n",
    "\n",
    "http://matrixmultiplication.xyz\n",
    "\n",
    "A nice visualization to assist in understanding matrix multiplication\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[10, 40],\n",
       "       [20,  0],\n",
       "       [30, -5]])"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n = np.array([[10,40],[20,0],[30,-5]]); n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[140,  25],\n",
       "       [320, 130],\n",
       "       [500, 235]])"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m @ n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([140, 320, 500])"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(m * n[:,0]).sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 25, 130, 235])"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(m * n[:,1]).sum(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rewrite #2 `fit` - matrix multiplication from scratch\n",
    "\n",
    "https://github.com/tensorly/tensorly\n",
    "\n",
    "#### The original master model\n",
    "\n",
    "\n",
    "```python\n",
    "class LogReg(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.l1_w = get_weights(28*28, 10)  # Layer 1 weights\n",
    "        self.l1_b = get_weights(10)         # Layer 1 bias\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = torch.matmul(x, self.l1_w) + self.l1_b \n",
    "        x = torch.log(torch.exp(x)/(1 + torch.exp(x).sum(dim=0)))\n",
    "        return x\n",
    "\n",
    "net2 = LogReg().cuda()\n",
    "opt=optim.Adam(net2.parameters())\n",
    "\n",
    "fit(net2, md, epochs=1, crit=loss_fn, opt=opt, metrics=metrics)\n",
    "```\n",
    "\n",
    "#### The first rewrite:\n",
    "\n",
    "```python\n",
    "def get_weights(*dims): return nn.Parameter(torch.randn(*dims)/dims[0])\n",
    "\n",
    "class LogReg(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.l1_w = get_weights(28*28, 10)  # Layer 1 weights\n",
    "        self.l1_b = get_weights(10)         # Layer 1 bias\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = torch.matmul(x, self.l1_w) + self.l1_b  # Linear Layer\n",
    "        x = torch.log(torch.exp(x)/(1 + torch.exp(x).sum(dim=0)))        # Non-linear (LogSoftmax) Layer\n",
    "        return x\n",
    "```\n",
    "\n",
    "### We want to focus on re-writing the `fit` of the model\n",
    "\n",
    "We want to pass the fit function mini batches at a time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "net2 = LogReg()#.cuda()\n",
    "loss_fn=nn.NLLLoss()\n",
    "learning_rate = 1e-3\n",
    "optimizer=optim.Adam(net2.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "dl = iter(md.trn_dl) #Data loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = next(dl)\n",
    "y_pred = net2(Variable(x))#.cuda())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 4.1750\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Compute and print loss.\n",
    "loss = loss_fn(y_pred, Variable(y))#.cuda())\n",
    "print(loss.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the gradient using pytorch inherited method `backward`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Before the backward pass, use the optimizer object to zero all of the\n",
    "# gradients for the variables it will update (which are the learnable weights\n",
    "# of the model)\n",
    "optimizer.zero_grad()\n",
    "\n",
    "# Backward pass: compute gradient of the loss with respect to model\n",
    "# parameters\n",
    "loss.backward()\n",
    "\n",
    "# Calling the step function on an Optimizer makes an update to its\n",
    "# parameters\n",
    "optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What's the stepping doing?\n",
    "\n",
    "<img src='https://sebastianraschka.com/images/faq/closed-form-vs-gd/ball.png' style='width:400px' />\n",
    "\n",
    "<img src = 'https://media.licdn.com/mpr/mpr/AAEAAQAAAAAAAA0oAAAAJDUzMTBlMDdjLWM0ZmMtNDJkNS1hODk3LTAzYTllMDUwZmY1OQ.jpg' style='width:400px' />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's do a single step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = next(dl)\n",
    "y_pred = net2.forward(Variable(x))#.cuda())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 2.3914\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Compute and print loss.\n",
    "\n",
    "loss = loss_fn(y_pred, Variable(y))#.cuda())\n",
    "print(loss.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's loop the steps\n",
    "\n",
    "Note: in pytorch you have to zero the gradients."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:  2.4622957706451416 \t accuracy:  0.859375\n",
      "loss:  2.4914746284484863 \t accuracy:  0.875\n",
      "loss:  2.5012717247009277 \t accuracy:  0.90625\n",
      "loss:  2.438032388687134 \t accuracy:  0.90625\n",
      "loss:  2.41378116607666 \t accuracy:  0.90625\n",
      "loss:  2.336564779281616 \t accuracy:  0.875\n",
      "loss:  2.5167903900146484 \t accuracy:  0.921875\n",
      "loss:  2.4636924266815186 \t accuracy:  0.890625\n",
      "loss:  2.624217987060547 \t accuracy:  0.84375\n",
      "loss:  2.524811267852783 \t accuracy:  0.90625\n"
     ]
    }
   ],
   "source": [
    "for t in range(100):\n",
    "    x, y = next(dl)\n",
    "    y_pred = net2(Variable(x))#.cuda())\n",
    "    loss = loss_fn(y_pred, Variable(y))#.cuda())\n",
    "    \n",
    "    if t % 10 == 0:\n",
    "        accuracy = np.sum(to_np(y_pred).argmax(axis=1) == to_np(y))/len(y_pred)\n",
    "        print(\"loss: \", loss.data[0], \"\\t accuracy: \", accuracy)\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}