pytorch/Workshop_Classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Workshop Classification.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/caocscar/workshops/blob/master/pytorch/Workshop_Classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aa28NQ4b50Wk",
        "colab_type": "text"
      },
      "source": [
        "# Classification Problem"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WK-LdzWl5vH6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        },
        "outputId": "0f560fe5-5a78-4942-950e-8f5661f81fd9"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import TensorDataset, DataLoader\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "print('Torch version', torch.__version__)\n",
        "print('Pandas version', pd.__version__)\n",
        "print('Numpy version', np.__version__)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Torch version 1.3.1\n",
            "Pandas version 0.25.3\n",
            "Numpy version 1.17.4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AKCLbDM754c0",
        "colab_type": "text"
      },
      "source": [
        "The following should say `cuda:0`. If it does not, we need to go to *Edit* -> *Notebook settings* and change it to a `GPU` from `None`. You only have to do this once per notebook."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a2RWBSbo53bz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "81fac650-e814-4b79-f433-a47d4d089dce"
      },
      "source": [
        "device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n",
        "device"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'cpu'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DhBlj7GI6Npt",
        "colab_type": "text"
      },
      "source": [
        "Read in dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VIjSGCNv53fT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_train = pd.read_csv('https://raw.githubusercontent.com/greght/Workshop-Keras-DNN/master/ChallengeProblems/iris_training.csv', header=None)\n",
        "df_val = pd.read_csv('https://raw.githubusercontent.com/greght/Workshop-Keras-DNN/master/ChallengeProblems/iris_test.csv', header=None)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0p_SPGXQ6PaD",
        "colab_type": "text"
      },
      "source": [
        "Construct our x,y variables along with the training and validation dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V5R_cuLZ53ib",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_train = df_train.iloc[:,0:-1]\n",
        "y_train = df_train.iloc[:,-1]\n",
        "x_val = df_val.iloc[:,0:-1]\n",
        "y_val = df_val.iloc[:,-1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "enL0Q9306QBM",
        "colab_type": "text"
      },
      "source": [
        "Preprocess our data to go from a `pandas` DataFrame to a `numpy` array to a `torch` tensor."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ASnI4ZrW53lj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "xtrain = torch.tensor(x_train.to_numpy(), device=device, dtype=torch.float, requires_grad=True)\n",
        "ytrain = torch.tensor(y_train.to_numpy(), device=device, dtype=torch.long, requires_grad=False)\n",
        "xval = torch.tensor(x_val.to_numpy(), device=device, dtype=torch.float, requires_grad=True)\n",
        "yval = torch.tensor(y_val.to_numpy(), device=device, dtype=torch.long, requires_grad=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ttYvEnkb6Qkb",
        "colab_type": "text"
      },
      "source": [
        "We'll write a python class to define out neural network."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RBZtZhgy6TCk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class FourLayerNN(nn.Module):\n",
        "    def __init__(self, D_in, H1, H2, H3, D_out):\n",
        "        super().__init__()\n",
        "        self.linear1 = nn.Linear(D_in, H1)\n",
        "        self.linear2 = nn.Linear(H1,H2)\n",
        "        self.linear3 = nn.Linear(H2,H3)\n",
        "        self.linear4 = nn.Linear(H3,D_out)\n",
        "        \n",
        "    def forward(self,x):\n",
        "        h1_relu = self.linear1(x).clamp(min=0)\n",
        "        h2_relu = self.linear2(h1_relu).clamp(min=0)\n",
        "        h3_relu = self.linear3(h2_relu).clamp(min=0)\n",
        "        y_pred = self.linear4(h3_relu)\n",
        "        return y_pred"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M54pEgD06RoL",
        "colab_type": "text"
      },
      "source": [
        "We create an instance of this class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iVdKpsuh6TS0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        },
        "outputId": "b83a76b4-a989-4f10-a52a-2f4857de6ed1"
      },
      "source": [
        "model = FourLayerNN(xtrain.shape[1],1000,500,70,y_train.nunique()).to(device)\n",
        "model"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "FourLayerNN(\n",
              "  (linear1): Linear(in_features=4, out_features=1000, bias=True)\n",
              "  (linear2): Linear(in_features=1000, out_features=500, bias=True)\n",
              "  (linear3): Linear(in_features=500, out_features=70, bias=True)\n",
              "  (linear4): Linear(in_features=70, out_features=3, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SFRVmzOR6SB7",
        "colab_type": "text"
      },
      "source": [
        "We'll define a template for our `fit_model` function that contains `train`,  `validate`, and `accuracy` functions."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gZnvxqPu53rs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def fit_model(model, loss_fn, optimizer):\n",
        "    def train(x,y):\n",
        "        yhat = model(x)\n",
        "        loss = loss_fn(yhat,y)\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        return loss.item(), accuracy(yhat,y)\n",
        "    \n",
        "    def validate(x,y):\n",
        "        yhat = model(x)\n",
        "        loss = loss_fn(yhat,y)\n",
        "        return loss.item(), accuracy(yhat,y)\n",
        "    \n",
        "    def accuracy(yhat,y):\n",
        "        probs = np.argmax(yhat.cpu().detach().numpy(), axis=1)\n",
        "        actual = y.cpu().detach().numpy()\n",
        "        correct = (probs == actual).sum()\n",
        "        total = y.shape[0]\n",
        "        return correct / total   \n",
        "    \n",
        "    return train, validate"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qCrMhx8Q6TLd",
        "colab_type": "text"
      },
      "source": [
        "We define our *loss function*, *learning rate*, and our *optimizer*. We pass this to `fit_model` to return our `train` and `validate` functions."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XFBR4YbD53oz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "loss_fn = nn.CrossEntropyLoss()\n",
        "learning_rate = 0.01\n",
        "optimizer = optim.Adagrad(model.parameters(), lr=learning_rate)\n",
        "train, validate = fit_model(model, loss_fn, optimizer)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ME_plDOp6Slt",
        "colab_type": "text"
      },
      "source": [
        "Define a `DataLoader` for our mini-batches."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5SS1NgRs6Syz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_data = TensorDataset(xtrain, ytrain)\n",
        "train_loader = DataLoader(dataset=train_data, batch_size=60, shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XNVzkMZI6Tam",
        "colab_type": "text"
      },
      "source": [
        "Here is our training loop with mini-batch processing. We have to move each batch onto the GPU. We also should have a `DataLoader` for the validation dataset but we'll skip that in this case since it is so small."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AKk5nZjM6Ths",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 353
        },
        "outputId": "adecf0cb-e8fa-4f20-d18b-2ac258a72d8f"
      },
      "source": [
        "epochs = 2000\n",
        "for epoch in range(epochs):\n",
        "    # training\n",
        "    losses = []\n",
        "    for i, (xbatch, ybatch) in enumerate(train_loader):\n",
        "        xbatch = xbatch.to(device)\n",
        "        ybatch = ybatch.to(device)\n",
        "        loss, accuracy = train(xbatch, ybatch)\n",
        "        losses.append(loss)\n",
        "    training_loss = np.mean(losses)\n",
        "    training_accuracy = np.mean(accuracy)\n",
        "    # validation\n",
        "    validation_loss, validation_accuracy = validate(xval, yval)\n",
        "    # print intermediate results\n",
        "    if epoch%100 == 99:\n",
        "        print(f'{epoch}, {training_loss:.4f}, {training_accuracy:.2f}, {validation_loss:.4f}, {accuracy:.2f}')"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "99, 0.0790, 0.97, 0.0645, 0.97\n",
            "199, 0.0817, 0.97, 0.0577, 0.97\n",
            "299, 0.0537, 1.00, 0.0652, 1.00\n",
            "399, 0.0497, 0.98, 0.0516, 0.98\n",
            "499, 0.0403, 1.00, 0.0566, 1.00\n",
            "599, 0.0382, 0.98, 0.0541, 0.98\n",
            "699, 0.0382, 0.98, 0.0578, 0.98\n",
            "799, 0.0355, 0.98, 0.0596, 0.98\n",
            "899, 0.0338, 0.98, 0.0643, 0.98\n",
            "999, 0.0385, 1.00, 0.0620, 1.00\n",
            "1099, 0.0339, 1.00, 0.0672, 1.00\n",
            "1199, 0.0327, 1.00, 0.0677, 1.00\n",
            "1299, 0.0293, 1.00, 0.0716, 1.00\n",
            "1399, 0.0293, 1.00, 0.0717, 1.00\n",
            "1499, 0.0290, 1.00, 0.0738, 1.00\n",
            "1599, 0.0267, 1.00, 0.0826, 1.00\n",
            "1699, 0.0280, 1.00, 0.0815, 1.00\n",
            "1799, 0.0274, 1.00, 0.0912, 1.00\n",
            "1899, 0.0253, 0.98, 0.1166, 0.98\n",
            "1999, 0.0249, 1.00, 0.0899, 1.00\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1AudSc0uAqt9",
        "colab_type": "text"
      },
      "source": [
        "### nn.Sequential\n",
        "\n",
        "If we wanted to user the simpler `nn.Sequential` function, our model construction would have looked like this."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JlIMzvDyAq3U",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "outputId": "d4987403-3fbd-48ea-bcd7-06fbbc112df7"
      },
      "source": [
        "model_sequential = nn.Sequential(\n",
        "    nn.Linear(xtrain.shape[1],1000),\n",
        "    nn.ReLU(),\n",
        "    nn.Linear(1000,500),\n",
        "    nn.ReLU(),\n",
        "    nn.Linear(500,70),\n",
        "    nn.ReLU(),\n",
        "    nn.Linear(70,y_train.nunique()),\n",
        ").to(device)\n",
        "print(model_sequential)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sequential(\n",
            "  (0): Linear(in_features=4, out_features=1000, bias=True)\n",
            "  (1): ReLU()\n",
            "  (2): Linear(in_features=1000, out_features=500, bias=True)\n",
            "  (3): ReLU()\n",
            "  (4): Linear(in_features=500, out_features=70, bias=True)\n",
            "  (5): ReLU()\n",
            "  (6): Linear(in_features=70, out_features=3, bias=True)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}