00. PyTorch_hands_on.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PyTorch hands-on Tutorials"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 02 - Tensor Basics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initializing a tensor with values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.0078]) is an empty tensor and not initialized\n"
     ]
    }
   ],
   "source": [
    "x = torch.empty(1)\n",
    "print(f\"{x} is an empty tensor and not initialized\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.1719, 0.9470],\n",
      "        [0.6675, 0.7632]]) is a 2x2 tensor of random numbers\n",
      "tensor([[0., 0.],\n",
      "        [0., 0.]]) is a 2x2 tensor of zeros\n",
      "tensor([[1., 1.],\n",
      "        [1., 1.]]) is a 2x2 tensor of ones\n"
     ]
    }
   ],
   "source": [
    "x_rand=torch.rand(2,2)\n",
    "x_zeros=torch.zeros(2,2)\n",
    "x_ones=torch.ones(2,2)\n",
    "print(f\"{x_rand} is a 2x2 tensor of random numbers\")\n",
    "print(f\"{x_zeros} is a 2x2 tensor of zeros\")\n",
    "print(f\"{x_ones} is a 2x2 tensor of ones\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensor properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.float16 returns the data type\n",
      "torch.Size([2, 2]) returns the matrix size\n",
      "cpu returns the device used to perform operations\n"
     ]
    }
   ],
   "source": [
    "x=torch.rand(2,2, dtype=torch.float16)\n",
    "print(f\"{x.dtype} returns the data type\")\n",
    "print(f\"{x.size()} returns the matrix size\")\n",
    "print(f\"{x.device} returns the device used to perform operations\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensor Element wise operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Element wise addition, either by x+y or torch.add(x,y)\n",
      "tensor([[1.5366, 0.7826],\n",
      "        [0.7602, 0.5329]])\n"
     ]
    }
   ],
   "source": [
    "x = torch.rand(2,2)\n",
    "y = torch.rand(2,2)\n",
    "z1 = x + y\n",
    "print(\"Element wise addition, either by x+y or torch.add(x,y)\")\n",
    "z1 = torch.add(x,y)\n",
    "print(f\"{z1}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Element wise subtraction, either by x-y or torch.sub(x,y)\n",
      "tensor([[ 0.4398,  0.4125],\n",
      "        [-0.3239, -0.1546]])\n"
     ]
    }
   ],
   "source": [
    "print(\"Element wise subtraction, either by x-y or torch.sub(x,y)\")\n",
    "z2 = x + y\n",
    "z2 = torch.sub(x,y)\n",
    "print(f\"{z2}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Element wise multiplication, either by x*y or torch.mul(x,y)\n",
      "tensor([[0.5419, 0.1106],\n",
      "        [0.1183, 0.0650]])\n"
     ]
    }
   ],
   "source": [
    "print(\"Element wise multiplication, either by x*y or torch.mul(x,y)\")\n",
    "z3 = x * y\n",
    "z3 = torch.mul(x,y)\n",
    "print(f\"{z3}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Element wise division, either by x/y or torch.div(x,y)\n",
      "tensor([[1.8021, 3.2294],\n",
      "        [0.4025, 0.5502]])\n"
     ]
    }
   ],
   "source": [
    "print(f\"Element wise division, either by x/y or torch.div(x,y)\")\n",
    "z4 = x / y\n",
    "z4 = torch.div(x,y)\n",
    "print(f\"{z4}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensor slicing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.4604, 0.6807, 0.7354],\n",
      "        [0.4883, 0.9258, 0.6992],\n",
      "        [0.6147, 0.0610, 0.4863],\n",
      "        [0.9785, 0.1011, 0.8472],\n",
      "        [0.7676, 0.2988, 0.8057]], dtype=torch.float16)\n",
      "\n",
      "Slicing the second row\n",
      "tensor([0.4883, 0.9258, 0.6992], dtype=torch.float16)\n"
     ]
    }
   ],
   "source": [
    "t = torch.rand(5,3, dtype=torch.float16)\n",
    "print(t)\n",
    "print(\"\")\n",
    "print(\"Slicing the second row\")\n",
    "print(t[1,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Slicing the 5th row, 2nd column item, then extracting the value\n",
      "tensor(0.2988, dtype=torch.float16)\n",
      "0.298828125\n"
     ]
    }
   ],
   "source": [
    "print(\"Slicing the 5th row, 2nd column item, then extracting the value\")\n",
    "print(t[4,1])\n",
    "print(t[4,1].item())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Resizing Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.0478, 0.4924, 0.1920, 0.5781],\n",
      "        [0.7363, 0.1883, 0.7280, 0.3224],\n",
      "        [0.9769, 0.9024, 0.8114, 0.6090]])\n",
      "tensor([[0.0478, 0.4924, 0.1920],\n",
      "        [0.5781, 0.7363, 0.1883],\n",
      "        [0.7280, 0.3224, 0.9769],\n",
      "        [0.9024, 0.8114, 0.6090]])\n",
      "tensor([0.0478, 0.4924, 0.1920, 0.5781, 0.7363, 0.1883, 0.7280, 0.3224, 0.9769,\n",
      "        0.9024, 0.8114, 0.6090])\n"
     ]
    }
   ],
   "source": [
    "f = torch.rand(3,4)\n",
    "print(f)\n",
    "f1 = f.view(4,3)\n",
    "print(f1)\n",
    "f2 = f.view(-1)\n",
    "print(f2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When using -1 as dimension, the multidimensional tensor is transformed to a 1D-row vector that fits the appropriate size automatically "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Numpy Arrays to PyTorch tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n",
      "tensor([1., 1., 1., 1.], device='cuda:0', dtype=torch.float64)\n",
      "tensor([1., 1., 1., 1.], device='cuda:0')\n",
      "tensor([2., 2., 2., 2.], device='cuda:0', dtype=torch.float64)\n"
     ]
    }
   ],
   "source": [
    "from matplotlib.style import available\n",
    "a = np.ones(4)\n",
    "b = torch.from_numpy(a)\n",
    "print(b.device)\n",
    "if torch.cuda.is_available():\n",
    "    device=torch.device(\"cuda\")\n",
    "b = b.to(device)\n",
    "c = torch.ones(4, device=device)\n",
    "print(b)\n",
    "print(c)\n",
    "print(b+c)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can move a tensor to the GPU by using   \n",
    "a) tensor_name = tensor_name.to(torch.device(\"cuda\"))  \n",
    "b) tensord_name = torch.tensor(size, device=\"cuda\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 03 - Gradient Calculation with Autograd\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient Calculation with Autograd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.2271, 0.3016, 0.9031], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "x = torch.rand(3, requires_grad=True)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are creating a graph that keeps track of the gradients\n",
    "of x, so we can run backpropagation later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([2.2271, 2.3016, 2.9031], grad_fn=<AddBackward0>)\n",
      "tensor(12.4571, grad_fn=<MeanBackward0>)\n",
      "tensor([2.9695, 3.0688, 3.8708])\n"
     ]
    }
   ],
   "source": [
    "y = x + 2\n",
    "print(y)\n",
    "z = y*y*2\n",
    "z = z.mean()\n",
    "print(z)\n",
    "z.backward() #dz = dz/dy * dy/dx\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we demonstrate that variables that depend on tensor x\n",
    "carry the gradients and each operation has a relative backward operation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([2.2271, 2.3016, 2.9031], grad_fn=<AddBackward0>)\n",
      "tensor([ 3.8601, 12.2753,  3.8824])\n"
     ]
    }
   ],
   "source": [
    "y = x + 2\n",
    "print(y)\n",
    "f = y*y*2\n",
    "v = torch.tensor([0.1, 1.0, 0.001], dtype=torch.float16)\n",
    "f.backward(v)\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If z is not a scalar value we need to pass the vector of values that need to be packpropagated"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Disabling the gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.4776, 0.2055, 0.5134], requires_grad=True)\n",
      "tensor([0.4776, 0.2055, 0.5134])\n"
     ]
    }
   ],
   "source": [
    "x = torch.rand(3, requires_grad=True)\n",
    "print(x)\n",
    "#x.requires_grad_(False) # in-place operation ->NOT PREFERED\n",
    "y = x.detach()\n",
    "print(y)\n",
    "# with torch.no_grad():"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([3., 3., 3.])\n",
      "tensor([3., 3., 3.])\n",
      "tensor([3., 3., 3.])\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(3):\n",
    "    model_out = (x*3).sum()\n",
    "    model_out.backward()\n",
    "    print(x.grad)\n",
    "    x.grad.zero_()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When oprimizing our weights, at the end of each iteration we need to reset them as to not carry over previous values that will interfere with our calculations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 04 - Backpropagation - Theory with Example\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Backpropagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1., grad_fn=<PowBackward0>)\n",
      "tensor(-2.)\n"
     ]
    }
   ],
   "source": [
    "x1 = torch.tensor(1.0)\n",
    "y = torch.tensor(2.0)\n",
    "w = torch.tensor(1.0, requires_grad=True)\n",
    "\n",
    "# forward pass and loss computation\n",
    "y_hat = w*x1\n",
    "loss =(y_hat - y)**2\n",
    "print(loss)\n",
    "\n",
    "# backward pass\n",
    "loss.backward()\n",
    "print(w.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When backpropagating tensors that require gradient are optimized for, using the chain rule, whereas others that don't have a gradient are treated as constants"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Manual implementation of a NN including: forward pass, loss, gradient, backward pass using Numpy.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([1,2,3,4], dtype = np.float16)\n",
    "Y = np.array([2,4,6,8], dtype = np.float16)\n",
    "w = 0.0\n",
    "\n",
    "# model prediction\n",
    "def forward(x):\n",
    "    return w*x \n",
    "\n",
    "# loss = MSE\n",
    "def loss(y, y_predicted):\n",
    "    return ((y_predicted-y)**2).mean()\n",
    "\n",
    "# gradient\n",
    "# MSE = 1/N * (w*x - y)**2\n",
    "# dJ/dw = 1/N * 2 (w*x - y)\n",
    "def gradient(x, y, y_predicted):\n",
    "    return np.dot(2*x, y_predicted-y).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction before training: f(5) = 0.000\n",
      "epoch 1: w = 1.200, loss = 30.00000000\n",
      "epoch 2: w = 1.680, loss = 4.79687500\n",
      "epoch 3: w = 1.872, loss = 0.76953125\n",
      "epoch 4: w = 1.949, loss = 0.12255859\n",
      "epoch 5: w = 1.980, loss = 0.02015686\n",
      "epoch 6: w = 1.992, loss = 0.00318527\n",
      "epoch 7: w = 1.997, loss = 0.00045776\n",
      "epoch 8: w = 1.998, loss = 0.00006032\n",
      "epoch 9: w = 2.000, loss = 0.00003529\n",
      "epoch 10: w = 2.000, loss = 0.00000000\n",
      "Prediction after training: f(5) = 9.999\n"
     ]
    }
   ],
   "source": [
    "print(f\"Prediction before training: f(5) = {forward(5):.3f}\")\n",
    "learning_rate = 0.01\n",
    "n_iters = 10\n",
    "\n",
    "for epoch in range(n_iters):\n",
    "    # prediction = forward pass\n",
    "    y_pred = forward(X)\n",
    "\n",
    "    # loss\n",
    "    l=loss(Y, y_pred)\n",
    "\n",
    "    # gradients\n",
    "    dw = gradient(X, Y, y_pred)\n",
    "\n",
    "    #update weights\n",
    "    w-= learning_rate * dw\n",
    "    if epoch % 1 == 0:\n",
    "        print(f\"epoch {epoch + 1}: w = {w:.3f}, loss = {l:.8f}\")\n",
    "\n",
    "print(f\"Prediction after training: f(5) = {forward(5):.3f}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 05 - Gradient Descent with Autograd and Backpropagation\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stochastic Gradient Descent with Backporpagation using Autograd in PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction before training: f(5) = 0.000\n",
      "epoch 1: w = 0.300, loss = 30.00000000\n",
      "epoch 6: w = 1.245, loss = 5.91015625\n",
      "epoch 11: w = 1.665, loss = 1.16113281\n",
      "epoch 16: w = 1.852, loss = 0.22888184\n",
      "epoch 21: w = 1.934, loss = 0.04577637\n",
      "epoch 26: w = 1.970, loss = 0.00926971\n",
      "epoch 31: w = 1.986, loss = 0.00183105\n",
      "epoch 36: w = 1.994, loss = 0.00034070\n",
      "epoch 41: w = 1.997, loss = 0.00006032\n",
      "Prediction after training: f(5) = 9.984\n"
     ]
    }
   ],
   "source": [
    "X = torch.tensor([1,2,3,4], dtype = torch.float16)\n",
    "Y = torch.tensor([2,4,6,8], dtype = torch.float16)\n",
    "w = torch.tensor(0.0, dtype = torch.float16, requires_grad=True)\n",
    "\n",
    "# model prediction\n",
    "def forward(x):\n",
    "    return w*x \n",
    "\n",
    "# loss = MSE\n",
    "def loss(y, y_predicted):\n",
    "    return ((y_predicted-y)**2).mean()\n",
    "\n",
    "print(f\"Prediction before training: f(5) = {forward(5):.3f}\")\n",
    "learning_rate = 0.01\n",
    "n_iters = 41\n",
    "\n",
    "for epoch in range(n_iters):\n",
    "    # prediction = forward pass\n",
    "    y_pred = forward(X)\n",
    "\n",
    "    # loss\n",
    "    l=loss(Y, y_pred)\n",
    "\n",
    "    # gradients = backward pass\n",
    "    l.backward()\n",
    "    \n",
    "    #update weights\n",
    "    with torch.no_grad():\n",
    "        w-= learning_rate * w.grad\n",
    "    \n",
    "    # zero gradients\n",
    "    w.grad.zero_()\n",
    "    if epoch % 5 == 0:\n",
    "        print(f\"epoch {epoch + 1}: w = {w:.3f}, loss = {l:.8f}\")\n",
    "\n",
    "print(f\"Prediction after training: f(5) = {forward(5):.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 06 - Training Pipeline: Model, Loss and Optimizer   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Complete PyTorch Pipeline: Model, Loss and Optimizer   \n",
    "Prediction: PyTorch Model  \n",
    "Gradient Computation: Autograd  \n",
    "Loss Computation: PyTorch Loss  \n",
    "Parameter updates: PyTorch Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 1\n",
      "[('lin.weight', Parameter containing:\n",
      "tensor([[-0.8817]], device='cuda:0', requires_grad=True)), ('lin.bias', Parameter containing:\n",
      "tensor([-0.8608], device='cuda:0', requires_grad=True))]\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[1;31mSystemExit\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Harry Soteriou\\miniconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3449: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "1. Design model (input, output size, forward pass)\n",
    "2. Construct loss and optimizer\n",
    "3, Training loop\n",
    "    -forward pass: compute predition\n",
    "    -backward pass: gradients\n",
    "    -update weights\n",
    "\"\"\"\n",
    "import torch \n",
    "import torch.nn as nn\n",
    "\n",
    "X = torch.tensor([[1],[2],[3],[4]], dtype = torch.float32, device=\"cuda\")\n",
    "Y = torch.tensor([[2],[4],[6],[8]], dtype = torch.float32, device=\"cuda\")\n",
    "\n",
    "X_test = torch.tensor([5], dtype = torch.float32, device=\"cuda\")\n",
    "\n",
    "n_samples, n_features = X.shape\n",
    "print(n_samples, n_features)\n",
    "\n",
    "input_size = n_features\n",
    "output_size = n_features\n",
    "\n",
    "#model = nn.Linear(in_features=input_size, out_features=output_size).to(device=\"cuda\")\n",
    "\n",
    "class LinearRegression(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim):\n",
    "        super(LinearRegression, self).__init__()\n",
    "        # define layers\n",
    "        self.lin = nn.Linear(input_dim, output_dim)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        return self.lin(x)\n",
    "\n",
    "model = LinearRegression(input_size, output_size).to(device=\"cuda\")\n",
    "print(f\"Prediction before training: f(5) = {model(X_test).item():.3f}\")\n",
    "learning_rate = 0.01\n",
    "n_iters = 51\n",
    "loss = nn.MSELoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n",
    "\n",
    "for epoch in range(n_iters):\n",
    "    # prediction = forward pass\n",
    "    y_pred = model(X)\n",
    "\n",
    "    # loss\n",
    "    l=loss(Y, y_pred)\n",
    "\n",
    "    # gradients = backward pass\n",
    "    l.backward()\n",
    "    \n",
    "    #update weights\n",
    "    optimizer.step()\n",
    "    \n",
    "    # zero gradients\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    if epoch % 5 == 0:\n",
    "        [w,b] = model.parameters()\n",
    "        print(f\"epoch {epoch + 1}: w = {w[0][0].item():.3f}, loss = {l:.8f}\")\n",
    "\n",
    "print(f\"Prediction after training: f(5) = {model(X_test).item():.3f}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 07 - Linear Regression "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implementing a Linear Regression model from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 10, loss =582.9234\n",
      "epoch: 20, loss =341.1050\n",
      "epoch: 30, loss =332.8644\n",
      "epoch: 40, loss =332.5780\n",
      "epoch: 50, loss =332.5679\n",
      "epoch: 60, loss =332.5676\n",
      "epoch: 70, loss =332.5676\n",
      "epoch: 80, loss =332.5676\n",
      "epoch: 90, loss =332.5676\n",
      "epoch: 100, loss =332.5676\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD7CAYAAACCEpQdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjYUlEQVR4nO3df5RcZZ3n8fc3TYIEUKHTMpikuxkmIEFHRtoMDqPDLDogZ53wY9nBbWIUnQyCjrLOEZieszs7sz2Dx13GOPLDqMiPtLIcfkjOkVFBVA4MCM2IkACRAOnQkgMhYYaYQH50vvvHvZW+t+requrqW3Wr+n5e5/TpqqduVT30Id966vt8n+cxd0dERIplVt4dEBGR1lPwFxEpIAV/EZECUvAXESkgBX8RkQJS8BcRKaBpB38zW2hmPzGzp8xsnZl9Lmw/3MzuNrNnwt+HRZ5zuZltMLP1ZnbadPsgIiJTY9Ot8zezI4Ej3f3fzOxQ4FHgTODjwDZ3v8LMLgMOc/dLzWwx8F1gCfB24B7gGHefmFZHRESkbgdM9wXcfTOwOby93cyeAuYDS4FTwstuAH4KXBq23+zuu4DnzWwDwQfBg9XeZ968ed7f3z/d7oqIFMqjjz76irv3lLdPO/hHmVk/8HvAz4Ejwg8G3H2zmb0tvGw+8FDkaeNhW1X9/f2Mjo5m2V0RkRnPzMaS2jOb8DWzQ4DbgM+7+2vVLk1oS8w9mdkKMxs1s9EtW7Zk0U0RESGj4G9mswkC/4i73x42vxTOB5TmBV4O28eBhZGnLwBeTHpdd1/l7gPuPtDTU/GtRUREGpRFtY8B3wKecvcrIw+tAZaHt5cDd0bazzOzA83sKGAR8PB0+yEiIvXLIud/MrAMeMLMHgvb/hq4ArjFzD4JbALOBXD3dWZ2C/AksBe4WJU+IiKtlUW1z/0k5/EBTk15zjAwPN33FhGRxmiFr4hIASn4i4gUkIK/iEi5kRHo74dZs4LfIyO5dOPWW4OfZsh0kZeISMcbGYEVK2DnzuD+2FhwH2BwsCVd2LoV5s2bvL+vtx/7h+FM318jfxGRqKGhycBfsnNn0N4Cl1wSD/zrOQbbFH4AZfgNRMFfRCRq06aptWfkl78EM/jKV4L7/4v/gWMcwzNBQ8YfQEr7iIhE9fYGqZ6k9ibYtQve9KbJ+7Nnw9Y9b+ZQtldenOEHkEb+IiJRw8Mwd268be7coD1jZ54ZD/y33Qa7d8OhfYcnPyHDDyAFfxGRqMFBWLUK+vqCPExfX3A/w8nWZ54JXvrOOyfb9u6Fs88O77TgA0jBX0Sk3OAgbNwI+/YFvzMM/GZwzDGT99esAXfo6ip7/yZ/ACn4i4i0wE03BXG8ZNasIOh/5LWUNQVN/AACTfiKiDTV7t1w4IHxtvFxmD+fXNcUaOQvItIk55wTD/yf+lQw2p9fOrswxzUFGvmLiGRswwZYtCjetndvWV4fcltTABr5i4hkyiwe+L/3vYQJ3ZK00s0mrSmIUvAXEcnAd74Tn9CFIOgvXVrlSS1cU1BOaR8RkWlImtDdtAkWLky+PqY0qTs0FDyptzcI/C3YQC6rA9yvM7OXzWxtpO1vzezXZvZY+HNG5LHLzWyDma03s9Oy6IOISEOmsX3zuefGA/8FFwSj/boCf0mTSzrTZJX2uR44PaH9n9z9hPDnLgAzWwycBxwfPudqM0vKhomINFep1HJsLIjaYwm7ZyZ8ODz3XJDiie61v2cPfOtbrf4PaFwmwd/d7wO21Xn5UuBmd9/l7s8DG4AlWfRDRGRKapVaJnw42PmDHH305OW33x48dECHJdGbPeH7GTN7PEwLHRa2zQdeiFwzHraJiLRWrVLLyIfDzfwZhscuc4ezzmpmB5unmcH/GuBo4ARgM/B/w3ZLuNYT2jCzFWY2amajW7ZsaUonRaQA0vL6tUotN23iDQ7EcD7KzfsfHqMPT4xanaNpwd/dX3L3CXffB3yDydTOOBCdDlkAvJjyGqvcfcDdB3p6eprVVRGZyarl9WuUWs7xNziIN/Y/tIwbcYzevqQxbGdpWpbKzI50983h3bOAUiXQGuA7ZnYl8HZgEfBws/ohIgVXLa+/cePkNZFSy3uPHORUA5iz/ym7mMMc9rSsDr/ZMgn+ZvZd4BRgnpmNA/8TOMXMTiBI6WwE/gLA3deZ2S3Ak8Be4GJ3n8iiHyIiFWrl9QcHY+WV5Qu1/v6//JK/eWQpbNoLvX0tq8NvNvMOSVwNDAz46Oho3t0QkU7T3598LGNf3+TIHzjuOHj66fglHRIeqzKzR919oLxd2zuIyMxWI6//2mvBaD8a+O+9d2YE/moU/EVkZqtyKpYZvOUt8cu9r58/PnXqq307jYK/iMx8ZVso3LB3sCK3/9o3/h8+9+Dqq31nkA5bkyYiMj3lQX/WLJiYAPovTa8KmgETvOUU/EWkEMqDPpTl9XM8WCUPSvuIyIxWmtCN+va3EyZ0czxYJQ8K/iLSuGlsh9wKiRO6Dh//eMLFOR6skgcFfxFpTD3bIefkqqsqR/uvvlqjfLNKVdBMpEVeItKYOhdPtVrN3H7BaJGXiExfNM2TFPgh2wnSKaSVzJLP0C1y4K9GwV9E6lOe5kmT1QRpnWml7dsrg/6Xv6ygX4vSPiJSn7Q0T9TcudnlyetIKynFU5vSPiIyPdXSOc2YIK1Sd//1r1cG/q1bFfinQou8RKQ+vb2tneBNeT/zfXBhvE1Bf+o08heR+rS6Dr7s/QxPPENXgb8xCv4iUp9W18GH77dj4Tsqgv7wcI2g3+aLz9qBJnxFpG01NKFbqhKKbtKW5UR0h2nqhK+ZXWdmL5vZ2kjb4WZ2t5k9E/4+LPLY5Wa2wczWm9lpWfRBRDKW4+j5K1+pDPxbttSZ4ql2Zq/sl1Xa53rg9LK2y4Afu/si4MfhfcxsMXAecHz4nKvNrCujfohIFlq1dcPICMybN7lCa948zOCSS+KXuQeX1aVgu3M2KpPg7+73AdvKmpcCN4S3bwDOjLTf7O673P15YAOwJIt+iEhGWjF6HhmBT3wiqNEknNDd+krskoYmdAu2O2ejmjnhe4S7bwYIf78tbJ8PvBC5bjxsE5F20YrR89AQ7NnDaxxaMaG7jBvxvv7GvmkUbHfORuVR558whUPiZ7uZrQBWAPTqU1ukddJq+rP8d7hpU0XQB/BSiBgjSDXB1CZqS9cODQUfVr29QeAv4GRvNc0c+b9kZkcChL9fDtvHgYWR6xYALya9gLuvcvcBdx/o6elpYldFJKbJo+fly8PFWhFj9E4G/pJGU01lZ/Yq8FdqZvBfAywPby8H7oy0n2dmB5rZUcAi4OEm9kNEpqqJNf1mcOON8TbH6I1lgyM0UdsUWZV6fhd4EDjWzMbN7JPAFcCHzOwZ4EPhfdx9HXAL8CTwA+Bid5/Ioh8ikqGMR8+JWy6vHsG7a5TxKOXbFFrkJSJN9R//AW99a7xt4cKEAb0WZzVF2iIvbewmIk0zpRW6mqhtKe3tIyKZ+/M/rwz8Tz5ZR82+JmpbRsFfpChatF2DGXzzm/E2dzjuuPz6JJWU9hEpgvJ8emm7BshsdD3lTdha0CdJp5G/SBFkvV1DZMS+vff4isDf01NHikcbsOVKI3+RIshyu4bIiN1wysvz6y4gTDsPuNY5wZIJjfxFiiDLzc6Ghjh9520VWzM8wbvw1VPI2XelbOab1i6ZUvAXKYIMt2uwsY38sGwHd8d4J2unlrKZSFnbmdYumVLwFymCDLZrSFyhG56su9/YWP1VO319U2uXTCn4ixRFtRr6KiWXr72WUsmTuEEv9R/8oq2Xc6XgL1J0VU7tMoO3vCV+ecVoP0k9VTutPhBeYrS3j0jR9fdXVNj8Pg/xML8fa/sJp3AKP6v/dc2CbxmSK+3tIyLJyso9Ew9Y6eufegmmduNsa0r7iBRdGKQtTOhE7T9DNyk/X41y921PwV+k4Lb/zZfSj1MsTf5G8/NpurqUu+8gSvuIFFhQxfNnsTa3WZPLdMv32xkc1L77M0TTR/5mttHMnjCzx8xsNGw73MzuNrNnwt+HNbsfIhIaGeHUgx6oKN+8664wt19eBFJeuaMqnRmhVWmfP3b3EyIzzpcBP3b3RcCPw/siM0crtipu5D1GRrDzB7n3jZNjzb56hA9/mPr3ANK++x0vr5z/UuCG8PYNwJk59UMke1Xq5vN8DzOw8+NBen/Nfmlkn+UeQNLWWhH8HfiRmT1qZmHykCPcfTNA+PttLeiHSGu0YqviKbzHjh11rNAtjey16rYwWhH8T3b39wAfBi42sw/U+0QzW2Fmo2Y2umXLlub1UCRLaamT0r43WaSC6kzPmMEhh8QvSVyhWxrZK59fGE0P/u7+Yvj7ZeAOYAnwkpkdCRD+fjnluavcfcDdB3p6eprdVZFspKVIzLJLBdVIz7z3vZWj/Vvn/LfkbRnKR/bK5xdCU4O/mR1sZoeWbgN/AqwF1gDLw8uWA3c2sx8iLZWUOjFLrqI5//zGvgVUSc+YQflOKN7Xzzm7v1v5Ol1dGtkXVLNH/kcA95vZL4GHge+7+w+AK4APmdkzwIfC+yIzQ1LqpNoeWknfAmpV8pTeo7t7f5Pt3FE5oVtaoZuWJtq3T4G/oJoa/N39OXd/d/hzvLsPh+1b3f1Ud18U/t7WzH6ItFx56qTWHvXRydqkSp5ly+Ciiyqf9/rr7OSg5BW60aZmVfG0oqRVmkLbO4i0Qj1745RG50mVPO5w7bXx4Do0hO3cwcHEr/XuecFirWhAbkYVTytKWqVpFPxFWqGevXFKo/C0FI178MEwMsLiOc9gYxtjD3+DTwUTulu3Vn5reOCB7Kt4WlHSKk2jvX1EmmlkJAiGmzYFwb000k7aG6f0WG9v+vbJY2MVeX2ocqoWTH5rOPnkIAWVlXpXA0tb0shfpFnS0iJQfRQ+PJy4Kitxy+V6TtWCyW8NWdJq4I6m4C/SLNXSIqUJ4ZtuCtqXLYtvn3zhhfs/AN7gwPQtl6P6+mLVPxWyHpFrNXBHU/AXaZZaaZFqE6ZXXw033YThHMQbsacnjvb7+oIPk5Urk/dygOxH5FoN3NEU/EWapVZapMo3g3e9q3ITtqu4qL4VuklVRc0akWs1cMfShK9IM4yMwG9+U9keDcIp3wzKq3ggZULXbHISOe2QFQhSQStXKjBLjIK/SNbqDcJlVT115fVLSmmeqKRvEhDs7KbAL2WU9hHJWr1B+Hd+B4BdzEkO/Fbln2dSCkellzIFCv4iWas3CN97L4bzJnbFmt1mBVszpM0ZdHcnj+RVeilToOAvkrW0YHv44fv3wTl69hjm+2IPf5EvBWme0qY8aaWUK1cmv75KL2UKFPxFspYUhOfMgddeC1bo+j6e2xvf5sExvlR+lPVUSylVeilTYF5tq9k2MjAw4KPlm5SLtKvybR1+8xts6ysVlyVO6B5yCGzf3oJOShGY2aPuPlDerpG/SDNE6t/3PLOx/sB/wAHBPjwiTabgL9JEZkHGJyq2Qre7O56muf56pWmkJRT8RcplcEDJccdV7rLw2QOujo/2S5O3pRWyw8NBqkgHo0gL5Bb8zex0M1tvZhvM7LLazxBpgQwOKDGDp5+Ot7nDV69/S/pkrA5GkRbLJfibWRdwFfBhYDHwUTNbnEdfRGKmcUCJWeVo321WcKpWabfOtH1wmnEwio5YlCryGvkvATaEZ/zuBm4GlubUF5FJDayS3bs3eSPN/TX7pVH8RRelB+OsV+fqm4TUkFfwnw+8ELk/HraJtF50hDwr5Z9EysItM5g9O97mff2VlTw7dwZVPGnBOOvVuTpiUWrIK/gn7VZVseDAzFaY2aiZjW7ZsqUF3ZLCKR8hT0xUXpOwSvY976kc7S9fHi7OrXYGb1Q0GGe9Olf7/EgNeQX/cWBh5P4C4MXyi9x9lbsPuPtAT09PyzonM0itvHfaJmxdXamrZM3gF7+IX+4eVGkCUxutl4Jx1qtztc+P1JBX8H8EWGRmR5nZHOA8YE1OfZGZqp68d9pIeN++ionZpAndffsqB/SJo/hWna5VrQ/a50ei3D2XH+AM4FfAs8BQretPPPFEF5mSvj73IDbHf/r6al/T3b3/kr17ky/xuXPdV69Ofu/Vq4PXNgt+f/rTwfXRF4g+f/Xq6o83orwP03kt6VjAqCfF4KTGdvxR8JcpM0uO2maT16xe7T5nTuU1s2e7r16dHPSTPkzqCazVgnE9H1QiDUgL/trYTWau/v7YSVn7lZ+CNW8ebN0au2Qp32NNWfXxOdzKrZyb/F5z504vRz9rVkL+iCBdtG9fZbtInbSxmxRPvXnvbdtidw2vCPze158e+GH6ZZSaoJUWU/CX9tfoStVSBU1392TbQQdVXhcGWAu3XIvaP6Gb9EFSbjpllJqglRZT8Jf2lsVK1ddfn7y9dWvF8/1/Dyefobt6ZLJIJ1qKmWY6o3QdxCItpuAv7a2elarVvhlUe/5IENxnLYsHWO/rx1eHrxF9XQjmClavbs4ovdrePyJZS5oFbscfVfsUTKkyJqkCJlqxU6tEMqXi5yK+VtF8Qdf19ZdeqoxSOgSq9pG2U37U4fBwMNotpXqSVt6WlCp2alX0JDyemOIp7ThSel5CBVDscZEOoWofaS/VcvlpWy6URFMstfawiUykJk7oRk/VgqAfaYG/2vtp+2TpMAr+ko9qufhqVTPlE6G1SiQHB/GPLU8d7VdsumCWHvjT3k/bJ0sHUvCXfFQbsacF9FLKJToRWqNE0gxmXXt17GEvH+3HHqyRBk2a1NX2ydKBFPwlH9VG7PXWvEdTRF1dQVv4zeCiBwYr9lL7ON9OD/r16O5OrsDR9snSgQ7IuwNSUMPDlZO6pQBfCrBJk8El5ZPCExP7n2/nVwbouoJ+V1fyfv6lvq1cmfxYb2/ypLNW50ob08hf8pG2qAmCCdNly4LbN92UXPOekGqxnTsqAv/ERLBYq+bq3Llzgw+TpOu6u6svuNLqXOlESfWf7fijOv8CSKqtNwu2Qy4Xqd/fB8m7b5a/dqkuv7vb/eCDJy/s7p5+/b7q/qVNkVLnr5G/tI+kiVP34Ozb8sqZyH48s8oqeUpRPaa0evamm4LtHnbsmHwsuv3D4GAwYu/tDVJO4UrgmrQ6VzqMgr+0j2pn355/fqx+/u8G7qwo3/yvXbdObsuQplZljso2pSC0wlfaR9pq3ag5c7Dduyqava+/clI4Sa198+s9A0CkQ7R8ha+Z/a2Z/drMHgt/zog8drmZbTCz9WZ2WrP6IB1meDj9rFvCFbplgX/v3jCW15tqqbUoTGWbUhDNTvv8k7ufEP7cBWBmiwkObD8eOB242sy6mtwP6QSDg3DhhYkfAIkrdH2yvL9utSpzdKiKFEQeOf+lwM3uvsvdnwc2AEty6Ie0g/I9cU4+OZiUDffNT9qPx7GaC3FT1do3X2WbUhDNDv6fMbPHzew6MzssbJsPvBC5Zjxsk3bWjI3L0iZXga/+940VQf8cbg0Wa0VP5mpEtcocHaoiRZFU/1nvD3APsDbhZylwBNBF8AEzDFwXPucq4PzIa3wLOCfl9VcAo8Bob29vc4thJV2tve0blbJff2LNfvROtC5/qv8d3d3Tfx2RDkJKnX9LFmgB/cDa8PblwOWRx34IvK/Wa2iRV47SDlXp66v+vFoLn8oOWkl6iz03jMQDdqMfPqtXu8+eXfk6c+boA0BmtLTg38xqnyMjd88KvxEArAHOM7MDzewoYBHwcLP6IRlopAJmZAQuuCCe0rnggni6KDKJmjih29fPAV0OhxxS+fpT3TVzaAj27Kls371bu29KITWtzt/MbgJOABzYCPyFu28OHxsCLgD2Ap9393+p9Xqq889RI7XvaQeidHfDK68Et0dGam/CNndu+sEupdr8eqTV90/1dUQ6TMvr/N19mbu/y91/193/tBT4w8eG3f1odz+2nsAvOWukAibtQJSw/Uc/oiLwf5EvVe6+Gd2uudxUyi+rXasyTikgbekstdWzxfIUJK3jqrrl8sQEzJ4dT9tMtfxyeBg+8YnK1M+cOSrjlELS3j5Sn6luXJZQjplUs7+HA+rba9/CEs9Gyy8HB+Hb3473q7sbrrtOZZxSSBr5S3OsXBkbaSdO6FqVPHy53buDid/SfEEjBgcV6EVCGvlLtkqLwZYtgze/OX2F7tyD4fDDp/ba2l9HJDMK/pKdyIrdR/xEbGt8lP5/+MJkiqdUwZM0kZy2glcTsyKZUfCXZI1s5xDulW84S3gk9pBjfIEr49dv25a8lcLKldpfR6TJlPOXSuWHo4+NBWmcBx6Aq69Ofdq7x+7kcd4da9vDARxAyqHovb3V8/AZVReJSCUd5iKV0hZ1mQU7biYE4arlm93dwVGJ0cVac+dqwzSRFmj5Ii/pYNWOUyzbCsGsMvB7OM0LBEF+5UrtlCnSZhT8pVK1idXwg+HxxyuD/urVBGfoJgV5HXAu0laU85dKw8NBjj8pJdjbm5zi2X+paulFOoFG/lIp5TjFP+Kn2NjGWNuePfWv0xKR9qGRvyQrVfVcey145UItUNAX6WQa+Uu6u+7CfF/lCt2+fgV+kQ6n4C+JXnqJihTPLZwbVPFomwWRjqe0j1SoueWytlkQ6Xga+ct+X/taZeBP3HL5jDNa1ykRaYppBX8zO9fM1pnZPjMbKHvscjPbYGbrzey0SPuJZvZE+NhXzZLGmdKwRvbkIQj6n/3s5P1zzw3P0E3amuGuuzLpqojkZ7ppn7XA2cDXo41mthg4DzgeeDtwj5kd4+4TwDXACuAh4C7gdEBHOWYhaU+eFSuC2ym191Vr9mc1cHC7iHSEaY383f0pd1+f8NBS4GZ33+XuzwMbgCVmdiTwZnd/0INNhW4EzpxOHyQi3FUzZufOii0ZAF5+uTLwP/hgWflmWm5fOX+RjtesnP984IXI/fGwbX54u7w9kZmtMLNRMxvdsmVLUzo6o6SNyMvazeCII+KXuMNJJ5U9r5GD20WkI9QM/mZ2j5mtTfhZWu1pCW1epT2Ru69y9wF3H+jp6anVVakxUr/mmoQJ3WordAcHtSGbyAxVM+fv7h9s4HXHgYWR+wuAF8P2BQntkoXh4XjOH/aP1MuD/plnwh131PGaOvdWZEZqVp3/GuA7ZnYlwYTvIuBhd58ws+1mdhLwc+BjwD83qQ/FUwrSkUNQ5ow/y57zu2KXaXWuiEy31PMsMxsH3gd838x+CODu64BbgCeBHwAXh5U+AJ8GvkkwCfwsqvTJVrh18isv78PGNrJnYjLw33+/Ar+IBHSS1wxUfctlESkSneRVAHfcURn4d+9W4BeRSgr+M8HICGZw9tmTTZ/7XBD0Z8/Or1si0r4U/DvcZ/5kPXZ+vBrH5x7MV95b37YOIlJMCv4daseOIMVz1d3H7m/7FYuCTdhSVvWKiJRoS+cOVJ7XP5oNbGBRvFH774hIFRr5d5AHHqgM/Ht7f7sy8IP23xGRqhT8O4QZ/OEfTt5fuTKY0O36h7/X/jsiMmUK/m3u85+vHO27w1/+ZXhH+++ISAOU829TO3bAIYfE255+Go49NuFi7b8jIlOk4N+Gykf6fX2wcWMuXRGRGUppnzby4IPJWy4r8ItI1hT824QZ/MEfTN6/8sogt3+AvpuJSBMo+OfsC19IntC95JJ8+iMixaBxZU527oSDD463PfUUvOMd+fRHRIpFwT8H5SP9t78dfv3rfPoiIsWktE8L/fznyRO6Cvwi0mrTPcnrXDNbZ2b7zGwg0t5vZq+b2WPhz7WRx040syfMbIOZfdUs6eiRmccMTjpp8v6Xv6wJXRHJz3RH/muBs4H7Eh571t1PCH8ujLRfA6wgONd3EXD6NPvQ1r74xeQJ3b/6q3z6IyIC08z5u/tTAPUO3s3sSODN7v5geP9G4Exm4Dm+r79eueXOunWweHE+/RERiWpmzv8oM/uFmf3MzN4fts0HxiPXjIdtM4pZPPC/7W3BaF+BX0TaRc2Rv5ndA/xWwkND7n5nytM2A73uvtXMTgS+Z2bHA0lfEVJPmDWzFQQpIno7YIviRx6BJUvibbt36yhFEWk/NYO/u39wqi/q7ruAXeHtR83sWeAYgpH+gsilC4AXq7zOKmAVwMDAQFsfQ16e+briCrj00nz6IiJSS1PSPmbWY2Zd4e3fJpjYfc7dNwPbzeyksMrnY0Dat4eO8Nd/nTyhq8AvIu1sWhO+ZnYW8M9AD/B9M3vM3U8DPgD8nZntBSaAC919W/i0TwPXAwcRTPR25GTvG2/AQQfF2554At75znz6IyIyFebe1tmU/QYGBnx0dDTvbgBBbf7ExOT9ww+HrVvz64+ISBoze9TdB8rbtcJ3Ch59NEjxRAP/rl0K/CLSeRT862QGA5HPzuHhILc/Z05+fRIRaZQ2F6jh/vvh/e+Pt3VIpkxEJJVG/ikmJuB974sH/o0bFfhFZGZQ8E9w223BpO5DDwX3b7klCPp9ffn2S0QkK0r7RLz6alC5U/L+98NPfwqz9BEpIjOMwlro8svjgX/dOrjvPgV+EZmZCh/annwyqOS54org/tCQNmETkZmvsGmfiQn4wAfgX/91su3VV+Gtb82tSyIiLVPIkf/ttwcTuqXAf9ttwWi/IvCPjEB/f5D76e8P7ouIzACFGvn/+7/DYYdN3j/5ZPjZz6CrK+HikRFYsQJ27gzuj40F9wEGB5vdVRGRpirMyH9oKB74164NFnAlBv7SE0qBv2TnzqBdRKTDzfiR/5NPwvHHT96/7DL4x3+s44mbNk2tXUSkg8z44P+Rj0ze3rYtPvqvqrc3SPUktYuIdLiZnfYZGeEHb5zCwyzB+/o57K4pTNgOD1eewD53btAuItLhZu7IP5ywXbR/wpapTdiWrhkaClI9vb1B4Ndkr4jMADP3MJf+/uS0TV9fsEObiEgBNOUwFzP7spk9bWaPm9kdZvbWyGOXm9kGM1tvZqdF2k80syfCx74anuWbPU3Yioikmm7O/27gne7+u8CvgMsBzGwxcB5wPHA6cHXpQHfgGmAFwaHui8LHs5c2MdvohK0WfInIDDKt4O/uP3L3veHdh4AF4e2lwM3uvsvdnwc2AEvM7Ejgze7+oAf5phuBM6fTh1RZTtiWFnyNjQVLgUsLvvQBICIdKstqnwuAfwlvzwdeiDw2HrbND2+Xt2dvcBBWrQpy/GbB71WrGpuw1YIvEZlhalb7mNk9wG8lPDTk7neG1wwBe4HSUDgpj+9V2tPeewVBiojeRtI1g4PZVOdo/kBEZpiawd/dP1jtcTNbDvxn4FSfLB0aBxZGLlsAvBi2L0hoT3vvVcAqCKp9avW1abTgS0RmmOlW+5wOXAr8qbtH8yJrgPPM7EAzO4pgYvdhd98MbDezk8Iqn48Bd06nDy2hBV8iMsNMN+f/NeBQ4G4ze8zMrgVw93XALcCTwA+Ai919InzOp4FvEkwCP8vkPEH7ynL+QESkDczcRV4iItKcRV4iItKZFPxFRApIwV9EpIAU/EVECkjBX0SkgDqm2sfMthDsyt8O5gGv5N2JNqK/R5z+HnH6e8S1+u/R5+495Y0dE/zbiZmNJpVOFZX+HnH6e8Tp7xHXLn8PpX1ERApIwV9EpIAU/BuzKu8OtBn9PeL094jT3yOuLf4eyvmLiBSQRv4iIgWk4N+gaofXF5GZnWtm68xsn5nlXsmQBzM73czWm9kGM7ss7/7kzcyuM7OXzWxt3n3Jm5ktNLOfmNlT4b+Tz+XdJwX/xiUeXl9ga4Gzgfvy7kgezKwLuAr4MLAY+KiZLc63V7m7Hjg97060ib3AF9z9OOAk4OK8//9Q8G9QlcPrC8ndn3L39Xn3I0dLgA3u/py77wZuBpbm3Kdcuft9wLa8+9EO3H2zu/9beHs78BTNOr+8Tgr+2YgeXi/FNB94IXJ/nJz/cUt7MrN+4PeAn+fZj5pn+BZZg4fXz1j1/D0KzBLaVEonMWZ2CHAb8Hl3fy3Pvij4V9Hg4fUzVq2/R8GNAwsj9xcAL+bUF2lDZjabIPCPuPvtefdHaZ8GVTm8XorpEWCRmR1lZnOA84A1OfdJ2oSZGfAt4Cl3vzLv/oCC/3QkHl5fVGZ2lpmNA+8Dvm9mP8y7T60UTv5/BvghwWTeLe6+Lt9e5cvMvgs8CBxrZuNm9sm8+5Sjk4FlwH8K48VjZnZGnh3SCl8RkQLSyF9EpIAU/EVECkjBX0SkgBT8RUQKSMFfRKSAFPxFRApIwV9EpIAU/EVECuj/A1al/yqklajSAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\"\n",
    "0. Data preparation and pre-processing\n",
    "1. Design model (input, output size, forward pass)\n",
    "2. Construct loss and optimizer\n",
    "3, Training loop\n",
    "    -forward pass: compute predition\n",
    "    -backward pass: gradients\n",
    "    -update weights\n",
    "\"\"\"\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 0. DATA PREPARATION\n",
    "X_numpy, y_numpy = datasets.make_regression(n_samples=100, n_features=1, noise = 20, random_state=1)\n",
    "X = torch.from_numpy(X_numpy.astype(np.float32))\n",
    "y = torch.from_numpy(y_numpy.astype(np.float32))\n",
    "y = y.view(y.shape[0], 1)\n",
    "\n",
    "n_samples, n_features = X.shape \n",
    "# n_samples -> how many rows/groups of data\n",
    "#n_features -> how many numbers/features in each row\n",
    "input_dim = n_features\n",
    "out_dim = 1\n",
    "\n",
    "device=\"cuda\"\n",
    "learning_rate = 0.1\n",
    "n_iters = 100\n",
    "\n",
    "X_test= torch.tensor(9, dtype=torch.float32, requires_grad=True, device=device)\n",
    "\n",
    "# 1. DEFINE MODEL\n",
    "model =nn.Linear(in_features=input_dim, out_features=out_dim)\n",
    "\n",
    "# 2. LOSS AND OPTIMIZER\n",
    "loss_fn = nn.MSELoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=learning_rate)\n",
    "\n",
    "# 3. TRAINING LOOP AND WEIGHT UPDATE\n",
    "for epoch in range(n_iters):\n",
    "    # forward pass and loss\n",
    "    y_predicted = model(X)\n",
    "    loss = loss_fn(y_predicted, y)\n",
    "\n",
    "    # backward pass\n",
    "    loss.backward()\n",
    "\n",
    "    # weight update and reset\n",
    "    optimizer.step()\n",
    "    optimizer.zero_grad()\n",
    "    if (epoch + 1) % 10 ==0:\n",
    "        print(f\"epoch: {epoch+1}, loss ={loss.item():.4f}\")\n",
    "# plot\n",
    "predicted = model(X).detach().numpy()\n",
    "plt.plot(X_numpy, y_numpy, 'ro')\n",
    "plt.plot(X_numpy, predicted, 'b')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 08 - Logistic Regression "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implementing a Logistic Regression model from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 10, loss: 0.2423\n",
      "epoch: 20, loss: 0.1766\n",
      "epoch: 30, loss: 0.1469\n",
      "epoch: 40, loss: 0.1293\n",
      "epoch: 50, loss: 0.1175\n",
      "epoch: 60, loss: 0.1088\n",
      "epoch: 70, loss: 0.1022\n",
      "epoch: 80, loss: 0.0969\n",
      "epoch: 90, loss: 0.0926\n",
      "epoch: 100, loss: 0.0889\n",
      "Accuracy equals: 0.930\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "0. Data preparation and pre-processing\n",
    "1. Design model (input, output size, forward pass)\n",
    "2. Construct loss and optimizer\n",
    "3, Training loop\n",
    "    -forward pass: compute predition\n",
    "    -backward pass: gradients\n",
    "    -update weights\n",
    "\"\"\"\n",
    "import sklearn\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "device=\"cuda\"\n",
    "learning_rate = 0.1\n",
    "n_iters = 100\n",
    "\n",
    "# 0. DATA PREPARATION\n",
    "bc = datasets.load_breast_cancer()\n",
    "X, y = bc.data, bc.target\n",
    "\n",
    "n_samples, n_features = X.shape # samples -> rows, features -> columns \n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1234)\n",
    "\n",
    "sc = StandardScaler() # mean = 0, variance = 1\n",
    "X_train = sc.fit_transform(X_train)\n",
    "X_test = sc.transform(X_test)\n",
    "\n",
    "X_train = torch.from_numpy(X_train.astype(np.float32))\n",
    "X_test = torch.from_numpy(X_test.astype(np.float32))\n",
    "y_train = torch.from_numpy(y_train.astype(np.float32))\n",
    "y_test = torch.from_numpy(y_test.astype(np.float32))\n",
    "\n",
    "y_train = y_train.view(y_train.shape[0], 1)\n",
    "y_test = y_test.view(y_test.shape[0], 1)\n",
    "\n",
    "\n",
    "# 1. DEFINE MODEL\n",
    "# Logistic Regression: f(x) = sigmoid(w*x +b)\n",
    "\n",
    "class LogisticRegression(nn.Module):\n",
    "\n",
    "    def __init__(self, in_features):\n",
    "        super(LogisticRegression, self).__init__()\n",
    "        self.linear = nn.Linear(in_features, 1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        y_predicted = torch.sigmoid(self.linear(x)) \n",
    "        return y_predicted\n",
    "\n",
    "model = LogisticRegression(in_features = n_features)\n",
    "\n",
    "# 2. LOSS AND OPTIMIZER\n",
    "loss_fn = nn.BCELoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=learning_rate)\n",
    "\n",
    "# 3. TRAINING LOOP AND WEIGHT UPDATE\n",
    "for epoch in range(n_iters):\n",
    "    # forward pass and loss\n",
    "    y_predicted = model(X_train)\n",
    "    loss = loss_fn(y_predicted, y_train)\n",
    "\n",
    "    # backward pass\n",
    "    loss.backward()\n",
    "\n",
    "    # weight update and reset\n",
    "    optimizer.step()\n",
    "    optimizer.zero_grad()\n",
    "    if (epoch + 1) % 10 == 0:\n",
    "        print(f\"epoch: {epoch+1}, loss: {loss.item():.4f}\")\n",
    "# plot\n",
    "with torch.no_grad():\n",
    "    y_predicted = model(X_test)\n",
    "    y_predicted_cls = y_predicted.round()\n",
    "    acc = y_predicted_cls.eq(y_test).sum() / float(y_test.shape[0])\n",
    "    print(f\"Accuracy equals: {acc:.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 09 - Dataset and DataLoader - Batch Training "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "epoch = 1 forward and backward pass of ALL training samples  \n",
    "batch_size = number of training samples in one forward & backward pass  \n",
    "number of iterations = number of passes, each pass using [batch_size] number of samples  \n",
    "e.g. 100 samples, batch_size=20 --> 100/20 = 5 iterations for 1 epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import numpy as np\n",
    "import math\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WineDataset(Dataset):\n",
    "    def __init__(self):\n",
    "        # data loading\n",
    "        xy = np.loadtxt(\"./data/wine/wine.csv\", delimiter =\",\", dtype = np.float32, skiprows=1)\n",
    "        self.x = torch.from_numpy(xy[:,1:])\n",
    "        self.y = torch.from_numpy(xy[:,[0]])\n",
    "        self.n_samples = xy.shape[0]\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        return self.x[index], self.y[index] \n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.n_samples\n",
    "\n",
    "\n",
    "dataset = WineDataset()\n",
    "batch_size = 4\n",
    "dataloader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "## WHEN CHANGING NUMBER OF NUM_WORKERS >0 IT SPITS OUT AN ERROR AND I DONT KNOW WHY\n",
    "num_epochs = 2\n",
    "total_samples = len(dataset)\n",
    "n_iterations = math.ceil(total_samples/batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataset module is not that important as long as you can transform the data into tensors. The important thing here is the Dataloader module which allows to create batches and data shuffling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total samples are: 178, number of iterations are: 45\n",
      "epoch 1/2, step 5/45, inputs torch.Size([4, 13])\n",
      "epoch 1/2, step 10/45, inputs torch.Size([4, 13])\n",
      "epoch 1/2, step 15/45, inputs torch.Size([4, 13])\n",
      "epoch 1/2, step 20/45, inputs torch.Size([4, 13])\n",
      "epoch 1/2, step 25/45, inputs torch.Size([4, 13])\n",
      "epoch 1/2, step 30/45, inputs torch.Size([4, 13])\n",
      "epoch 1/2, step 35/45, inputs torch.Size([4, 13])\n",
      "epoch 1/2, step 40/45, inputs torch.Size([4, 13])\n",
      "epoch 1/2, step 45/45, inputs torch.Size([2, 13])\n",
      "epoch 2/2, step 5/45, inputs torch.Size([4, 13])\n",
      "epoch 2/2, step 10/45, inputs torch.Size([4, 13])\n",
      "epoch 2/2, step 15/45, inputs torch.Size([4, 13])\n",
      "epoch 2/2, step 20/45, inputs torch.Size([4, 13])\n",
      "epoch 2/2, step 25/45, inputs torch.Size([4, 13])\n",
      "epoch 2/2, step 30/45, inputs torch.Size([4, 13])\n",
      "epoch 2/2, step 35/45, inputs torch.Size([4, 13])\n",
      "epoch 2/2, step 40/45, inputs torch.Size([4, 13])\n",
      "epoch 2/2, step 45/45, inputs torch.Size([2, 13])\n"
     ]
    }
   ],
   "source": [
    "# training loop\n",
    "print(f\"Total samples are: {total_samples}, number of iterations are: {n_iterations}\")\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (inputs, labels) in enumerate(dataloader):\n",
    "        # forwqrd, bqck3qrd pass and weight update\n",
    "        if (i+1)% 5 == 0:\n",
    "            print(f'epoch {epoch+1}/{num_epochs}, step {i+1}/{n_iterations}, inputs {inputs.shape}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 10 - DataSet Transforms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The takeaway from this tutorial is how we can create our custom transform methods and use the dunder __call__ method to apply the transform without creating a specific object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WineDataset(Dataset):\n",
    "    def __init__(self, transform=None):\n",
    "        # data loading\n",
    "        xy = np.loadtxt(\"./data/wine/wine.csv\", delimiter =\",\", dtype = np.float32, skiprows=1)\n",
    "        self.n_samples = xy.shape[0]\n",
    "\n",
    "        self.x = xy[:,1:]\n",
    "        self.y = xy[:,[0]]\n",
    "        \n",
    "        self.transform = transform\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        sample = self.x[index], self.y[index] \n",
    "\n",
    "        if self.transform:\n",
    "            sample = self.transform(sample)\n",
    "\n",
    "        return sample\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.n_samples\n",
    "\n",
    "\n",
    "class ToTensor():\n",
    "    def __call__(self, sample):\n",
    "        inputs, targets = sample\n",
    "        return torch.from_numpy(inputs), torch.from_numpy(targets)\n",
    "\n",
    "class MulTransform:\n",
    "    \"\"\"\n",
    "    Example of a custom transform to show how to write a transform function\n",
    "    \"\"\"\n",
    "    def __init__(self, factor):\n",
    "        self.factor = factor\n",
    "\n",
    "    def __call__(self, sample):\n",
    "        inputs, targets = sample\n",
    "        inputs *=self.factor\n",
    "        return inputs, targets\n",
    "\n",
    "composed = torchvision.transforms.Compose([ToTensor(), MulTransform(2)])\n",
    "dataset = WineDataset(transform=composed)\n",
    "batch_size = 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another takeaway: transforms.Compose(), can take a list of transforms and use them to apply various transforms sequentially"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 11 - Softmax and Cross-Entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "import torch.nn as nn \n",
    "from torchvision import datasets, transforms\n",
    "from torch.utils.data import DataLoader\n",
    "from torch import optim\n",
    "import numpy as np\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "softmax numpy: [0.65900114 0.24243297 0.09856589]\n",
      "torch softmax outputs: tensor([0.6590, 0.2424, 0.0986])\n"
     ]
    }
   ],
   "source": [
    "def softmax(x):\n",
    "    return np.exp(x)/np.sum(np.exp(x), axis = 0)\n",
    "\n",
    "x=np.array([2.0, 1.0, 0.1])\n",
    "outputs =softmax(x)\n",
    "print(f\"softmax numpy: {outputs}\")\n",
    "\n",
    "x1 = torch.tensor([2.0, 1.0, 0.1], dtype=torch.float32)\n",
    "outputs1 = torch.softmax(x1, dim=0)\n",
    "print(f\"torch softmax outputs: {outputs1}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Good prediction cross entropy 0.35667494393873245\n",
      "Bad prediction cross entropy 2.3025850929940455\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Now let's code cross entropy loss from scratch\n",
    "cross entropy = 1/N_samples * -Sum(y_actual * log(y_predicted))\n",
    "\"\"\"\n",
    "def cross_entropy_loss(actual, predicted, n_samples=1):\n",
    "    return -np.sum(actual*np.log(predicted))/n_samples\n",
    "\n",
    "# y must be one-hot encoded\n",
    "y_target = np.array([1, 0, 0])\n",
    "\n",
    "# y_pred has probabilities\n",
    "y_pred_good = np.array([0.7, 0.2, 0.1])\n",
    "y_pred_bad = np.array([0.1, 0.2, 0.6])\n",
    "l1, l2 = cross_entropy_loss(y_target, y_pred_good), cross_entropy_loss(y_target, y_pred_bad)\n",
    "print(f\"Good prediction cross entropy {l1}\")\n",
    "print(f\"Bad prediction cross entropy {l2}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accurate Cross entropy loss 0.3018244206905365\n",
      "Inaccurate Cross entropy loss 1.6241613626480103\n",
      "Good prediction cross entropy tensor([2, 0, 1])\n",
      "Bad prediction cross entropy tensor([0, 2, 1])\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Now we will use the cross entropy loss offered by PyTorch\n",
    "nn.crossEntropyLoss applies the nn.LogSoftmax + nn.NLLLoss \n",
    "(negative log likelihood loss) -> No Softmax in last layer!\n",
    "y_actual has class labels, not one-hot enoded!\n",
    "y_ored has raw scores (logits) not Softmax probabilities\n",
    "\"\"\"\n",
    "\n",
    "loss= nn.CrossEntropyLoss()\n",
    "y_actual = torch.tensor([2,0,1]) \n",
    "# n_samples * n_classes = 1 x 3\n",
    "y_pred_good = torch.tensor([[0.1, 1.0, 2.1], [2.0, 1.0, 0.1], [0.1, 3.0, 0.1]])\n",
    "y_pred_bad = torch.tensor([[2.1, 1.0, 0.1], [0.1, 1.0, 2.1], [0.1, 3.0, 0.1]])\n",
    "l1 = loss(y_pred_good, y_actual)\n",
    "l2 = loss(y_pred_bad, y_actual)\n",
    "print(f\"Accurate Cross entropy loss {l1}\")\n",
    "print(f\"Inaccurate Cross entropy loss {l2}\")\n",
    "\n",
    "_, prediction1 = torch.max(y_pred_good,1)\n",
    "_, prediction2 = torch.max(y_pred_bad,1)\n",
    "\n",
    "print(f\"Good prediction cross entropy {prediction1}\")\n",
    "print(f\"Bad prediction cross entropy {prediction2}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now lets classify the MNIST dataset using PyTorch\n",
    "going through a standard training pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download training data \n",
    "XY_train = datasets.MNIST(root=\"./data/mnist/\",download=True, train=True, transform=transforms.ToTensor())\n",
    "\n",
    "# Download test data\n",
    "XY_test = datasets.MNIST(root=\"./data/mnist/\",download=True, train=False, transform=transforms.ToTensor())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "XY_train.data.shape[0]\n",
    "XY_test.data.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X_test [n_samples, channel, height, width]: torch.Size([64, 1, 28, 28])\n",
      "Shape of y_Test: torch.Size([64]) torch.int64\n"
     ]
    }
   ],
   "source": [
    "# Create DataLoaders\n",
    "batch_size = 64\n",
    "n_epochs =50\n",
    "total_samples = XY_train.data.shape[0]\n",
    "n_iterations = math.ceil(total_samples/batch_size)\n",
    "\n",
    "train_dataloader = DataLoader(dataset=XY_train, batch_size = batch_size, shuffle=True, num_workers=0)\n",
    "test_dataloader = DataLoader(dataset=XY_test, batch_size = batch_size, shuffle=True, num_workers=0)\n",
    "\n",
    "for X,y in test_dataloader:\n",
    "    print(f\"Shape of X_test [n_samples, channel, height, width]: {X.shape}\")\n",
    "    print(f\"Shape of y_Test: {y.shape} {y.dtype}\")\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cuda device\n",
      "Classification_NN(\n",
      "  (flatten): Flatten(start_dim=1, end_dim=-1)\n",
      "  (model): Sequential(\n",
      "    (0): Linear(in_features=784, out_features=512, bias=True)\n",
      "    (1): ReLU()\n",
      "    (2): Linear(in_features=512, out_features=512, bias=True)\n",
      "    (3): ReLU()\n",
      "    (4): Linear(in_features=512, out_features=10, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# Get cpu or gpu device for training.\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "print(f\"Using {device} device\")\n",
    "\n",
    "# Define a custom Multiclass classification NN\n",
    "class Classification_NN(nn.Module):\n",
    "    \n",
    "    # Custom classification Network\n",
    "    def __init__(self, n_inputs, hidden_size, n_output):\n",
    "        super(Classification_NN, self).__init__()\n",
    "        self.flatten = nn.Flatten()\n",
    "        self.model = nn.Sequential(\n",
    "            nn.Linear(n_inputs, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, n_output)\n",
    "        )\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.flatten(x)\n",
    "        z = self.model(x)\n",
    "        return z\n",
    "\n",
    "# Initialize model, choose loss and optimizer \n",
    "model = Classification_NN(n_inputs=28*28, hidden_size=512, n_output=10).to(device)\n",
    "criterion = nn.CrossEntropyLoss() #Softmax + Negative Log Likehood Loss\n",
    "optimizer = optim.Adagrad(params=model.parameters(), lr=0.001)\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define training function: calculate error, backpropagate, reset gradient \n",
    "\n",
    "def train(dataloader, model ,loss_fn, optimizer):\n",
    "    size = len(dataloader.dataset)\n",
    "    model.train()\n",
    "    for batch, (X,y) in enumerate(dataloader):\n",
    "        X,y = X.to(device), y.to(device)\n",
    "        \n",
    "        # Computer prediction error => forward pass\n",
    "        y_pred = model(X)\n",
    "        loss = loss_fn(y_pred, y)\n",
    "\n",
    "        # Backpropagation -> backward pass and reset gradient\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if batch % 100 == 0:\n",
    "            loss, current = loss.item(), batch * len(X)\n",
    "            print(f\"loss: {loss:.4f}, [{current:.4f}/{size:.4f}]\")\n",
    "\n",
    "\n",
    "# Define test function: evaluate performance \n",
    "\n",
    "def test(dataloader, model ,loss_fn):\n",
    "    size = len(dataloader.dataset)\n",
    "    num_batches = len(dataloader)\n",
    "    model.eval()\n",
    "    test_loss, correct = 0, 0\n",
    "    with torch.no_grad():\n",
    "        for X, y in dataloader:\n",
    "            X, y = X.to(device), y.to(device)\n",
    "            y_pred = model(X)\n",
    "            test_loss += loss_fn(y_pred, y).item()\n",
    "            correct += (y_pred.argmax(1) == y).type(torch.float).sum().item()\n",
    "    test_loss /= num_batches\n",
    "    correct /= size    \n",
    "    print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 2.2987, [0.0000/60000.0000]\n",
      "loss: 0.4629, [6400.0000/60000.0000]\n",
      "loss: 0.6247, [12800.0000/60000.0000]\n",
      "loss: 0.3352, [19200.0000/60000.0000]\n",
      "loss: 0.4305, [25600.0000/60000.0000]\n",
      "loss: 0.2999, [32000.0000/60000.0000]\n",
      "loss: 0.2348, [38400.0000/60000.0000]\n",
      "loss: 0.2274, [44800.0000/60000.0000]\n",
      "loss: 0.2413, [51200.0000/60000.0000]\n",
      "loss: 0.3357, [57600.0000/60000.0000]\n",
      "Test Error: \n",
      " Accuracy: 91.8%, Avg loss: 0.294055 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.3286, [0.0000/60000.0000]\n",
      "loss: 0.3340, [6400.0000/60000.0000]\n",
      "loss: 0.4320, [12800.0000/60000.0000]\n",
      "loss: 0.4747, [19200.0000/60000.0000]\n",
      "loss: 0.2134, [25600.0000/60000.0000]\n",
      "loss: 0.2961, [32000.0000/60000.0000]\n",
      "loss: 0.1372, [38400.0000/60000.0000]\n",
      "loss: 0.2787, [44800.0000/60000.0000]\n",
      "loss: 0.3369, [51200.0000/60000.0000]\n",
      "loss: 0.2976, [57600.0000/60000.0000]\n",
      "Test Error: \n",
      " Accuracy: 92.6%, Avg loss: 0.260540 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.2342, [0.0000/60000.0000]\n",
      "loss: 0.1720, [6400.0000/60000.0000]\n",
      "loss: 0.2311, [12800.0000/60000.0000]\n",
      "loss: 0.1123, [19200.0000/60000.0000]\n",
      "loss: 0.3157, [25600.0000/60000.0000]\n",
      "loss: 0.3273, [32000.0000/60000.0000]\n",
      "loss: 0.2469, [38400.0000/60000.0000]\n",
      "loss: 0.1731, [44800.0000/60000.0000]\n",
      "loss: 0.3691, [51200.0000/60000.0000]\n",
      "loss: 0.2929, [57600.0000/60000.0000]\n",
      "Test Error: \n",
      " Accuracy: 93.3%, Avg loss: 0.237238 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.2040, [0.0000/60000.0000]\n",
      "loss: 0.2829, [6400.0000/60000.0000]\n",
      "loss: 0.1886, [12800.0000/60000.0000]\n",
      "loss: 0.1927, [19200.0000/60000.0000]\n",
      "loss: 0.4057, [25600.0000/60000.0000]\n",
      "loss: 0.2165, [32000.0000/60000.0000]\n",
      "loss: 0.3983, [38400.0000/60000.0000]\n",
      "loss: 0.2860, [44800.0000/60000.0000]\n",
      "loss: 0.3810, [51200.0000/60000.0000]\n",
      "loss: 0.2032, [57600.0000/60000.0000]\n",
      "Test Error: \n",
      " Accuracy: 93.6%, Avg loss: 0.223771 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.1827, [0.0000/60000.0000]\n",
      "loss: 0.2604, [6400.0000/60000.0000]\n",
      "loss: 0.2018, [12800.0000/60000.0000]\n",
      "loss: 0.1697, [19200.0000/60000.0000]\n",
      "loss: 0.1795, [25600.0000/60000.0000]\n",
      "loss: 0.2614, [32000.0000/60000.0000]\n",
      "loss: 0.2350, [38400.0000/60000.0000]\n",
      "loss: 0.3335, [44800.0000/60000.0000]\n",
      "loss: 0.0814, [51200.0000/60000.0000]\n",
      "loss: 0.1248, [57600.0000/60000.0000]\n",
      "Test Error: \n",
      " Accuracy: 93.9%, Avg loss: 0.213138 \n",
      "\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "epochs = 5\n",
    "for t in range(epochs):\n",
    "    print(f\"Epoch {t+1}\\n-------------------------------\")\n",
    "    train(train_dataloader, model, criterion, optimizer)\n",
    "    test(test_dataloader, model, criterion)\n",
    "print(\"Done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving models\n",
    "\n",
    "# torch.save(model.state_dict(), 'model.pth')\n",
    "# print(\"Saved PyTorch Model State to model.pth\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading models \n",
    "\n",
    "# model = Classification_NN(n_inputs=28*28, hidden_size=512, n_output=10).to(device)\n",
    "# model.load_state_dict(torch.load(\"model.pth\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
      "Predicted: \"7\", Actual: \"7\"\n"
     ]
    }
   ],
   "source": [
    "# Make predictions -> NEEDS FIXING\n",
    "classes = [int(item[0]) for item in test_dataloader.dataset.classes]\n",
    "print(classes)\n",
    "eval_model = model.to(device=device).eval()\n",
    "x, y = XY_test[0][0].to(device=device), XY_test[0][1]\n",
    "with torch.no_grad():\n",
    "    pred = eval_model(x)\n",
    "    predicted, actual = classes[pred[0].argmax(0)], classes[y]\n",
    "    print(f'Predicted: \"{predicted}\", Actual: \"{actual}\"')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 12 - Activation Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "# When defining activation functions there are two ways\n",
    "\n",
    "# Option 1: explicitly define self.activation_fn_name\n",
    "\n",
    "class NeuralNetwork(nn.Module):\n",
    "\n",
    "    def __init__(self, input_nodes, hidden_nodes, output_nodes):\n",
    "        super(NeuralNetwork, self).__init__()\n",
    "        \"\"\"\n",
    "        We first define all different layers and activation functions that we will need\n",
    "        \"\"\"\n",
    "        self.linear1 = nn.Linear(input_nodes, hidden_nodes)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.linear2 = nn.Linear(hidden_nodes, hidden_nodes)\n",
    "        self.linear3 = nn.Linear(hidden_nodes, output_nodes)\n",
    "        self.softmax = nn.Softmax(2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # In the forward pass we essentially assemble the network architecture\n",
    "        out = self.linear1(x)\n",
    "        out = self.relu(out)\n",
    "        out = self.linear2(out)\n",
    "        out = self.relu(out)\n",
    "        out = self.linear3(out)\n",
    "        out = self.softmax(out)\n",
    "        return out\n",
    "\n",
    "# Option 2: use activation functions directly in forward pass\n",
    "\n",
    "class NeuralNetwork(nn.Module):\n",
    "\n",
    "    def __init__(self, input_nodes, hidden_nodes, output_nodes):\n",
    "        super(NeuralNetwork, self).__init__()\n",
    "        \"\"\"\n",
    "        We first define all different layers and activation functions that we will need\n",
    "        \"\"\"\n",
    "        self.linear1 = nn.Linear(input_nodes, hidden_nodes)\n",
    "        self.linear2 = nn.Linear(hidden_nodes, hidden_nodes)\n",
    "        self.linear3 = nn.Linear(hidden_nodes, output_nodes)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # In the forward pass we essentially assemble the network architecture\n",
    "        out = torch.relu(self.linear1(x))\n",
    "        out = torch.relu(self.linear2(out))\n",
    "        out = torch.softmax(self.linear3(out))\n",
    "        return out\n",
    "\n",
    "# Other available activation functions\n",
    "# nn.ReLU\n",
    "# nn.Tanh()\n",
    "# nn.Sigmoid()\n",
    "# nn.Softmax()\n",
    "# nn.LeakyReLU()\n",
    "# For more check torch.nn.functional"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 13 - Feed-Forward Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([100, 1, 28, 28]) torch.Size([100])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD6CAYAAAC4RRw1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAeOklEQVR4nO3de5QUxdkG8OcVgSiggLDILYKAwEJUEhTiZzQGb5AAonCUE81+ikENeMTLEQSikEiCGi4mmuAmctEoBm8BvASQICZRwMWAwsdNBOWyigiICwmC1vcHY1nVTM8OMz3dXT3P75w9+9bUzPSr727RW1PdJUopEBGRe46JOgEiIsoNB3AiIkdxACcichQHcCIiR3EAJyJyFAdwIiJH5TWAi8ilIrJORN4VkRFBJUXRYl2Ti7VNFsl1HbiI1ACwHsBFALYCeBPAQKXU/wWXHoWNdU0u1jZ5js3jtWcDeFcp9R4AiMhTAPoC8P1hEBFeNRQTSinx6WJd3bZTKdXYp++oasu6xkrauuYzhdIcwBajvTX1mEVEBotIhYhU5HEsCg/r6rb3M/RVW1vWNbbS1jWfM/B0Z3BH/IutlCoHUA7wX3RHsK7JVW1tWVe35HMGvhVAS6PdAsD2/NKhGGBdk4u1TZh8BvA3AbQTkdYiUgvAVQDmBJMWRYh1TS7WNmFynkJRSh0SkaEA5gGoAWCqUmp1YJlRJFjX5GJtkyfnZYQ5HYxzarGRYRXKUWNdY2W5UqprEG/EusZK2rrySkwiIkdxACcichQHcCIiR3EAJyJyFAdwIiJHcQAnInIUB3AiIkdxACcichQHcCIiR3EAJyJyVD63kyUAP/3pT612eXm5jh9//HGr7yc/+UkoOVF6tWrVstqDBw/2fe7bb79ttV977bWC5ESUD56BExE5igM4EZGjOIWSRsOGDa12586ddez9U/rWW2+12ubdHbds2QKK1rnnnqtjb60uu+wyHYvYN2d85ZVXfJ+7f//+4BIkygPPwImIHMUBnIjIURzAiYgcxTnwNK655hqrPWHCBB3fddddVl+zZs1CyYn8mZ9Z9O/f3+q77777dHzCCSdk/Z49evSw2itWrNCxd4nhE088oePdu3dbfa+++mrWx6Tc3HjjjVb7wQcf1LF36WhVVZXVfuaZZ3S8ePFi32P84x//sNobN2486jwLgWfgRESO4gBOROQobmqcctppp+l42bJlvs/zTq/Mnj3bau/bt0/H/fr1s/q8S9Oi5PKmxtOmTbPaXbt+vddraWlpTu/pXUaY6++FWX/A/rPce+VnZWVlTseoRlFsanzTTTfp+De/+Y3V941vfEPH3qW8LVu2zOl4M2bMsNrXXXddTu+TB25qTESUJBzAiYgcxQGciMhRXEaYUlJSomPvcrOKigodV3d5/NatW3Ucpzlv1w0cOFDH3s8W6tWrl9V7fPHFF1Z7+fLlOu7WrVse2X2tTp06VrtXr146NpesAcDll1+u448++iiQ4yeVOecNAPfff7+OzTlvABg2bJiOf//731t9Z511ltV+6KGHdNylS5d80wwdz8CJiBxV7QAuIlNFZIeIrDIeaygiC0RkQ+p7g8KmSUFjXZOLtS0e2UyhTAfwEIDHjMdGAFiolBovIiNS7eHBpxee73znO1GnELbpiHFdL7jgAqs9ZcoUHdetWzfr91m7dq2O77zzTqvvxRdf1PGkSZOsvqFDh1rtY47J/4/V7t27W21zWmDMmDF5v79hOmJcWz/mXT8BYO7cuTo2pziBI6dNTOZGKt5pM+9VtN5+11T7U6mUeg3ALs/DfQF8tTByBoDLgk2LCo11TS7WtnjkelrRRClVCQCp7yXVPJ/cwLomF2ubQAVfhSIigwH4bz5ITmJdk4l1dUuuA/hHItJUKVUpIk0B7PB7olKqHEA5EO9Lc9u0aZPV81xcanQUIq2rObd9xx13+PZlYs55A0Dv3r11/N577/m+zrtbj3fO25yvrlGjRla5VGfEiBE63rBhg9Vn3uEwIFnVNsrf171791rtU045xczL93UjR4602p9++qnvc727KY0bN07Hzz//vNW3ffv2tM+Lk1ynUOYAKEvFZQBmZ3guuYN1TS7WNoGyWUY4E8AbANqLyFYRGQRgPICLRGQDgItSbXII65pcrG3xqHYKRSk10Kerh8/jTjL/1PZ64YUXdNy2bduM77Nrl/fD/3iKY12fffZZHV944YVZv+7999/XsbeOmaZNMrnlllt8+7x3ojv++ONzOkbNmjV1/Nhjj1l9+UyhxLG22Rg9erTVNu8QuXLlSqtv7NixOv7rX/+a8zG/+c1vpj0eADRv3lzHH3zwQc7HKCReiUlE5CgO4EREjuIATkTkqKK9G6H3suYWLVr4PtdcTtS6deuM77tgwYL8Eiti3ppky5w/znXOuzrmnPjDDz9s9Zk1z/RzRJl5lwOanwN458D37NmT0zG8dxodPvzruwl4lyrOnz9fx+3bt7f6Vq1ahTjgGTgRkaM4gBMROapop1COO+44q21eXbdixQqrz9zQdOLEiRnfd/Xq1Tnlc+yxX5fCu1TRe3VhUpxzzjlWu3bt2jr2LukymcsGAfvuc2FYv3691TavGPRu4tGjR6xX7sXKzp07rba5IXRQvFOgzZo103FVVZXVd/311+t427ZtgecSBJ6BExE5igM4EZGjOIATETmqaOfAO3XqlPVzGzdurOM+ffpkfK656XGrVq2sPnOO17ss6corr9TxJZdcYvU1atQo61xdcv7551tt89LyTHef8152vnHjxmATy4M37y+//DKiTCidm2++2bfP3LwciO+8t4ln4EREjuIATkTkKA7gRESO4hx4GmeeeabVNi+brV+/fsb3NXf18F6261177mfevHlZPc91ue5uZO5WHgXzFqQAMGvWLB2fccYZOb3nr371q7xyovROPPFEq33NNdf4Pnfq1KmFTidwPAMnInIUB3AiIkcV1RSKOYXhXapn2rdvn9WubtrE1KRJE98+c9PWpUuXWn1m+4EHHsj6eC7r2LFj1ClkrWfPnjr2bnCb67TJsmXLdDx58uSc3oMy+973vme1zaWqgH3bjAJsJF1wPAMnInIUB3AiIkdxACciclRRzYGb82Hey9wPHDigY+8tQC+99FIdjxkzxurz7gBz9dVX+x7f3Nknrrtch+nJJ5+02vfee29Wr5s2bZrVXrdunY6nTJmSUy7eZXzeS+LNeW7vPGq2Xn/9dat9xRVX6PiTTz7J6T0pM+/SwEy77riIZ+BERI7iAE5E5KiimkL5/PPPdWxOmQD2n1Lm8i4AaNOmje97vvDCC1Z7yZIl+aRYVCorK622eee+Y47xP7coLS31bffr1y+nXLw7AGW6G2Im3iWor776qo5vuOEGq2/Hjh05HYMy69Chg469d/LcvXu31XZ9+SbPwImIHMUBnIjIUdUO4CLSUkQWicgaEVktIrekHm8oIgtEZEPqe4PCp0tBYV0TqybrWjyymQM/BOB2pdRbIlIPwHIRWQDgfwEsVEqNF5ERAEYAGF64VPNnzkeeddZZVp95x0Gvbt26+fY5vPwr8rpOnz7dag8ZMkTHp59+utV37LGF/bjmaObAP/vsM6tt7p4+ceJE374QJeL3NVePPPKIb595t1AA+PDDDwudTkFVewaulKpUSr2Vij8DsAZAcwB9AcxIPW0GgMsKlCMVAOuaWAdZ1+JxVKc1ItIKQBcASwE0UUpVAocHAxEp8XnNYACD88yTCoh1TSbWNfkk2+VSIlIXwGIA45RSz4nIHqVUfaN/t1Iq47yaiOS2Nitimzdv1rH3Zv7eG8S7ckczpZQA8a1r//79rfbIkSN17F1GmOuVkSbvssWtW7da7fHjx+t49erVVp85NRcDy5VSXeNa10LwLhXctGmT73PbtWtntR2aQlmulOrqfTCrVSgiUhPAswCeUEo9l3r4IxFpmupvCoCLWh3DuiYT61o8slmFIgAeBbBGKWV+QjMHQFkqLgMwO/j0qFBY10RjXYtENnPg/wPgGgDviMiK1GMjAYwHMEtEBgH4AMCAgmRIhcK6JlNdsK5Fo9oBXCn1TwDi093D53GneedDvUvMTBUVFYVOpyDiXtdnnnnGtz1ggD32lJSk/TwuL3/84x+ttnkbhpir+urzjTQir2sh9O3b12rXqVNHxwcPHrT6vLtrOTQHnhavxCQichQHcCIiRxXV3QizdfbZZ1vtli1b+j7X22duLkCF8fTTT0edAsXIqFGjrLa5NNp7d8i1a9eGklNYeAZOROQoDuBERI7iAE5E5CjOgaexa9cuq23Oo3l3UVm6dGkoORHR14477jgdB3ErBVfxDJyIyFEcwImIHMUplDTWr19vtevVqxdRJkSUTvv27XXcrFkz3+e5cnfQXPEMnIjIURzAiYgcxQGciMhRnAMnIufs2bNHx/v37/ftu+2220LKKBo8AycichQHcCIiR3EKhYicY240XszLfHkGTkTkKA7gRESO4gBOROSosOfAdwJ4H0CjVBwHxZjLKQG/H+uaWZi5BFlb1jWzyOsq5vZDYRGRCqVU19APnAZzCU6c8mcuwYlT/szFxikUIiJHcQAnInJUVAN4eUTHTYe5BCdO+TOX4MQpf+ZiiGQOnIiI8scpFCIiR3EAJyJyVKgDuIhcKiLrRORdERkR5rFTx58qIjtEZJXxWEMRWSAiG1LfG4SQR0sRWSQia0RktYjcElUuQWBdrVwSU1vW1collnUNbQAXkRoAHgbQE0ApgIEiUhrW8VOmA7jU89gIAAuVUu0ALEy1C+0QgNuVUh0BdAcwJPX/Iopc8sK6HiERtWVdjxDPuiqlQvkC8F0A84z2XQDuCuv4xnFbAVhltNcBaJqKmwJYF0FOswFcFIdcWFfWlnV1p65hTqE0B7DFaG9NPRa1JkqpSgBIfS8J8+Ai0gpAFwBLo84lR6yrD8dry7r6iFNdwxzAJc1jRb2GUUTqAngWwDCl1N6o88kR65pGAmrLuqYRt7qGOYBvBdDSaLcAsD3E4/v5SESaAkDq+44wDioiNXH4B+EJpdRzUeaSJ9bVIyG1ZV094ljXMAfwNwG0E5HWIlILwFUA5oR4fD9zAJSl4jIcntsqKBERAI8CWKOUmhhlLgFgXQ0Jqi3raohtXUOe+O8FYD2AjQBGRfDBw0wAlQAO4vAZxiAAJ+Hwp8cbUt8bhpDHuTj85+jbAFakvnpFkQvrytqyru7WlZfSExE5ildiEhE5igM4EZGj8hrAo77UlgqDdU0u1jZh8pjUr4HDH26cCqAWgJUASqt5jeJXPL5Y18R+fRxUbWPw38Kvauqazxn42QDeVUq9p5T6HMBTAPrm8X4UD6yr297P0MfauittXfMZwLO61FZEBotIhYhU5HEsCg/rmlzV1pZ1dcuxebw2q0ttlVLlSG09JCJH9FPssK7JVW1tWVe35HMGHtdLbSk/rGtysbYJk88AHtdLbSk/rGtysbYJk/MUilLqkIgMBTAPhz/dnqqUWh1YZhQJ1jW5WNvkCfVSes6pxYdSKt18aE5Y11hZrpTqGsQbsa6xkrauvBKTiMhRHMCJiBzFAZyIyFH5rAMnSqzSUnsD9qFDh+q4f//+Vl+jRo10/MADD1h9w4cPL0B2RIfxDJyIyFEcwImIHMUplICNGjVKx7/85S+tvnnz5ul4wIABVl9VVZXve55//vk6Xrx4cb4pUhonnnii1Z45c6bV/ta3vqXjPXv2WH2fffaZjrt16xZ8ckQ+eAZOROQoDuBERI7iAE5E5CjOgR+ltm3bWu3nn3/eardp00bH3tsUXHzxxTpu0KCB1WfOgbdq1crqM+dVOQcenLp16+p47ty5Vl+nTp2s9ssvv6zjIUOGWH2tW7fW8aZNm4JMkSgjnoETETmKAzgRkaM4hZJGrVq1rPbo0aN1fOWVV1p97dq1s9qZ7u544MABHX/55Ze+z6tfv77VnjRpku9zKXcTJ07U8bnnnmv1jRkzxmr/4he/8H2fzZs3B5kWUdZ4Bk5E5CgO4EREjuIATkTkqKKdA/deOt2xY0cd33TTTVbf1VdfndMxzDlvAOjXr5+Ot23b5vu6FStW5HQ8ysxb12uvvVbHTz31lNV33333hZIT+Tv55JN1bNYKAPr06aPj7t27+76H9zMpEf+NqObPn2+1zbtOmrdLiBOegRMROYoDOBGRo4pqCqVHjx46nj59utVXp04dHderVy/nY0ydOlXH3qVomaZNqDA6d+6s43vuucfqq6io0PHkyZOtPu/0FxVez549rXZ5ebmOmzVr5vs675Jc8/fs3//+d8ZjnnfeeTq+8MILrT6z7b3iOi54Bk5E5CgO4EREjuIATkTkqMTNgR9zzNf/Jl1//fVW36BBg3ScaU7taHjvTLdgwQIdc847fLVr17ba5mcdJSUlVt/AgQN1vGzZsoLmRYd5l/G1b99ex08++aTVd8IJJ+jYuxzwwQcf1PHvfvc7q2/fvn06/vjjjzPmM2zYMB1PmDDB6uvSpYuOvXerPHToUMb3DQvPwImIHFXtAC4iU0Vkh4isMh5rKCILRGRD6nuDTO9B8cO6JhdrWzwk093zAEBEzgNQBeAxpVTn1GP3A9illBovIiMANFBKDa/2YCKZDxYA8wrLXbt2FfpwR/xJuGXLFh1PmzbN6vMuK4zY+XCortnyXrH36KOP6tj8sxsAbr/9dh1nujukY5YDuA0B1LYQdTWXdQLAypUrfZ+7fft2HY8cOdLqe/zxxwPJx7w6+umnn7b6zN/tRx55xOr72c9+Fsjxj8JypVRX74PVnoErpV4D4B0J+wKYkYpnALgs3+woXKxrcrG2xSPXDzGbKKUqAUApVSkiJX5PFJHBAAbneBwKF+uaXFnVlnV1S8FXoSilygGUA/H6U5vyw7omE+vqllwH8I9EpGnqX/KmAHYEmZTLWrRooWNzJx/AXgo1duzY0HI6Ck7WtUmTJjoeN26c1WcuKfvLX/5i9SVo3jsbsaitOedcnbvvvlvHQc15e5mX2h88eNDqM3fmMjeuBuzlylH+HOW6jHAOgLJUXAZgdjDpUMRY1+RibRMom2WEMwG8AaC9iGwVkUEAxgO4SEQ2ALgo1SaHsK7JxdoWj2qnUJRSA326evg8Hinzz5l33nnH6uvQoYOOa9asafV9+OGHOs509Vbz5s2t9kknneT7XO8SQ+9myVFyra6ZlJWV6djcBAAAfv3rX+t4yZIloeUUpbjVtkaNGjru1auX7/P27t1rtb1TXrnw/g4OHTrUaptLezP9fv7gBz+w2uZ/k4tTKEREFDEO4EREjuIATkTkqMTdjdDcfPTMM8+0+szNievWrWv1mfOjmTYVNnfwAIBOnTpZbfOS36DueEiZ3XnnnTr23iXOexc5itZ//vMf374//OEPVnv//v15H+/mm2+22pMmTcrpfcxbMgBHLjmMCs/AiYgcxQGciMhR1d6NMNCDFcGluebSRe/0iumUU06x2uZdDMOglJLqn5WdqOu6c+dOHXt/nhs3bhx2OlFLe9e6XBSiruYdIAHg/vvv1/Fzzz1n9Q0YMMD3fczfLXOzcu/runXrZvXt3r3bav/973/X8RVXXGH1mUsFL7nkEqvvlVde8c2tQHK7GyEREcUTB3AiIkdxACciclTilhFGzbysNtPnC23btrXaYc+BJ9VLL70UyPsce6z9qxGXTWxd9+KLL1ptc9mt93J1c9Np7/JD8+6Exx9/vNVXVVWlY3OTcQC45557rLb5mdWPfvQjq89838rKSsQRz8CJiBzFAZyIyFEcwImIHMU58DzVrl3baps7dWRy6623Wu1FixYFllMxy3QrYC/zcwjvJdfdu3e32ps3b9bxF198YfVNnjxZx8uWLcv6+MVo7dq1Vtu8HbP3cwfzWolTTz3V6lu1apWOvfPcv/3tb3VsXiMAAPXq1bPas2bN0rF3Lt28LUemWwBEiWfgRESO4gBOROQoTqHk6brrrrPapaWlWb1u/vz5hUinKHgva65fv35Wr/v2t79ttc0lbebGyACwcuVKq925c2cdd+zY0eq7/PLLdXzjjTdafeauMnH9MzwuvEs1N27cmDYGjpw2ydYZZ5xhtb1LB00VFRU6fu+993I6XqHxDJyIyFEcwImIHMUBnIjIUYmbAzeXEPXp08fqO+ecc3S8ffv2QI43bty4rJ/78MMP63jKlCmBHL8YvfHGG1bbXO7Vu3dvq++OO+7QsXf3cPNWouZtTYEj58DN5aLe2wSbOwL96U9/svrM+fqysjKr7/PPPwfF19KlS6NOoVo8AycichQHcCIiRzk/heK9q5/5J2vLli2tvscee0zHEyZMsPpef/11HX/66adWX61atXR88sknW33eJWyZ7kBo/qnPu9vlbu/evVbbvDLSu9NR165fb2JiLgsDgB//+MdZH/PAgQM6fuutt6y+q666Ssc33HCD1Wdu1PvnP//Z6vPemY8Kz7sENRPvz0sc8QyciMhR1Q7gItJSRBaJyBoRWS0it6QebygiC0RkQ+p7g8KnS0FhXROrJutaPLI5Az8E4HalVEcA3QEMEZFSACMALFRKtQOwMNUmd7CuycW6Folq58CVUpUAKlPxZyKyBkBzAH0BfD/1tBkAXgUwvCBZZvDuu+9a7Z///Oc6njlzptV3wQUXpI0BYPHixTpesmSJ1VdSUqLja6+9NvdkYyTudT0agwYN0rF3nnns2LE6HjVqlNW3YsWKQI7foMHXJ7NNmzYN5D3zcFAp9Rbgfl0LoV27dlGnEKij+hBTRFoB6AJgKYAmqUEASqlKESnxec1gAIPzzJMKiHVNJtY1+bIewEWkLoBnAQxTSu0Vkaxep5QqB1Ceeg//JRoUCdY1mVjX4iCZlr3pJ4nUBPACgHlKqYmpx9YB+H7qX/OmAF5VSrWv5n0K/gNhLuv75JNPCn24IzZwMK/2W7hwodVnXhn63//+t7CJVUMpJS7VNVs9e/a02qNHj9bx6aefbvX97W9/07F3aeKmTZustvl74t1c4LTTTtNxq1atrD5zw4C7777b6tu3b98R+QdgOYDvImF1zVXjxo2t9rZt26x2jRo1dOy9Ort169Y6jsGy3+VKqa7eB7NZhSIAHgWw5qsfhpQ5AL66NrgMwOwgsqRwsK6JxroWiWymUP4HwDUA3hGRFanHRgIYD2CWiAwC8AGAAQXJkAqFdU2mumBdi0Y2q1D+CcBvAq1HsOlQWFjXxKpSSrGuRcL5S+m9qqqqdHzeeedZfffee6+OzaWBANChQ4ecjuf9DOGll17SsXe3nqjnvYvByy+/7NseMcJe+mzu0ONdHur90M+ss/cS+KlTp+rYu4zx4MGD2aRNBfLDH/7Qaptz3l7eusZg3rtavJSeiMhRHMCJiByVuCkU88+ef/3rX1afefVlvXr1rD5zudfFF1+c9fFee+01q23etH/Xrl1Zvw8V3vjx46NOgULmXfKZydy5cwuYSWHwDJyIyFEcwImIHMUBnIjIUYmbA8+WuTsOcOQl2ERUXBYtWhR1CkeNZ+BERI7iAE5E5KiinUIhouTz3oEyaXgGTkTkKA7gRESO4gBOROQozoETUWK9/fbbVrt3795We8yYMTp28W6hPAMnInIUB3AiIkdltalxYAdLwCapSZFh15ajxrrGStrNb3PBusZKbpsaExFRPHEAJyJyFAdwIiJHhb2McCeA9wE0SsVxUIy5nBLw+7GumYWZS5C1ZV0zi7yuoX6IqQ8qUhHUBy35Yi7BiVP+zCU4ccqfudg4hUJE5CgO4EREjopqAC+P6LjpMJfgxCl/5hKcOOXPXAyRzIETEVH+OIVCROQoDuBERI4KdQAXkUtFZJ2IvCsiI8I8dur4U0Vkh4isMh5rKCILRGRD6nuDEPJoKSKLRGSNiKwWkVuiyiUIrKuVS2Jqy7paucSyrqEN4CJSA8DDAHoCKAUwUERKwzp+ynQAl3oeGwFgoVKqHYCFqXahHQJwu1KqI4DuAIak/l9EkUteWNcjJKK2rOsR4llXpVQoXwC+C2Ce0b4LwF1hHd84bisAq4z2OgBNU3FTAOsiyGk2gIvikAvrytqyru7UNcwplOYAthjtranHotZEKVUJAKnvJWEeXERaAegCYGnUueSIdfXheG1ZVx9xqmuYA3i6+08X9RpGEakL4FkAw5RSe6POJ0esaxoJqC3rmkbc6hrmAL4VQEuj3QLA9hCP7+cjEWkKAKnvO8I4qIjUxOEfhCeUUs9FmUueWFePhNSWdfWIY13DHMDfBNBORFqLSC0AVwGYE+Lx/cwBUJaKy3B4bqugREQAPApgjVJqYpS5BIB1NSSotqyrIbZ1DXnivxeA9QA2AhgVwQcPMwFUAjiIw2cYgwCchMOfHm9IfW8YQh7n4vCfo28DWJH66hVFLqwra8u6ultXXkpPROQoXolJROQoDuBERI7iAE5E5CgO4EREjuIATkTkKA7gRESO4gBOROSo/wfiD3YVHT9V5AAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# device config\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# hyper parameters\n",
    "input_size = 784 # 28x28\n",
    "hidden_size = 100\n",
    "num_classes = 10\n",
    "num_epochs = 2\n",
    "batch_size = 100\n",
    "learning_rate = 0.001\n",
    "\n",
    "# import the dataset\n",
    "train_dataset = torchvision.datasets.MNIST(root = \"./data/mnist/MNIST/\", train=True, transform=transforms.ToTensor(), download=True)\n",
    "test_dataset = torchvision.datasets.MNIST(root = \"./data/mnist/MNIST/\", train=False, transform=transforms.ToTensor())\n",
    "\n",
    "# data loaders\n",
    "train_loader = torch.utils.data.DataLoader(dataset = train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset = test_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "examples = iter(train_loader)\n",
    "samples, labels = examples.next()\n",
    "print(samples.shape, labels.shape)\n",
    "\n",
    "for i in range(6):\n",
    "    plt.subplot(2, 3, i+1)\n",
    "    plt.imshow(samples[i][0], cmap=\"gray\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, ('', NeuralNet(\n",
      "  (linear1): Linear(in_features=784, out_features=100, bias=True)\n",
      "  (leaky_relu): LeakyReLU(negative_slope=0.01)\n",
      "  (linear2): Linear(in_features=100, out_features=100, bias=True)\n",
      "  (linear3): Linear(in_features=100, out_features=10, bias=True)\n",
      "))), (1, ('linear1', Linear(in_features=784, out_features=100, bias=True))), (2, ('leaky_relu', LeakyReLU(negative_slope=0.01))), (3, ('linear2', Linear(in_features=100, out_features=100, bias=True))), (4, ('linear3', Linear(in_features=100, out_features=10, bias=True)))]\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[1;31mSystemExit\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "class NeuralNet(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, output_size):\n",
    "        super(NeuralNet, self).__init__()\n",
    "        self.linear1 = nn.Linear(input_size, hidden_size)\n",
    "        self.leaky_relu = nn.LeakyReLU()\n",
    "        self.linear2 = nn.Linear(hidden_size, hidden_size)\n",
    "        self.linear3 = nn.Linear(hidden_size, output_size)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # because we are using crossentropy loss we dont need to define\n",
    "        # a last layer activation function because it is applied automatically\n",
    "        out = self.linear1(x)\n",
    "        out = self.leaky_relu(out)\n",
    "        out = self.linear2(out)\n",
    "        out = self.leaky_relu(out)\n",
    "        out = self.linear3(out)\n",
    "        return out\n",
    "\n",
    "model = NeuralNet(input_size, hidden_size, output_size=num_classes).to(device)\n",
    "# loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "# training loop\n",
    "\n",
    "n_total_steps = len(train_loader)\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        # 100 x 1 x 28 x 28 -> 100 x 784\n",
    "        images = images.reshape(-1, 28*28).to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        # forward\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "\n",
    "        # backward\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if (i+1) % 100 == 0:\n",
    "            print(f\"epoch {epoch+1} / {num_epochs}, step {i+1}/{n_total_steps}, loss = {loss.item():.4f}\")\n",
    "\n",
    "# test\n",
    "with torch.no_grad():\n",
    "    n_correct = 0\n",
    "    n_samples = 0\n",
    "    for images, labels in test_loader:\n",
    "        images = images.reshape(-1, 28*28).to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "\n",
    "        #value, index\n",
    "        _, predictions = torch.max(outputs, 1)\n",
    "        n_samples +=labels.shape[0]\n",
    "        n_correct += (predictions == labels).sum().item()\n",
    "    acc = 100.0 * n_correct / n_samples\n",
    "    print(f\"accuracy = {acc}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 14 - Convolutional Neural Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import libraries, define hyperparameters, configure GPU.  Define data transforms, import data and visualize dataset batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAB4CAYAAADrPanmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAADoA0lEQVR4nOz9Waxta5bnB/2+bnar3e3pz703MiKbiOyqyunCLlWZggIh+aEsEAgjIZCQ8okHJB4o8VZvfkLikURYwpIlQMIISiqBkGWXjFFWpdNZkZERN9rbnna3q5vd1/Lwzb3PzSQzI1yZVlTKZ4TOPXH3PXuvdeaca3xjjH8zREqJ9/E+3sf7eB9/9UL+vN/A+3gf7+N9vI9/sXifwN/H+3gf7+OvaLxP4O/jfbyP9/FXNN4n8PfxPt7H+/grGu8T+Pt4H+/jffwVjfcJ/H28j/fxPv6Kxl8ogQsh/jtCiB8IIX4shPgHf1lv6n28j/fxPt7HTw/xL8oDF0Io4IfAfwt4Afwe8G+nlL73l/f23sf7eB/v4338WfEXqcD/VeDHKaVPUkoW+D8Bf/8v5229j/fxPt7H+/hpof8C3/sE+PIr//4C+Jt/3jc0TZPW6/Vf4CXfx/t4H+/jv3rx+vXrq5TS2Z/8+l8kgYs/5Wv/f/MYIcRvA78NsFqt+O3f/u2/wEu+j/fxPt7Hf/XiH/7Df/j5n/b1v0gCfwE8+8q/PwVe/ck/lFL6HeB3AB4/fpwA4u6C2G2QArQQSAQohYuCdvRs9iNJF5R1iTYGYzTGaIQAaz1KKaSUgCDGSNM07Ld72vbAOFqSgKKqqU4eYeYPELpGCIkQkELAjgNSSrQxlFWJQBBCoGtbdGyZqx5I7Kt/yhg9WinqomRWVtzc3CC1oCxrjCkpipL5csbFzYbYJ5qqJAnJq9sDY+yQRjBYj7UR7yValAjpcCkQowA0UtSEvuPsSIFKjGOk7QIhalL0lCZgjEQIxeAEo/X8xuq3MLLgeFHx6KTBW8+LLz7lxYtX7HYHCm342vPnnB4d451DCEFKkeA9kogQgna/oywUy8WM+XLJGBKXl1fY0RFCQkjJ6njJfL1iHEbcMBCDRxrFbmxp2555OadU+RoqI9FFye2wx0VLFBovTgAQ7Su0BK0NSitCCCBgtANFUVJVJSJBuz+gtSaEQAKM1sxmM2KMxBinp0qQAFUYBmuJzpNChJSQSoEE7wMpRVJMOO+w0SOUzPddapTQuNHhxhGzfozQFTbWXPvnSJny85IkCUlMgpAkMehcokQHcQQGbFSQJNpUGFOidEFCkpJAKvHuuUtASsTpdwAhEgiIPpBifr+CgJQglcRaCF4QEygsz5d/CMDbdofzPRpPUWi81IQoSDEQQmAMjsEFhsOOSkuaqmBel8xnNbtuICaY1RWV0UigMpoxBPZdN71vTRAJlMzPTYh453E2sJzNSd4x2hGkRCjFaEeM1vle6SWlrHFu5Nvf+U+AhJQCpfO1TxHwEUggJVJJlFYY/S4VJRIp5V9SKLQ2CJE/7+8wOzldW4lAkGIihgBCIoUCkafDIiWEyNVmAmLMz8SHX/8mIPjg8RHnR7PpdcF7jx0tdrQMg+VwGLjdHNhsD3TDiPP5uW2ahrIqSRGMKSiKEuccXdsRQsR7j/cO7z1aS4zWCCEIwROjZ9YUPH50xno9p64LqrKgrisKbYhTbfzm+sCXbzZ/bhL+iyTw3wO+IYT4CHgJ/A+B/9HP8o2xvSHcfgkCpFQoIUlFTYqaoQtcvLqhmC+oFw1VXZGqElmVxBgZBovWGqUUQoh8UaTgcHPNfrfLD5ZS6HpBXD2lUGtUuUZIjZKC4D1DOJAALQ1RNygBNo5cD5EmDcwbgMQ4+xGt7zBKo6oZVbng6uUnzOYVqjiiaBaY2Yzlw2Mu40uCdsjZjCAUt+0l2/EapQWtd3QxMY4SwxxpRsZk8UEQY4UUc9Jhiz6riCqyGx03o8c5Q/I9q1mgUSVIw+0Y2bUD31r+NQwFs1rz8Kjk6tUOd/Ulh5efcXu9oS4q9LrmaFkgJAipCN4x+g4RPSl49LBjIUvOlOB0seRq33PYv0aOkRgFQmvmi8jT5QmH2NP1HS70lHVJGG84HK6R4xqtGiSSslQ0R0v6tCGlHk8FUwJnuERpQSlrSlEw+JEoEuOwx6iGmVyikYzjBbWsCTEAUFByXGpC8DjnIOWkmIRAFgHpOwKWGDwpRrQ0kAQ2WGII+BBww4ALI0IrpFAIZVCqJHYjrm1R81OUrnCp4sp+gNIRIRQyGcCQUAxREEMFMSF8B2EP9LShRiRNEWcUosGompgkMQpkukswEFOCBDEmUkykFEEkEIngLCI6pPBIEdA6olDse8E4CryHQvb3Cfx6PDC6LY1yzE1FHw2DF6QQ8MHSjj270XF78YplqThdNKCXzMwRPQdsSFTFElWXECNlVeJDYBg2uJCQQuNkAq3zZ5PAaC2DtZh5JMae/dCCVgijOYwdDSW1LiipKKlx3vL9H/xnJEJOYEajjSKFBDbkFKUk0ihMlRPYXaQEKUViTEipKMsGJTUgIeZEK5AIqZFSI5BEH/DWIYTCmBIpFTFESBE55YmUEjFGQgh88AvfRAh4eDLnG89PESIRQqTvB/a7A9uN47obCO2O9vqG68sN+0PH6D1SSU5OT0jzOSFEyrJGVIm+H9htd4zDiB1HxnFgHEeUFlRliZQS7y3BO9bLmnWlOGpgvlhwvKw5Pq6pqwohC4SUxJT+y0vgKSUvhPifA/8vQAH/bkrpuz/jdyOI+eQUAo8kiopYrzCFQW4Tpw+OUSZRliVlWaG1pu8HqqrKp2hKSCVZLVbsbjZ4O7JczKnrU1Sh2QVNPT9GmAKpBVortCmxLqC0oe8HRjtitzt0igxdx83tFansoQEQVM2K6CRKQAiKy0vP688cHzw4BreCfk4cKkZVULoG1AGbPGMIoMD5QLQJJcAoiUMRxoARklIViAjWg4uOmYZ5WWKjY1BQlQItEjE61kWunoIw9ENLrwJiGmD1g+XV60t+9z/+/7J9/QWFLnm4OqXQhmG/Z3N9wdn5GUpLhpDwdiTYgXa/4fxkxWxWg9K4mLi+uGRze0tZLTBFQxSKy4sND59LvBMMg8c5i5lpCqkINvDm5gqsQEVJbSTnH5wQ55JEzEl4gsljSiQhkVqhCkMcepz3+BCwzuGcRZcVkXxfy7JACIlSCq0VafoggkBMlZ/znpQSQgqkyg+8lAKBQEuJjzHnSCFQSKIXuOCJIhEVeBdJKf/M6bEkBAFCkkhoFEaWSFPggkXqEhktUnqEDyQ1x8VTpFC5K9SGKDSRhEsRLMQQCCEn7Nw1QvD5YIkhEFNEhJ668JQmoYqEwiGSICTN6MCOgqjcu49PEShKybwqmVUl3iucTUghMDGRKkew0A+KykhmS8lqpTk5KZE1dOPIo/OGeV0yDgPL0lCKml55dv2AixGhAB3RWiGCQhQKlEDXkUBC2USQgWAkSShkKSkbiQoCAggEUsr7g/iuqhYkpBa5IpZToSwgAkLmViVN/1FKSCnnCGSurIWWxBhz4SfzM5KSICVQKueFqtAYbRhHi7MBIfNzBLkCF+IddyPGXC2nFBjHke224/pqy6uXF3z54i1XVzt2+wHrIj7mZ7A0mlldUpUF3gdECoxDxzgMGK0YokfgUSIg8djBI2J+H2k6wPdbxw++/yXXlzc8e3bGk6enODdS14b5Yo3WFd6Hn5pJ/yIVOCmlfwz84//i3xmJKRKTICnN29uW3juef/2c3/y13+Ts7AHOH7B+ePehFAKtNU0zo+97nHNIIVFScnt9Q1kXPH/+jKOjJZvNhvbigFGaJBNaJgolUFIhS4UvCqQuUH3H4bBjc33D7dUF7dBSH0s4aUCAqmck4Rk8DANcv2zZXys6V3L4ccvY7/HBUy8Ljn9pjjp1DLKnixYbe+pCYApAggiRnbO025Zls6SqazqV2A8RQuJ0XXG+LtkO0FlPEgGhYT6fMWtqKlXgA1RSU6h38MPu+prDJ69or654eHrMs0fPqKsa7wP7/S2HtqfaH/Aun/wxBFzfUpea2bxh31teb94w244M7cCjR4+o6xUxafadJXQtQzvS7lu6rid4i+oUWmgKCl5fvebm7Q1xsHzt6TnPfvmMsRL4kHAu3dOcdF2iTa7YUApZFAgPOpYgJS4GipRAClwIVGWFMWZqO0OunGIkhUQiEAU4EkorBIaUBCGBnF5RSUWSiSgjpTbTGETgYyBF8DHiXELrAjGdhikJvDWEqBFKI0sIRSAZiw4Fipp4eEPqPkG4HXr5CzTzNVqXSJmAPCKJgFCJ6AUxgIq5NQeIKSC0wGhFijmZq0JTGIfWAaE8eYYC82VJUSu80xiG+3uudQDlURpSHIlBYkdHZQRVJannFXqwNOqUhRGcLxserOdUtaIQhmpuWK1rZnWJswYRBTFKnFY4rYhIZk1F2x5o93tkhOQDMQauri8Zdi0gEFVJENCFgDKRmkgij7kSubtQSqNUTub5nMwjECklSTId8IlEQgrJ/bxJkF8DQUwxV90yj2GEyllf3B3oIh/qUmmid8zqitViQQyRz798iVQF5VQBx5ir7Hc5DGIMOD+w3W55+eKKzz97y6tX11xcbeh6i/cJpRRVZTCFQhuBFIHgRnbbHXZ0pCTQ2gCC6EcKIzDK4N2AHRxjiEilcueAJBDZ7wYIESU1MSTsMPDgwRpEQVWBd185tP+M+Asl8H/RUNqQTMHgE19e7HmzHWiaJbe3t1x8/mOO5jVvNgNd3+Y/rwxaF9R1g7UWKSXz+RwBfPrpp+hC88HXPuT0/ASjBaPrKIRDBIvCY2Sk0AKjBV5qXBTTc1PifUWvNG60pBi4LxkTXG06bg4Hei8ZBuiHgWqmmdcL2l1CtBFsYnfj2OyvKL+e4NSgFpJ5rVjMFoyuJQUPyrOXAacldhywY8foJR6DrkvqSqKlJyVLSj7fGCUYInz+ZstcS0qt8UGQbLifo66bhufrZzxbrbm6ekU/brm+ueDQ9gzjyGy+YnvoKFWiMopCKfpDx+J8xcXNDTcHy35I6F3k0ckR82WJs5Htds/bqx0XV5cUi4rS5IcTNCJWrI/XDCeC5eIE9euJRkEjHUOZcKUAUaB8Rerz5VysVtSVpjAFUkqk0YQY8N5CimilMEXJbLHAW4vzHinlfRK/++Wjx7lASJCMJMWARqAArTRKyjwHTwaLRcSEQqCVISAZrcVaj/MRHxxlUd4ncKUVq3XDrg0kofEIohcoL2m0YmGuSeKHHLpvM7QHitkKWWuSkGgRpkJDEZLG+0RQKRcqMaGDIMWYq82UpsQB0SvAEEWBkykXHEaidMpVrJJ4JZHh3aFttEBXhqZSyCgppIC2BSSlKZktG4RWzI6XrJuS9bxiVhus81jfsWgKZFEiVIEoDPtDz6ubPdvO45NCG0WhKpp1zeWbtwz7Ns+FnafddiTrKUtDKRVJa9q2J9nA0iji9PERiCkhJ4QQyOmXUJK6LKnrkn7sGb0npIQi4x2klFP8XVM0jUySyJ8HofQ0YyHP4IUkRYFQguQDQgjGvkfOG54+fsjtZsOhH9G6YbVao7Tm88+/ggemSPQe24+M/cjV21u+/OI1l5dbBhsomgbpLE1dYQqFEJEUPYKMJxmtGfuRtm0RZFxNK4kUgkhCK0FVlcxmi1w4eE+I+XlPIWFt4ub6kAsra4khEZNiuUp5ZPhT4ueSwF1K7Kzn4nbg7fWALBvQmm2758dffMrxokE0BYVRCPKvhMb5PAMP3kPqSDESguf5h885PTuhrkpIkVnTUFea6Ht0ChgpMEpOQKCEKBBJk4JCi9xaJZGrP+RUjQG7m472MBC0IiFRVaB5plnWJbuLDcNoEckwLw3720S81qilBDxdN/BoUYGOHJc1j+SSZYj83u0NfZ/QBBSCeQF1HTluCoypGELHaAPaScrS8OWwQ1lBpRRGKXzy+YMxPeFKgpGRvtsxjkMeDY0O60ZG2xP3gkNIrOclojJElccRQzfQ2pGu97hokKXm6uDYbDdEH0lokjYMIbHdHTg5WmN9ZBwcegjE2z37wwEXPMRIlyJNIVHrCldMFVWCuyawaCq0kSAlCYEsNBKNSSrPlcl4SFFVaKVQQsGfmFuGu5m2d4zWEaygqDJwKKVESYGaro3SEikKJDCMA9F7qsIgkoYYiTFMiVLeJwslE00TsUHg0WhZYJRESzCyQ6kNQVpgAH+LGF9TlCOjjxAsSkhMOSMFTxc0QeZEI6dxQkp5cJgmIE3KfP9STEQkISVSFCQnkPcgLMQkMKj7z4+pSlQjkJVCJ03hoLAjSkSEySOqIoKSEkti7z3OSdAaazSt98ym8dO+s9zsWm5bx6G3JJEoguQQPU1VZWBYZDAwWE90kVIZCq1QIhKDpZA5UeU7+46IdjdCyTPtRIyQouf47Iz5fM6byws6a9Eij8HEdABO9XvGDLibd0/VOXkAm5FJCULlEZqMCKUgJZy3dG3LMPScHp/QvnxDjFAUBcvFnMPx+p5CF53D9gPttufNy1tevrhkd9vm0QsSIyVCCZTKSVcIiYsCO+YuEJE7gpQiPniUEqAKUgQhNU0zp6oy9hGTJ6ZACB5SvK/GfYS293B9QGhF1eTu0/vIT4ufSwI/9I7Lbc/ldsBGwXFVUhkJyTGOAV9LTubrfCI5CGiiKokpf5CHfiAEj5KCo/Wa80dnNLMKJTPIUVU1s1lDm6fOaJEfZq3yLC0FAUkSnESSiNHnE17eDeVyqCARNlHphG4kqW5opOdoVrH/omTYWcZuoDKaoVWEAyjrEFhECpQKBhtZmxJd1IxlnoXVuqEUComgKgXnC8260iBVRrDHiOsFKkYInhmaplCYWiPHhLb6fgbuvaP1LV3fMQ6OFKFpSqpaw87TtgN4ga8KQpQoJTFFTUyKGAQiSYiJvu1pezcBdBGhDEIZUIbdrkcJxdhnUMZFgZCJ0VmKwpCQdOPAYCNzqfBFQiRQ7l0CF0oR7+agAqRSKCXyAR0zg0TERFEYhNZ3+B4gcM7hnCOGiJISow2DdaQEUmm01mghkfGdMk0rhZb5Gjtn6eyYcRAFRaEIRFAKIdP9hzklT3A7tJqREhRaUhpFoQMkR1QC6nPU7BFqaAndF9S+ZZQNftygo8RUEiX3WNY4VSKSvGdBQMxsFhFB5mSnokSkhCcSIxBz0koxIVK8a7S++liiSoMoIRYSF3JVaGYFmogoDVFJAoLeOnwSjDGgg0fXNZZEdJbOKZzz3B5a9sNINwb2hxYhIJSKaAXJuzy+SokQp4rTaOqioKok0ggsgUImSq0QxPvOMN+5uyFIHk8x/f2Olwuk1veFiEAgUv5d3iXn6S+dpy6ZEUQS+eckkELeJ8D8MxIISZAKnwKHYeR2u2exPKK+2edkrBRHyzni0dn9a4zWst1GLi5u+eyz17x8dc123+NcRGpNcB4pIXqPFxlUJUmG0eGSy+CkykyZEMY8E8fjyQeY1hopE+PY430G4kMIeCkxOhcrSUh8FHSjZ3sY2e1a6qrC2XeH9p8VP5cEfrMfeHPdYYPg7PiEs/UKLQNaQV0Zzk/XPDo/5/bygi44PBrRzOjHFmLCe4cAmmbGBx8952g6UfOBqBCmYL1aMu4jioASTKcnaAFJSURSeCkhRryzRAJfxbMAal3RS8OsVqyOa4qiIdqOh8dz9C9U0ElefPoGmTw6GYa2Q3QjZQzMZ4qyVMgtGCEhgLceEQLn84ZZKfCupzGJbywrBqMYoyP5SBgDhzbQdpGTdcGjOlHPFG6mGYiIoZw+FtANI1fdFusSg03M6oKToxmmlCgN+8M1SlV4DC4VGGHQZYHQAhkkJjnG3nJ7e40sKuZ1QUhg+x7rW7Qu2G1bukN/XwnH6z2IyPpozWK+ypVW2LHvW0QSJK0ok0RJzThdyxBDZgRIidYGqSRSCdQdmhUjyUcKY3KrHSCGnDTGcSSGgBCSsqwoK42NkpGENg3GGDQJ4X1OfNOBLWVmKmklsUOP1gJlCrSWlEKjksD7d21q8J59e005K0giUBhPWQYK43DeEZmjFscUURGix779LsvtW9TxU0Z3i0sOVXka+Rld+gajfoQQJTKmd3OACEnmw0ykRCEkSiQ8gRgEKZBHLQRklNx9q/mqZlorkkl4xcTMSahaoZVBKo2NicNoif3ArNYIL++reiEF0Ts6q1AJDv1ASAlnPYfdASEhhRKnBThLcBFrHaNzhBSYNzOqUjNvNIgAzk1dLqTgSSr+sc/QXZq9S9BaGU4Wcy5vd/csEzmhmHd/RkxJXEqdqX9BQBQkL0DlxCiFvk/q+bDPH14vIKBofeByc+D0/CkPzs9pD3sqLVk1BY055lWezrJre2xr+eKLN/zwky94fbnB2gRJYnQGhpu5xlqLdaCUpjAVgxvprUcgUUJRlDXOepz1eJe7RSkFVVVlauc44pybRigBrTXGVLmzuKOqxkwUuN20FGWJFRU/TSz/c0ng3RCxTvPowQP+1t/4LVaLJf32LWHcoJWnWs5otMIrgapKKFeUJw948+YFl4cDRkmOT0948vwJDx6eMvWZ+YenDJwdH604uJYoPCJ5hIgT8yWzQhAgiEQ/4sae6D1JKd49fYlXVwcSipUpWFSGmRZUsxUfPvuQB8HAvmTz5pboAyTFuO2pDpFF0pwclYjakCwchgFrPTdbi3CJWRx5dnLCrrcEf6DUlr5ouLI9IUViSOxah4zwd37lGU9Fhy8llzHRbnoWsrq/rdYGNtuB29sD202PjIauUJS+gFATU4UpF+x6OPQ9hRqZVQVlqXHJZ4BISI7mNU1TURQVPkZGF+hHR9sPuCAnBkrm/9rRQhK0+8TmZsAYCdKDBn/oqY8b5kVFWTQcpvfpw4jSktIUNFVNcIEUIklFlBDT7FqghMqHmJTY4AnBEqOfqi5FWc4R5QyZGrqdpz+ArwQLHagQE3PhHc9aqpTHKVOy8ynDbJ6ErvL8O8qJmiYEWhrqUoIWGGMRjIzDAetGVsszdFFgyl9AGcHN7VvGT/5DVqf/PW6Vw7VvGC5/xKL4PFeXxXLiUgMJUpBolee9EUBKSqkAjxJiIioL5MQVTx5iygnRKLg7DV30iBTRSAZnaYQkqUTSEhsT9tDx9uqGWijmlUEmGK1ndC3zymC0IgSBdZ7RerRQxHYgHAaSloxSI0qNjJHd9kA7WlJM1MbQNPlw04VAkplAcXTY1BFlSSpT5qSRSMHn7iOpPLbUksWs4ngx58tXbwnhrqLNc/JcR+fZv5oO+hgg3A3FM06MTBI5HQ0xZvqfjAGIpImtQlJYl0AW/O3/2m/x+Y/+CIlnONzy2csL6gffAgQXl9eM+2tevLjg9cUV7ZAQwqCVRkhFjAnv756nDJT3wdKNnt55gk80VY1WEucC0efxnHOWRGQYMmB6p21w3pFILOcr6rphGBwuREzQKC/p+p4UHAiBnq1Bzv7cXPpzSeCmVBSVhiQ4OXrCL//yL9NuX7K7/oL97i19TNzc3FCVJY8ePCDVK672A7dXVxSF4OGjRzx68ojT8zOEhBA8SSRSjPf80qaqeHAsuO164niLKA1SrUgpn/ghOpIb8WNHcH1maPBOKJAS/PjFK7759RVLrWjGgVkYEKz50R/tePudPS9+/JZ+jIxxBAmzVBBfe25Vog+ez2aJ3aeBmzQSvOVtO5IGQRET+25DxJOk4tPtll4lojYIURJw2KHjtNJ8cLJkpRva6Nm2HYrE6Wo5ndx5ttYPI6/fXlCVDW0Pw9AjkiUmw0wd45xHaYWU4KPnZruhqas8h1a5CVUi4ccBkSI+pCyE8YlZXXHoPclI6qqiijO6tmMcLELkD1lZFUiVEBIuXt2gFxK9ril5x+0VCopKZyTfSKJz9/NRyE2wRqHIs8M8LlGUVUWIlhQVWs0YnGawgU0n2Y4V7W7PVnacL+DJcYUWAh88w9ChjUJKgdaCppkx2lxJIyWqMGitUCRGkSe32hScnj9ktljhmUQXQSPEjBANIRi0B2VmVCe/xPGvl7z9z/5d5j/5Dzh+8lsEteT2sz9ic/kHHH3rQ46bX0M2OleUMZKCzFVtDEShQBVIqXA24GPKWIAI4AdS6nOhITN7odD+PoHHfONxo6dvW7QqMYsSowyBhI2Z5VUXFf2hxRhN0preDsRhZL5c4m2uBlVS+M7jNz3G507Aj0MWnc0b/OiQSIpKMy9zd9Q5C7IipoSfaqcYA0qKieqZk91iViFTwjpPAOazGb/xq9+kMgXdocN7n6mD71ic07jlTsiTvy6VyOOl6eukhBZyAoITKUQiESmgNFUeywWBCJKhHzlflvhVwRdfvuGTT7a87gTfOs9TFClAS5Ak7NBj+4A2M2TKYygnJNpFUgoYY5BakpKckrrE+4h1HkFBWTQMoSfeUSdjuiddxCk3LRcLmlmDqQsurzf0rUeQaahFoZkvGmxQjLmV+KluVT+XBD5vGuazin7Y890ffofj0zk6tiAy73QuJYujY2arYygWXO0Hvnzxgt32hifPHvPw8UNWqyXqTmEl5URbmgYLKRHwLGpDSo7OtdjdJUKYzBpIkLwnektwI9G7DCylNPGC8819/LDmwYOSUjqEc6yWa4ax4YuPr7j8dMvt9Y5x9HhncxtZzPC3mp3zXA8O8bxmc1Ow8xkh70PC1CWDEoQUqQpNXZQUdYV2JbiAs6C8ZK4EXzsuWBjNbrBsvWc3BgYh2WM5n8CimAIBTz0rmdU1pjS5jbOJSgqWcwVSMvhE8AESlLMaKSRJCbyfQMEQqKuKOI4orSmmhD+GkF8jxYl/K6lmJc1sGltITwgCo0vOTs+hv0T1IIoA8h0IE1KkdyNuwi4IgaJoSDG3wUpKVFmiIozdQEwOqQTGVAQEhzZyfeMYvccLhZWG4DN4GVLMqgKjiN5n3vfE+00xP1Naa2yIuW5TCmMMIQSUVPc9l5QK08xB6zw/T0VukSflr5ASTQayAgFT/CLhW/8mH//eP+K4Ezx8/i2e/NLf4ePLH1LtXnEmPFpDwEGacBbaDGjJCkxxPyrQQYL3JB/wbiC4wzQCCoAghfH+WsYY8ONISgEZoR+HnKRlLjx8nKp8bxm9JXpDUoH2sCc1FV1oSaMFEqYqCZ3FH3oKBFoJhAIDROsRZAJAYXKCUVFQFOU9l1pqgykKtJTESU2ar6Xk9PgIQ+LV20uUNjw4f8Bf//XfYP/lT1Byuj8wgZByStATHxyRE6xWBJ8yWySmaQ4eEAYW8znOBg6HlpgESmuUKfMYL0YkCUMk2IHr62sur2/Y9x5pVvfX8snjRxzPnvLBBzvqquGf/bPvEMOIloq6KEAkRmszUBkSRoLRKnMdIpOoMDOR0Bq0xkiB8ALvLC4EEAZTVsznMwqjCT5w/XbPsG/xwZGUpKxrqtmS9fKEFCzeOkzw+Ub8OfFzSeBFUVBVFQd34Is3n/HxDxrOljWVzjQggqUoCkw1o/WR/WFP2+5YruacPzpjsWwojL5HvKUQoMQ7etGE5hdaMq80UiRa22HbSxQC9IzoHd6NBD8QgiWlkHldXwFhzk4MykCSmba1ODqivD7lTW8R3kK0RO9wNs+1UvRZmDQmOhvQZaIfJP0QSQGiUpRF4G1vWcwNqRRoqShSSRwiJkHsItImVqXmowdLCq3ZHBy3w0DrHUlItnZPUpkrq5SiqMoslW4KmkXDxc2WQzegFzX1UU2lI3a0eJf50efHa7xzeA/D6OkHS9tbpNBoGalKg1ASF0G6RFKKgCCElOfSiPyB1gohFEoqdCFz8nQJ7QXCRaL099cyV2uRSEAm8thE+CzYEJIkBVFEbPTYOJJE5scmIQgU3OxbPntxQzf4jOiZgqga5vMyJ5BSIrQmOvdObp0gMrXYSk2KQ58TsVRIQEvJcAeaSYkqqtyGa42e1H5KapTKisTsBZEQQSBFyezRX6M5+4Trm0u8+CGPP/gmloZx9wbh9khOQQRE2tMOI4UYGAdLknNK3RCYPvzAnVQ8xizwyZX23d/nXQIPIeLxiBRIKXHYd9RCk4IgSPDOU6oMwCXvSDGBDIRhxAnF4Aekc7nCj6B8YlHPOPgRr2LGilLmIacYUVplQLfQqCBIyPwsxAhJUFcVMkRiCPm1VK7A68JwNGvY7g6U8zXPnz7nwckZ+y9+wmI+ZxM8LthJEn/H676LfA+zpiEx9NnGIStY8/07Wh/R9yPDMBICSFVOFgh5VCoJaDyXl5e8udmzHRNBlVkYNEXdVBwd10gpOD1eYkRgcI6yaThez1GF5tXFJUQBSaCFRBnDcrlg9AEhVaatCoHQinqxIAXP0B2y0IyENAWL9RF1qbHjyO5w4Hq7Q489tU6owlDVhuOTNecPnzBu32DUBP7+lFz6c0ngUmbARWnJ7e6K7/3gu3zw4ISzowWL2uCGgeLQMYgdu27k9voakRyPnzzk7PQoMxUm4YS4w6EnqD+lRIo5qScpqUuBkhEpAjf9NTYJZBnwLuDHDm97nLek6IE8J4dc2R8tFTFFVF3QHC+ol3PkTcPSeIYaehPZx5BlvD4R/ICUOs9Ux4RbDlDA0Cd8FIhS4r3n1c7xaC3RSaC8IPSeYTuyqAvGgweXmFcFR4uKwXo2e8uuHRmFx1SCw3i4I1qhlKIuC5ZNxbqpWKwabrYb9mNHKGc0NCyakqYSEARaCp48WhKsI4yJvnfsO8tWD4weSJ5KS4SR6Ilrq8uCJMC7wDh4xiFQaChLjUgGqQTaQN/t8dZTJQkhEKN9d9OFIGXoLleiWuRkLvMMPojEGCIxOBwWicAH8BY2beL1dcdnr2/puzF/WKRAlTOeP3/IvFQkZXBJkkLIH2CRmToxZNAbQRZzeElKKVMVlUYJwWHqU4UQSK0n64FcGQuhptHr9L+J15z794RZPePZN//b/PDb/5iLy5fYmNjve2Zxg+svMOEMpTXC3nC4vaVUlr53yOIYU53ipcrS+5DuVXoBRZKGKARxStJCvGMkRO+JBBCZZ344dHgHdvBQSFx0lErj+jGDukRETOiYZexjtMykpBKSwidAo9Zrxu01ETsdsHc85Cw6Ko2h0LkbyVROn7GfmGiKguRcfl9fwaJCcNSlYbVYcPzwER88e86sLInes1jMKbqWzucEnmmgMmMyKeXRCPmeaZVBVoefnnuBkoLlYp6FfDeamARS1QiZkMmCSJmbHwa+ePGCq4OlpwJTklz4ymM5VfokjAJNAmvRRBazmtXZMde7PYOLeQ4oNUprFsuKJBRdP+BCzH41QlPVNd5arBtBKpSRVHXFvKlIbmA4bLi9uWR7GDhOnqUpKYuKelayWtU0qzmMCqMlSk7jsj8nfj48cOdxIUzSds/l9TVuaNnulhwvFwz9iL61RFVMVVuiqSoeP36YqwAp/tiJLUj387G7X4gMegGUIua2HcuLqy9x8haXFGM/4sc2P4wxoFL4YzzWealIWvDgdMbDszkxjFxcfMm8bHCNoK8kGwVdiMQQ8T4iVcynchBsPttgzpfoIPDJMXpHVCWjDazOYb1QjAN8+ck1doAnD1d0YyBNarMvrrZctiObNuIdGCOpmogs3kGtwXu89RzNFzw6XiNkYKE1yQX+8Puf8OJizb/xax/w7NjQKEEMnsPumlUzQ5USGRKEhBElo4OLyx19EqhSgTZ4J0Hl2WdlNJUuGbUHmYGtvu/RmjwflCVVVUOAMHqEtPdv1EwmUzAZKSRFRFAUguAdve3pU6LQhiAD3gq6znK7cVxeO97eeDovsFOSJnjsfqAqJd6WkBqMbCjGnkVd0jQNMTra1mKtI8SYK26VZ9JG6yziSSB9BsgECZV14Pm9SjkVCGpituRKXApJKkGFiE2CR7/4G1TLgi9/8J/yo9//f3L9o3+O+WDFw5uPqY9O0GqN33/G9YvP8W3PfH3K4kQThiuCUQTqPD6JgSQkomiQoiZlCWTGKBjBvbvnqJTnscAwOLqbK8pZg5mVUCjKssTHNDF+BColFkVmGDnnmR2tOW8aZki6znMVBHYTcd5TKEUyisFZqnqeLSpKhUgWKRVDO+SiabqmJYmQEoObKnCyRP3ieoNOsJjPeP7wnGfnawrhiLajEBElcgEmpKIyhkobur7HxURQAmUSIeQxTjb4UqQkkcogZUClDoPFKIENBqULilIhXULEAR0svrvh1ZuOIdVQFjnRpztoPY/Ykg8QIjIkGlNghcd2A4fDnmdf/4inz59zcXVDTFn5i4CyKjl98ICrq2t2+5bROpSQaKOJIc/2TWmoq4p5UzNsr+k3l+y317jtFjFYlNbU8xNmpUYZSddu2X36PWZ0rGarSd3758fPJYFDukdqq6rEjSOfv3rDi1dvKIqCYcwn/IPzM5TK6qbj42OePXtMVZ6BzpRtpcQf/6l/YrtQSinPLicV2KIynC8OfP+Tb7PdO3pf0McqA0jwFb5u/peT9RFFU6DSyPXVNTIWbLrIo0JRqgGDwyAolMYHR5zaX4lAC03dG8a3jigElCBryXAbsCmxvUosTMID221gcxtZlCNFoZCVpLeOj7+4QEtYrk+pTE1lCuZlolpF1G2mXs3nc06rh2yvrglKUUvJ1x4+wKgawets6hUU1kmaoqSuFJvrS8besWxKQgok4SbFXOL8bIWLHhsiY4iZGhY1/Zg9QMrSUDWG5XLNzeWB/dAzBkv0gqY6YnAS63eYVc1yWUGVL2cpNUJqklAgNKDQmbMGySFkzHRArWnHPTe7kdevd7x6tSGEBlUsqJqauqkwKoOPNzc7NjdbNreO7aYhujOeL+H58+ecnp9yfX1F17f3fhyDHamKkkJrFBKcR8h3la0gUsiRhCaECZ1TCaESSpUombtGoYBMVKGc2BHLb3yTp88f8yu/+Zt8+5/8I77/u/+I9p/833j4yQ9YnjzkcPMZP/jBT3j48GtAjzACVVUcHZ/ixn5iDossRlFzklK5a58qUun2sMvv01tH7w6MMlfsUmuGrkOknuA9UUG1XFGUFVKBFBEZ/TS3jfTWsm8PnJqC9WzJJ9//Ad9vW/bBUpaZnz+mgA2Os1nDfN5gZMSNmUtvjCH4gFaSSkq0gCEmvIv3FXhMcN2OHA5vmJmSdnugvXzJX//WRxSFY+yuEX7MwqukmZcNx3XFy33L4DyhkEif2O93iBgIQSCEBlkiqwYpLam/hCGiUkDqGUoFnpwdQRcJ25YUI3Y8sO0slDOENPmQll9Je5NfjlIyjyLXK0IyuAjXm1u+/6Mfsjp5wJMPnmVHzOCJ3tH2HeVuy2I+o6or+mHg0A0kEmVlODk9yrRZobh++5Zhd4vyjlVR8/DIsBgCFCX9asahabDB4V6/RO57Hn3rGecnC0I547L/8zPpzyWBl2XBcjknhBpE4rNPv2DXZnEOCNo+04/aYWDWNKxWK6p6xueff8lsMadparQWCOG/Yiv7leo7ZQ5xNq6Z1FspAyHzSrGuYXt5w/7W0qkTEBOokeI9CAPgoyTZgv4wEFxHSB1NdUKJQurcIo7DSPRZNpyYaHkpIJJgbmpsO+BEwLnczmOy18XF556ZrNDHJUXT0PSCMiXmUjCERB8iplrhxp4QE7d9h+sCsz7x+KP5PdiaUsTagR99+ilS/yLniwYhNUeLBf/Kt5YcbKQ/bHHNArmoMaYgpg3bfUtVZiZGIbLS7Hhd03WW0RfYCGVMmNIxBkNIAmstXedIomKwisMQGKMiREW0uaVvrZvaWINRX2GhhEipK1xQ7A+WQ9tPopzJ30Kq6RBxbHeet1c9by4GbraB05MlpigJ+w3jcKBQipP1CWenx2y2W3YHy9VNx7za8mvPPuLJk6coLbidvDIKU+AB14/ZplYqhARnLUYZkHdYSsJIy75tQdWoIpJEwEcx4RsVKel7ccqdR4+U02ilrlk+/Sb/+t9/xEff+A2+/Pg/4fblD3j7w/+IYPfI4hEH31EmyVF9yvr060BCqA5d5KltFNkfKMDEG5eICHxVSq8U29FzGHt0FMQY6McRQUJ7hwuOzRhojucII0nJI72nSnm0GALsO8freGDYWH74+RX7WUEoFCiD0gapErqIlKUmRo+NHgRURUFTwa7tssOiUmijaYoCpwXKfOVAjIASjNHx4vItN7srbrdv+fVf+RrPP3pOJ99irw/YJBldQM4VZWHovSc4C1VNSoruMCBVgSoM0ghMUxCj5OLigLce5xTSOJZK8qhwtG3HretofaAdAtYJpE5IEUgi4zB3IREoIbIKeFYjZyVhyF0tIjGMI2boWawqqrrOnXz02LFHG81iOUcqzegc82FEqYIUHYfdlt1mw+3tlv2hoykXJDHQh442Rba1xqgKo+YEl20O3DBwMit48uSMRw9O2PqSy/7Pl9P/XBL4nbez1mpiUYANEe+yx8foPAiB8z6r/nRJuevQry948PAB6sHpVFkz8bnfPTR3SfzOAOmrwEgioVXBar5iVt2i5UBIDtTEYkkT0XSKoizYHHqGtsdZi0BQiY750QMOc0NdKkodGUTCOU9vI5HsY1EKWNZLnICu25OGgBDZZ8KLhO8T7S5S6cDoAqSsEA02j2KSFSxmJxyGa8pmxqEduL5tub12zFYVD3Sa2oVEJDulvXjzlqb+CCUMwsBcCYRwXG4HxrGiHyNSJPohEr0giQJTFlQIovPM6jrPYYeAt7kClTJR6qz4QxhSBCUKRivYjYnea5z3BDuw2e8QRUFyHXMt0WYOx9PFlArrJZvNyKvXt2wPPSTQUlMUBVIprLN474kI9p3l0Hp8FJR1nQGd4BFAWRWs1kv6znLoWhAC7wLD4JGyYhwdcRgZhxZSoCxL8IkUR1JIRBXxKeHcmMFCk69lCAF32LPfD9RziVSZFeGcQwhNVc4RospWs0ik1MjpNsQokdpQ1QWLE8PT07/Fot7xQl0zFDecnT+ibT6A5a9y8uiXWZ59RLWYE5NEi4kjHMCT/UJCEvgh5PubIjG9wxOCi9jB0/eWIkCyIlMRRUCRFa3DoQcjkKXOCTwGirKkKkti5+ltYJNGfHRsuwFbFlktpDWyMCgZELLKrBYXsp9P9ITCYYzCFAZCZv/4CaBmsni+C5mnc1ndKfLs/vPLa84fP+D05IzjI8fBCrZDYHCWbXugrEoWQqD8nbe3puscSkElC+pGIQtNGiQ3N7cQPbJuWMwKjioN+xv660t22z2dyKreJDMAmSfK4U7iC0y4hxQUheTkeMXp6TH9mChcoijL/OykQGEUs6ZGa4WzI9YOaJMPr7KqqcWMmfWECO1hN2EAOZ8VZYXUmqIqCbOa0fYEQMkSJwx2cLgwIEXg+Oyc0/NTlqsVwz5xPzf7M+LnNELJidZ7TyLbLEYgTCIcJQURiU+R0Qe60bJvO3z0vHr9hvliRlEWSElW8QnICH6cGChxMm6fkHxx94+EEJr5bMVqueK2dez7+zx4X73fv8cY6fuefTsyjhGVJMfznmKlOTpbc3Zl2W06pLJgJaENjC4fIDHkefqirtiPA84lvBWIKJEmIlKibyO3wmWL2WAYx1wNOA/RS3QqmZVzyrrGuABRMWwHtpcD6WH+K92pI+v5jCgFu9FSaJOZFgrqpCBGDoeeQpcEnxhtBp98FCT0JKLIXhVaKaRIWaHqA6rMfiFVldWN3gtClPR9pB3BJYUNkq532GFgNm+ItCy0oJmv7xO4i4K2HXlzceDLlxsOw0D0AYmiKEuUVLRti/OO2XyBj1mk9M4bOlvoVmXJbD5ntpjjwh5TGsrCZBVfkrig2W4PxHig79tMtdNFltmLLNQKMRJSxHqLi4GoIqhMz+v6Fu8DIYzEkJMh3hIieZxiEiIWEDS5acic6yQLlBCUAha1p14ccXN+jHh8hll7/vq/9q9yZQ2b8DXK9RNEtcBGhxKGWZXNr6yP2dhJR6wLWALRJ+ww0rmb++fS27xcwdtAHDypT0SbqXVaZkOvfnD0+w4dTAb0ZCKWJUJqfEwkIqMMGAGerCAsJg8OU2kQoKPOwHOQJJ+ILuBHi57XFKUhOE/wgW4cGbNMkT8xxSTEhFASpTRRJTaj5bO3V9TLY+rZnJOjRNr3WO/ohp5Sa4wxNLoklRWFLoheEkPE+JgpnSr7pQy9pS4Uy/kMVWnC2LHZbthtdvTWY5s6H7gTKyajL3c0xSmy6BMlYTGvOFot2OwGvIeybtA62+gWRrGYN5RlxTAM+OComyofilWFMQXOBfa7luADMWTKbVlWgIYUqatMSyz8DBcTOLBdT/AjIjlmc83Dx6csV8sMuHf/kppZFUWB1obtboM2kqLU0/aSTHNrSoWdGBBCZrVXN/TsDjt+8ukXnJ6fMpvVGJXNdu4zMGkyAAr345D7LS4isx9EklT1jJPjU1qb2F4MHGy+UDF9BcJMsN/viT4w2kjbZ5R51yTcXHP+tTOMmKFl4uJqyz4aLjeO201Pux/wY2DY7VmcnHBcL0n0bP3IOECqBMZA33mG1kFInM01h96iTImfaFrd4cDJ8YxkYNmUiOWCXZtob3rSecqbZ0LEhcDp+Rnnjx7z9vVbRjcyq2vKosSYAik0t5sOZ8EfR7wXRB/orUMKke1OrWO7H6gKk7uBnMOpigaPoCryrLNPkX1r2XmB9ZlRlKRktJ7dYUeKPVoG3LzCh3CPKewHz4tXO1693HN53RFSwNkRkqAw+fpvN1sSiTNRAIIYAjIldtsbSFCYgrqpWCyW6LIEeaCqS9ZpAcFTFCX9mLjeHCDeMgwdkZhb/cnoX0jwMeDsgA+W5CBW+RmJIdB3O3TRMA5bos9USS0lztm8iMAkoCbFIs/KvcMFjygqpDBIF0ndwM4l/O6KVW149tGv8N/4+/9dvvzhH/LxF46dv6brLDGUlEVNU9U4PIWYrJuS4GBb6qLCicDt/pb25hNmEwjsbCDZiPBgO0t/3ZJGiV4YqrJGSOi7DbadzKnKzIOPMWVsw4es7DSSqixoKsmhPbA8aphXBc2sYAyRQhtUSOiUMjAdBdE5CtNgoiTFfF0yzXekaWb44iu8CZmtgEVUSK1BCmyI/OjzV5TNitPjY45Pj5FVRzcMdLsNh37AmJpmvqBYzElRYkydFYxJ3IPQkkBlEqdHM87Ojnm97fn8s8+YxwmELhtEvSDqGhE8yEkjIgRSfnV5RMrukDEgIbNm5jUuKnRZTZRDQVkY5vMZ88USHwLVrEIrSTFt49HKAJNXkHcorWiaBq0rytETgscU2YtoJuaAZnuzoW93SOmoK8WjB2s+eHpOVRbThqF/Sc2sslF/ftgW85oYEofNjkMISJFXXg27AyEltNEEKRjtiA+RH3/yBeujFUoKnj15SF3m+XMIPgM4d0b5MRJCvGekCDF5pYmIkILV0ZpHSA72gvZiRxezjWP8ygw8hIiIhuQynWx5esouDHx8dc2vHy94/q0HPH444z//vX/O68selQRzpbB1hfCSt1c7+ttLmnKOM5qDHbBdINUCITNin5xAU5DqEmkELkDbD9xsLG4WKGcCezlwXBesFgUvrwR85QGczRc8OSm53mz4zd/6G3z32/+cw26LuDNFQlDNlnT2wOAVY9BoMwOt8LKktYlk8yHSu5HKeKqqppw1xNLTDY4wzXyD9YyDZXsYaGM5SZbdvZL17HTFg/WSQmmaSuFCutdiXl0deP36hqvrnnFI+BQJPrMYjM4Vkp3sM1OCrmszl7iuefniBVJKPvzwQ9ZHa+qmISG4vr5CC8nJyQkSiM4yROhCwg15I08gMQwdhZkxWzR47+j6kXboCMGitbkHv/fbS/7ZH/w+88Wa7eEWKRV1M2c+PyKlyHwxZ746o1mcUs+PqetZ/oBWDbIw+CC52Tuurm44LRNHhUedVazOZ6hipKwST472rK3lcvOWT960fP/FhhAgOM/YH+jbW9r9Dd3Y8fRr3+Rw2HPx+gX727f82//W3wNyBS6jYKZrfAHtsMMPnuCyUEvJ7NdOdJgoKaWmECqv1tMRG/JWGWkk80XFr37tEf/093/M2vacpZqVUoxKsmxK0tBRkMFcGwXDMDK2LdtuQIhsJBalYn8YCU7gGg91vudJSVKQxADR53lzRDGGxE++fMkYPCdHK+aN5vToFPHkAaODGEsQBQ6PGz2CbAiVV5dVGAwi3XB+qpC+4yff+wG/+90fYwR86+u/hDQlwZQIMwelQE96/CRISZG+MnKNU6K8q/NMoVgsa3xSoErawVJWNUVV0Y0jNm6zQZU2zOYziqIgpcQ4jGx2G9Ap2/mqNLmbBpAKrZvcDaosMvNeYEpFXUmKpuRk3fD1Dx5wuqjRU0Ga+BPtzJ8SPzWBCyGeAf8e8JCcE34npfS/FUIcA/9n4EPgM+B/kFK6/amvSNZhlKVGpjn7my397sDRfIFKcLvZMcbsm6EnqbYUgnEYMqG/s3z88Y+zXFXAs8cPqWP22E2Re+P/GOI0u5ysPEU2hZdKZzVhVXG8hqdDz+32ln0X6IMixnfeA1pLhl1gbAXaGM6ampsbz/e/fIHfb/nNxx/xza9/g789/zf48cef88lPXnIpN4zlSFMovvErH/Hq7Z5hgHofGWxis+ugbHAuA51SaKQy3Fzvaeaafozs95a2i5ydzeg3Awd7IC4MhVRshoCpK+489IqiYL6oONiRzW7LL//atxAChq7n6vUF3/69P2DfeRYnRxRFTZCS+eKEelbj/UDwFqFGAh2iVCQZ8SpgtGI1m1OPljFIrHMMg2UYRlJKFEbTD5a+a4nesl6tODk5xugC7yy70WI7x90a7bevd+x3E+CrNCLcsU++Aj6T3RWFEHkWHjMzpa5rLi8vubq6wofAYuU5PjniaLmg0IqqbPA+0AXPm82WxUoifEIiSAg8keR6qrIkpAzyWW8nC4bsSAnQ7m74zu/+Y6QsiIykyX1OqRLINDZTVhRVQ1HNKKqGxfKE1dEJ69MnrI4es1ycUosNf+fvPWL+oKQqnjJfLajSlmdPn/P4qWKwnq4P/OrNyD/73e/wv/vf//t8+eaCvm+zsMx1IOB73/5d3Djix4HFrAZyAh/7jqFvqbVmUTXs65rL3YZ+7FmEiqYpMFrixohBkcZAP3lyzPWC4BNeZQWuUoG/96/9JtzeMp8LHtaR88ZQmhnCBzpn0XmuyaGW3ISCVyP0rSMJR2lMBu6cpB063PFX2n6RZfCk7Pjpg8wmalLSjyOfv3zJzeaG4/WaUpfs2p7l8ojjozNm84rtYc+XX3zKfr9huVyxqAvmWtEUBr0yXH62483rW7YHz+NHNR88eoqLS3ZjwFtoPJRaErV4R1CI8JUxPVpnyqWW2TL6+HhFlDAGQaAiKTPp7cXkRRbxxNzN6ZIkFc5Zem/xKW+gyr8LdGFIIovgyqqiLDUpBca+ZzxsELHjwfmMs5MFTx6c8OzxQ47Xq/x8lSXF/qfn0p+lAvfA/zKl9J8LIRbA7wsh/t/A/xT4D1NK/44Q4h8A/wD4X/0MP4+yKPC6oLUd+/0eJSXVbJb5uuEWF3w2QE8T35ssWMkeuYlx8Lx+dYUQkt12z0dPH1PX9eS5zDQCeIc051VO3N+5lHJFXhUFZ+sV56s5V/sd1mU+6P33IZjVS4bRMvqB69dXfPJ5i28sm7bFJU1RrPmNDz5kVlUURlLXkt2+YzZbUMxrRjvS3lrwErtq2LjAZpB4LxEqkWTA0uGcZXTzvBOxVBw3DUdHBfK2JXnF6GCbEttkkMM7nwihFEVTMnNLmvmMs4cP6PuOrm3pDnuaUjEOjrKQLNZzZrMF4+C52Y3EmDAI9LQapTASabKHQxR5lZU2JaJQKK0ZR0dZeoz1hBRJcaQwksJUCCEY+pFehsngcxKOTPHm9S1dF0jJ3I9V8gKebH6fyLLkcRyyR4bINrKbzYbT01PKsuT6+prtfsdqtaJQiVkhKZSgLhWxLJBK0g4jr9/ecLzIq6+yoUQi4Tm0I97Z/BrBTx7jA1XM2EtKEW87YAThgUQQAk/uBhIwiMxbvlPgaVOgi4JmtmI+O+L85Iy/+6//AvW4J/obUjSIOtFvXjF0ElmvMdWS9aqm0iW/+OEZBQf6wwXdMEzTwPzs+v1NZi7FiEjV/bXM7n2R6BzOJYQSmKpAGgkyIWVEKtBGEZwjuowhJKCvhrwoWGnkpAc/Xlf8K996RJEsy7lmVSUqk/AOdkWR1bLGcLIwmE5w6wxKjPR+IMURTUkKEMZ3PHDE3dadvDoxPxQTSWAiH4QYaIeBuNlSVzOGvme72/DixRcYU9A0muR6vv7RKUerBYt5SVONLGqFHXo2WjJbrpgdV3z44RG1qHl9q0nO46PFO0tZmAwoppT9yv8EziXEnQ+5RKA4Ol7jUqBzCZ8qhE644JFS4WM2rWuqmvl8mScFQ4+1lpQSVV0xupGU4uQ9pLJGQkeWqwUxOnbbG/bbG4TveXDccHIy5+xszcnxmvlqSVHnqj7rKn6KEQo/QwJPKb0GXk//fy+E+Bh4Avx94L8+/bH/I/Af8zMm8OAC3trstasl8/kMJRX7wwEpsleKkiq3NZNPgpIKqbK/rpCKQzvw8uUl7aFn6EeePnnMcrGgLAxSClJ695e/F/hMlK/8EOU1SbNZzdFyxtzs2fcDuHeSZRsSPuUuwMeRdr/j4tqCkCQdaG53LJrXPH34iGfPH9FZi6w0h0PPen1GMJL20DMzgWoWidXIxRg57AeSyCu/0r36U+E9rFYGozQCk1WFMlDVBb2P7IbAEBPVV+5rVVesT0/og0eXBTEl+kPL7uaWYb9n3VRooQhaoGVEK8EoJLfbAzFGKpmoVEDKAIVkVhRIBSkKRje1ltM6LGNMBlfaiBssWuXN8DGS7QScRxSZjZFC9j2+i/2ux/nc1eRFDdllT2mFKYp76822bbNSctpS3nUdbdtSFAVd3zPsB2Kw7NcNpzODjBJTlzC1/dtu5GYzMjMzzKTuI0aEFoSUHeKEFCidsZV78Jp8vmv9FQHzVwERMX3wY8rzACAgsF3+eWLcMxcHZqcD5/MT+luD6/cMlcGP2XPn0Dl0c4au1zivub468Mlnn7I7bHG2J/hsmcs9RdRP9hDijzXTKYS84gzwKVNulVEgEyl6Ysge1rNZDSGbkmXWjIQQ771NpADrHEInvvZ0jRgO1EbQNBKtBWNhCH6Oc/l6FTND5SLKGZQsII15c5II3NvAfiWkgDQlITEl8EScpPMZTEyADR7sSIiOYehw1qOERrJmXjc8PFnw4HjGvJbI1NOYyEigP14yXzTIcsFyKTjctISkJzpqgOiyiVhKkwoYBNl//S7u3AzDxFhr6pp5XSNUwlGiCkPbtSQyMUHphCkMRuvsphlDFrkVJV5KNnEDSWC0yWNbLZC1oq5KtpsDY3cg+Z6jRcGzJ2es1zOWyxmzWY3UKuN0k3blT1zOPzX+C83AhRAfAn8N+KfAgym5k1J6LYQ4/zO+57eB3wZYrTLfum87DtssB1+frDk+WtH3PebWUJUFJ0cnXN1sCAmMKjJwIUCIlAFPaYhRctiP7PYdt9s9vY08f/qI8+MjmrpAiExviyHcy29z4ZLnaXclrNSKujbUOiLcgeTejVBGF9gPPYnsDdxZz+gTZTIECReD5w/fXvDk5jUPP3jM7OEZ5wqOrePk+CE3LrJ6fcPR2YxVn+DtDa9by8vblqQ1XkSEgaqAJPJihKNVQ6kVh13k5qpjVkR0VeKsZewdgsB8Vty3gWVdc3x2yvUmK8WuLq+4evWK24tLkh05amoWdcXtEHCupW8F0DDakXHoCUqQDGjlSVKwKiqMNvgU6axjcAElPYXMoqiyKEipx7mBpi4wpsCOnsF7lC6oSkNZlNjB3tsSQAbeXMhLhIWaKGKS+xGJ1hprHbe3t/dmVHmLt+f6+pqTk5P82sERxoHxcAuqAiVRdU1I4AdL1w0USeFshFJnQ6iU96oqLVBKI1Red2WnmfEd9c1oycm6nghLk7T9rly8W8Kd0rR0IHdoPkaKouDxgyM+fHbEL/7CMedHkdvLW8Z2QMrI/nbP0O3Zjx3oKwZXcHnT85NP3/KDH77k4vo2y/KlzFzw+83rk6udSHn7yxRhdMTREaXMaUmm++QdvCW4vIZusZzhh5FoRV6WYQqsEgQklclc+G4cGZLj6VFDaB2lFlR1SVKKlCqUD4x9QCiBMAYvLMkrtCqy97hzeOdQSpCMumd4ZJ58vrb3+VLkuW7GBad7orKRWYy54zFa58NG5Nc4WR+znhkerA3rRhDdiIyOVDSU6pghrRioeXvzllc3LT5pIKJkzFRIEafkHad7Gf+Y4C9M9OUYY16mrQ1NWWV3w1Siq4rR2izsQk7aBQ0pEJwjhYCWCqUl3jqCzaNRow1GG1IUlKbE2Z795paxPVAXiiePT/jow6doozBmWrqcIiE6Ysz57U8KE/+0+JkTuBBiDvxfgf9FSmn3x41n/uxIKf0O8DsAjx8/TgB939N1LfPVnI9+8QPKyvD65Stm84qT4zVKag5dR0wwmwmKwqCUuld5KaUzmGEMCbje3vL7f/Bdbq5v+cWvf8TXv/YBRqt3p1i6a5WyQXyYDPNDCHTdgdvNNeN4wA9bolvevXOCHbHOoynQJlIuJU01sIiKhVDI5Hjb7flPf/wZ85ua1nUM44HkPPrtDd+7GlgtFnz04S9yXsxZvLzihy/fUgQLQ8rbYRaKsyczLg59roq8pm8Dr18e2B5Gzp/N8N2eODjmBJqF5uysnNpf6Luei4sLrLX0bceXn31Ov9tSRM/52RGyH7jebDBKEX2g2w3o+piHT0+5vrhCe0+hEoVKtPuBWwaaWfZutlGy60dEjCTXMWsapNEIlVWbRkpEDEgSRaEJSbBYLKgKjSsjyXzl8ZLgrc3eMsnkVWhKo42hKDKSP597jMkMlLzEWlFOm7/zUtmSulQ0RrBuDA9WNSYFBCPbdmRzvacdAk+Pn1DIEpXyxptS6Vx9i2w+pIXO3PeUbUrvrHlXi5q/+ze/Ptk0yAlGilPSEffeKkJkOX1IkaqpeP78KR88OWFRKVy/o7vdM4oSmRIpWMauhyTYWMubi0u+/PKK129uubzcc33Tc1YbzmbLfCCkTPN0PoO8uXqE2eydLV3oHa61qNJgZgUmZCO16BIpFPmASfkZbhoDhUCERFVWdG5EaMG8mubkceSm7fj60hBTQ1UVNIsZMQlGp2gvb+iDQwlNipKNz0dZU9SMacT5SUBUqYmO+a49vNsRekfhy8uJ07Rd6K47y59FIfLnMuvtEj55+vHAYv2AeQVH84oHqzk6nbIfDfV8ycInXm063rzd8GZ/oE/l1ChN3v8yIqaxzR2lWEzsprtQUlIU+fMUY07kTTMjkFf2xTQZnlnPbFaznC+oq5rSGK73B/q+u9edvHjxgr7vWS4XFEVBXVWkIBEx8ebqisN+j9aKh+cnPH36lEePHuUFGunOxTF3ATFGrLV4984M7s+KnymBCyEMOXn/+yml/2D68lshxKOp+n4EXPwsPwtAFZrF0YKnzx7y/OlD2m7PalFhz9Z4F/ij731K6wJKQRoHXIoUpswfdKFQMsNTKWQg83h9hLOWL1+8pW17hFR8/WvPJ+qQnBK4RAiN1ALvBtpDz3a74+riik8+f8mriz3WhXfKTaBrE63NMy0dJaqA9QxWyvLs5IxYZmn97qbj2y8/5tXuinJmmBUV/S38e//kO+iF5uhsztFsxVLMCbXka7/6S7x+fcntfs94ERhqSTEvWC3m7K8Hui5XV48+WnH+fMmbL29ZlIIHS8PJoxkvWs9df+9DYBwtWiheffol17e3nBwvWFaK6y8+pYiKUcg8q02e5DvGccP6wZrDUKFdxESHG3bEILjae5Ymokl454hRMo4x75+sNWVZUjUlw8U1ofccr+cs5jWz1Yqrw0BvHcN4oK4EzXx+f88//OCYFy9fs993BJ8FL1WtKYzGh4iwgaqcoZXmcNhhjCGrFBNVUbFerjlez3h4MufhUc35ouCkNLih52A9dduDiawHR12OxOhxTiAIxOhzDTa1ymkSB8zmDc463NTKa5lYVgFtFEoDKcv+hRRozf2iZSHyfHl9vODv/t2/zfnxisPtNdevL3izGRn3npRGrBtIMWK0JlCwHQMXl7eUteajj854/vSI6BMhJFyY6sRE9mMPkXHMG4ZiSkhd3l9Loyd7V6Np6gahNX3rKXSaEqHPSZW8yFqXCiUETVExDyVtirko0hKVBBeXO9SDcwqjUaog6gZlDLQWUzfYNGa6qDIEelqfBXChd4Q+YMoKbTytHe5ZXEIIqtrkjouMQsQk8t9FTVuTlETrjAXdferuvLQRgn3f85MXn1B/8Iyb/YgbBHhNqI7w+8Tb7UvebK+42Le0o8LHbHGcGTYFpqmpmhofHIhMbMgajXedodIKY/QEnENd13nrUx3Qrad1guADTS0oq2xnQYioQhImnUjXdXRdx+3tLavVksVyQdM0SJEXNHz83Y+5uHjNclazPF2xWCxYrVa58zR36z0yyOp95v5n/5e/hBm4yBnt/wB8nFL633zlP/0/gP8J8O9Mv//ff+qrTbFaLdDLgocPz7M4oypZrVYMvaUw23zBIYt5IiQX8SkQCQiRV3sVRUVZ1hRTFV4WJdFLxtHxk598woPTY8pqAjXJM3AlImM3cHV9w9s3l7x5e8PF9W4ykNIcHc9ZLd+NUNqbftpOnj2x6yR4+miG7x3BJ0LK/s2qiGxvNwzjmOGyPnJ7PaJmClVnU6AvXr+m34wsTc3pYsnpN5bIK8ntRcfbn7TMH1d0RSCNmbO8XhsePTGU1UBfJhopma0UixNDux3vZ6LZEF+y3Rz47JMvePzBE2Z1SbQtKSZG7+iGEek93kNIguPjI/p2hx063OhJREqTV2/tDgOqKmhkpnKNbsAFRUqa0YE0mZaoyg2zsmK+XFCUhiFEqsoQXI8Snrqas1zMGKb3+Y1vnCBoefFiw36f0BIKk7ISN0aMUhQm89b77kCq67zBpy6wY2DsR1aPT3l2vuKjh0seLBukBztaxGFDMJFiIQgCkvcUSqDIXi7j5KqHUKQYJnopKBXgK7LqhCBKPVnnhly1pamLG/PzV2hDUWiqpubpB09ZHy9I0WP7Ib+X/FPo+xEfspOgdx53cYuZz1FKMjp7rxSGRFKCEAM+TOC0zMwYU+TlxzHmpPfujUaid9gU6bREF1nMpEREidzmL2YVy6YmuI5ItiwVBuZVhYwBXej77fOjF3hd03dZYUgraVaanYXm+AyrWvq+Z+yzIdvt7kA/Zv63tSP1UQkq87zv5rZC5AMya+vywRmiJyGyAErkcYk2GqlUFr9MCz6kVChdIIxh9InrQ09VtNgqoJOgbo45P/+QaEZ2w4F0tSOOudIviwJTFahKo6rMw1btPu+1jBnEFF8xiZJSTuSHjLvkzTk539RBE0jcWEdVNzRVxWw2YzafI6Qkxfw9KWVTNyEEpydnPDg/w3rLm9dv+d53PuYnP/wRzazk+GiZRWizGWLiyCstJ0+nu1zlv+Lh9NNz6c9Sgf8t4H8MfEcI8c+nr/2vyYn7/yKE+J8BXwD//Z/hZwGwWMypdZ1tYUW+CPP5nGFlmS82FIVG2gzo3LUnCEeICuvzgls1jFSVZb5YUFdZyRfJa5y2my0XFxc8efIgzypjIkRPN7ZcXV3z8vUFF2+vubrds+0cumx49OgBDx+dsj7O0kEBPJzVdAi60eFsQFkoG0ksNfvWEXVEFAozEwTlGYOn2zpiTOx6i5wH6qUh9JFgPZLI+lixWFmauWRxXrE6Vrz+QYu9HHnrPEJCNVMUi8R6bpChoxEFSMcgPL1wHOy7ZbcxRIZ+4O2bq4nRI6Y9n27im2Z/lhgiLmYeboyJFAcqA3070ntLXRUkKfHRs+8GXAooJRhGz+gi3o345Km9g6Kmmi8R5IQptWLR1LTW0vmeZlbQ1Iay0PcJ/PR0yX635nDwjEPHrC5pZiWHgycGR1AO5/MsMMvXPTEbSxNj5PZmw/a4pj2SuLVELQqaZkEC1CApKoXRFXJa6pDHF4HgFdpkMGnwI1iLnBZNKKXzNqXpOEwpA9dGvFtufecnnsgLlYUUrFYLHj95wNnpCcSAn/AEO46Tn88d6pI7RaLAjp4u7Bmto7d2csDMykKlFDZlA7cQIkoZrM17X8PkdCm/shQzLzJJxJA38hQx0+AEd3TMTPNUAvykFBVJMgSXWT9k8wk9mbwdesfFziL7wP5gae1IcXCkyXN/2znsGNAiv9++39MeLO2uxVrHvJkhVGaK3S16EFJQzctcUUcIPhtSxSTypZ0sfbWUSC3JmZ7JZtogdUFMMtsXu+y8WNcSQ+64VycP6MJI8faK4C4AOS10UCg1LT1OZA9xkRBSZs+aFBFfARSElCitJr3I1AeIMPG4I0qLDCpKiTElTT1jPpvTde0EeN9t61GcPzjn7ME5UimuXr/mkx//hE9/8iOGceD47Ij10RGr9Yq6rjBacld1pzSBlryT/KfJRvinxc/CQvn/8Gfjof/Nn/oKf0rUdUVjPCH6DCqq7Jm7Wi05PT1mtVrQu/zg5uWfmbuNCFibH3QQ9OOQpalmjZ5GKyEKhn7gxYtXrFZztFHEEBiGkdvbW168eM3bi1tutwcOncWLPOL4+tc/4OmThyAV+3YEBMezGSY4Sh+w3jG2IvtNVDnxCJ2YVRppAk4HrAi0o2Owjh6HrALNTNIPgUJJ5suaRw+XVHXL6jSALDk6LjAh8erHLbeXPUlpXID5caCUCtmC9IoOB8FT+ZHWvaP49/3AxduO16/eojTE6PA2EGxediu0BCkJgAuJ0UXaQ0tR5W3io8jdhTLZ/S4KQTf02DCgtcRawegC1vaMLjEmx7woaRYLxkPH4DwViaYykBxGeeazOVUh/xiIOWsq1qs5i0XL7c1IU5ccrRf03TXBDzgn8c4jpZiwjckje9oyczi02Zi/sRyVI6eNYb04Ro4jKFBIZKUwRpGivtvzBTFTHG30yKFFqew7crdEoGvb+/cYYqTtHWWZATgpufeZSUApJWVhODs94unjc5q6Yug6lB9x1mKdw1l3LxyTIqfxlCCExO1hzyASNgScD4QYMEZiBLiYsFN3oIVidH46gPPXlHh3LbXOftHBB+wwElNAyCZX5jHPgNXEfYZEiAkfAt67aQORQsdI8h4fwHYjn726ZaUit9uOt5uBYHbUqznRCwabD/PVrKQsDNGNjMNItx8YR4c99lBLVB5Y5hBQzjQhCkQUJK8IXmYgO77z7M86jcwXF+S1dqYoEUrTjx4/JoLPnuRNrVA+0FQLdn3EphLUDKUrJAotNcg8fhJjREUQOmQWm8qsthjzcpH7tynICyjInWyMcfrMkLUjQqLLkohEKk1RlFRFyWF3i9FA8mgjWa2XnJ49YL5c8vr1az775DM+/clP2N7esDo+YXVyyvr0lNV6xWyWuzjEXQLPFTgopNSg0kT7/JdUiZkIhOjwMSJkQSkrtFQsV0uePoPnz5/SDZaxHTg7XXNyvELLxKdfvOCqs9n+Nd0p9yJGCmIdqcoKJSVd5/j008+YzUrmiyzy2NzuePHqFbe3HTZIegdozdFyzl/7jV/k1371F2iqis1+mBI4XHUQlWSeShZecbNzXI+O5bMSGQYgolxgvNnSeo8vyRVuAUF4zK1mJjQYjayyIU45m1MUlvOTmiHkkcQvH5+yftjw3d+9ZL+NhDYwtBG7S8w2iv3esq89TYisgyd8pYK4vrrlzdUr3r5+y1/7G88QWPyUDLthYL5YEmWPtYFD59keLIjAo7JmaA8YpWhW89w6JkcSEXCElEhe4mPCxkSYLGcdkqACxbym6wZscPRjjzhEnOs4Pp5xvJ4hifTdAe5HtxFjBGUpKQqBMYKH58fcXt+wHwes9yDyoo75fAlofHSMrkUrg64MXdvy9k3H0vQ8P1tTlgWilyiTObdoCNz50EBpCqrJwGmz3+atPIXLuy4naulht59WXWdbgttdhzEapTPlNFvHZh9qrwLPHx5zdrxivZixP7QI11OrdJ8srbMIUSCFwN0taIgC6xP7bsQXmiQ1UeaFFdGnvEUnCYLQJAVRGnStJtp03gQk1DsQsywMVWGwKeURWIx4P0IKEwU1Wy0LkatK51M+wIRisSxJNuQ9lTZXesPo+c6PXvK1kwXbfcenb2+57B3lvCYMDl01HJ2sUPUps6KgVBJCXiDRt5a+HSl1AS7cLxcXAkypkQJUUoiQiF4yWMc45OJCipS7BjJvHQRKSUxhEEYx2AFSXr4SfMSPlqHrSWrLP/3xj5hNSxVOHz/i1eUGjSJ4jx0GUghZJVnXVFXNGHN3ldLd6Gp6KpMjJjt1DrnjlyqPdZIAGxz1fE7bjrmbIVvK9t0+M10UrFdLFqsj1senXFxe893vfZ/vffxDbt5esZg3nJ6fcvLgIcuTI+brGYtGUlZZ1BNCpl9qnZO3KjUxgHXpL22E8l9CZDK/FHdWsNkdTmvN0dGaX/+Nb/HpZ18ytC1VoXlydsyThycYBfsffEEcHEnk1sZHGJ1Dm0DAooTAFBVu2PO9P/petg0NgX5wHFrL4AVITVFVPHl4ym/9jW/xzV/5WgZMg8sboclN8MXVnroqwSuOqxm/8Ysr/qOPf8CryxGZErNSEreBT95ukKFi9fiIovHE1OfNP9eC8m1kGRS+UPhFQjYHgoy8vLKMQ8DUhme/dMLYJB7uF1SfdLS3jt1Pen5kb/ib3zhnVV2DiegQiTYig77nKL99e8EX3/kO5+fHHB/NKIwkjbnC0UoiCsVh35OBd0VZFfgY2O/27LYbiAW2bOicoR8jIXYcHc84f3DC8dGaL798xZdv3+JczECW0Bw6iy4azh8+JtoNtrth++YCpGSxOqKoJMm7e4UjgO176tpwtK65arZsbt9Slx9ytCgptUAIQxIlox0xZs56fUJVG0Ia6PuB0HtqeeDRkeLRwzXr5ZKiypx1tCCmiPUOAlRFiZIybxGKYRqDJKTU1HWB9z57VjjHfL5ikzQRcD5wtWlBJMrSTF46CT/xsB+sDA8enHO8XhOcZ3+zRywq5suKWTNjaBwbeWAc8/gqGvA+4mJgZy03/ZhZEiJXWN7nZBLbAedyVwkS7/pcZUumWXwi8o6RUBidD6+YwGSOem9HjM6CK1MVmFJlr3cEHsGYMldZNA1ITxos1vbYsSOguNoH9Lhh33dc7jt2lKjWEVwg2o59SiQN3zhacTyfcXUz5LY/wXa74+HqDJO+uoM3V7SmyvsyRUwEC14qfMjFGzIDmHcNvhRZiWvwlEbnNX0OutGCNsyPTyhPE4NSfP+TP2S9bnjw6ITV8YKDdwyHkZDyLtY8EknEACJKCm1IOuCd5SvLjXBuYOj22RYiCgQZi1EhIX3isD0wWx3Tip4kIwFHZ3dEkSiqGY8ez0lJEmLi7etLvvOH3+Xbf/BdtpstpqiZr5Y8//BrPHr8mMV8TlUYtE6TmV/IYxhtsspXZw3LMORDJfxl0gj/MkMIOSnB8u93kVJGhT98/oS//jd+nd/7vW/z5iqv0bq52ZGkYTWfo0WPC5BUiSoaPApRFCAVPkaCDyznCySeYRxwg8/VwuCwKVE0kg+en/Pr3/wlfuUbX5t8I+74oXc8VsHDB0u6W4tLiYPy3JYdH374C3zy9i2pykZQPghmvubmMiLFgQ9+YcXyvOFVF+m+u0V8uaMC/ELSK0OSoOaJGGvOjk4pq4rXF3uu3Z5WtDz6hQX+kHjzascPf3ANXsJRoDktmCvwY2Lc2fv5mJRQVpKHj8/QaEJMJBdQEdaLhl5CVc5IMTK4yBgjnoLt6MDUDB3sbj1CJGYLxdNHR6xWDQ8fHPHoySM+eHrKd77/Mbt94O1Fx83eI6Xh2ckRlQnsuoAQgZkwBDS9GzHWc3Z6gp6d8snu7uZGtIycHs+JX3vKp5+9YOx2uRoxmiQMsphxddtCMhijKQpFFJIwrSp9/vghH53//9j7s1jbtjS/E/qNbnar3e3pb3+jy8yIyIxMbJebyiqXKAu7sEBgeLCEKCS/AQIhqHr0A5IlXuC1xAuSETJPgFQSQipwS2baEZGR0ceNe++597S733t1sxsdD2OutfeJbCJSWFwL5ZRunBPnnL3WXHPN+Y1v/L9/U/L28YT9g3kahmmJjZbGJd91FSXzao5RirauWW1WeJ+sVJXOUoCITXxzay15XiD7lF2VRCUBpQTWJQivt4HWBXrX8fb9x0zHYySBermk3TToCDfekWVqiGETtE0NGNquxwVB6yJXy5rzmxWy7JKAJQS8sylfUyZoQZB44G3TEWNyd1WDRWvubztwrSSjqmRUFNS6pvcdQQ7deyaxBKKMqACEgCalKWmtWK42ZBhc0+Prlth7jE5WFZdRs+4sGx/wOqDznF6BCyL5r1yt2TcKZRR5pZkdFpQTjcgM0/kYYTV5PpxnBIJAadBZ4qGjwERFjBla+5TlEdIiKUUS4MTokNLvdmmdjLS2p/MQdUFWaV6cvuLBozGjQmFUS9v6xBpSDqF9co2EZCMroHUJDpSlRCtJ7G9b2/Wm5+pakBuTfHmGsBAVJVo4+nbF8YP7SLNHUVVEIbABJnsHFGVFjEkN/vL5K37845/xwz/8Ed5G9g/uMZmMONwbM50fcHB0zMH+hEpbCOukT4mpKQohDhbbPkEoUqGzHKV/SaIxX7Cd7NYKNfkQDHQpkiLv1772FTbrhhfPXrBZbXj++hRlNHmWpS1RH6j7gNA5QmsmVc6kKvDOcXnZ0tnI4d6UzORIWeNDTdY7sjzj7ffe5te+8gHvvvOQPFPDrk9yl5EAYAqJygVFlEwKSVZGimXg1w5LyBVWQe00y5ni5dMrxjM4LhX3D0qsyDidQX4pyJXBTCVyLBlNJIt1y7xKAqW2s7x8fsP0UPLg8IDjaYlterxpsXXO09cLzEryUJVUe1my2F1tdh14UeQcHu5xfLRP0/T0ISA7RyUi43HJxXKD0oZ609H2EGWBqeb4viYvyhQplZ4kRuOcvb0xUgS6rqVvGw4P96nyEVeLNZQZZVVSHc0pR7C6PEUWlmk5ophN8RTkJvDo3h77e3M6SlimzlGQKMJlmXGwn3F+nlNvFsm5MPb0zg7fVUq/8d7RtC02rLi5uYbO8f7bJdXYMBrnlGWRjJRInXfvHdEIMq3Z0qCFSG6WPgaMSl3rNlE9/bpBKrlTEEqR3PmkSMnnWkpErpBGIPt0PQUheVnUdWKCdIqF6yhLjettCs0OCcrxLrLpHFfrnmenN1yuWnTrU+CFSMHOhTFJfu4HMlmMOBex1tP4rZ1upCxv70shUsHRWiGFoG0F0gSsSMM+FwON6yiFSsPgmKLAjJS0XUfvHa5uiZ0nFzLtEvoOCLTOEyKI6JOk3yQzqtY7Lpdrrkcm7ZZzw97RFCUUjYvJdjXqN6hvIUSigCg9UoOWAu8kkGBF3/uk4HVhh1N779MAT3i0AWVk8hIXGpWXZFXG6emn2O6avaNjlAosmg0xGqpJRrsZGD42wThBhfT+RqONRFJAE3dTPes1nc+RWhNdQ9e3+NCn9B7h0crSNkuyajrE05k0Rwme1SYFVJ+dXPD06XOef/aSrrVMxnOq2YTxpGQ0LlBZRllWjEYVWayJTRx0DmZH1NgG0iTGkcYI8UZz+6cdX0wBvyNf9j45dEkpbtWRUvLowT1+/de+zLgsefXqhMuLK64WK6rxiKrMiEZivcMLgdYZh/Mxe+OCvu9pmw2bTUuIYDJDFUsQAp0XTI/2+OrXPuC9tx+xNx+nLe3AC97Gr22PrDLoxlIKxShTFFpiTeBwklEoQesDFx2IqUEDI6OYZIJJBuNMMZ5BnCu81uR7mupYUuaS6xtFJnJs17Osa9pFzZN7Ew7u7TEaw3rjmWwM99ZzPvv0gvbG0myyZAYkBL53u+tXljnF4R7TUUXbLvFCkcWIUAGlNH3rQQjaPtBaAVmO0BWh9xgpyUwylUJYxuMCY3TaVjYNy+WSvb0pdW2pnac4nDA7mlHuVRRtx3KxJq8UpcooXc5kfp+q0EyL1DX23UDfg8Q6kDKZ8VeK8agkBkeZZ+mBtR6jU5EP0WNdA9IjlcP7hmazAWlROqK1TB16bmCASXwMKJWUoQmSIwUyG0NEpJ1J3HrlJD+TEEEMQh0YmBO5QcRI9KkzRmpsFCidurrNpkEHi+06ehdQ0uF7B8Hje0ffeJra0YXIurZcLGteX615dr7ARiijSCITIkbpIXAh0juP9w7nI7119ENo8G3O6203tsW4TaYJMSNGC3IIc1YSpKB1PUJm9EMB18Zgokjv41JoinDJsyMGl+xaCdjhOqiYWComV3gXsa2j6XqWTQs6GTwVRZES6VubRDoDZ314xAkxQZExpSSnoaUGGQRKp2IfARcsUqbvNBLScA+P0ilXUqIQ2iBNgckz2m7NbKQ4nOU4l0KdgwuMZnuJ7eIS/U8KhSpE8kmvFHllkAbEOuxYGRFDIMdHhQ89bb2i6zqyPCMiKQzUmwV71TS5KfpkDmVd4Oz8ipurBeenl5yeXlG3jrycMNnbp5hWlKOMrNQIbUj4vkZGjRuEYmqwxt2GZHsfEMKnWijEG7XoTzu+oCHmlrYVd524EOoNrktZ5HztKx9yfHjA8xev+fnPn/L9H/6YzWaDQKCUIdOKHqiqkqODGdNc0baS9mBO25/TdC2jImM0LhlPx2TVmLfff5snbz9gXOVDR5BuNjnImJUeOggB5ayisz06JCuM0EaKsUIVikkLI9eD76krxdFewdF8hBLJgL/UktlBxvlNZCMj5ZHk+GFOHXom+YhMSpr1inq5YERgQuTewyKZX20cIoPH7+2xWK5ZX24Sv3jg+KbbIR1VWTDSe0g8m9ZSzWZUsqMQ7WCVqYgDnuajhCixDpwnBblCykHMIuNJRhiMlLz3bDZrrq6uuLi8xhU5x+8dcfzhA6Kw2Bev2ds3CC0RFqKLPHz0kKODPZ7//KdcLm7YWAmDH2EIkBmDEIbgItNpRVlopqMRVVWk/E1VYcyCm5slWmjms4LJbILtlmyuL2EbiRV9KrZVgcnUTtaulMaYHO8jLjiUFCid8hu7PjnvyUGBG0PEx4AbChwkDNYoiYwBqVOnFYRC+uRKul7XvDq7Yl3mxBCGRB6FiS7BU03P4qbh4rJh2UfOly2vL5ecXq9YNB1FVVAWOSIKgo+00WGbJNdv+56+s1jnk44wxmFIJgYjrbsPcxgYCnEH2+hkc5KCPIymj45gLd45ZIio3qG2NAyhhvzWgHSgg8eFiO06hDFJRBWgyjSizHCdQ/oAXrK2Di0UffBI0oqrTOr0bW/xOwOz9PoJlUyGW3LrhyLCkCmawOjU8QuySiNsQOUCqQI6U2TBoGLyy9E6R+mc6WTEowf3OZyWLBdrriXo6BmPSpx1KURCR4zOKGY5soB8nFGOc1Qu0BsBJ8NpCkVA03sgGtYbx0Z4qsKT54Yyk1yu14S5IyiPtx3O1XTW8uLFa05eX7BcbGgaTz7eIx9FxvMZptJkhUQXEmE01np8SGypKJIiXAqD0mqwaADnEgVVq6RliP5NROBPOr6YAj5gPtvtlrUOpZJichu3FmIkL3IePrrPdDZlPp8z3z/g06dPmY7GjKoJSudsOs9oXOFDpO4tQUA1yvjg/ccc7s/SMMTk5GXFfP+AyaRIPNGQBhwBgZAxFW7BG1vAfBKYCsVqEXl50nL28Q1P3p1Rhn3an18xo+PBfcPUwNc/2EdNMi7XjpX0PHhwwINvPuafXn2Mo2cy1YyM4dUnCx59aUrTL6hiilu6udlw/nrBh795xOZM0C8Fp9ct89ENk0PL0f0JRw+rVDCN5P796e48R6OKIxV49ewZF1drnhRHHB+OmBrJetGSmRFVlmCCq6XlplmxWfYI71gslxS5Zr434uBgRlEatJAIUaK1pCgyrLVM9wrufekhxdtz5CTib3pMFyiEIYbEty2VxmD57JNPefn0c4o8Ixvvw5AEJnWOCKmzld5x72iGMTlGKlCRiZJ4kVGWZ5yeXPDkwSHvPJow3y/YLM65zDRBRFpnaW16UVNodKZTEDAKgiZ66G0yeEJrYiJMoKROzn3RJpWiSsO+ZZ9CH2CA9ZxDCkGWmwF+EehBq9A2jo+fnpAapsi4LHjvwRGy35BJgXeRurN4KXn66hWfvLhC6JzpZMzefMrV9RWFClRFRozQtD1t16e9fnRoTepMtUmp5kMzEUK4xZZJ96iIEi8iq7Yh2B4jFFqkGVJRjuhtS1M3Kedzm7jUu2SLHQI+puF/ay2llIzygt42qMxQlBW41AlqY8B5tNEUeYFTKgmPhKDuO1IAXipCrfNpOEnaTHe2x7Qpqk1IkZwStULFIaxDSnSuqLQkRI8eB0YmJ8tztDbYxhPyjLHKmVSGUZaxP77P13/9LwMb+uaGrK45nJbsH+yz/9bbPCtfcHJzSRcjk4M5Bw8nhMLis4jIRIJy+op4mnA27xxtXeNcT6YFeZVyaNd1y6Zu0FqjpGCxuES3HSFIVqs1r04vOLtY0PWR3gYcEplnZLlhvDejHGeYPEFbRudok6G0QURHRA5GbwKjMyANNJ1LwQ+t77B9N8xS/uzjiyngMSZJM8mLQIgkCY8IjJBJeELCBLVOHr1Hh0d88ze/xauzCy5OTjk/v2C9bvnKV9/m6uKEH//kp1SjkvlsRJ4JvvKl9zk+PkyDgCENXUSQJO8OghwMlQQhuuTRgOBuXp5zG6oycvbSsbwK7O3NefBon5fXPau5IRQl2f2KTVjz1l9/m5PLmp9+8oLF8xu+vJzwt/7SV/mNLx/y4uqG62XNq5MlrfNwVoOUGKsQrSLGnOWy5XIhWFz0XL1u6V73yHcKvvZkShN69BxiJSFXxJxdC16WBXsjzY+/94fM9h7gQ1KFVdIyns4oLawX10wmBdVkzIEXZMWIvqlZrRV5rhmPS4oyp6wKgktqPq0lUqVrdPhwxvz9Q5qRZN2uicsef1ZTiZLWe3SUFErz4pNPENmY8WyGktDfmaJnpgDbYrtk5yqUTEyMENOioSEGhxQBgcPoSJEFchMo87TzCF4QgsSHgU88iFK0UCgU0UdsbzGD9fD2XpNKU5UT+qZPBCgpkEpuBcx37kvoHejcEEWyvZUxouKQrRolJ+fL4TUV69LT1R1jDfeODyiLiug7+mWDzBWzecl8OuXRgyP296Z8/OlTTKl3TnxaTdLgMkaarqXrHX3v2LQt1kWkzug6mx5sdzdsO9KFgAyRDnAxJjm4hRAE1gba3pMXJRKFdIGoNTZC5yLO9lg70CmtxfpAtTemKiaoPNnjiiiwpO8nuEBwDidhg0Tg8UmHQuhTVmZZVPTev8GccBaadXLajEFgcokyKeov+JjmBSSM3JTJuSArkvBKCIltNaIt2Vc5pmxp+3OuNoKVu6HpTvGhxpY9JlMc7RfMD1uyfcm4ztjEgJ7A5EjR60hLj5cBIcF4wTbsXQyLdt+2yMIQCdRdT9+2ODt4wpcTrq/PWdevqBuLs5Hx7IDjh4+4vl5TNz3KRayzZOMcneudlKssSqbjKfv7e5RFga03tF2D1smULg5zN6kgG0z7vAOpDNp44M8u4l9MqLGI+CHqKrm7peIqYkxm6MFhokcGBSRTISECmYbjwzn9ZsFqqejaiJKeUWl4+PABWW6YTUcc7FUcHh+RleVu+ykGySxBMpjNDcPLRLnz0UMUO8MsIpyftnz4zj73DiELKd8yrzKq9Ro9HZMZiVMaPclpTcvzsxsublr6PrLqAuu2Q6nA3mxE5yOX1y3L1jHuDNZ5pI0YD6oyKKP55NUa1QImUpUJgxznI2wvkYN7YVP3OJG2zwB9b9mIlsPjB8zvP6BpWoRwSJmhxxOOyjHnVy/ogbxQjEcZWgtkUJhshhrwZGMMSmqkTkNNFz3RgTIw2z/i/uFD1rSo5oYgPOtBwdb1LY6IVQLpoK1risLgoqNzt9Q3NXh8RCJKJ0dA5zxqcCQ0eZa280rSd91gr9ohhSbP0oBws2rxNmJUtlNahgEPkiEgY/JvKYcwWuccfd8nNolqhxmLwPs0pE3f+270gvOB5brFuYCqWzKj0TI5eUgh6MPWrySCCMmaoJVstGDVX6P0Cts7VpsNMsuZzEAbQd2u4bonqsRFHvIRUARyk+hjRkpc5umtJ1MptaaxAic9Xqo3doYuCPoAwsf0d2VF7wa3PR9Zbzo2fUcxn2CKAjE0R0FrbOfpejcYkBmkEqwWG1yImEHN3HmHkAYvJE3b07U93nqUEbTOMhrpFJEWI9GDdT1hs6Gz/Y5jLQTk2ZZPLYhSIHXaCSgp0DJltUY8IgtUewIUoHqUTj4pPuZoWzGynpivWLpAs7zm9foUJzeYUiDGGqUMdX5JCJfYiaeYJVMtZzb40SBQwxEIiEE5vKtFweKso2k2dL3EeYfznswUTEZzPJEmBGazCeUMrJNIkVFWM66WG7zMyTu3w/uDGCwBBJgsYzafsTfdYzqdYozADWpeMQjFYnRsg5W3sv4tM0Wbf1c7cAY1VBxc3gZELylstyncaQ2LWyVbCNSbJV29gW5NKR2xANyaqsp48PCYvCiYTkr2piXVqEIOU9z02KauK4o0xQ8RZAgJfxOKuPWluHOewSmWNx6cJDOBum8YxRHjMsNJhdaCbCwws0jbNixvaoINFCYZIJ0u1rQhcngwRWrDat1DoXEOglUYBFUhmYwMvvdcnK2ZKAMmoscSCZxftfRGMpImGd/4NKHfHn1vaULHbO8gucj5Fi3TkC5KPeT3TUkQpkDi0INwwKhELTODI6AxGu8tvR1i6WJEO8d0eo+j2RFZt8RS07gGo+QgUR7cP5REBUVucqKMtE1D3Vu2mWp92+Fdvxuo6SLDDYPUIs+TgIHE/HC2I3iHICTmSm7IjGG9agZL3QQJua7DWwsuIHwEF/A7OXmCHtwgT3erBVpogjbE4Km7lhgF3set/Xbyjhkk7CF4Mj1kYqrUEcYQ0CYNnkIEawObELBOsOzWOye94B1RpYVRKol1jtXapoXCx8R/DiKJeERAiJB4wEohZMCHiPIR5x1Ba0SM6Dsuf87H1KFLiFEhlcb1HUpoQGI7P6hxA2SGKFNj4jxYG+k6S6lSFq0OZvCnl/goiG5436HoNtHhumQAFqXA+gAihUGIgYzQB8u6adLgdVfABcVYoQuDGQnykSQvI4gMKQJKRERIYQqykOTzQJRJKCO1GBgfaYEoe1C6xypPy4q+XBCMRYwMMoeoBbWsabG4HHojsVrjpCDKGi9kahjFYCgb4q4qxBAIPvHyO9un4p3nVKMp89k+AThfL6gmcya6AJEhYkYUmsZD3fmdYKusCkK09F1HmRlGo1GKJ8yHbFqVYGMpkmCJocJt/ZqSR49KVrL84tzjTz6+mA48CnxIydKIITeP5JOgokg2oN2WFxmH1BZPs16yvjrHtS1GxlRIbUc5mTMrRpRVRTUMtjoXkTvF1fYipaVCbqk7MSYcLMbBTD9Nl7dHIY65eeWRSELvub5p2R+PKI0hqIjUkSyHKvOcXDQYP2Y/zykrGGWa82uF8VMeFsdoYVkeZ0y9Y70BlGJsJLNKMZ/mtNcdm26BLhNFbipzSplzc7Mmn43J1YiR1sSsZ5rrneeE9YF1iFTVBBsVqAwrPBtviC1crFfkkz0667G+x7ou7Uh0ngqANmByMBlepvT4bjBWwgucCIxjjm0kvpH4jaDdBDwG6z1B5oNYJIkR8mrGul5Td9D0cRff2bQ+OdNJg1E5JqtQKqUTCqUGpzpHWY6SJW0xRqiKEHLycsLh0X3A0FpJ3Uuua0c4W3C96umsxDmd/Esk1G3ER5/SmzqB99A1LUYbMpn4t733RG0IIduZMCltmM0OUve33R3o1CEjBEoEijzfNQDD3YySgt6lQiAUCfscYBqlJEoIvLdoBifDbRpO3KbWiKRalMk+QGaWEAUyS7sB590bnOBMVbgIShm0TEUh+g6jFCBQPhCjIY8FhSgQKhJEQJJTSYFQkXGZM8oyVAz4VqYAXpWeBWJS3AokeXQolSN0sjjQMjLRY4QMQzJQRLuM2CwwusCI9IULIZiM52TjAlMJdBHRWUzBIcEluIxk0iUzh8ITQw8EhFcgCoglKhZoCUFAGwO9DwgzQuiIR6U64hXWS8DQe4+TEq8MUWWomAJEvEj0UoHA+yQghPS8d6QBf+8sPkKuMnRWoUxyFBSyJUSFUTlK5QSfdiaCCNHhfYcPASUytBKoqKjKnDzTBGtp6w2bbIVTka5uaXqPjAGxsel6ALsAdqEgpoahbn95Kr34VQxT/m0dDx8+jP/gH/yD/5+9318cf3H8xfEXx/8/HP/wH/7D78QYf/sX//yXG87+xfEXx18cf3H8xfHv5PGFQCinn/+Q69PPEVKyv79PqQzLzQorHapIW+k+SpRMsUSZ0Rgz4JFycC9LFAGEDARtsbLHixS26xzUG/A2JHe7TCRprhZkucZoiYgiDbF8ovAEHEE4Mp5QxK8ihOBL7/1WytcUsMWr2GGsybnMOkfb9ljrcN4OJvUpqVupJA8OIZlubXGurrcYnSOl3A3Utqk2MYYBC0uOeVobpDS3pP7BQ+bk9GNCcDThJQv/40HBppAahEle5et1R9doxqN9slIijENoxySb4jYeipqm3WA7R5mNcX0kOoEVDq+SK9s4nzASFW3XEg2YUlMWmq5eIhWUVUWRF2htCD5ydr7EZDpRNe0Ec/MhAP/RP/mX6IN9/DsPEId7yE9PUPtj2n6TrE3rlvDxZxSrFVHFYSg0fODtEYfvQiY/kTgY7gW2LiLJL4ZiTPzmV/HjMUEo0AL/o4+Sq9zj4ySxvlyhDifwvZ/y+3/1N7jZm9L1PaeXl1QiMD+Ycf+ttyhGYzarFZ9/9jm5MXzzG99kPJmhtGY6m3H/8JgyywlDZqYcfJ3DwGbahkIFEUn5xINZ/xDWzTADkoOLQ2JC3YYgpCsgWK/X/OP/4z8GwF59h9Bfs3VAfWMXPYQmxLjNgBxYN4Mw5K5dRHp7kZ4lhtTI3UuJQQU6DP6kSLm0UrKNltt6xSAGuEgqGH2JmN1jUzf84//T/y0NmyVDvlpyQowxIIwBkRTY0SUhj9BDGtPgoxJjBNcRXXdnRhUh+BTakCU6VvQO37eE6FOEnkgQBDGFJQTXJHLZ1v1QCf7T/+HfTyEOZh8rKqy1tJ2nCZF+yDZPc7jhuf9TgAohhzTQmBKbpqWGkAgRs3HJe4/3ub8/w3vPYl2zqhvqITcgINBKomRSE3rv8TEFh3gXod+g7PJPfuPh+EIKuOtb+maJkIJ6CUhN1zU4FRFB4nH0MaJ1RgwZMmZIDBEFWiJEcuqSpAIeZY+RnkwOGJeU+K5l3SfDnShEqvg6IoxGbsnzHoIMSAJRWIToIRyy9Q0SIkseGio9TttU7e1DEAcrgDxL5khd3wKJtqZ1jlbDIAYD3PJjC+tT1BRpuGR0RgwOYzR13dB1Nc45xuMJWVYh77AQ0gN1qxh1oaV1l0nF5hWtT/7fLjhWTctmmSh3U12gdVJdVkYQc0/HDZ1oCDripKf1yV7URoeMisJUZLkAZ7Fhg7ceryUqz+ncEuUFxji8sWgyfAxgOoqxobctXZs+OcDsckmxv4882EO9/ZD6+RnlZMKaHE1EsaBbbZicXaPU4CqI+GPPTWT4LiEJeGKabyTrz4DwlnJPwuEecf+QEA0x1/RXSzIH/sE9qArkfE12OKX91z9GD2MPkxkOjw959fEPKEaGrKioxlPavsfkBePJhMOje+wfHGCyjLKqmE9mzIbkoRgHgzYBlsQxdtbS2Z6mt3TO03YWiGipKHLDZDRKQ1qtUUNs21DG3yjgRXGbSh/dmtAvhutxe2wVfWKr3rz9i8QE2f78nYIvpSDsFH/Dz2yJWEMBT/E5EqF0oiWKxOSJIYVeICAGldROIXH0QwhcXi3SXEIwNEGCnXGVzlLEYYgE1w6LmWRXwBHDAu1SEd+uVlJA8KkGDIsJIRJcn0RoQqQinlzOUsn37XBd0udSKi1AQogk2NIlHk3X9tQxmZe5QegU79j4/uIhxDBOHxZcKSCvKjIlCC4JgbTJkWYEyoP2BOkIMmIyTaYznLfJITF6PMk6OAjww+zvlx1/nkxMBXwbeBlj/DtCiH3gnwDvAJ8Bfy/GeP0rvtbuP9u3dHJYlaMk+qTuitEOwQOk4VcUu85FCoGSiTsplCBKjY4G4dINJoQi5DnOgpAerUBr0CbZPyoF25tVRoZFAYJMVDiGMPBN2yO0JhMapeTg8R+GVOuUci9Eov0ooRFIXEheFylUNQ4MA4ZWKf37wpgk7Q6RENL5WJuuR8rDc1hrKct4p2ti+P2bk+kQt8PeFKAaBpWl8xCDTLzp1E4MD4YEEZAm4K1NQzelh4UuTeqD9YiQDPJNyJLpkJcEZxNlUDtcHxFIrAioYBF5pHMuGfLLDNvXtHXPNlTNKk+WC1SpUSNDGCmYlEiT7AzoAk5qQtKqJWYL/MLuB7ZFYFvi0idKcV1OCKyAUmrEZIyYThFeEcsMjg4Q6w41m8CkQukMsz+mzTK2vp1ZlrE/3+fpj1e0XYdzkYhGm4LZ3gH3jo6ZzfeYz/fIsowgSMyLmLBIwdaW1rNqWzbrDcvVipvVmkXd0nnBelOjpKTMM6ajkuP9ObNxxagsKfJk4qWVGjr0O9/13Y1IvG0itn8u5VD2hvtFxNufTnLtW7rk9jXSr9tn8s5b/MK/TV16oripwW5imzHJbqGI/OI8LTGhhisj2DmIprBoECEiwvA8hQjBsf22EYnZlLrfuPO1iUoMRdkSvRsaoVSwb2vtsFAkwj1EMwwch/O9cy19BBchColFJOdJMVjxyu2tMdxtd+9FIVDDPxCkkA0tBKMqYz7KwDmUgOA9q6YlRk9nXfKWFynMosgzNrXF+eQ5733A+eQTn2wefvl88s/Tgf9PgZ8A29Tf/wz4r2KM/0gI8Z8N//9/9Su9UuJbQZQ4b+mFT5JcrRFKE5Sh8+uklFSDCbuH6AUosSUboETijhbSQK1pNhFvBZkxTCZjXBbp4gYpHUaBVjF5Wsht1xYZdmuDzPe2uQNY1R1RCKoqp8i3KsXbvxckPrCSHpTDm4gcqFFp0Uidihx2DGK4IYzWhCBwziUbUSFAKLq2R0pJMQT+Bh9xLhXoFGxhdgvf7bVkeK2A7Yduf+hO8lwiZ4ZxVZBnEq1B5CnJPIrk2qaVQguDjBKVJTWss8mDuekaNnIDSERUiZffe1wXCFEhhcb1njY0OCtobUs23We13rBc1vg62/mBr6fg7Qr1/Bm6WeNw0PcEXyXD/14RVIUtpzgZcTI5htzufAZp153PL2KKTJOERGeUkuDHdKIkXm4IzTnRBcQoozk7w7YW87pAXubYqyV2v6LH7x5opSTVKCcvklJusViTVTPG0zlVNeHtx0+Y7x8yHk/QWrFum2QO1Vv0sENyMXKxXPDs1StevTrh5ck5J5c3LDtPVk7ZrNfMJiP2piP2JhVn5+dMRyWz6YS92ZT9+YzJZExhho78T3h8vB8U8dviLdK9eNsJJhqaIDUJ2xtlF8wc7uxc4rYbBSV3GUJvVvOhGP6iV1CM26z31HhIxNDcbN8xdbHEJEpSWQE6IzqLbxpisLtFJ33Vw87hzo5BkAq3lAqhNFEKnO2ht4PadmBRyYgIgxW0T7RHoQLJYz7bNvcQwvCMp8+xbBzraJEx0oXBjoB0PVIeUdKiCMGQ95k8ZzA5avDW2S5wuRbkWrI3zcl1SXCepnVcLlJf60OyEDBGJWh2l4Pp6G3yk7E+RRyKONz7v+T4VUONHwN/G/hfA//z4Y//LvC7w+//D8A/5Vcs4EpE8sGoJityrO3ROkvbDZVjEXQxw4vBIzkKYhgwORKWltDGVMwrqXj6+pLPXlxys+koy4r7R0cc3R9RlAFMi1aOPBNDVFJMvKTIkMZB2o5tsbPhcEKxqDtaaxlVOfPpKEm/Rdw5+BEt3tbE0KeinRmkytEqS9aQSPyAgUu5JewnzYIc+MVdb4lAliuEzCnLEqV08uAWArTepdAnF7Pbr00KjRIF3oLtIuP5ASpTuNjQtDdkezmFkoAbHgzNEBqTuu3giaRFcDQeYS10XcDWji5z2APHpu6ZZiNKXSW5d6bxRYkOnr7fYKNDAKNZTk/LctURvKIoZrfb8f/wd1FmhpJjQszonrzPVQ6ZD2RCkJUF1d89QHQtvR640kKmayUBmfjSMTiCs0Tbs7w65+bynCo3zOcz9g+OEXKK1BWryfBQi4DIYDJ9D3KDEgXKScRsSjuVFH/zW8hBph5CoLMto3HJvXvHjEYVeZ6zN5txfHjI44ePGJcVUkDf93QDHJK6KsG6rnl+8prv/fCHfPzpZ1wtNixbT4fGjPcwwaNRHBwf8/j+MVWhaVcLrlYbFpuG12eXlIVhbzbh8YN73D84wGj1xi4snWcq4kLGAd7bPae7Lf0A8hEIu2K9G6MI3mwCYGfyxYCJh6HBT0lnySXPh+F+hJ0ILzW2cfizkGDN7WsCQmbIbdSajxD61M1nBd4l+EDKIXZJa8RAp4sxAWgp3FcTgiW4FDAOIDODkgZlCkRWgjb44LDtBkIL0SG8TwJBDSorEBgIAq3jboHa2MiVC6jhM8g/dm2GHZ+UCNvh1pe4xQnFW19B6cnu2kkRkSqpZF2MVFqiVPIy39TJl0gMjotKpXfrrcM6T28tfd/jQ5IbIUAGkv/MLzl+1Q78fwv8L4HJnT+7F2N8nT5AfC2EOP4VX4tqVGKO9lKWnpRcXN2gjEHnBiGhW9UoD0KmdJEQLS5IXFCYQYSopaSQGt10nJ/e8PzlDRerni4kz+z+5IouTDl6klFWEWWSuRFqgD+GcxEwbJV2X+HuPKUp6Jsa2zcJo+87DvfnmEwNW9JA9D3e1hA6kFni5iqN0RnJcUzggySE/lZ1RbIJFTGiZCQzqXMTDqTSiCgRUpPnaoBM4hsCCa317mFUOiMXE/q+oSgTJzvLEse3zALCuLTIOEdUGmkyvJfYztJ5i48OER09kb3xEQcHh0y7jk2oaW1L7DqOZlOqMEXLiJMdy3pJQ0+VqYQ7y0iUgd7W6KIkopMjXIi7O2z2pfeZHb9FGB/S6IpC5eQyEJVNHhlCpgEcYDXoKJPgSIDUEWNgs1nw+sXnXLx+hq+XLDE0umQyKrh/vI98+wH7xw8oJ4fkkz1UMUKZIjnh6UAwGtWrZCZlBH0F5o9+hvk334HlkizLeHD/IT+MmuvrS7weMZnv8ejBAx7dv894NEJFQdcnD5Ou75Fa0QVP09S8eP2aH330M56+fMlyvabpLFFqsrwkKwts3XIwr5hPKiajPPn0MMaHSFlW2L6j62oWqzVCRDIp2ZvNyPP8tvqS7v8QQ4IghBpCguUO5kjPTDKSklK+kUCz273s5gfxjYKVdopJRJayQEmdcUj+6GFYTMLOP39YHIbB5i3QTirYvsH1Pt33UqOKEoQZvJB06pCHRsy5lsB2xR6mAMHjiakLHyDG9HAqQnQE2xBtC8qgTPIcEUYio0eF7RIARVEynUyYzabMxsUdeCn9ryPx/FViR+zgKzHApCiJjoJcdoRwg9EqiayGZVKQBHbLTUO5FChRUGQKkwvKWCTbXJ8WJSlSnFyWaZrm1lY7xIgfotQUYmcs92cdv0oq/d8BzmKM3xFC/O4vfcU//vP/APgHALPZDIDJbIYaefLM4FyfZK8EOu9pu54oAlqKxCCIw7ZmO8WPYsDLPUZE9lSkb9fk1pFJRRjkqF4KNo1n3hdUMUPJFiH/uNoynSQMOlB2qX4CTF7gnMX1lt5aVut1ilCaT8l0sgWN0RJ8T3AtOk+p1pnJUFIn7P4u9hYj8s4wk2GrthuSEkkj+9SxhJBupuD97iFLBv+3py6VIstyTCEgalxMXiIiCGSQWNtTjDXCCXz0BN/gLbi2I0a1u2EijnbdcNOuEQtBUWeMREawPYdmhL2RtL6mMUtssaGxNolyTEoWD8pR9xsy6XB+RLAkN7XhDvPtGVaOseMJm9EMwZgKQTBpaCNI8wAAaRQyyrTHEkOnmcP19TXPLi95dfoSSY/QClsYNn3H5vqS2gjejg17zQrpGnQ5Iy+njPb2MZM5Ns8QPhVwKSOmkFBNtkMRiDF5UwuDEFDkmtlkxNH+PqOqREtJ9Cllvut6nPPkRtN7z+uLcy5WN4z25/zWvXucvnjNx588xUtDNT+gmOzz6tPPmeaSdnnFZewYj8dMRqOkPjSGTAmUEhRGsre3hxvUgUJK3B1nusitMC2EOAQXiDeLPDtU4u6zuPs1/fNbLF0MPy+GGVNqyG8xxVvL03QeYQfBDKpqGZG/MJ+JYTDqGs43BoiuR5lteMNgQYvYzviH9054oxBpo7zF10WUaUe1BVekGSwyFAiZPNY15EVFVWRMioLj4yOePH6Le/eP2d+bM52OGI8yPvn4E7bbEimHIk1I73sH296FDQtBqQXjaUlRPeRaQSO3OEC6lpCsEpabHiUFVaFQIlLkmqJI9g+JsUJqbobmZ7vqJURY4AdZ+p8MoL15/Kqp9P9NIcR/AyiAqRDiHwOnQogHQ/f9ADj7k344xvhfAP8FJCEPkGKWQp5WnAjluKAPHtd0xI5kUerjrS/usDELQ4fmARcdXijKyZz9Q8Feu6DZ9KjBQxmVAo5jUEOnm/C7wBaKEbejscjwBUbuTjiUzjCmgBDwFnrbs1it0SoyGRkUlug7gk+htqZIlqZCylQYY7oR042ebozU22xvlgF/F6nT1gw36fBZtykxt+k7ia519wjR44MjyxKurmJMWY5ADJHWteSqTDeY88mUyEasdUQviUEShUNpiJ1lvaoxvaESOVVWYFRF1Rs2NjnIWSyxTEW/9x3SpBiq4BxN3+NiJLocGTJkvD1X217Qd2NsmOPVIRJFHCh+UiTmhJAJLzVSIXfDKYEQAWUETduxXC1YrhdUlWE8niNkYH1zTbtY0XmH8hs20xVmtUTlE4pqxmH3iHuTLyNEQfLqHAZPpAV/e3jv2SxXTMczdF5wdLDPvcMDJlWVzokhos07YvAYJTHGsFivOL+5pHU9R/ePeXD8AOEj55dXWBSj6ZS8KimNQPQNZy8uuTKKw6NjZl/6MgSXPqOWSQ4fA6PpFNfUnF9eEaN/I3w5hiEYeKjZcRih3oUzGO5r7nTF2257W+t/cUC+7Ta3xToNRrcNjdi9xjZ4INz5VYg/qTESA+SxReQD0Vqi6NEqhXH4GAa4Rg5BGtunMllMC6WHZzSxXgTJYCw1PjpRG1WiGU/HBUfH+xzdO+ToYJ+j/T2ePHrIk7feYW8+o21bVqsFea6205XdZ1ZSEYUfmB+3TVz6Jd2DpRHMyzGjrKReB5wewrR31yb1X50NrBtHCFAaQVFAlikypVAIoktU4n5gnyTKZ/qexPb9fvFa/inHr5JK/58D//nwJf8u8L+IMf59IcT/BvgfAP9o+PX/+iu+Jz5YbL+h6XuEgNKkdJrCgDOBJjqCt8PAUqf/RMq8Tr0idNHTCoW+9w6zacFR/pT2/AK5WtK1XfJTVnI7gkmSWKGGTljubvqUBES68iK+cemUVKmADwXeWdg0KxQNmoLCBKJrhlT1ZACEkLgQ6HuLlJpcpURr0EgkSqSw1CA8EFLnohTBJ8qd3PLTgyfEZC4Uh4dJDkyYu3ios47G1ShZ4lzEmJRBKAV4EWl9T4lGyeTD7L3HOYELELrkuyCVxGiBdoG2s3gt0EYTC8E8O6S7ST4fKIHQybpV64gLDdYLfBRY1+JdxNuAtqCERt+Rf4fQYrsFoV2h+jS0tkogY+o6pBh8MQbnP5Eia4gDr1dpAfhh8BXIi5zp/kHylmkaFoslF9drShFYbVr01TXK5FTVmL69Yv7oXorEGljjYhi4WXGL2TrrWF/fcHx0Dxs9Tx7c5+HxMYXJ0gKDGAKRk8x+XBbIzPDZy+cs1kuysmBvPmc2neFDxJQVzks6L3DrFhEDy6trTl6/IARH/dbbfOX998H3iNijtE6eP80GlKTzgRfPn3F5cUazWd9ey7g1/99yuLkt3neK+C1bJb5RqLeDNyHiG7OVLVec4ZlJuaJv4u/p+U1+LVvb5xgGt0Y1FMXhBaQ0eJ/k8cDO+U+KjjzT+CgSYzfEbZLH8BnCbS8lzW4nIUR6Hy0Tf7osSoxJHj6jUc4H77/Ft37763z4wXs8fvSI46Nj5tMZ+RCj94Pv/5DPPn2aqL13rodUCjUMwaUUw+D3DqxEwAjPqDBMq4yqNBTdDbKQO+gvQZ3Jy0ZFgXOCjoCKEqM9XnuEVighQQl64fDOYp1N8Im/tTQWw8L3q6jk/7/hgf8j4P8shPgfAc+A/+6v+oMXi0vWy2cIpZjmJZktGJczyqykQHMVN3iVJW8JkYJ0gzTILW4XBjhFGorD+/TiALXcoNsOM3hF294l6k7XYftIiRr2k2IY2Nz6IcSYOvNU7m87Mi1F6tC0RqATlBMCUjiCD3g8wbX43iKEIURF01pcTNPwqhz8F7YGNjBMnxkenJC4rsP2TYiIFgn3Cz4NSRN8JNNNptPWru/dbbcTBdEL1osU0jsZC7QwaKMxeUnOiBjBCZ9M9LWhJeCFJIiUjh5Ey6Z3BN8RZUWtO6xKFMP9zT1uXvW4PUeYB1Ru8LbDB09LT8AjYsRbS5ZVeCeITuE9OHl7AxazA0Q2IkSF8IGoUtivtAE/FGylNHIr5lADPVOmDj1KgTQZQUoa65GbnnmE/ekYX0+IwbKqLatY0PeSfrlgXJUcCs16ecPZi2e8NT1Em63BWcTahn69JoQECyghKE1GJsas6hVFXlCVFVrroXAHrOtBBIrSIFXG+WLBi5evKMZjqmLKxekN1+crfvyzj7led3hV4NcrltfX+KtT6ssXXF6ckWWG/b19fN/jugYVx2gUIQbqpuH88hKN5PXZGZ/8/KesF1dMi9lwv4bdOUuphlzJBHckxOHuwDN1d9tu8y4eDsM2fei6b7vzWzhm26VvMe8tXutDvLNA3LJabm9MgVAGbQyBRJsRLhCcRykwRiCcQERJZlLiTlO3aFMQSFbDIboUMh5CoukpidaaXGvGpeHJk0c8fvyQe8eH6Dzjb/z1v8I3v/HrjEYVUqjEXgseaz0hRB49fkRZFSgl+C//y//7rjFSA6SmjKTINM45XLi9ZkpEZkZyb17waK9kOi5prUBXI9og6H0kxIwi12TSE/rte6bBe5GV+OBx1iXIJAp8CLRtR9/29DYJAbexasPlQ0v/S7Xyf64CHmP8pyS2CTHGS+Bv/nl+fnuk1JCMUVZwf75HaYdUDq3Q44woItd1TRdTdJgSiRooJRgxTHyFQChS1qWZ0jcN9WpDPXQ6MSZaadt4+k4Sg76djkcSP3EHlyTVlogOYrokAtAigLT0cYN1C1x7gZaWg/17lEbgbUPfNnQ2okxO1zlcbEAoqqoangGflKMxFWjEFsIZHqRUwdHSkGcGfJ8w0RiIXYvAoLNqgHgSPrium90X3beWq/MNEY3talZZy3w+Z7o/pxiPUI2myDXjKqPMcrRSXLRLlhFqL1n1ntqB1YLWR/xySYiRTBuqMmd+NKe98SzzC5yOBK0oxJTNuiYoRds5ZIRMFGRxwqbbELxDSY9Ut4XkonbMjiqyYoTKIJhIkAJFHOiVkiCSqETo5MscYor/8jFS5DnVdE5RTdGmoCpKHu7PuF86HpSBk5Hm2dmC0eEjvvblD2mXl1yfvsS1a8Z6P6XmLG/o7Q1N3bJcLvn02SeMnj5j03ZAsv88vnfMZ5/+jA++9GXeee89Zvt7eCI2BLx3XFxf0fYtQkoyXfLP/8XvUTs4NHtsNjWfffYj6rphtV4TEJhckOeS/arg2dNLLs7PmUynvPX223zt13+dKCRd71gta7SxWNtTrxvOTy+YjyeslivOz865vjhh+n4q4NtiG0IclMTDrmXAbm/r952Is6H5YRg4bv9RIl1tAUV2r789QvD4bciKD7vuf7vZfxNof7NjjAhUlqXn0TqSRVYkRsHNcp0yAaJMKTt7Y/77/72/zfvvvc3l5TV/+P0f8e3vfh8ffYIvypLReExZFCwXVxw/ecJv/qW/xJc/fI9HDw453J/w7pNHGJMlPcQvsEm0MhweHHJ4OE9ztzufVUmBFgksKkREF5oQE57tvCdTkSoPCBkQWjGbjignObmGsdTDDltQZAJrPU6DD4kGLCJM8xK0YN00rNuW4CO9d/Quok1OERVK2oHZ1KUGDnD633IB/7d1jPKSihmFyoh9wEWJyQtMUSK0QkiobQdSEqNECoWWAaFAKIeSaTsVQs9ms+LB/XepTE5hcpZI1ut1krMLmTpBm+xbVcZ26jLwX9VwM8sdhHJ366RI2Yxtfcl6dUq9OaMoJV07RpOBj4Sg8F6SVQVS5eTKJGYAkq5p6VsGeb1MCSsqbUu98ymLU0i2arW2tUnJFi1ReITsBwK8TtAJCmsDddPuCriz0G6S7aaQOTIrsL1ifdPTN569bM77+R575ORNINQth61nISNXRE59y2Uv2LQBj8ZbBzYl0kwnM2b7JRfnGbrIiLKhcxbvJfWqxZZDGAOKGDIChjIr6ZqeGPshYTEdn706ZyJzDs2U++UBSm3YND1G+CRc0RlCaZSqhkFXunNTtyewzjCa7jE/uMdoPKetl5y/fsH8uKDQMJ+NWTmB05rVasXDg332C0m/vmacCVTsefrzjzg/v+bq+obFzQ1nV2e8taqxswMwBucdi+UK5wMHB0eYvMT6QOta1qsVq/Wa65tL+r5LW3mVs9nUNE7y7LMX9H1gs2kQRMZlCUiUNiiVdhevouXwYJ/3PviQx0/eYjyeUjcdIKnXdSrKMSCCoF7XdOsNZ6en3Fxf0w+LDAyiHZlmJGGwg1AyJrtUGXf1dNcVQ4Ikgh+gQzcI4hLeu61zadueWCsCdhBJsndOv991prviPbz4Dlhn94bBW0Kb4u+IDLCYwkUIUhK1RqmMsprw5MvfoLr3Lk/e/4Cv/obhK7/xFd754An/8l/9a5Ca8XiMlpF6fcN0Nifff8L3P7/ipydLpuOct45n/Lf+49/lg7cf0jQbVqsVbdvy6NEjlFJY5+htsjTeJh1tr6WSyTEyebx7jBw0HzKg8EiR0ouu1w4ZPMJ5+qZhPJEJ1zZJE9F1PYvlGpOl8HWdG7QQ5EYQjYI20rmernM7mKuqSjITcHlK5GnbLgVueIem4xeD1n/x+EIKeC4NOh+RmwrhLNF5EArv05ceAxTaUBb57sZz3g1Oi3Hgag6DDgST0YT5dMZsNme9SWG8zjuiVHQtNBtNszboQhPFEBQ7dBApF0SyJYvcCeRJLJPQE2yN79dEt6ZvAvVmgREp5NQHiVB5sp8cqIPBe+q2wXk7xFKlRHTyjG1AhfcJChKAVIlH3PeW4C1KOpQMSBkhdPheErUnCk3XJexsd45RImOGMWOUHqxN+4gInomseDg54NAWZNcdbrGhXdwQXctkVqFMRq4mzLOUrN57hdWO6CNjUXIgC4z0eHp619Frh40e7wK28/QRVJl4rUIK6rqhqAbbC3EbFgxws6rZnF3gijF5ViKqlrN1Q6kiVVGk9BilkDrHeUde5ujMgDAolbwqlM4Zzw+Z7N3j/FXN5WLDQRGYT0c4XRJygYuK6+WGw9mY8XjKuNBo4WnblucvXvLy1TmLm2Xi3UpP1zaEqQcMbdPyySefUJWauul48fI1N+uWoiy5uLjk4vKCrmuI+JRapHKKPKNzluVyQdc5EuKWmoHgHbFzuK5FqEiRaYrckJkMax3X19c0db0bZie1YSqsrutZ3Fzy4sULmrpmXFW33/kduAOG4VkceNrDnbs9tjNMISR5VTGePeD6+oxg6+S/PrAitkX4NkQ57hSBdweXO8jk7rtI+cb53B6B6D0xDCwPIRBKEYUkEtBKMd+b8+TdD7n36DEfffQJ0XW8//4THj444j/+j/4G47Lg/GrFxdk5F+ev2CyvefzhNxnNDnh9ekbdNUgleP6q4L23HnP/aJ/F1RWnp6d473nw4AFb6+jELU9S+921HFhrqVuXWFLSkR5CU4axGT4K2t6xDDXGeby1xFxhQ0cMKa9TSkmmJbkxKC3IjGRUFoyLilXXIQS4EIZIwPQdKsRwDim2ryjAG4WyFuU8+O4XL+obxxdSwLXQ5KokL6aEdkNn6xTm2qfhn4spabgyeeoXXE8bLSlQRyQhjkiTb6U049GU2d6c2WLFYlOTXV9h+5aIx9pIsxZsFppyrIm5ZGh9BxbKID2IPiV28KaN+hBXgBQeJT3Otmw2C3Jt0DJ5A6ew1QKESFvgzZrr62ts76iqiqqqCCMQgydxBILbDn+GwalKknc/bFOFSFz3PrQJfwyOIDS2HYafW6hMSIzMU5xXdATRY3uL9pK9+YT7ep9s2dCcrlmdnbO8vsSFnsm9I/K54l5ZcGxyGulYO0Gbd/joKXXJgVRI36G0w7qOru/ptSPpkiS+DZCJgYkSaW2N6DV5tjULuy3gymRY51ktrrl4+RSXXXJaOyaZZDoeMR6NMEYTfKDrW4pRSVaOMMUUU+zjQ0qf0dUe1cEj7MUNN85y0UmCz+mEYa3yFLvW9FzXDj0tGJWGEBJ76NXJK05OL+janiwzTOdTuLk1C2qahhfPPuWrX/2Q169PeX2+ZLp3wL0H93n54hUvXr0kL1JjUWQGLSzTUUWkZb1c4bs6sVxEMlHynUuRaL1NUX4h4BBcXlywWq3RSibD/zJHqTRMllKitaJrOp49/4yXz1+g8cyOD2/vyWHgGMTd4hp28MkvqnXDMCTU5Zy9+1/muhZYd4L0m3S62zt+W6wZAp+9G3aqb/7dm7xxMQwsbz1B7v5dDEnzgCT5k8jU9SsR2JtNePLWYz748pfoLfzsj37M+fkJq65jPJvxO1/5kKPZjO9974f8i39+xrOba5xzTPcOEcEhQgpS7kNg03S8Prui7Xo2dc1ms0n+QHdZNjJ14qs73/n2z9PgVRFEwMbUUKlBMxh82uEGItangOxMC5SArqvpbbsTIWZGkGkBItlpFEYyyhTrJsHBqYBbREydv0AmJs8wY0hCv0R0kKhf1oB/MQVcCYWOBoKkblrqVY0xHu+hbjpWzQZtIHiBkZIYA56QcCqtEDoS5ZCfKBR5WTIej6nGFePJiP2DPVaLa8CjhcJZxeo6UI4C1fHgxTAMLj0Mct9kaiVivIWdRDI4yvMM22UEp4kB6s2aXJcUmUDKAp1V9F7g+4bV4pLTk1d88vGndK3j3Xc/4OHDx0hVovQWEgrJXyQO8vAQU6xWlhP7CFgiAaMyclXjQkfwPpnXO0GhDc0AYyop0FrgbMNms0RPJN45RlFykI3IG8nqfMnmJi1uV5sW2/cs3SXlqmVvNmZvMuKoqOiUodYNve6QUjIOgSwE7j3Y43p5yVXnqPsGvKDKR/iuRbiQmCEmkI0SNGTyDN9JnL0t4B988CXCeEyPY7O84nT5ClvOkaVG+A2xLxlXBZvlDW2zRmUKmVeQzfDZEeXhO4RiRE1BP39Cf9AQqbmuNI2saIPiWnmqTNH3G076gs1KUupIJgxXiwsurm9o+wTNZUVBOaoGWmYqO9571vUG7wMfffQxNmruPXiMVoannz7l888/5+Gj+4weP2QymuC7HqMF8/kR9XLB4nLF8mpDNSqRQmKtpa5r1qs1m+WKvm1oeotzHqlUyjPd22cynZAX2bAzS8Ovm+sbXr16ie0ajg9mu4xP2EJyg9NeSI1MiAG2CTfcYSoJkZhbHm46gVgFLrsC10hKImW1JfmxG3Zuu3kfImGQpQ+UaaQaSG7itmAnKqxka6C147HoHJFoJsNQdMBcnGU8K/n6b36Ddz74Cn2Q/OxH3+btJ0eMp2MOJhnROz5/dcVf/fpXOP/kJ2RuTZHn3H/v1+iuXnNycsL4/rvMZ1Naaznem/A7X/8a46pg8vZbPB6gE20S1TCgWLcrPnvxin/+L/45alfY5RCKnQgTSgQyGTBKoFW6np3rcLYmy2A8qbi/f8CkzMhNxsnFGevNBruuaZuGoigoioK8yIk+Z+ks8eqSIDPqumHdtDRNj0CiZUrUElvDrxixrkssMql/JTLhF1LAu97StkvszYIXTz9lVE4pRyXWetZNhzIKFaBd3+DyHJPl5CZHCot3knKkQYM2itlkjKZDCUumoSwy5vMpuRH4vkUPbBbvOlbXLdVB+rISBz2mqLEgkkXl0CVtjwBpyFCO8XaCtwticHRdy2azRpJTlmN8kLRNh7Vrzs5e8cnHP+V7f/hDYjQQDWUxZzQ6hHEF0RBij3M9wVmIjqAdQmuE0BhTAhkKi9aBMtvgosUFhyDi1ZAfOpyj1JBVqUOna9BKcTCbcDyaI5Tl7OIasVnioydqlTrEoCiqEbkusJ3lvL1MdrSqoGka2k2DVJrR3gaXFYgHBxgvwAVcG5CUlLlgYiR5CTrzxGQFhDYFMUDf9bi+3Wl333ryPtmjt+iVZH11hvj8KefrNd47FrVl1a7JN4qxUuA6QvBsupZzHJ9RcHJxwZW9YN311E1DW0uED2SvrlDuAoVGq5JRXvKloz0W9QzdSoTwFKaH9Q0na8961aEi9F6wanoerTxhlh5mbTTz+R5KG05fvUSpitm0odnUnLx8xeLqhm/91m/ytS9/hXtHh/i+I8sUp1eX/N6/POHF008wOiOXRyxWa1zoB7glYBScXi/o+x6ISCXxfYsInqZeIbXEuttBVtd2CY9Vg3jrzvOzZXFIKfEu7HJc03CelNdNKshSpAzK4DquXj7jn/2/P+Jq3XJ/pnnvXs7+HWiG4WfCYBMbAwNskv5uIF+i1N2MztuO/A3WmxAIpRGmSFTY4GCwQJAy4z/423+Px+9+gPM9Ny8+5r/9d36X3/nWNzg8usfNuuHF6RXPr9f8X/7pv+Hjb/8RQije+fKvM3/vt1iffsrl5QXr6xNmB4d842vv85/81/8Gv/GVd4nW4qNHaUVe5kSpeXpyzh/8m2/z7W9/mx/98Edc3iz4+3/3byUIY1gM0zLnmRSaqZEYGYneUddrLm9eYl2PKwzzkWL/aMxX337Ci5enfPf/8Ud89PGn1E1yEFVRkBnNl7/yJd7/0gccHu7jLq+wxZjaBrq+H6iDA5U4JqMupVJEI8PuR2HIhETxZx9fSAFvu55+taL3Nm2bCbRdDSIyKiXFeJzyJmVINDcdCZkjEwKsIvYKKTPyPKcwks3yHAZ+sJIiZSyKSL2BzCQc3faCtlnhvSbqMMAngkBI8WEhoITfTe2BwahHo3RFVkww3RgfukQp8w7bt2jVYEqD1gUhaqJQWC+wLqJ1CnSVOqKNIDcy0dgitM2a1WpF0yyJMTCdzpjN95HKEKPHhxTBlVyuBcg0zE1c9Ts88NDRugVVOePowYwQaqpSkBcCKy1gyZXChkjvA0EosmpMNZmihMD5NknD2xYtPSKkrEff91i1xp2eoUsH48h0PsHrnNXCDyZdEhUVpSkpSsEiLmjbtC0tihxjRrvzPD27wOgRVCXYQJnnZKsVsfXpU0qBaCOtChArNjHjQlQ8UzN+wpzTRiBtpK17NvWaxtYo26L6HuU9GovWnmlh+fB4n4gioglB0nYto3zCaP8BrRNEZxFFhVfgpdqBZkJITJ5Rjkbs7x9gHRA99XrF+ckJRZHx+ME9puNxCl7uOmbzI57+6+/w4vPn3FxdMRlPefnsOev1Bhc6lII8NxR5hVaKLoYEl8gEDa7XS9abJeHOd+oH5e2oyNBGkhVZUlsOh5TbTMXU+Trvbu/bHT4uBqgwFfNMSw5milwv2GwarkTO/tjQ2UhuEsYRGSTzd6CvW2rhNstx8PPZoS6/gLffLeI6G0R0IhnTOY/Riv/g7/wnfPhrv0VUhkxHfuPLb/Nrb99HK0mzSbTcMs9onOTqYsGFM3SyYL1a8PoP/yXN6pqHBzO+8Y2v8+GXPuTdd97i3UfH5AKc1iBLNr3j409f8d0/+D1+7w9+n7ObmtVqw/pmyfL85PbzSZAq7WKEh1IrhO9o1ms26wWbeo0pFKPJHGLyCl9uOqaTfapqnfQhIUURBp8shLe6FSXSbn9NpG+vCeQQEwzjepti7oQCSaL1osl0Upd6lRFCzy87vpACLggpF893lJMKj0irjkyy8LIqyYocXEdrm5SfJ3KsT4Utth4tUmBDvVnysn3BarUkeIfRijwf/LUzg8mygWjvuFn3A686nUccXNlCHOzdUv9x50zTsFOqDG0qTDbG+4bgW4gB2zfEKMhCpJwZijxnOt3j4PA+R8cXZKbg8ZNHHBzOKUtFDC3eBpRKE+uLyytOzl6wuLlkOh7zG7/xTeazPSDS9Q1ETSYFUWVJyBQUUdo79Mc0MCtGGSZTVKMc2wWEElgZCIVGT8aEfkGQiiA1Qcph95Ec37oIvRRQ5DirUhJ7noJtG+8JfUus14Q8yaJzk2GLSAwdbRtwNqnXqrLC6JyQBZRKzBzhb89zsVwii0tkV6JsTbNeYl0LIaKiRPikiqyDoFYZ5/qAUzHnJWNOrKERBh0aWtfS24ZoG0TbI1xaSJAqbfNDzzSXzDLIBt93vKHM9igej3DBcHVxxuW6YTwbIcrkxwOJJZHlGT5G2q4lBEHY5jeKwNHBnP35jFFZ0FtL11tOTs753h/+gNevT1itVtjeYp2nbVoQgSxTOJvRtz3O9gTvEwyxVQbHRM9LRkfDMC1E/BCMnOcFRVnsznH7BG3pb4GIjHIIPLj9eynlG3RCiCjhqXKFkJGmdyxrx7qNlPn236Zho5SDm58Qw0BaDmrFtPBsu++tTS1wB2u+8/R4n1hUOEaZZv/oHk/e+RK/81f+KllesKXeZSbj9HLF1fUCIQST2QSdZ1ycveTly9csyAj5mDxuUHh+69e+yTe+9CFPHj/i6OiAvfmMUTWi7QPnVzc8f33C088+52c//QlPP/2Uq9WGTdOxWS7Z3FzR18s7qtSAlIl1I4WkrTfcLM5oVjfYrkHnJjU7RhF8oPeOm/UapKIsRuih4IqhwUoir4ElBEQf6KTi5voEp0ZIb5BCoDNFCAIpM0TyukZlGUoV6b4QYpil/NnHF4OBy5SOE6UmmDwloLsU0mqKgqrMkTqjt23iiEhJpjRN1wGpuw2dxDaGer1k0/RcbRr6PqTE62SNkHw69OCEFrIdRUpLENttJyH9F5NMN96loSTJDUIk03eTjYixSwONgZbkrCcgKWd7aJ0zmcy4f/8Rmw8sRVHy7nvvUI1K2nbNerWkMCNmewd459hsas5OL3j+/FNyo7h3fI8iT1ztrmlRMiMbm/QwssUYQxoGDQ+KMpqiKkmy89RBRiS9iFitUKOcuO7QzmP6HtEYrLX03iOFohdglURlSUCDUKiyQDlHL2ATPc72iK4HLdBKk2WJceJdmsj2jcB1AonG6ORw6K1D3vHv2Gxqws0Vqi/I6Gnbmj7YxPP3MoW1K8FlgGulOdEjTuWYC1vQ2EDQntY1dL7GhTpN551NafQ6ddF+cPsrZEhpJgSKLCPTBUJLqlFBXUc2TeDs9CWaAl0ZhNLDfakoq4oQIm3XoaRGikieafb359y/fw8poetamrbj6vqak9dnfPzxp6xWG5xzrN2atu3oug6jFZCjZFJ5xuBRSpJlhizPBl+bZCuc5RlZliGFpGkarq+uIEb2sjlFUfyxwaEUcsd89SJ1xXeLqBjkkMmZcOvJEDBakRkFUtFFydoK9qNKW/UBpt5aIAxOPrsCvu28d/4gw89sxT5vpP0MqstMRfbnU548us+7777Pkw++zv2Hj3j1yaeMMo2JGeuLNWulubhZMJ1Mk+GV83z+7BmvT8/Jc8Ph8T3GmWA80vzmt36TL7/zPlmmUTp9lqtVw7MXp3z00Uf87Ocf8fTTT3n58iWbxoLStJsVm8U19eoG7tBbo2vAbVCmQijB8vKaq9fP6TYLlBTsHR3s/EqiSN3zcr2hs448L1NAtdiybCRKpl2KGPjxMUZiliOlxnQ1xkkyMrLJHtZDECYFGyidKKci53Ya8e8oBq6UQJWGTBdcdj0OkRR6WmOyjFGh6eoWv1pTlTllVSG1po9rdAZtb/FW0jeOel1ztmi4rteEIIYi7REykBcGSNNeXRVMJzOIK6RIKd07o6mh8w4MncT23hym70IolMzJzDSFjnhHu14OggCPjg4RAjEEyqLgwYMHTCdzJpMZ+/tzXrz4jJNXL1nfLDk6uM9XvpYnhWGUuM6zWqy56Nacnp5ydHSPsqyStaQPmKyCGOhtolJGFEKkzwWgpMZkBc7Cel0zHplESYyw8o6RyplUIwqtgcimbXFAHwJGapyQWCRBCrLMJAvbPMcUOX0MrMRg79lZjEiLBCLS1Gs0aT6hpKKuHUKnLmazrvGdRMjbB6WzHd16hY4WkclE8YzgY5o7yJCoiGchciM1N5mkjh7X1lRR0PWelV1gw5pAP6Saw7CfI3oH3mGqCd1mw+nmjHE15nD/kOn+nNaC0GMO9t+hbSU3SwdmTFEKpFwBHqU1s2KOlAqtFEZrijxjNBrx6NEj9g8OeP36hLPzC5qm5fT0jD/4/W9zc7NEaUWWZbRti7U9fd8BhiKkBVgpRV4ociEpRyVFWZBlGUII+r7fsZWklFxeXnJ2eob3HqNz8qzAuzcdBbfc6sAQLHAXqxbb7nkIX4gJHiFEQugZjUbkRUE5q2hUSYNkrBxSeJLJ1DCUVDuHqWFgeeuXchcqSYvHADnudgpph3C0P+Yv/+Xf4Ru/9S3efv/LdKHk4uqCk49/yINJgapKlq1n/OgJWZlz79E9rA08/ew5T5+9gtDyzvETvvLBO7z31kMe3ttjNp9hhKHtOurecb1e89Enz/lX/+pf8IPv/gGX56epqZofIauCi2cfU99c4Po+zVaz27Jn1+fYtsHsPUHmBZfXp1yePif0DaPJGCH38N4ifOqGfXDUm5q6rjHGoLVGywSXeCXIpSYbhHtbi4GyrNh/9D6Lzz4m1jcEXbK/9w6rIKndrZXHNo7vz3N8MZFqQuCEokey6RP2tFeV7Fc5s0KifUNZGaQf0YTAqreMjODe/gHe9tS6pbWBbmN5+ewVvjqidz7J50NMLINMDUq1hOkpKXj0+DFBf4KSLRAIyZZg8JuOyHDLSIAthVAACikL8nyfSIbr1/RtgwoJ9skziXVdutGloixzJuMp8/kBl5fnfPe7f8jTj39O6DsO9g6YzSYcHt1nVFRkOsfbmNguZUFvHYge5+Igj86xXU/TOto+AIaU9ZcOJQyFHNGLnsZ5cBlGS6KD86srWrfgt0YHlDrH9yXZaEzIM6b7c6z30IPtU9SXC55SFGRZiVCChh4xApMJgtCJ7SA6gu+RsSdER1VmFGNNLDzrrkFsBO0yWaHm1e22fzafE/f2CUWO8D3rWtMsLDkVRTairOaY2RFCZxxPpkQzokOztJEbJ2k6j9vMWK2XXG0WXHY1zgZ812Gix3hHGTz3x2MWJyfQrOiKCpxlWmYEK2iaSFaVvPP4XR48eEgTPfOzV5zXHfhmsPg0CKn54P330VJSVRU3i2s++ugj/uiPvn8nJzEpaPveU1YlWitWYsVms0YpyaNHD5MiVaWGpShyZrMJymS3g0KfBDJt21LX9c76dblcMp1NqYqSMivomx7bNOxN94EBwpAyqZdFxEiDiupWCCnucsV3+uPBkxq+9OUPuH//HrPpFGM0k8Jgbz6D7hJCjyASY9rlyS2sw5s6y1uXIXZ/L9LJAZDlGX/1r/11/jt/69/j69/6TVoyPnt9zfPPPuGH3/8uD8cZ2JpXn3zGzz75nHe++S3e/mv/Pp3vuTo9Y31xwl/71q/xja++zTsPDhiNKjKTkRmDF5qrRcvHTz/n+z/8Ad/97nf40fe+S93UFPM9sv17BBdYXF5w8SI9d3EL+wSI7vaTrE4+YtH2HDxeM7r3BG+vsf2K2FtEKJmMR0R5G+NIiLR9y9X1ggdHR0xyw4NxyZ6WnK1WTPIsuTiGQHAuhXJISV5MECZjkhfsFQVFaOljRiOTT7qIyczD2ogyGqHkDob5s44vpICvnWVRb+hjQATNLM/YqwpyKaibnj6kYeZGG4Q2FFlGVRjWixtKkyd3OGtZbtZcnV+SHSb/hDiEABhjEie7bbGuT5200oiiIstHxGELJUWKQQq4YYuo33DQ2471042pUDrHujbFfimF1Inn2nU1RTVskXXCkfOsJMbI9fUNi8WKm5sFttkQvefq+pz9g33u3TviN8RvUJQGYXrefuddirwa7CQNPkhWmy6JTnwkCgM7N8N0GJNR6BFXV5c4G2jqhlGeUxUVRmeEtsHLOokQmg0X3YaFa6ktOGt3EU/KGEwv6GPA9R2dFKy1p5MQmzXSxxRjlwvKSqLimLruWa6vaGPOtByjS0NYR4xSlHlGUd5S3+4/eEA8PGIdYXG9gPGY/fkHTCdHqGpKyCtaU+B8oLEC5xUuQNSRCYrDmcQQsF3Hqmk5qxue3VzQ12uOteS+VhyJyDSsOSTy4PAxeTVCZSVFqXGtpRjYSw7oY4Ycj7mvWj5/9pKNh7qp+eTpS5rOU2bpM4zGI5TW/OAHPxjsS5Mfe5ZlGJPjXMKrnbe0XYOUgqPjwwQFKJHUiMGhtObm+oreeaRWgwXqLXacZdmAWydO8Gg0YjIaJ5fHzYY8u120s8yAzPEh4nzY0QbfMGiSt/4mO0ph9EymOQcPD/mv/fa3+LWv/DrHx08QSnLy4mM++dl3OHv1EZvr18joBqhuh8ckfcJua3/LQmHbnUsIShKB8XjM//h/9j/hwVSwXjc41zFVkZWTvFoLpqOcA+kZacl4nDM+vMf9ew95NC35+uNDRn/t1ymKMXXb46PFRYW1gtfLmj/64Y/4vf/Xf8WPvv89Tk9PaHvHaP+I4ycP2Kw2XL14webmgr7ZEKK7M/RiR5PcHiJ22M1rzj9esbr8iG5VE+smzc18RAmFdQHn00wj+oCPkuenp8ynUx4fjLj/1ceMtOIHn5/ys5evmewdcP/4PveP7zE/SFCJFmDzMaic/aMj3n73XaqLaz49veDV6QnL1QqApvdMpzNUPmEk3DbQ6k89vpAC3rtAaz2CyPGk4qgqUcHR957eB6IwBAcmpGiyAhA+pM5DSPo+0NmU4CyEwG5qYjFKOPCA120Nf6JPOGsg0FtFuR3WDAOYlPgiE2dwsHzdHTECfncDJP+rIdxVK6QXpGQehxRxmIomBWIIgaZp6XtLnheU1Ri8ZzKbMRpXKCMpxxXFuGB+MMFTc3BwgBA51ga6LoW0tn2ypUyPzpuRVttTjESUESAVUpUYoSFIYuPwq4Yu1tgoWHvHje+4kA03bYsRyYNFS0PuFFoZMmOo3VC4q5xybnjx6hXGKIqgKdBkuSRIj9SSaMF2HfVaUMwN+WzCOm7QRibzreHw2rAOknMnuQpT1iqnNjNeuopmEViHJSt3w6L11NZjY0QJSSU1UxSF8CD7ZITlArILjNqWfaN4dzbi7WnFk8pwaGr21IT5tEJlOSiDVIK6rtFG4EWg84FNb1nWa1Y3r/CDN0bwyWBob/8eT+4fQXTUbc35xSWbekNZjHfJTVt6XRJgkJhJwWOyBKUoLTEqZTiGwTgtyzKkAaUTdrs1lzLG7OLy+r4nyzLKsmQ+n9Bs0mC0zMe7a5lXOdiczvrh9kz2sgw8bAR3nAW3SHb63f68YFQZEMlrxhhJFFBNZhw++gAbBS5qNpcvyAaDJ0TabYgQB9f8NNzfDT53sEl6fiKp2J83jsMR7E8LMm1Ydp7q9RqZVbx+ccb9PcFb9x7w6Ne/QfHeN/jqwwPmIwNEms5xcrnG9WmIvLh5xcsXL/jxT3/M9//w21xenONRqPEeI6nxzvPi6ce06wW2rfG2fzMQId5uT+52tkKkVPu2vSHEluhLqskIrXOqyRhre1KeVqoFycpX8vzkhFwbOiRFNabSkodHBzSjim9+47d55623mU6ntH3Ld773Q0ajitYoehHAdUz6jrcfPUystGaNaFY47zHCkXdLXLMiZBrK7M+spV9IAU+BnYJSG46nU0oRaVtL7zwuSsZVgQEyKZL3r7V0zmJ7B1FgnceGlLBjypxl02KjRmYF0qgkRPCeECwxJqesJGbY3mhq2GbGXWRXihtTsJVKkjDwrWQ/teEBZEoWkUoh1TaFxA2KMz84CTps7Ol7cM4xqibs7x3SZDlHx/cpRxVCgykUhS4oxxm926B0joiGZLQv6LoW64a8xzfq9h3KWUzXQucKGSI+pAUuWIuJgAs0vSUIwY3tuHYtdRHIc4kNASMUBoghWbnmRtBDcmEQgkxCFAHnIn0DQkZ8EPTe4WN6uEMI9G1HGTKq0Yx63Q6ZULfnOR4XdKMS32oWsePFVc2r1TkuCFofqJ2jdp6690SpyccTxuWIPMuRIlDolsgifRU+I/cVk0IxLQ0TWTPFsl9NuDdO1q1NZ6FPFMxAoG1S2osHbAjUveV0sWD56XO6Mk+p61JiTMbR0REfvP8etm94ffqa1yenw6fYNgfgnEcIi3epnG3tQLVOAdh5npEbg/eW3ib4YjSqCEMA67bAOpe68+3v27bFOcd4PGb/YI9L19I1PZm5XQyzzBClxkUGGfvgDz7AO6lHvh0ywpYKGJjNSoQMBG9xvieE5IJX1w1aF4yn92g7h/dg6wtEdDtMXcgUlrx9lyjYiXrSc31LcHU+8NGra949OGQ/z5G+p1/doDeXjOyGbnmJnR8yvf+Q9771W8T5MdWgWGxsYN313NysuL444fTFM14+e8bL58948fI5l9dXaWE2KjF2ug3dasnq+hzXNzv6JEK8sSvZNT93mqAU/K2T2ZeDvBoxKkaYvKQsS0CgRLKwBZkSqJRksVlzenmJLMdJOm97ppPAwWzG47ff4t6DB0gpWLxe8rOf/Zz9/QPq2OKEpyYwXi6IylAHiygM+XSM8mnwL4XAWfurzDC/IBqhFBitmVcle5MR68UCGyIuShCavekY6hUSgfWWvgt0LtL7jjxPnYNQkGVpyHR5s6DuNpRThTZm8Ly2qfsObijgITEFRfrYqQNP4Qdy8M8OQiLuQCjpIfBDJU83glQSqXT6GSmSkWFID4SUhqDSe9soaGpP1zuKsmL/4Ag3nXH84D55WRBlBDV0OMoQhMH71Gdvt8AIgR/cGOMwbP3FpGrnPa1rkVoQvKNpWvzKY3rJrBghlGATLHUInHc1130DY83efIxtOvyQUCNiJLY9RZRYIekR9CHiu5ZqnNHXDt97muCh83gNIgxbbGKKbAsRZUqQAo/Dx9ubcD7JaCclSmg6HM9fvuJ7P/yDJO6IQ5AtAuc9s/1Djt/7Gsdjw1RF9lXNg/ESGV7hHVg/I4pH7M32KBScvPqMzbplKfaZUvH8s08HI6aEMzvvsC7JmEOMO0n02eKa/c9PkB9+GUZjtFLMplNGozHz2Yy+M9wsblBSkg8Qh5KaGCJ9b/E+lcq7UV9KKpSSTCYTyiKnbTaETQqvHo/HBATW+6ScHHaQW/VlChxYUdc148mY/f057foGETqKO4M3YxQBjVQxJVAJtfMqgRSOsYUKtoPHtN2EyUSzCR19v6Fta6zt0KqgrmuCdxRFxd7BY3Q25uTzH+P6KxSDL48APxTvEMOOcKuG13d+6zCUCvjPX13yl790n0Ntac5fc/LJU7JXJzyWDeuJwRwckD16wr133saLyNl1y8uLDau6pesb2uU5H/3wO3z/2/+a1y9e0dQtWVkwPjgiyozVYsHm5oJ2eUno6vSMiESP/UVS+i1t8M2qqGRJns1AKYwylAf3MFVFluVkKhnixWEIvSU7xBhpu4a6azje2ydvW/z1FXlWUJUlDkHrLN71vDo94cc//gnHxw8wswJRKArrmKxWvF5vaGxHryLhYJa6+xjTNXYe2h5ay591fEE88EimJZOiYF233LQ9zoNSOeOyYG9WsKyvuVm3bFpPZ9N2VWnItabIFNoopDEopXFNR9PUaJk44MIoXLCEmJJOwCVeqxwijGLy6lCIRBuUMRXUcBfbg2REMFBRBkMIIZIhz1bYEOVQ/IJL+ZhdwAuHi47V2nJ1dcNm0+KjJC8njKdz9MA+2PVLQkAQtE3CCmNMxSwKUoe79QsfnIninZvQOsemqUE2LFfX1JscsVEU3pCLDIznzNdc1i2n7Zo6esZRsV9W9GpM26fhrxfgVMeiabGhJOQjlMmJfoExno4e1wliJ3B9xBeeXKcEIEEgXUZF30dcDBA6OtvAsAN8dbHioxvJ513GsrYIt6J9/RGx3SQOc0zWmyEE3tr/S/x7D+bcf3BMdB0HI8+XjiJztcZbRd3PqP2IqjA8+/xTnj7/Hq9ePucneN5+5wO6vieZFm0f2ERTFFLinaXvW3rbggKTriwRyIzh3tEhZ69f8G9WV3S2YdPUOO8pyxJn0wK6fYhj8AmyG8yR0tclyLOMe8dHFEXGep2DiFxfXWO0ZL1p6G2Pcw4fAluv92ygpMUYWSyXCYLJ9Y4KeweNQhuFizrdswAiKTXFkNSzvV9jSBv/5F6Ydg+ZjDRdzcsXHxG8oyxLfvMbfwWhHefnL7m5uSIKyXhyHxF7Tl89xbsm+b6LiHcbAhbnIz6k5yHLFFJI+mjRRDSpYHZ1w3f+6CmheYm5PCNcL/l6kfPbf+s/ZDGeoMYl5XTMi7rn5LKn3rTEZsHZ86f85Aff5V//q/8nL559znTviPnRQ/Yfz3DBc/LsGTenr3C2gejTMy3VsBv+RTXR7RHDwDC7I1R68vhtHjx8SNM2uC4Qp/Mk7PMMOZWezlrcAMkG7yB4qqrkeDajzBXdquby7AWj6YyJ2ePy5AQdkx5jsbhhU69ZrxfsVVnK7AwKJzVntFg1KKRRhBBwbvBCyk36GP8uFvBMSnKtmZRFcrTTGTF4plXB8d4MYR11kLQomhiwBIwR3L9/SK7S4MbHIZmbQKkFvQzJGGutKGcjxkbSC03fO3yIyBgxeSCKVJTVkG4fowJcYoNHEHewiogHtm5tsG0npVDJAEcO3SuCGDpwgohLCkCZkkt662g7S9dZQow0bUfbdpRFjvAR4Qc6Hwbh+9TNExPWrjUosxsbie2Ccuf+9M5SNxtCWA3mSBlaFuTBEIVk3dc0wHXf0RDQxhD7wMWLC/JyhhuGZsYYqDS27hC5QuYaFFSmonbrxEYwBiky8lzTm5Y8V0gPInqicCxv1rTqBO89WiU12vb4Zz/f8F0vedZLmvWa9vSK6cMPwfdEn4ZD9I6+7Vi6jN//yfcxn/2cGCXjTPFYXfCNo5bH+2PKfM2y/iGnbcu3//C7WNszKyXCVISoGU/nd0J306IgUWSZQRJo6xWr5SVSRu4fHnJhknGn0opRVXJ+fsYnn50QvKcoS2bTGdPJhIuL63TNvUuRedpgTOrMg0u+2XmmKPKCvb09ptMR0+kEJSU3VzcQ00xmNhkzGo2pqorJZIKPUFQj1nXNs+fPeHXyik29wfmAygxZUQw8/XTYEHAx4gf4YrsvS9nqg/MeAWQ26ByGjhxJjJ5CS5puyfPPf8jV1RnXiwXeN5y8fsri6pS+2SBFapAClig9LiTFciDinWe9aRLXXyalaFnmLNcNs7FFK3C259Mf/D6z/rcRLz9mpCF79Bh+6xvsPTxikucwJBCdn6159slHfOcP/hk/+vbvc/bqNb31TI8f8u7v/E269ZrF1TmvP39Kv76h77skdGFL+f0VsIbdM30nNAFQVY40BTGU9FGQl3OUTP4pSqZg7TDw271z2L6j7xJMU2qDbzpOry745Owlb+eKJ+N3eeutR0ynM7wPbI4fMK5yXr9+ThSe2f4+KtN4ETFZhuvvns+QikRiy4hfgVP4KxVwIcQc+N8Dv56uGv8p8DPgnwDvAJ8Bfy/GeP2rvF6ZGWJQ1HWNyycpeiuSFITBcn5ds9xYnA9UmUZXOUVh2JuO6bsGkWnMkETSrpZMRjlBSDa2Y319QXAdX/nwHbyYcHF5wXqdrDF38uI4eHCTvKwTzJ3MlVOO5fZMh2K5FS5shQpCDlFvihQEkV5fRgthcDU0GVWecf/okCrP6fseqQRVldN3LbY3OCOT2b2PdJsO36fJv5SKIBRCZsMAKSBiSArM+KbYSClBaTQu5DjX4V2b5L0SQtdB5wjDQrPN7/NINhZE1qVMQQHCOYRQ+Ewhc5Hsa7EEK/CdwruQFjupyETK7RTDd0ZM6SfRRtbtGVoWaGkg3hadzclzVq2h7h22XWDPPiXXAjtQNZUUqCxi+g6/OmdtPLKbEGVJK9NW9nF5RKEMRjoubs7puppVvQIimSwodOLHb9b1roDHmBJkjMkHs7KAdxYlBPP5nH1TcqMNHYnRc3x8jx//5Kcs1ms26zVZlrM33yfLMppmAwz2o1KmhUp7QCd/C+ehKplOJ7z91hO00VxfX3FzfY0xhoODfd57732k1ljnaeqal69e4kNkNt+n6VqWi1SgnHd455hMpsynU6aTMetNGsp1jiEWb+vlPXDCY0y5sREQhvn8COf9AB/IQSHrMBGKELAuYp3kJz/5KXvTgmbVILwm1+Mk8VaSwgxpMiF5BzX9muVqyXrTDuk6gtZ3dK5N2ZBiOzCNFN2SH71+yWr+kPGowpRTnl9a9ppriszj6ituTp7z8qMf8fMf/xHL9YaQVUzf+yZ6ekTMcs4//TH1q4/p6xXe9UOzkJajrcHWVroxlPS7dWv3+52T4i8ELy8itEIRMoVPKi1kbNFKkRuduN5CUuQ5uS4YqxlGKpSAeTHC9y3RpdQdpVKouIsR60MKdNCG+WzOs88/x1nP1c2C6XqF3J8g789hsJTeGlzvzvcX7Jj/tONX7cD/d/x/2juzGNuy867/vjXs6Qw13LpjT+4OJiaRgVjACxAhIgHJi0EiUt4CiuQXiMIDEoa8JG8BCSSekIIARSgiQgSUvIBAESQICzttx3E80LbbPfh237mmM+xhTTysfaqqr3uKbN/bfX3+V6WqOnXuOXvtvc631/q+//f/w39LKf1tESmABvgnwO+klH5FRD4NfBr4R+/r1ZLgfeKo68EX7E0sboi03lF0ivViwXLVYo2irGsmTU1dlfRtT9c7mp3MBui7lugjRd1QpZbW9bh1z4nLxcTdeU23KOilw8VA8IEYx9z2uSMsIcjYNj2utt+1g3UsYipNVPpMAQ4hi/GPhU8hURaaS7tzJmWJD44kkbLWWDs6nbge8Y6h7emWLS4pVFEhRYU2dtS/yMd4ttIn5dvz5lTGSAw+f+Bi1psI4nFRkYaBuHJYa0kq5yr1KFXpUvbIzNkZyapzIeKDw0SHShZJivXKsep6Ahtmj8PHHimG3MBgcytxluXNUgeKguAV8cJ5tL5luPMG6+UJKZ7gj++hm3kO32MaQitFqR2pP8WdgKxbYtL0zmGN57UwYzmxWJ1NFgqbmMwbVssV/TCgzIANDtf7bMc2cmkLY7FGYyTifY93WSxrujNjaicon1M4VVXx7PPP87G79zg6PuZLX/wSd27f4eRowcHlfYrCZPMEnXdfzjlEoO+zlgzkQJHV6Cq6rsX7fCzWWoJ33H/wgHXXsVqvWSwWHB0dQ0o0zQQfAyeLBUPfc/jgkPv3H7C7M2e6u8fO7i7L1ZtATo0Gl+dtrjAyBnBGBkh254lJn1dNRCPaEsMwtn1HVAqkEDg+ekBtdnGDJ6Y8v7M7O/iQcEHG1470vWPwLufEdX6MJHjnMXpjW5j7K27U8Prtr/Dmap9yskNx1HDn1suURMQtcKtDutMj+tNjoq7YefpZ2L2GtxM6F1jcepnF7ddwyyOCd98R5M6D+Ft9O8946Rv2WIq8E5zSdNqCEqKC3q3p1312mbKGsiypi4LKO6yxGK0xY2d4BGzXMZBoplOayYQ+eP7fK9/iqWs3qIuGo+MTyrJkNpnldOfpgqQ168MT5gfzbE232UmkUZOFkbz8UL7+7fCeAVxE5sCPA39nnKADMIjIJ4G/Mj7t18hWa+8rgPsQWfeBro80yqHmZbatcp4+JdbLVXanqcqcKhlXFYvFEucDtvIoJblbMyrEFlg7UBpNB6xXLafLJTvTEougkmQTAh8p06ZVeLQoS6MITRBCkDOH6TzWzR0+vWVSqDOjZZXtlFQ6uxmc02aztm/ZVDSFJURPkICyIJL9LqN3hJRw61N8u8JFUGGCUTNsofMWOanRUHfMxsuGxpURY8Q5N/ZPGIRAitkKyg+O0A85xzv6aSqliCoRiKBzEE+5jS4X+4IneY8RDwG6bs26H1CFHc9JIAwDOkbQhsLqXMRjvJEg+UaQAl5FqPNx7jcVxbAgnLxBjMeEtiOZGi2Zj59ZPaP5cuiJyxPglOgc3eoYMfBav8PRpKGqLGVpuHy5Yf/SLkKiawckeRS5fduYrBOjzSgnGhyuX9O2q1ywSxFbFFTNBLVcgc+dmLv7+/zoxz/Oql1z8+YtvvXyq9w6vUMzqZlMGvrBIShChK7rsueq83gfswbPOFeGYaBtW0LI2jcicHx8zJt37nJ8umC1bum7Dudd1gDXY9E65pB7dHjI0eF+lpvd2We2dwlezwG8HxJjKjbrbI/L0E0A37SftX3PxrxESXZ/d4M7e34IIafg1gNto+iHjpQiSjjTPVm0LYPLRXylPIuTUwQorCaoPP80Aj4zrjZsHBFhVlv277/OyfIOx1gimuj7TByIA4pEWVbs7h3w9NMvUB48z9pMuP3giMOXv8bRK1/FHd8Zudxj/eehtMLDRcmH/5aLxOlspf4wotKE8foguYDYuSFr/LRCYTuqkVGkjR4dfKA0Bau+YUdAGU0znzOdz+hE+No3v06IcO3SFZbLFUoU89mcZdvSOUe3WjGcLtEBho2M2Vu2Dg+xZt4F72cF/gJwD/h3IvJngM8DvwBcTSndAkgp3RKRK+9wEj8FfApgZ2cHgN5Flp3DBzioCqqioNcdRfBY5+mXawY1IQbN+rTFrDqaqgACyUXccAdSJKRI1cyRwaGAvfmMsii5d3jC7buH1Bb6LlO9Bh8gBGIySFKkpLN3XhSCV1mfOyYu9vE8fIdPOX9xpjaWPd42zvYxd8SJIaFz/jJlXRZtBB1zMdKUBj9uBVVK6BSwsSexZLVa4taWMuxhrEfrBkklkgwJITPn37raSCkH69AFJtOGqsyrM+8CLsRsYUUeX5A8Pms0gZ6UYt4a+4DE3CKvR91p1/d41xHCQBTBDQNam8yZT0AUggtIzPSyGAba9Zq6mmbTaedx2XMcgOc+8jxPHZ5wqz3k6PSYWEyxk0uUjNtizrfeOX2wAD8grqU/uYPDYAtDLGr2K0uphLqp2ZuVzOqSru0Z+oBYKOcT4tg3oLUipsD9wzscH97nZLHMOhZNQ+cGiqJEpAUCzg3cu3eX+e6c2c6cq9euUjcNN799k1dffZVnn71BIlvftW3ParWi6zpSgrKssusSObDfu3uX2WzGdDKjazvWqxUnx0e8/MprDM5nlUpRVHVFXZcYPfLLEUICYsDagr29A65ce4rpdH52LvsBgk9j2nHT/CjEs/RATh2tlidnc3j8NALnhiEp5Zv/etFxaFakdL5t11owFm7fO6TtepBEYRWrhefa1TnejTFVhBSzFVnfeXZmOc0TE9wNJc89dYWjN2/x+huv8u179wlJ2Lt0hac/+lGuP/dDXHn6BQ6uf4TZ7iVELHfvH9K++U1eufkV+nuvIzEH75FD8LY4X2mnt/wOgjEl4EaFx80ce2gdv9nIiNBMa5SGvu3o2p5Vt2TZQmGyVk1R5sVDjImYPNOdOUVTUZaG+c4OhWq4e/8+s8mMWT1h0jT5Ju89k50ZMvS0/UC7XOW06GjjeHEXcXZ03yMaoQE+Afx8SumzIvIvyemS94WU0q8Cvwpw48aNtDljRjRFmU2MutUSCXlG9ClyjKJTVd4KC1gNbQzMmoLZNKsUut7hg6dKQzZ/0IoUhUaXXGGHNhlQEzAQZIULDptstpiMgsS8+g5xTFNsRLbf5qRdXIFDLgaJsoi2EPQZTdEzlkfF5iYS0bmAurFOE0GhcrrGZVGqUqlRAhcKXfDVb7zC4ctfpZrv8vEf+wTa7iJqQhA75u0fokFpTWFLVv0aUsG0aTg+WnC6WDCsB3anu5SmYnADMQ1oI9RNwRAjIcnoq6hQadzmR4cRnSd+oTlZubM8P8mDaOqmRkRRFpro8yerMAXeBOqiwfcx54Mv9FF4UyLTPcL0GitvMJducCQW3Z0ifoDkUSmgos/EfO/zziMpUr0HuuTS08/z8T/1J3nu2iXi+gHHR/doW2Fx2hODwxrFm99+Ce/DSOczGGupqppZYzH7BatVi4uO3ckuTx/sM68K9NjYEWOka1fcu3ublGA+nfD0U9d57ZUsjHTnzi1AcfXada5du87Vq1e5desW3geGwZ11UfZ9z+np4qyIe3Jywp07d1icHtE0JTM9oShKprMp169f58b1a0xnDU0zpZ7MqJsZCcXVq9e4cvUG9XTGarU+O5cxKkIYbckuLDjemgNOgPuODk3RguTOsNzoljxNI2g7ECXLBBMFW1owA/UUfAwMzjN4TUT49usPKIwwndXUTclitSICqrBnjByUIV76YV56fcDHFcVVw8eff54b15/muY/+eczuFepZw8H+nBduXObGwZw//OZrfO4zn+PrX/oMRze/Dmn0jnzvVPB5Lnxs5DsLighK59pISiNL5+LKdrPSHc+jNobppGE6qQgpMHjParFitVhxvFoST7NJzLSu2ZtPWJusDxRT4M7hCcvhiDfeeIOha/HDmsu7eyyHU24Px3z0Yz/KbtOwXK9Z4OlVRI+NT0AWdBs/35uy83vh/QTwm8DNlNJnx9//EzmA3xGR6+Pq+zpw9328FjC2f1cNJI8bPA/6LHKvJWuXtJS50uyzeLwkxVqE0AVqm7DGZMeMFEkm0KuIWA0+kGyknBq6xcDhYcu6X3GyajMfOxTEoJGYaXspKVJUBBJR3MggeTcPow0LxWJsBbGEYAhxXHXImCsc+360AmLKjvQpIjEykHOUfggs2o4hOoxfY8WTfEA6h6w7RK8puo7oT6ASxNQIxUjSeitXPXf0CVoVmSEjCq0tZWM4Xa0xMuQbiEnYIvPfvXcEW2GUQRtwXW446t2Cfj0QgaaesrO7x7Jb0Q9rEjn3mc1wC9qVI6aBohCaiaUqKjQG9MjBviDAtFpns9f9/X1SWZB29jkcYCgbJOV8rEqO5HvEralSQvoW17akEJhOpuzUhomFQjytH2jKkrIo6a3Fp4gV4cbelLKwlFVNWVQUtqSsaqqmAqP52Mc+Dmhmu/u88MM/yvz+En3v8OzqWiVICnRtS1koLl3a4crVA954YyD4rCbYdT2Hh4copUaZg3hWNA0hEEJgb2+Psqw5OjrKeW5yI09dVUynU3b3drly5QpPXX+K/Ut7VE1FUdUUZZZ7aCZzjK0pqmqk2p+fS6UixuTVm7G55hLjW3sEsrb3yFqSvLMRpYhEQnJ0XWAYAj5E0BG/SEymFXVtsTYbiz9YrBlCLtJ6B6u2xxhh0pRcvbxPXRtIHquzHLTzCTs2HNVlwU//1U9wePdZ7j44AiXM5hOuXL5EM9mhUJGDnYrL84aduubVO0f8xm/9V/7gM7/Hg5vfwq+P2awAHk6bnM39CyvvlDarr00Az64WYexQVVpDUtlE4SKUjCauuTEpxNyULYm8wKoMpiiYzKYMQ0/fdrSrNWvfEU57tHdMtaXWmi4mki7Yu3aFuqnpiRy2K+bPXuf6lSnV1UsUZYmOM5QtztOzSObwj3Mor8Y3lNB3x3sG8JTSbRH5toj8cErpJeAngK+OXz8L/Mr4/bfe891G9M6z6gaUJHrvs57DKCxFAh8zPS6G3CARo0InwWqLMZm5gdYoYwiqJ+pIZQuCikSd0JXG+MT9wyOGNNAOHu+zr50khcRsbUbKgkCRTWPChvf9ljPw0O8CaLQuiLoAbSG4vJpPQhpfO5E98xIR5T34sett5OT6dYsfWnxyzAwka5EI8+kuylQUdZN3CzjSsARAmVycSherrKJQYlAYhjZQNwqrC6qqRlCcuFN8yFK5WfdFCB6GPpJiR13WGGVzTndwFJUhDBHvEr3LHNud+Q79YPG+J6aIEoNRJUlbXFCQAs5FEA+pJTlNivkmskGlIs9dv4TMS+4fH3LcD6j1kqEfMh80hpFS2KPDQBHWiPakWqOVZXdaIN0p926+wuqOpludMJs2XL18mXnTUO3uMJtUVCZRlwVFUVMUVW7IKCpUVSG2AGUQZbFlzezSZYrTN85awb13nBzdIzpPtzqlbxdYnbhx7YAUHe2qI6a80OjbVU5r+UCM+aamJItRLZcr+sGhdMFiseD09ITZbMrOrGJ/b4+Dg0vs7e4xm8/Z2dllMmlQhcHYiqJsqOo5VT3NJgEJ8D535p1d8oCSrCtu7ZgWiZytttOYIkGPzT0jTSMRxhtOoO0Hun6Tf816PzsmM5K6ruP4ZMUQHN7nGpEWRVMUlHViUtUoo+iGAe+y/V4MUJcm5/LJOfK//KefZbW+zOFph4+CLSyzOnPbw8gwK7TidLXmdz/7Il/6wosc336dsD6B6N82af2OeeE0Bm+Vu6kTMNZXYcOYIi9e3vISImPgGatim1XwaHPGeOOzdYEqNLYqqCYl3udaUVlPscrk96xr6smUp55/jmlVMW8mJBKTy/vUK0UqDV4DOjccZju8fHwb7ZqNmcf72XXA+2eh/Dzw6yMD5VvA3x3H+x9F5OeA14Gffp+vxeAcq65Dj8WvEMeW9jEv6EM6UxKESAiCjRpdZbphDrO5+aZPHZUSlAoMeAYVsFqjmoQ77Qkq31V9iBCy+L2knC9MY1EkjnnB7wzd5zmz862cICmnSURb0shGkY2ULZqEJUUDyUDw0A7EviX6gSSCMZphtUS5LmuXzzMFKSnDfHcf6z1SlHRRKFRA+fXIdFEYpQmoM256ihC8oClwbaQwCUFRWJvZD4XG+/HGZCXf9IIQvaL3Q87bl1lqtu3X1LMKWxcEUTgfUYOnrEoKW+YWX++JPmuBKC2oZIgxC2Ml7VHakIJCYTEXAvhObXlhPmc37nJyUnN0esozE4NbrwhDT/AO3weEEp0seA1hiqREZQsmTUNdjLlisqPPzmzG/u4Os0nDbDphZz6lKS3W2Kwtb3LjhDYFSWfRfGVtFqXSFtNMUcaeBYUQPMvTQ7TShH4FvqMpNdev7GEkcHKyoO8HnA94HwkpgRb8WZcfBB9YLFbcvn2Xup5w/8EDhr7lyuU9nr52wMHly1y6dMBkMs3Syc2EoqoIJJQuKIqKpplgy4Y09jDEEM+Kg3leBpCAiM75b8kuJdnadVxxjtm2TfOxjKtR5z2kSEoRkWy+ISanY3IREtbrjuOjNXVTkHzuVG5qi9YFIfVEIm3fEYLD++yq5fpAUVY5y0fW/H/q8hTvK25cTmRpeMGq7G/bDpnNs257Xrl1m8987nPce+NVwuokF3E2lN2Hq3tvi5zE3uw+NxnRSF5Vq00wftvXHF9h/D9FuwZRRKvxoseejKwAqbTCWg3VmAN3gUk9pRgL99FodFWwd/kS07KmtiVDcNS7cyoGlBk7uEVhTTGme87ZJrnZUH3Hsb0b3lcATyl9Efhzb/Onn/hjvduIMDo7A4Rs/YGQxYu0yjniFHLuLcWxpp4iVhrWg8tFMx0Yup5lPOFGOSWENQOOTiWCFmhgMikZsPQ+EPya5MEklbdbI7MlEs9oRt9Z+T13r7/4qIqKKBrEgJisyRxHXrlYoEAoUarEtwN+0RLaFYTMnkmS6BbHFDrBpMLaElNatPY0WlAx4pRhlQRLwoQO8eT3PHM9z/A+0rUBRQle4/osrqSUgpSyoJTKYvTGWkxREmNAU+JDmxtoElk+tluTKKmaBidCt3RE51icnowOKmQ3epdYd0sKq0cNsIgqEqrMbJKoLDoVKDkP4Jf3Zly6epmnrCL0O6QUGHxP6hYM3Zq+a+m6FqMyE6gfcpAUhKooqWwx1nsyBzvnKmt2JhOqsqCqSqq6Rpkir56SGhuusmyCj4Ek6uwmqyUziRhXbAApBoZujVEKiY6mVLBTY9Uu00pzNC1YLle03UDfe3qXJR66PguOESPeeRanS1566ZsUpSWGnsLAc89c54Vnn2Znb596OkMbA0mYzXcwZUHnHAmNNpaqnqBtTRSftcXjBRspcvExjZSzGNM4HTbiWtlFHZV1O3zKaoV6E8CHQGWhLnMNQ1uDtULb9iyXLcF7hiF3LjdlwTp2GKMo64Ky0Ny537I4PcWWFqNGsYmYre6aqc4SASOEhJHEvLrQ8CZw2HZZOoPEneMlL37l63zxDz5PWB2R/DB2Sp6rHZ5x3bn4/TuhjMnaMBst7jH4GqPyTSuOXMu3BPB0Vv+SELF376CahmE+wxXmjFk2koRgFMwDKCpDXVSIjwzDQBc81g/YssxNUFqIaKrphOnQjvIbGi0aayqsLfJNdiQiuOBHFtyocf8+VuGPR8wqpmzGOvb9AyMHMp6bpSaIPq/OQYgCXYKj+8dUVYEYT1Arou3xvsariDaRQhL4HlJib17T+xlDJyzTCu8TGpOr/GnDKj0/S0brMb/7MC5U8lMcd1cRRW7/TkPArwPG1jmm63yhJEB7ssSfLNDRU1qL956TkwcE36EnJSjwKWEBxBBSZpuINpSmJIYVWlJuEPI9IiXKVGfHHTwMnVDqgqeuP8vx6f28HUtw7+5duranmTeYwmQzWhQxRIyyRO/ySlkMPg74fmBxsmY6qUhJ8N4hKebGJNXndFaIJAwxgAvgQsClATGB/SszTo46pmXD0EfWvj87gwf7O9grB6S6wPiO0iiSteS4l1Xe8oct7wbC0BF93kYnl/nKAEpblM0rawUQHcFnJcCYhFRMstMzWb1xo+BYCpA8YbPTEnBuQAd/ofqfzT+s0aiywCihtIZJXWW382nNum1ZrVoWy3X+Wg1YqzM/esgdt4nAGzdfo6lLrl7d55mnbvAnfugFLl+6hKkqqskUY0pSEnRZ5Y5egRDyrjOvwhIpBZTKKoYbhkueg7lgH8j+pTrm1f8weAaXPzPiewbvCTEHHyUR5z2nJydMr+/QNIaUIISYUzg+sW4HUtwcy4BIYDqfMAyOtu1YLROLU4dVuXAvSqisyQulSjA6ns1LGamIIRWZ6zx2hzqfOG4Tt45X3HzzLl/+2tf53f/9f+hPTkl9bg4iSdYk2gjJXQjecqHod0aaHBt7xBoQTXTDaDEXzmoHMQRiyPNM9PlNYLMATgJF29K+/grFlStQavSkIkVBk8ciY3oojHOxLAq00pky6z22KIkq6zf1bkBU3vFKYZnPZmOKJBJ8ypRZnQugIcYL5yjL8m5SKu+FxxTAyTlpRnf4UZIz85QTWguQJ5/3mwKF4vhkzdr11L1jOjHMZg1V1TCrDrBEcD0xegIwJKEyCqOEvUlBchMmOCyGJJE45sU2MVwky59qckMMvLV4ImexPpJwxNRBHFDBERYdb770JrPda9SXrlFOs1Kh6juWb95G2iV1aWGiOD094f79u1y6soMqDU5gPfSIiqgiaw+fLk45Xa2ZzHfZqxWhzIW13ESzBtNcOC6FYOm7wL17R9y+/SaTaUFZWaJLBBfxnUeSoIq8Kh/aHIy71YBOGp2yy/jQ9ty/J7jeYlUJ5BtqWWVJ1pjAEenbgb2dK3Qrh7WJ3vesuzW3by4pGgVDh197vL0wvWIkr3UVBo32iUDeThL1mSxrCmasPylsEbNSZBHxfd5Wi9JZ/Gsjras1RkU23NkwNhUFH7N/aszniHH7qrVCdNawiUYh+jwnGlNicD4XM43NDJ+ioKqyPvfuzg69G+j6jnXbsWp71uuOYQisVtlmresHBheo6ornn32G5597hmeeucHB5X12dg9yz0JV5wYo8k7BGkOdEt4HQkoMQ4toT9e5zI9XKluHjdBao8cu3RizoXFhsz/mEB296/HrnpNFpjgWVrA6155u3zpFkmIyLdEmB5Su9YQktF0LKTMxZvOGvg0k1gRGtcMgaAxHp2vKUo8lw8S8L5nvT8h+KOdBRwm4Mc8bUqJ1gZNV4BvfPuZ/fe73efEPvsBrr36L9fER0q+J445rnNmb0b6PiDKqYroOrcss/ia5U1aA6ML5/OLikm38ZRTOrJcdB80UO5nhdMV6EHof8cX5uHJNLn8uXPCgLGVZUliLGc0ZdCQv4HRWv1yv19kH09p8nUb6sncu78QYNXbSpuC8+ffeeEwBPJ61+KaUcMmPBsP5sI0VROWuSR8gBUGi4yStCSRCSBitmdaWIlkWhz5T5HRAmTTqkAilTXi/pigMly8f0KglKSRIMr5f/oqbXtyHtiz573GsCucniCRC6HBuSRhWJO8wIULvSe1AWLcMZoW2IAvHcHiMiQM+VXRKc3pygi01zaTGFHkF2vYDyfcESazcwGKx5HSxwIqmaPYwSo/3Dk9KfV52j2iahqvXrtIvPYWq+chzz1GWBltoLl+6wvHxMXVTESXrndRNjXeeYRhYr1u0UVSVxhYwaSpO1w7vAsiA1Yr1akWkRFtBG0Fpoags+/t73F7eAZWwSqGScHracXDtOhIDWMFeMCHg5l1kiFCXWXJXhGgMYvI0lZhIMY1mrmTaopC7UVNEhTiKMuW0Rxo7BVPw47Y9fzQ3Yqo6gUojj19pktLZ8EPLKB0MUljkzj0YC4QheE5P1zRlQVXaXJhUCq3NmV53UZdMUsNOCPS9y9Z3LtK1AyEmtM4pNWsLrlzaZ3d3zmTajCqIirps0LbMxwTZR7TIFnvDMND2Pav1mqKs6YeNK7nQrs9phGWjIOUGlW7IK/WYNC4FhuhwfqBtB9brAa3AiEG0xijNwf4O83mTA16IObURI23b44ZsbJKSz0JfVlEoGEKmyCqlmM9rggpMm5IUEjEE6klN5yF0nlJHyjGqaJU/P+s+cLweuH/acvPuEf/3C1/mxRdf5ObNb7M6OSL2LbhhTHm8d977TBNoI4VxlvQOxNRnydsNpTAxpgnzHNmkVc4+4zGSfCBF6GyJvXYDV5cMKtElTyp0Zgq7eJbVyaoWgveBaBJGKZII7dBjtKUc9d0RGPqcDfAx5KCOGs9bRLQGPc5FlWWCvc8OYnGMj++FxxLAd3YPxjxlOisi5mA5VoN1dotm7JJMMXOo9XiilFbUpaFpCkypCQFS8iSdiFpGqRPBKUPUeRujlUGY4WNmYmQuScJKRBNAQCdBkYXzU0qcHN+FhzLgIhDjgPdrUhjAC17X1Fcuo6sGSgiSO+DEeOyVEo2FwhKsptQztJ0iOxOC0UTJFnM9uTvSGUtqphTKEquGJZYhZFZLIOEJsF6c5e0r23Awv4ErAgp9Zqqqxvb+aX2QK94pnK0oY8oFKzfLWs9aC9rApLjGbjswDC6zVrSin6yxpUFpOXNNT0mzOz0gXclFq5Ai/TCwXLdcvbRPDJ44DRQXdgpfWxxjw4DYzCkXkTGgXlgVPUSbOnc9z0snUefmsWcKEnEM3medHhcKz5uXk/wJOfOSZLykWiMPjln5fEOMSei9IYliCIwO7BvJYcH70S0bk28VukBpwahEpfO22pjsmpQ7XkuWbaAdVqBajk4DRX2cP7jjnBLRFCYb4/rgGZzDOY+xlsGNCcaU0z3n2AemQEKrkBcZKArtofSU2lFpT13kPorC5uJbShDnimltxp1vlp41WkhqoK7Hcz7OncIqrEARRwmKUQOoqgeKwowem5GitLiUG8Kszg1Hzjk+/4UvMrhI5yLr3nO6Hjg8XcLqLjdmit1nD/Bunhck3VVScGfX+hxvF8Q2N+xzvvfm5i1c7JVIZ5/l89dJ4zzKz6l7l/s0EhgfETuBBKpPlH4AbYgyymRcoC2K8hgBWSeiqDwPvSdqTTAG0eMccY7S9ZgY0SozTFLkbBWvjT6btSklvI/jdI4U/bsrEQLI+4ny3yvcuHEjfepTn3pk77fFFlts8STgl3/5lz+fUvoOIskfj7OyxRZbbLHFBwaPdAUuIveAFXD/kb3pBwMHbMf8g4DtmH8w8DjG/FxK6fLDDz7SAA4gIi++3VbgScZ2zD8Y2I75BwMfpDFvUyhbbLHFFh9SbAP4FltsscWHFI8jgP/qY3jPx43tmH8wsB3zDwY+MGN+5DnwLbbYYostvjfYplC22GKLLT6keGQBXET+hoi8JCLfHE2Qn0iIyKsi8kci8kUReXF8bF9E/oeIfGP8vve4j/O7gYj8WxG5KyJfvvDYO45RRP7xeN1fEpG//niO+rvDO4z5l0TkjfFaf1FEfurC356EMT8jIv9TRL4mIl8RkV8YH39ir/W7jPmDea0visB/v77IijQvk/01C+APgR95FO/9qL+AV4GDhx77Z8Cnx58/DfzTx32c3+UYf5xss/fl9xoj8CPj9S6B58d5oB/3GL5HY/4l4B++zXOflDFfBz4x/jwDvj6O7Ym91u8y5g/ktX5UK/C/AHwzpfStlF3tfwP45CN67w8CPgn82vjzrwF/8/EdynePlNLvAYcPPfxOY/wk8BsppT6l9ArwTfJ8+FDhHcb8TnhSxnwrpfSF8ecF8DXgKZ7ga/0uY34nPNYxP6oA/hTw7Qu/3+TdT8qHGQn47yLyeRHZCL9cTSndgjxBgCuP7ei+f3inMT7p1/7vi8iXxhTLJpXwxI1ZRD4C/BjwWX5ArvVDY4YP4LV+VAH87aRtn1T6y19MKX0C+Eng74nIjz/uA3rMeJKv/b8Cfgj4s8At4J+Pjz9RYxaRKfCbwD9IKZ2+21Pf5rEP5bjfZswfyGv9qAL4TeCZC78/Dbz5iN77kSKl9Ob4/S7wX8jbqTsich1g/H738R3h9w3vNMYn9tqnlO6klELK2r7/mvOt8xMzZskegb8J/HpK6T+PDz/R1/rtxvxBvdaPKoD/PvBREXl+NEb+GeC3H9F7PzKIyEREZpufgb8GfJk81p8dn/azwG89niP8vuKdxvjbwM+ISCkizwMfBT73GI7ve45NEBvxt8jXGp6QMUsWwP43wNdSSv/iwp+e2Gv9TmP+wF7rR1jd/SlyRfdl4Bcfd7X5+zTGF8gV6T8EvrIZJ3AJ+B3gG+P3/cd9rN/lOP8DeRvpyCuQn3u3MQK/OF73l4CffNzH/z0c878H/gj4EvmDfP0JG/NfIqcDvgR8cfz6qSf5Wr/LmD+Q13rbibnFFlts8SHFthNziy222OJDim0A32KLLbb4kGIbwLfYYostPqTYBvAttthiiw8ptgF8iy222OJDim0A32KLLbb4kGIbwLfYYostPqTYBvAttthiiw8p/j/duPnRZQ8wyAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 3, 32, 32])\n",
      "torch.Size([16, 6, 28, 28])\n",
      "torch.Size([16, 6, 14, 14])\n",
      "torch.Size([16, 16, 10, 10])\n",
      "torch.Size([16, 16, 5, 5])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# device config\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# hyper parameters\n",
    "num_epochs = 10\n",
    "batch_size = 16\n",
    "learning_rate = 0.001\n",
    "\n",
    "# the dataset has a PILImage images of range [0,1]\n",
    "# we need to tranform them to Tensors of normalized range [-1, 1]\n",
    "\n",
    "transformations = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
    "])\n",
    "\n",
    "# import the dataset\n",
    "train_dataset = torchvision.datasets.CIFAR10(root = \"./data/CIFAR10/\", train=True, transform=transformations, download=True)\n",
    "test_dataset = torchvision.datasets.CIFAR10(root = \"./data/CIFAR10/\", train=False, transform=transformations)\n",
    "\n",
    "# data loaders\n",
    "train_loader = torch.utils.data.DataLoader(dataset = train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset = test_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "classes = [\"plane\", \"car\", \"bird\", \"cat\", \"deer\", \"dog\", \"frog\", \"horse\", \"ship\", \"truck\"]\n",
    "\n",
    "def imshow(img):\n",
    "    img = img / 2 + 0.5 # unonrmalize\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1,2,0)))\n",
    "    plt.show()\n",
    "\n",
    "#visualize some random training images\n",
    "dataiter =iter(train_loader)\n",
    "images, labels = dataiter.next()\n",
    "imshow(torchvision.utils.make_grid(images))\n",
    "\n",
    "conv1 = nn.Conv2d(3, 6, 5)\n",
    "pool = nn.MaxPool2d(2, 2)\n",
    "conv2 = nn.Conv2d(6, 16, 5)\n",
    "print(images.shape)\n",
    "x = conv1(images)\n",
    "print(x.shape)\n",
    "x = pool(x)\n",
    "print(x.shape)\n",
    "x = conv2(x)\n",
    "print(x.shape)\n",
    "x = pool(x)\n",
    "print(x.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define model, loss, optimizer and the training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1 / 10, step 100/3125, loss = 2.3158\n",
      "epoch 1 / 10, step 200/3125, loss = 2.2842\n",
      "epoch 1 / 10, step 300/3125, loss = 2.2915\n",
      "epoch 1 / 10, step 400/3125, loss = 2.2865\n",
      "epoch 1 / 10, step 500/3125, loss = 2.2965\n",
      "epoch 1 / 10, step 600/3125, loss = 2.3010\n",
      "epoch 1 / 10, step 700/3125, loss = 2.3057\n",
      "epoch 1 / 10, step 800/3125, loss = 2.3093\n",
      "epoch 1 / 10, step 900/3125, loss = 2.2675\n",
      "epoch 1 / 10, step 1000/3125, loss = 2.2755\n",
      "epoch 1 / 10, step 1100/3125, loss = 2.2818\n",
      "epoch 1 / 10, step 1200/3125, loss = 2.2882\n",
      "epoch 1 / 10, step 1300/3125, loss = 2.3055\n",
      "epoch 1 / 10, step 1400/3125, loss = 2.3112\n",
      "epoch 1 / 10, step 1500/3125, loss = 2.2965\n",
      "epoch 1 / 10, step 1600/3125, loss = 2.2920\n",
      "epoch 1 / 10, step 1700/3125, loss = 2.2688\n",
      "epoch 1 / 10, step 1800/3125, loss = 2.2939\n",
      "epoch 1 / 10, step 1900/3125, loss = 2.2796\n",
      "epoch 1 / 10, step 2000/3125, loss = 2.2962\n",
      "epoch 1 / 10, step 2100/3125, loss = 2.2821\n",
      "epoch 1 / 10, step 2200/3125, loss = 2.3022\n",
      "epoch 1 / 10, step 2300/3125, loss = 2.2965\n",
      "epoch 1 / 10, step 2400/3125, loss = 2.2840\n",
      "epoch 1 / 10, step 2500/3125, loss = 2.2833\n",
      "epoch 1 / 10, step 2600/3125, loss = 2.3222\n",
      "epoch 1 / 10, step 2700/3125, loss = 2.3066\n",
      "epoch 1 / 10, step 2800/3125, loss = 2.3114\n",
      "epoch 1 / 10, step 2900/3125, loss = 2.2903\n",
      "epoch 1 / 10, step 3000/3125, loss = 2.2933\n",
      "epoch 1 / 10, step 3100/3125, loss = 2.3019\n",
      "epoch 2 / 10, step 100/3125, loss = 2.2860\n",
      "epoch 2 / 10, step 200/3125, loss = 2.2738\n",
      "epoch 2 / 10, step 300/3125, loss = 2.2949\n",
      "epoch 2 / 10, step 400/3125, loss = 2.2797\n",
      "epoch 2 / 10, step 500/3125, loss = 2.2938\n",
      "epoch 2 / 10, step 600/3125, loss = 2.3120\n",
      "epoch 2 / 10, step 700/3125, loss = 2.3031\n",
      "epoch 2 / 10, step 800/3125, loss = 2.3133\n",
      "epoch 2 / 10, step 900/3125, loss = 2.3042\n",
      "epoch 2 / 10, step 1000/3125, loss = 2.2961\n",
      "epoch 2 / 10, step 1100/3125, loss = 2.3259\n",
      "epoch 2 / 10, step 1200/3125, loss = 2.3021\n",
      "epoch 2 / 10, step 1300/3125, loss = 2.2848\n",
      "epoch 2 / 10, step 1400/3125, loss = 2.2884\n",
      "epoch 2 / 10, step 1500/3125, loss = 2.2834\n",
      "epoch 2 / 10, step 1600/3125, loss = 2.2937\n",
      "epoch 2 / 10, step 1700/3125, loss = 2.2795\n",
      "epoch 2 / 10, step 1800/3125, loss = 2.2882\n",
      "epoch 2 / 10, step 1900/3125, loss = 2.3089\n",
      "epoch 2 / 10, step 2000/3125, loss = 2.2861\n",
      "epoch 2 / 10, step 2100/3125, loss = 2.2858\n",
      "epoch 2 / 10, step 2200/3125, loss = 2.2943\n",
      "epoch 2 / 10, step 2300/3125, loss = 2.2919\n",
      "epoch 2 / 10, step 2400/3125, loss = 2.2798\n",
      "epoch 2 / 10, step 2500/3125, loss = 2.2769\n",
      "epoch 2 / 10, step 2600/3125, loss = 2.2931\n",
      "epoch 2 / 10, step 2700/3125, loss = 2.2858\n",
      "epoch 2 / 10, step 2800/3125, loss = 2.2967\n",
      "epoch 2 / 10, step 2900/3125, loss = 2.2915\n",
      "epoch 2 / 10, step 3000/3125, loss = 2.2791\n",
      "epoch 2 / 10, step 3100/3125, loss = 2.2837\n",
      "epoch 3 / 10, step 100/3125, loss = 2.2851\n",
      "epoch 3 / 10, step 200/3125, loss = 2.2782\n",
      "epoch 3 / 10, step 300/3125, loss = 2.3025\n",
      "epoch 3 / 10, step 400/3125, loss = 2.3012\n",
      "epoch 3 / 10, step 500/3125, loss = 2.3097\n",
      "epoch 3 / 10, step 600/3125, loss = 2.2860\n",
      "epoch 3 / 10, step 700/3125, loss = 2.2950\n",
      "epoch 3 / 10, step 800/3125, loss = 2.2785\n",
      "epoch 3 / 10, step 900/3125, loss = 2.2800\n",
      "epoch 3 / 10, step 1000/3125, loss = 2.2792\n",
      "epoch 3 / 10, step 1100/3125, loss = 2.2661\n",
      "epoch 3 / 10, step 1200/3125, loss = 2.2909\n",
      "epoch 3 / 10, step 1300/3125, loss = 2.2798\n",
      "epoch 3 / 10, step 1400/3125, loss = 2.2861\n",
      "epoch 3 / 10, step 1500/3125, loss = 2.2733\n",
      "epoch 3 / 10, step 1600/3125, loss = 2.2475\n",
      "epoch 3 / 10, step 1700/3125, loss = 2.2473\n",
      "epoch 3 / 10, step 1800/3125, loss = 2.2399\n",
      "epoch 3 / 10, step 1900/3125, loss = 2.2503\n",
      "epoch 3 / 10, step 2000/3125, loss = 2.2870\n",
      "epoch 3 / 10, step 2100/3125, loss = 2.2815\n",
      "epoch 3 / 10, step 2200/3125, loss = 2.2646\n",
      "epoch 3 / 10, step 2300/3125, loss = 2.2343\n",
      "epoch 3 / 10, step 2400/3125, loss = 2.2425\n",
      "epoch 3 / 10, step 2500/3125, loss = 2.2339\n",
      "epoch 3 / 10, step 2600/3125, loss = 2.2882\n",
      "epoch 3 / 10, step 2700/3125, loss = 2.2160\n",
      "epoch 3 / 10, step 2800/3125, loss = 2.2226\n",
      "epoch 3 / 10, step 2900/3125, loss = 2.3099\n",
      "epoch 3 / 10, step 3000/3125, loss = 2.1772\n",
      "epoch 3 / 10, step 3100/3125, loss = 2.2261\n",
      "epoch 4 / 10, step 100/3125, loss = 2.1988\n",
      "epoch 4 / 10, step 200/3125, loss = 2.1754\n",
      "epoch 4 / 10, step 300/3125, loss = 2.2969\n",
      "epoch 4 / 10, step 400/3125, loss = 2.0878\n",
      "epoch 4 / 10, step 500/3125, loss = 2.1875\n",
      "epoch 4 / 10, step 600/3125, loss = 2.1081\n",
      "epoch 4 / 10, step 700/3125, loss = 2.0910\n",
      "epoch 4 / 10, step 800/3125, loss = 2.1228\n",
      "epoch 4 / 10, step 900/3125, loss = 2.2043\n",
      "epoch 4 / 10, step 1000/3125, loss = 2.3820\n",
      "epoch 4 / 10, step 1100/3125, loss = 2.0975\n",
      "epoch 4 / 10, step 1200/3125, loss = 2.1253\n",
      "epoch 4 / 10, step 1300/3125, loss = 1.9923\n",
      "epoch 4 / 10, step 1400/3125, loss = 2.1647\n",
      "epoch 4 / 10, step 1500/3125, loss = 2.1807\n",
      "epoch 4 / 10, step 1600/3125, loss = 2.1057\n",
      "epoch 4 / 10, step 1700/3125, loss = 1.9973\n",
      "epoch 4 / 10, step 1800/3125, loss = 1.8948\n",
      "epoch 4 / 10, step 1900/3125, loss = 2.2042\n",
      "epoch 4 / 10, step 2000/3125, loss = 1.9247\n",
      "epoch 4 / 10, step 2100/3125, loss = 2.0253\n",
      "epoch 4 / 10, step 2200/3125, loss = 2.0061\n",
      "epoch 4 / 10, step 2300/3125, loss = 1.9973\n",
      "epoch 4 / 10, step 2400/3125, loss = 1.9608\n",
      "epoch 4 / 10, step 2500/3125, loss = 1.8685\n",
      "epoch 4 / 10, step 2600/3125, loss = 2.4160\n",
      "epoch 4 / 10, step 2700/3125, loss = 2.0914\n",
      "epoch 4 / 10, step 2800/3125, loss = 1.9191\n",
      "epoch 4 / 10, step 2900/3125, loss = 2.0731\n",
      "epoch 4 / 10, step 3000/3125, loss = 1.9519\n",
      "epoch 4 / 10, step 3100/3125, loss = 1.7674\n",
      "epoch 5 / 10, step 100/3125, loss = 2.1528\n",
      "epoch 5 / 10, step 200/3125, loss = 2.0946\n",
      "epoch 5 / 10, step 300/3125, loss = 2.1551\n",
      "epoch 5 / 10, step 400/3125, loss = 2.0475\n",
      "epoch 5 / 10, step 500/3125, loss = 1.9091\n",
      "epoch 5 / 10, step 600/3125, loss = 1.8979\n",
      "epoch 5 / 10, step 700/3125, loss = 1.7590\n",
      "epoch 5 / 10, step 800/3125, loss = 1.7877\n",
      "epoch 5 / 10, step 900/3125, loss = 2.2060\n",
      "epoch 5 / 10, step 1000/3125, loss = 2.0927\n",
      "epoch 5 / 10, step 1100/3125, loss = 2.4427\n",
      "epoch 5 / 10, step 1200/3125, loss = 1.9813\n",
      "epoch 5 / 10, step 1300/3125, loss = 1.8277\n",
      "epoch 5 / 10, step 1400/3125, loss = 1.8754\n",
      "epoch 5 / 10, step 1500/3125, loss = 1.9908\n",
      "epoch 5 / 10, step 1600/3125, loss = 1.9311\n",
      "epoch 5 / 10, step 1700/3125, loss = 1.6906\n",
      "epoch 5 / 10, step 1800/3125, loss = 2.0011\n",
      "epoch 5 / 10, step 1900/3125, loss = 2.1404\n",
      "epoch 5 / 10, step 2000/3125, loss = 2.0410\n",
      "epoch 5 / 10, step 2100/3125, loss = 1.8935\n",
      "epoch 5 / 10, step 2200/3125, loss = 1.6936\n",
      "epoch 5 / 10, step 2300/3125, loss = 1.7815\n",
      "epoch 5 / 10, step 2400/3125, loss = 1.9442\n",
      "epoch 5 / 10, step 2500/3125, loss = 2.1824\n",
      "epoch 5 / 10, step 2600/3125, loss = 2.1881\n",
      "epoch 5 / 10, step 2700/3125, loss = 1.8259\n",
      "epoch 5 / 10, step 2800/3125, loss = 1.9596\n",
      "epoch 5 / 10, step 2900/3125, loss = 1.6468\n",
      "epoch 5 / 10, step 3000/3125, loss = 1.9789\n",
      "epoch 5 / 10, step 3100/3125, loss = 1.7486\n",
      "epoch 6 / 10, step 100/3125, loss = 1.4991\n",
      "epoch 6 / 10, step 200/3125, loss = 1.9530\n",
      "epoch 6 / 10, step 300/3125, loss = 1.6701\n",
      "epoch 6 / 10, step 400/3125, loss = 1.7302\n",
      "epoch 6 / 10, step 500/3125, loss = 1.9255\n",
      "epoch 6 / 10, step 600/3125, loss = 1.6840\n",
      "epoch 6 / 10, step 700/3125, loss = 1.7783\n",
      "epoch 6 / 10, step 800/3125, loss = 2.1021\n",
      "epoch 6 / 10, step 900/3125, loss = 1.9318\n",
      "epoch 6 / 10, step 1000/3125, loss = 2.0043\n",
      "epoch 6 / 10, step 1100/3125, loss = 1.8307\n",
      "epoch 6 / 10, step 1200/3125, loss = 1.7596\n",
      "epoch 6 / 10, step 1300/3125, loss = 1.8823\n",
      "epoch 6 / 10, step 1400/3125, loss = 1.8190\n",
      "epoch 6 / 10, step 1500/3125, loss = 1.8880\n",
      "epoch 6 / 10, step 1600/3125, loss = 1.7985\n",
      "epoch 6 / 10, step 1700/3125, loss = 2.0021\n",
      "epoch 6 / 10, step 1800/3125, loss = 1.7717\n",
      "epoch 6 / 10, step 1900/3125, loss = 1.8977\n",
      "epoch 6 / 10, step 2000/3125, loss = 1.9817\n",
      "epoch 6 / 10, step 2100/3125, loss = 1.5241\n",
      "epoch 6 / 10, step 2200/3125, loss = 1.9491\n",
      "epoch 6 / 10, step 2300/3125, loss = 1.7041\n",
      "epoch 6 / 10, step 2400/3125, loss = 1.5786\n",
      "epoch 6 / 10, step 2500/3125, loss = 2.1650\n",
      "epoch 6 / 10, step 2600/3125, loss = 1.7598\n",
      "epoch 6 / 10, step 2700/3125, loss = 1.6514\n",
      "epoch 6 / 10, step 2800/3125, loss = 1.9058\n",
      "epoch 6 / 10, step 2900/3125, loss = 1.6725\n",
      "epoch 6 / 10, step 3000/3125, loss = 1.7127\n",
      "epoch 6 / 10, step 3100/3125, loss = 1.6899\n",
      "epoch 7 / 10, step 100/3125, loss = 1.6191\n",
      "epoch 7 / 10, step 200/3125, loss = 1.9790\n",
      "epoch 7 / 10, step 300/3125, loss = 1.6074\n",
      "epoch 7 / 10, step 400/3125, loss = 1.7903\n",
      "epoch 7 / 10, step 500/3125, loss = 1.8764\n",
      "epoch 7 / 10, step 600/3125, loss = 1.6672\n",
      "epoch 7 / 10, step 700/3125, loss = 1.8982\n",
      "epoch 7 / 10, step 800/3125, loss = 1.8129\n",
      "epoch 7 / 10, step 900/3125, loss = 2.0319\n",
      "epoch 7 / 10, step 1000/3125, loss = 1.8389\n",
      "epoch 7 / 10, step 1100/3125, loss = 1.8063\n",
      "epoch 7 / 10, step 1200/3125, loss = 1.7209\n",
      "epoch 7 / 10, step 1300/3125, loss = 1.5396\n",
      "epoch 7 / 10, step 1400/3125, loss = 1.6802\n",
      "epoch 7 / 10, step 1500/3125, loss = 1.5325\n",
      "epoch 7 / 10, step 1600/3125, loss = 1.6777\n",
      "epoch 7 / 10, step 1700/3125, loss = 1.8201\n",
      "epoch 7 / 10, step 1800/3125, loss = 1.7860\n",
      "epoch 7 / 10, step 1900/3125, loss = 1.6815\n",
      "epoch 7 / 10, step 2000/3125, loss = 1.8282\n",
      "epoch 7 / 10, step 2100/3125, loss = 1.6708\n",
      "epoch 7 / 10, step 2200/3125, loss = 1.6316\n",
      "epoch 7 / 10, step 2300/3125, loss = 2.0533\n",
      "epoch 7 / 10, step 2400/3125, loss = 1.5457\n",
      "epoch 7 / 10, step 2500/3125, loss = 1.5842\n",
      "epoch 7 / 10, step 2600/3125, loss = 1.4591\n",
      "epoch 7 / 10, step 2700/3125, loss = 1.8471\n",
      "epoch 7 / 10, step 2800/3125, loss = 1.7030\n",
      "epoch 7 / 10, step 2900/3125, loss = 1.4237\n",
      "epoch 7 / 10, step 3000/3125, loss = 1.4796\n",
      "epoch 7 / 10, step 3100/3125, loss = 1.8625\n",
      "epoch 8 / 10, step 100/3125, loss = 1.7764\n",
      "epoch 8 / 10, step 200/3125, loss = 1.8447\n",
      "epoch 8 / 10, step 300/3125, loss = 1.7250\n",
      "epoch 8 / 10, step 400/3125, loss = 1.3655\n",
      "epoch 8 / 10, step 500/3125, loss = 1.6721\n",
      "epoch 8 / 10, step 600/3125, loss = 1.5262\n",
      "epoch 8 / 10, step 700/3125, loss = 1.7991\n",
      "epoch 8 / 10, step 800/3125, loss = 1.4369\n",
      "epoch 8 / 10, step 900/3125, loss = 1.6318\n",
      "epoch 8 / 10, step 1000/3125, loss = 1.4707\n",
      "epoch 8 / 10, step 1100/3125, loss = 1.6495\n",
      "epoch 8 / 10, step 1200/3125, loss = 1.5758\n",
      "epoch 8 / 10, step 1300/3125, loss = 1.7023\n",
      "epoch 8 / 10, step 1400/3125, loss = 1.2516\n",
      "epoch 8 / 10, step 1500/3125, loss = 1.7187\n",
      "epoch 8 / 10, step 1600/3125, loss = 1.5889\n",
      "epoch 8 / 10, step 1700/3125, loss = 1.4113\n",
      "epoch 8 / 10, step 1800/3125, loss = 1.5848\n",
      "epoch 8 / 10, step 1900/3125, loss = 1.8359\n",
      "epoch 8 / 10, step 2000/3125, loss = 1.9994\n",
      "epoch 8 / 10, step 2100/3125, loss = 1.7790\n",
      "epoch 8 / 10, step 2200/3125, loss = 1.6401\n",
      "epoch 8 / 10, step 2300/3125, loss = 1.5213\n",
      "epoch 8 / 10, step 2400/3125, loss = 1.6120\n",
      "epoch 8 / 10, step 2500/3125, loss = 1.5511\n",
      "epoch 8 / 10, step 2600/3125, loss = 1.6260\n",
      "epoch 8 / 10, step 2700/3125, loss = 1.5501\n",
      "epoch 8 / 10, step 2800/3125, loss = 1.7413\n",
      "epoch 8 / 10, step 2900/3125, loss = 1.6736\n",
      "epoch 8 / 10, step 3000/3125, loss = 1.9737\n",
      "epoch 8 / 10, step 3100/3125, loss = 2.0345\n",
      "epoch 9 / 10, step 100/3125, loss = 1.7464\n",
      "epoch 9 / 10, step 200/3125, loss = 1.7254\n",
      "epoch 9 / 10, step 300/3125, loss = 1.4667\n",
      "epoch 9 / 10, step 400/3125, loss = 1.8815\n",
      "epoch 9 / 10, step 500/3125, loss = 1.6923\n",
      "epoch 9 / 10, step 600/3125, loss = 1.6613\n",
      "epoch 9 / 10, step 700/3125, loss = 1.4685\n",
      "epoch 9 / 10, step 800/3125, loss = 1.8480\n",
      "epoch 9 / 10, step 900/3125, loss = 1.6710\n",
      "epoch 9 / 10, step 1000/3125, loss = 1.9190\n",
      "epoch 9 / 10, step 1100/3125, loss = 1.6649\n",
      "epoch 9 / 10, step 1200/3125, loss = 2.0893\n",
      "epoch 9 / 10, step 1300/3125, loss = 1.3032\n",
      "epoch 9 / 10, step 1400/3125, loss = 2.0698\n",
      "epoch 9 / 10, step 1500/3125, loss = 1.7689\n",
      "epoch 9 / 10, step 1600/3125, loss = 1.4776\n",
      "epoch 9 / 10, step 1700/3125, loss = 2.0148\n",
      "epoch 9 / 10, step 1800/3125, loss = 1.5805\n",
      "epoch 9 / 10, step 1900/3125, loss = 1.5077\n",
      "epoch 9 / 10, step 2000/3125, loss = 1.7072\n",
      "epoch 9 / 10, step 2100/3125, loss = 1.6946\n",
      "epoch 9 / 10, step 2200/3125, loss = 1.6182\n",
      "epoch 9 / 10, step 2300/3125, loss = 2.0496\n",
      "epoch 9 / 10, step 2400/3125, loss = 1.5637\n",
      "epoch 9 / 10, step 2500/3125, loss = 1.8166\n",
      "epoch 9 / 10, step 2600/3125, loss = 1.4138\n",
      "epoch 9 / 10, step 2700/3125, loss = 1.3410\n",
      "epoch 9 / 10, step 2800/3125, loss = 1.6821\n",
      "epoch 9 / 10, step 2900/3125, loss = 1.3136\n",
      "epoch 9 / 10, step 3000/3125, loss = 1.5549\n",
      "epoch 9 / 10, step 3100/3125, loss = 1.4807\n",
      "epoch 10 / 10, step 100/3125, loss = 1.4094\n",
      "epoch 10 / 10, step 200/3125, loss = 1.3597\n",
      "epoch 10 / 10, step 300/3125, loss = 1.4889\n",
      "epoch 10 / 10, step 400/3125, loss = 1.7412\n",
      "epoch 10 / 10, step 500/3125, loss = 1.6092\n",
      "epoch 10 / 10, step 600/3125, loss = 1.7624\n",
      "epoch 10 / 10, step 700/3125, loss = 1.5502\n",
      "epoch 10 / 10, step 800/3125, loss = 1.2868\n",
      "epoch 10 / 10, step 900/3125, loss = 1.7918\n",
      "epoch 10 / 10, step 1000/3125, loss = 1.8027\n",
      "epoch 10 / 10, step 1100/3125, loss = 1.5734\n",
      "epoch 10 / 10, step 1200/3125, loss = 1.6924\n",
      "epoch 10 / 10, step 1300/3125, loss = 1.3381\n",
      "epoch 10 / 10, step 1400/3125, loss = 1.9231\n",
      "epoch 10 / 10, step 1500/3125, loss = 1.5357\n",
      "epoch 10 / 10, step 1600/3125, loss = 1.6719\n",
      "epoch 10 / 10, step 1700/3125, loss = 1.6099\n",
      "epoch 10 / 10, step 1800/3125, loss = 2.1284\n",
      "epoch 10 / 10, step 1900/3125, loss = 1.5795\n",
      "epoch 10 / 10, step 2000/3125, loss = 1.4764\n",
      "epoch 10 / 10, step 2100/3125, loss = 1.4536\n",
      "epoch 10 / 10, step 2200/3125, loss = 1.3562\n",
      "epoch 10 / 10, step 2300/3125, loss = 1.2881\n",
      "epoch 10 / 10, step 2400/3125, loss = 1.5366\n",
      "epoch 10 / 10, step 2500/3125, loss = 1.6109\n",
      "epoch 10 / 10, step 2600/3125, loss = 1.7976\n",
      "epoch 10 / 10, step 2700/3125, loss = 1.6008\n",
      "epoch 10 / 10, step 2800/3125, loss = 1.4664\n",
      "epoch 10 / 10, step 2900/3125, loss = 1.3567\n",
      "epoch 10 / 10, step 3000/3125, loss = 1.5686\n",
      "epoch 10 / 10, step 3100/3125, loss = 1.2712\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# define model\n",
    "\n",
    "class ConvNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ConvNet, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels=3, out_channels=6, kernel_size=5)\n",
    "        self.pool = nn.MaxPool2d(kernel_size=2, stride=2) \n",
    "        self.leaky_relu = nn.LeakyReLU()\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc1 = nn.Linear(16*5*5, 120) #channels * height * width\n",
    "        self.fc2 = nn.Linear(in_features=120,out_features=84)\n",
    "        self.fc3 = nn.Linear(84, 10)\n",
    "\n",
    "    def forward(self,x):\n",
    "        # because we are using crossentropy loss we dont need to define\n",
    "        # a last layer activation function because it is applied automatically\n",
    "        x = self.pool(self.leaky_relu(self.conv1(x))) #conv1->L_relu->pool\n",
    "        x = self.pool(self.leaky_relu(self.conv2(x))) #conv2->L_relu->pool\n",
    "        x = x.view(-1, 16*5*5) # flatten\n",
    "\n",
    "        x = self.leaky_relu(self.fc1(x))\n",
    "        x = self.leaky_relu(self.fc2(x))\n",
    "        x = self.fc3(x) # no activattion function applied as cross-entropy metric takes care of that\n",
    "        return x\n",
    "\n",
    "model = ConvNet().to(device)\n",
    "\n",
    "# loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n",
    "\n",
    "# training loop\n",
    "\n",
    "n_total_steps = len(train_loader)\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        # forward\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "\n",
    "        # backward\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if (i+1) % 100 == 0:\n",
    "            print(f\"epoch {epoch+1} / {num_epochs}, step {i+1}/{n_total_steps}, loss = {loss.item():.4f}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate the performance on the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy = 41.25\n",
      "Accuracy of plane: 59.1 %\n",
      "Accuracy of car: 48.2 %\n",
      "Accuracy of bird: 18.1 %\n",
      "Accuracy of cat: 34.8 %\n",
      "Accuracy of deer: 36.8 %\n",
      "Accuracy of dog: 32.3 %\n",
      "Accuracy of frog: 40.6 %\n",
      "Accuracy of horse: 50.3 %\n",
      "Accuracy of ship: 47.0 %\n",
      "Accuracy of truck: 45.3 %\n"
     ]
    }
   ],
   "source": [
    "# test\n",
    "with torch.no_grad():\n",
    "    n_correct = 0\n",
    "    n_samples = 0\n",
    "    n_class_correct = [0] * 10\n",
    "    n_class_samples = [0] * 10\n",
    "\n",
    "    for images, labels in test_loader:\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "\n",
    "        #value, index\n",
    "        _, predicted = torch.max(outputs, 1)\n",
    "        n_samples += labels.shape[0]\n",
    "        n_correct += (predicted == labels).sum().item()\n",
    "\n",
    "        for i in range(batch_size):\n",
    "            label = labels[i]\n",
    "            pred = predicted[i]\n",
    "            if label == pred:\n",
    "                n_class_correct[label] += 1\n",
    "            n_class_samples[label] += 1\n",
    "    \n",
    "    acc = 100.0 * n_correct / n_samples\n",
    "    print(f\"accuracy = {acc}\")\n",
    "\n",
    "    for i in range(10):\n",
    "        acc = 100.0 * n_class_correct[i] / n_class_samples[i]\n",
    "        print(f\"Accuracy of {classes[i]}: {acc} %\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 15 - Transfer Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define transforms, Import data, Write model training function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ants', 'bees']\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "import numpy as np\n",
    "import torchvision\n",
    "from torchvision import datasets, models, transforms\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import os\n",
    "import copy\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "mean = np.array([0.485, 0.456, 0.406])\n",
    "std = np.array([0.229, 0.224, 0.225])\n",
    "\n",
    "data_transforms = {\n",
    "    \"train\": transforms.Compose([\n",
    "    transforms.RandomResizedCrop(224),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean, std)\n",
    "    ]),\n",
    "    \"val\" : transforms.Compose([\n",
    "    transforms.Resize(224),\n",
    "    transforms.CenterCrop(224),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean, std)\n",
    "    ])\n",
    "}\n",
    "\n",
    "# import data\n",
    "data_dir = \"data\\\\hymenoptera_data\"\n",
    "sets = [\"train\", \"val\"]\n",
    "\n",
    "image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x])\n",
    "                    for x in [\"train\", \"val\"]}\n",
    "\n",
    "dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size = 16, shuffle=True, num_workers =2)\n",
    "                    for x in [\"train\", \"val\"]}\n",
    "\n",
    "dataset_sizes = {i:len(image_datasets[i]) for i in [\"train\", \"val\"]}\n",
    "\n",
    "class_names = image_datasets[\"train\"].classes\n",
    "print(class_names)\n",
    "\n",
    "def train_model(model, criterion, optimizer, scheduler, num_epochs=25):\n",
    "    since = time.time()\n",
    "    \n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_acc = 0.0\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        print(f\"Epoch {epoch+1}/{num_epochs}\")\n",
    "        print(\"-\" * 10)\n",
    "\n",
    "        #Each epoch has a training and validation phase\n",
    "        for phase in [\"train\", \"val\"]:\n",
    "            if phase == \"train\":\n",
    "                model.train() # set model in training mode\n",
    "            else:\n",
    "                model.eval() # set model in eval mode\n",
    "            \n",
    "            running_loss = 0.0\n",
    "            running_corrects = 0\n",
    "\n",
    "            # Iterate over data.\n",
    "            for inputs, labels in dataloaders[phase]:\n",
    "                inputs = inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "\n",
    "                # forward\n",
    "                # track history if only in train\n",
    "                with torch.set_grad_enabled(phase == \"train\"):\n",
    "                    outputs = model(inputs)\n",
    "                    _, preds = torch.max(outputs, 1)\n",
    "                    loss = criterion(outputs, labels)\n",
    "\n",
    "                    # backward +optimize only if in training phase\n",
    "                    if phase == \"train\":\n",
    "                        optimizer.zero_grad()\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "                \n",
    "                # statistics\n",
    "                running_loss += loss.item() * inputs.size(0)\n",
    "                running_corrects += torch.sum(preds == labels.data)\n",
    "            \n",
    "            if phase == \"train\":\n",
    "                scheduler.step()\n",
    "\n",
    "            epoch_loss = running_loss / dataset_sizes[phase]\n",
    "            epoch_acc = running_corrects.double() /dataset_sizes[phase]\n",
    "\n",
    "            print(f\"{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}\")\n",
    "\n",
    "            # deep copy the model\n",
    "            if phase == \"val\" and epoch_acc > best_acc:\n",
    "                best_acc = epoch_acc\n",
    "                best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    \n",
    "        print()\n",
    "\n",
    "    time_elapsed = time.time() - since\n",
    "    print(f\"Training complete in {time_elapsed//60:.0f}m {time_elapsed%60:.0f}s\")\n",
    "    print(f\"Best val Acc: {best_acc:.4f}\")\n",
    "\n",
    "    #load best model weights\n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrain pre-trained network with a different last layer depending on task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "----------\n",
      "train Loss: 0.6694 Acc: 0.5779\n",
      "val Loss: 0.6027 Acc: 0.7190\n",
      "\n",
      "Epoch 2/20\n",
      "----------\n",
      "train Loss: 0.6039 Acc: 0.6885\n",
      "val Loss: 0.5183 Acc: 0.8039\n",
      "\n",
      "Epoch 3/20\n",
      "----------\n",
      "train Loss: 0.5110 Acc: 0.7992\n",
      "val Loss: 0.4785 Acc: 0.8170\n",
      "\n",
      "Epoch 4/20\n",
      "----------\n",
      "train Loss: 0.4782 Acc: 0.7992\n",
      "val Loss: 0.4234 Acc: 0.8824\n",
      "\n",
      "Epoch 5/20\n",
      "----------\n",
      "train Loss: 0.4142 Acc: 0.8525\n",
      "val Loss: 0.3858 Acc: 0.8954\n",
      "\n",
      "Epoch 6/20\n",
      "----------\n",
      "train Loss: 0.4000 Acc: 0.8689\n",
      "val Loss: 0.3478 Acc: 0.9216\n",
      "\n",
      "Epoch 7/20\n",
      "----------\n",
      "train Loss: 0.3517 Acc: 0.8770\n",
      "val Loss: 0.3385 Acc: 0.9020\n",
      "\n",
      "Epoch 8/20\n",
      "----------\n",
      "train Loss: 0.3746 Acc: 0.8566\n",
      "val Loss: 0.3374 Acc: 0.9020\n",
      "\n",
      "Epoch 9/20\n",
      "----------\n",
      "train Loss: 0.3554 Acc: 0.8852\n",
      "val Loss: 0.3306 Acc: 0.9150\n",
      "\n",
      "Epoch 10/20\n",
      "----------\n",
      "train Loss: 0.3688 Acc: 0.8566\n",
      "val Loss: 0.3367 Acc: 0.9085\n",
      "\n",
      "Epoch 11/20\n",
      "----------\n",
      "train Loss: 0.3381 Acc: 0.9221\n",
      "val Loss: 0.3391 Acc: 0.8954\n",
      "\n",
      "Epoch 12/20\n",
      "----------\n",
      "train Loss: 0.3273 Acc: 0.9139\n",
      "val Loss: 0.3259 Acc: 0.9020\n",
      "\n",
      "Epoch 13/20\n",
      "----------\n",
      "train Loss: 0.3439 Acc: 0.8852\n",
      "val Loss: 0.3206 Acc: 0.9085\n",
      "\n",
      "Epoch 14/20\n",
      "----------\n",
      "train Loss: 0.3298 Acc: 0.9098\n",
      "val Loss: 0.3178 Acc: 0.9150\n",
      "\n",
      "Epoch 15/20\n",
      "----------\n",
      "train Loss: 0.3523 Acc: 0.8852\n",
      "val Loss: 0.3209 Acc: 0.9020\n",
      "\n",
      "Epoch 16/20\n",
      "----------\n",
      "train Loss: 0.3562 Acc: 0.8525\n",
      "val Loss: 0.3218 Acc: 0.9020\n",
      "\n",
      "Epoch 17/20\n",
      "----------\n",
      "train Loss: 0.3170 Acc: 0.9139\n",
      "val Loss: 0.3223 Acc: 0.9085\n",
      "\n",
      "Epoch 18/20\n",
      "----------\n",
      "train Loss: 0.3527 Acc: 0.8934\n",
      "val Loss: 0.3226 Acc: 0.9216\n",
      "\n",
      "Epoch 19/20\n",
      "----------\n",
      "train Loss: 0.3645 Acc: 0.8689\n",
      "val Loss: 0.3280 Acc: 0.8954\n",
      "\n",
      "Epoch 20/20\n",
      "----------\n",
      "train Loss: 0.3473 Acc: 0.8730\n",
      "val Loss: 0.3204 Acc: 0.9150\n",
      "\n",
      "Training complete in 3m 57s\n",
      "Best val Acc: 0.9216\n"
     ]
    }
   ],
   "source": [
    "model = models.resnet18(pretrained=True)\n",
    "num_ftrs = model.fc.in_features\n",
    "\n",
    "model.fc = nn.Linear(num_ftrs, 2)\n",
    "model.to(device)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.001)\n",
    "\n",
    "# scheduler\n",
    "\n",
    "step_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n",
    "model = train_model(model, criterion, optimizer, step_lr_scheduler, num_epochs=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use pretrained network with all layers frozen, re-train only last layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "----------\n",
      "train Loss: 0.7438 Acc: 0.5533\n",
      "val Loss: 0.6388 Acc: 0.6078\n",
      "\n",
      "Epoch 2/20\n",
      "----------\n",
      "train Loss: 0.6129 Acc: 0.6885\n",
      "val Loss: 0.5633 Acc: 0.6863\n",
      "\n",
      "Epoch 3/20\n",
      "----------\n",
      "train Loss: 0.5677 Acc: 0.7213\n",
      "val Loss: 0.5464 Acc: 0.6993\n",
      "\n",
      "Epoch 4/20\n",
      "----------\n",
      "train Loss: 0.5406 Acc: 0.7459\n",
      "val Loss: 0.5000 Acc: 0.7451\n",
      "\n",
      "Epoch 5/20\n",
      "----------\n",
      "train Loss: 0.5115 Acc: 0.7705\n",
      "val Loss: 0.4883 Acc: 0.7712\n",
      "\n",
      "Epoch 6/20\n",
      "----------\n",
      "train Loss: 0.5079 Acc: 0.7582\n",
      "val Loss: 0.4407 Acc: 0.8235\n",
      "\n",
      "Epoch 7/20\n",
      "----------\n",
      "train Loss: 0.4782 Acc: 0.8156\n",
      "val Loss: 0.4294 Acc: 0.8301\n",
      "\n",
      "Epoch 8/20\n",
      "----------\n",
      "train Loss: 0.4599 Acc: 0.8156\n",
      "val Loss: 0.4217 Acc: 0.8627\n",
      "\n",
      "Epoch 9/20\n",
      "----------\n",
      "train Loss: 0.4530 Acc: 0.8238\n",
      "val Loss: 0.4211 Acc: 0.8758\n",
      "\n",
      "Epoch 10/20\n",
      "----------\n",
      "train Loss: 0.4484 Acc: 0.8115\n",
      "val Loss: 0.4102 Acc: 0.8693\n",
      "\n",
      "Epoch 11/20\n",
      "----------\n",
      "train Loss: 0.4513 Acc: 0.8197\n",
      "val Loss: 0.4078 Acc: 0.8693\n",
      "\n",
      "Epoch 12/20\n",
      "----------\n",
      "train Loss: 0.4223 Acc: 0.8484\n",
      "val Loss: 0.4072 Acc: 0.8824\n",
      "\n",
      "Epoch 13/20\n",
      "----------\n",
      "train Loss: 0.4350 Acc: 0.8443\n",
      "val Loss: 0.4074 Acc: 0.8693\n",
      "\n",
      "Epoch 14/20\n",
      "----------\n",
      "train Loss: 0.4201 Acc: 0.8607\n",
      "val Loss: 0.4030 Acc: 0.8627\n",
      "\n",
      "Epoch 15/20\n",
      "----------\n",
      "train Loss: 0.4445 Acc: 0.8197\n",
      "val Loss: 0.4048 Acc: 0.8758\n",
      "\n",
      "Epoch 16/20\n",
      "----------\n",
      "train Loss: 0.4193 Acc: 0.8525\n",
      "val Loss: 0.3938 Acc: 0.9020\n",
      "\n",
      "Epoch 17/20\n",
      "----------\n",
      "train Loss: 0.4374 Acc: 0.8443\n",
      "val Loss: 0.3984 Acc: 0.8889\n",
      "\n",
      "Epoch 18/20\n",
      "----------\n",
      "train Loss: 0.4409 Acc: 0.8238\n",
      "val Loss: 0.4026 Acc: 0.8824\n",
      "\n",
      "Epoch 19/20\n",
      "----------\n",
      "train Loss: 0.4263 Acc: 0.8361\n",
      "val Loss: 0.3999 Acc: 0.8824\n",
      "\n",
      "Epoch 20/20\n",
      "----------\n",
      "train Loss: 0.4117 Acc: 0.8770\n",
      "val Loss: 0.4050 Acc: 0.8824\n",
      "\n",
      "Training complete in 3m 24s\n",
      "Best val Acc: 0.9020\n"
     ]
    }
   ],
   "source": [
    "model = models.resnet18(pretrained=True)\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "    \n",
    "num_ftrs = model.fc.in_features\n",
    "\n",
    "model.fc = nn.Linear(num_ftrs, 2)\n",
    "model.to(device)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.001)\n",
    "\n",
    "# scheduler\n",
    "\n",
    "step_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n",
    "model = train_model(model, criterion, optimizer, step_lr_scheduler, num_epochs=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 16 - How to Use the TensorBoard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reusing the code from tutorial 13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([100, 1, 28, 28]) torch.Size([100])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD6CAYAAAC4RRw1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAa+klEQVR4nO3dfXBU1fkH8O8joCKRGlAwIvLSUiBMOyIO+BvQoaUo0iqgbRVpGipTpBULLVqD9oUZRWnraKvFl3TAQEtpraFDqFXEiCAdpYBiDW/lZQRSUnl1gNopL57fH7kez7nsbja7d++95+73M5PJc/bs7n3MEx9uzt4XUUqBiIjcc1bUCRARUW7YwImIHMUGTkTkKDZwIiJHsYETETmKDZyIyFF5NXARGSUi20Rkh4hUBZUURYt1TS7WNlkk1+PARaQNgH8CGAmgEcA6AOOVUpuDS4/CxromF2ubPG3zeO1gADuUUrsAQET+AGAMgLS/DCLCs4ZiQiklaaZYV7cdVEpdlGauVbVlXWMlZV3zWULpBmCvMW70HrOIyGQRWS8i6/PYFoWHdXXb7gxzLdaWdY2tlHXNZw881R7cGf9iK6WqAVQD/BfdEaxrcrVYW9bVLfnsgTcC6G6MLwWwL790KAZY1+RibRMmnwa+DkAfEeklImcDuBVAXTBpUYRY1+RibRMm5yUUpdQpEZkKYDmANgDmK6U2BZYZRYJ1TS7WNnlyPowwp41xTS02MhyF0mqsa24qKip0/NOf/tSamz59uo7/8pe/tOZtNyilrswvs2asa6ykrCvPxCQichQbOBGRo9jAiYgclc9x4ETUgpKSEh3PmDHDmjPH5vMA4MCBA4VNjBKBe+BERI5iAycichSXUFJ49tlnrfGhQ4d0fPfdd4edDjnslVde0fHgwYOtuaVLl+r4Zz/7mTW3fj0vRUIt4x44EZGj2MCJiBzFBk5E5CiugacwYsQIa9yt2yeXTG5oaLDmfvvb31rj06dPFy4xir2HHnrIGpvr3j//+c+tuZ/85Cc6PnHiRGETo5RmzZql4x/96Edpn9e2bTxbJffAiYgcxQZOROSoeP5dELFJkyZZ45deeknH8+fPt+Y6dOhgjefOnVu4xCgW+vXrZ42XLVum43PPPdeaM5fjXn/9dWvu1KlTBciOWsO8GutHH30UYSa54R44EZGj2MCJiBzFBk5E5CjekSeFNm3aWOM1a9boeMiQIdZcY2OjNb7ssssKl1iAeEee1iktLdXxiy++aM19/vOf1/GXv/xla27lypWFTexMvCNPBgMGDLDGjz/+uI6vvvrqtK87++yzC5ZTlnhHHiKiJGEDJyJyFA8jTMF/NmWmw73OOeecQqdDETDPvgWA559/XseXX365NTd27FgdR7BkQq1QXl5ujTMtm7iAe+BERI5iAycichQbOBGRo7gG3koiknFMbjLXsQGgpqbGGpunyH/lK1+x5sy77hCFiXvgRESOarGBi8h8EdkvIg3GY51EZIWIbPe+l2Z6D4of1jW5WNvikc0SSg2AXwNYaDxWBaBeKTVHRKq88b3BpxcPS5Ys0fHQoUOtuTDPZA1YDYq8rjNnzkwZA0D79u2t8ahRo3RcX19f2MTyV4Mir+3HbrzxRmu8ePHirF+7c+fOoNMJXIt74Eqp1QAO+x4eA2CBFy8AMDbYtKjQWNfkYm2LR65r4F2VUk0A4H3vElxKFCHWNblY2wQq+FEoIjIZwORCb4fCxbomE+vqllwb+PsiUqaUahKRMgD70z1RKVUNoBpw9+pmH3zwQdQphCXRda2qqrLGDzzwgI7POsv+Y9R/Y+sEnCKfVW1drGsm5udXQOa77rz55pvWePz48QXJKUi5LqHUAaj04koAS4NJhyLGuiYXa5tA2RxGuBjAGwD6ikijiEwCMAfASBHZDmCkNyaHsK7JxdoWjxaXUJRS6f6OGJHm8cSZMGFC1CkErljqes899+j4oYceSvu8iooKa/z2229b44kTJ+p4+PDh1twXvvAFHc+bN8+ae+SRR3T84YcftphvEIqltkFYtWqVju+66y5r7l//+lfY6bQaz8QkInIUGzgRkaPYwImIHMWrEWaBd91xxy233GKNv/3tb6d97ssvv6zj0aNHW3O//OUvrXHnzp117D+s1Bzffffd1py5tt6nT5+0uVA0Vq9ereOtW7dGmEluuAdOROQoNnAiIkdxCaWVeEOH+BkyZIiOH3vsMWvu4osvTvu6a6+9Vsd79uyx5mpra63xmjVrdOy/gcO///1vHfsPVfzhD3+o4xtuuMGaW7ZsWdrciLLBPXAiIkexgRMROYoNnIjIUVwDbyX/HXgcviOPsy644AJrPGvWLB1nWvM+fvy4NX7ppZd0fMcdd1hzR44cySkf//b37dunY//p+RSM7t27W+NXX31Vx23atMn4Wtc/w+IeOBGRo9jAiYgcxQZOROQoroGncOmll1rjfv36RZQJpfLEE09Y4+uuuy7tc//zn//oeNy4cdZcrneX9/8+zJ49W8dXXXWVNTd9+nQdNzY25rQ9OtOAAQN0/PTTT1tzPXr00LH/swzzcw8A2LRpUwGyCw/3wImIHMUGTkTkKC6hpFBSUmKNO3XqFFEm9LEvfelLOr7pppvSPu+FF16wxt/61rd0fPDgwbSva9eunTXu37+/NR46dKiOH374YWvu6NGjOjaXTADgT3/6U9ptUu6++tWv6ti8lIKff9nqtttuK1hOUeAeOBGRo9jAiYgcxQZOROQoroFTLHXs2NEaV1VV6bh9+/bW3Nq1a3Xsv2SryX8K/mc+8xkdT5kyxZqbNGmSNT558qSO/YeeLVy4UMdc844X/x2SkoZ74EREjmIDJyJyFJdQUjDvsAIAO3fu1LH/xrSuX80sTswrx82dO9ea++IXv5j2da+99pqOM13xr2vXrtbYvHKg/448dXV11vgXv/iFjs2781A4/Gfb+q8emY554+ok4h44EZGj2MCJiBzVYgMXke4islJEtojIJhGZ5j3eSURWiMh273tp4dOloLCuidWOdS0e2ayBnwIwQyn1loicD2CDiKwAMBFAvVJqjohUAagCcG/hUg2P/y4ebdt+8mNK0B15YlfXXr166XjChAlZv+7eez9Jz/+ZhFmf9evXW3PmVeweffRRa85/9x7HxKquQfAfVmrWeevWrdbc2LFjw0gpFlrcA1dKNSml3vLiYwC2AOgGYAyABd7TFgAYW6AcqQBY18Q6yboWj1YdhSIiPQEMBLAWQFelVBPQ3AxEpEua10wGMDnPPKmAWNdkYl2TT7JdAhCREgCrAMxWSi0RkQ+UUhcY80eUUhnX1UTEifUG/wX7zTPv/H+iHzhwwBr7D1WLK6WUAPGq63PPPafjvn37WnPmksqgQYOsuYEDB+p448aN1tyOHTt0/MYbb1hzp0+fzjnXGNuglLoyTnXN1de//nUdX3LJJdbc3r17dVxbWxtaThHaoJS60v9gVkehiEg7ALUAFimllngPvy8iZd58GYD9QWVK4WBdk4l1LR7ZHIUiAOYB2KKUMj/pqQNQ6cWVAJYGnx4VCuuaaKxrkchmDXwogAoA74rIRu+x+wDMAfCciEwCsAfA1wqSIRUK65pMJWBdi0bWa+CBbMyRNfDevXtb43feeUfHHTp0sOb8d4DJdDW8OPl4DTwIrtS1SKRcK81F1HU9deqUjhsaGqy5adOm6XjVqlWh5RSh3NfAiYgoftjAiYgcxasRprBr1y5rbB7e5j+EberUqaHkRFTMysvLrfE111yj4yJZQkmJe+BERI5iAycichQbOBGRo7gGngX/DW6JKHhLl6Y/t2j58uXW+Kmnnip0Ok7gHjgRkaPYwImIHMUzMYsUz8RMrMSciUkWnolJRJQkbOBERI5iAycichQbOBGRo9jAiYgcxQZOROQoNnAiIkexgRMROYoNnIjIUWzgRESOCvtqhAcB7AZwoRfHQTHm0iPg92NdMwszlyBry7pmFnldQ70Wit6oyPqgrteQL+YSnDjlz1yCE6f8mYuNSyhERI5iAyciclRUDbw6ou2mwlyCE6f8mUtw4pQ/czFEsgZORET54xIKEZGj2MCJiBwVagMXkVEisk1EdohIVZjb9rY/X0T2i0iD8VgnEVkhItu976Uh5NFdRFaKyBYR2SQi06LKJQisq5VLYmrLulq5xLKuoTVwEWkDYC6A6wGUAxgvIuVhbd9TA2CU77EqAPVKqT4A6r1xoZ0CMEMp1R/AVQDu9H4WUeSSF9b1DImoLet6hnjWVSkVyheA/wOw3BjPBDAzrO0b2+0JoMEYbwNQ5sVlALZFkNNSACPjkAvrytqyru7UNcwllG4A9hrjRu+xqHVVSjUBgPe9S5gbF5GeAAYCWBt1LjliXdNwvLasaxpxqmuYDVxSPFbUxzCKSAmAWgDTlVJHo84nR6xrCgmoLeuaQtzqGmYDbwTQ3RhfCmBfiNtP530RKQMA7/v+MDYqIu3Q/IuwSCm1JMpc8sS6+iSktqyrTxzrGmYDXwegj4j0EpGzAdwKoC7E7adTB6DSiyvRvLZVUCIiAOYB2KKUejTKXALAuhoSVFvW1RDbuoa88D8awD8B7ARwfwQfPCwG0ATgJJr3MCYB6IzmT4+3e987hZDHMDT/OfoPABu9r9FR5MK6srasq7t15an0RESO4pmYRESOYgMnInJUXg086lNtqTBY1+RibRMmj0X9Nmj+cKM3gLMBvAOgvIXXKH7F44t1TezXgaBqG4P/Fn61UNd89sAHA9ihlNqllDoB4A8AxuTxfhQPrKvbdmeYY23dlbKu+TTwrE61FZHJIrJeRNbnsS0KD+uaXC3WlnV1S9s8XpvVqbZKqWp4tx4SkTPmKXZY1+Rqsbasq1vy2QOP66m2lB/WNblY24TJp4HH9VRbyg/rmlysbcLkvISilDolIlMBLEfzp9vzlVKbAsuMIsG6JhdrmzyhnkrPNbX4UEqlWg/NCesaKxuUUlcG8Uasa6ykrCvPxCQichQbOBGRo9jAiYgcxQZOROQoNnAiIkexgRMROSqfU+kTpUOHDjpeuXKlNTdo0KCc3vOss+x/H2tqanQ8ZcoUa+5///tfTtsgotz169dPx5s3b7bmlixZouPvfOc71tyBAwcKm1iWuAdOROQoNnAiIkexgRMROYpr4B5zjcu/5p3r5QY++ugja1xRUaHjw4cPW3MzZszIaRsUvb17P7nE9vDhw625nTt3hpwNtcbNN9+sY///53379k0ZA1wDJyKiPLGBExE5iksoefIfXnThhRfq+MEHH0z7uquvvtoan3/++To+duxYQNlRIQwbNswal5WVRZQJtVaPHj2s8fe+9z0di9gX6Dx06JCO16xZU9jEcsQ9cCIiR7GBExE5ig2ciMhRXAP3bNr0yZ2lnn/+eWvOPNTIz79e/Zvf/EbHmdbAr7jiCmtsrs01NDRkTpZyMmDAAGts1rwlpaWlOv7jH/9ozR0/flzH//3vf3PMjsLg/+ypc+fOOvYfGviDH/wglJzywT1wIiJHsYETETmKSyieF198Uccvv/yyNWdesezxxx8PZHu7d++2xuYhSxSciy++WMd///vfrbmnn35axy2dCfvkk0+mfE8AWLdunY737duXU54UDv8hoOahg/7lrz179oSSUz64B05E5Cg2cCIiR7GBExE5imvgKZw+fdoam4ebjRgxwppr3769NX7hhRd07L8jj3l1wo0bN1pzTU1NOeVKmV1++eU6Pvfcc625T33qU2lf17FjR2s8ZsyYtM/9/e9/n1tyVHDjxo3LODavQPjMM89YcwcPHixcYgHhHjgRkaNabOAiMl9E9otIg/FYJxFZISLbve+lmd6D4od1TS7Wtnhks4RSA+DXABYaj1UBqFdKzRGRKm98b/DpxZ//5sTXXXedjv03dNiwYYOOp06dWtjEWlaDBNa1Xbt21njChAk5vc8NN9xgjc8555y0z/WfuRsDNUhgbXNx0003WeOLLrrIGm/ZskXHf/7zn0PJKUgt7oErpVYDOOx7eAyABV68AMDYYNOiQmNdk4u1LR65fojZVSnVBABKqSYR6ZLuiSIyGcDkHLdD4WJdkyur2rKubin4UShKqWoA1QAgIrndXJJih3VNJtbVLbk28PdFpMz7l7wMwP4gk4ob8245jzzyiDV3yy23pH3diRMnrPHMmTN1HNPDBp2vq/+OK7fddlva577yyitp5zJdia6xsdEam1cjjDHna5st81DBsWPHWnP+Gxebn1Nt3bq1oHkVQq6HEdYBqPTiSgBLg0mHIsa6Jhdrm0DZHEa4GMAbAPqKSKOITAIwB8BIEdkOYKQ3JoewrsnF2haPFpdQlFLj00yNSPO486655hprbJ6h1adPn6zf58iRI9a4vr4+v8QClNS6jho1Ku3c0aNHrfHy5ct17D9McODAgdbY/NPb/6e2/32jltTaZusb3/iGjs877zxr7sMPP7TGrp9FyzMxiYgcxQZOROQoNnAiIkfxaoQpmFcUBM684mC2/Kftzps3T8cPPPCANffee+/ltA0CevbsqePZs2enfZ7/TkvmZxRPPfVU1turra3N+rnmFSn9V7JcsWJF1u9D2TPvoOU/bNB/uvzDDz8cSk6Fwj1wIiJHsYETETmKSygp+M+2vOOOO9I+d/PmzdZ4wIABOvbf/HbixIk69t/89sc//nFr0ySP+TMvKSlJ+7wnnnjCGpuHDt54441Zb89/w49bb7017XNvvvlmHe/YscOa4xJKMPxLlf3799exfwnF/1zXcQ+ciMhRbOBERI5iAycicpT414gKurEiuDxleXm5jt99911rzvxZ+9c/r7/++sIm5qOUkqDeK+y6fvrTn7bGb7/9to4zrYG//vrr1viyyy7Tsf8qhiL2jyfX/0/q6up0fPvtt1tzhw/777kQiA1KqSuDeKM4//9qrmX/9a9/teauuOIKHfvr5v9cyoUbF3tS1pV74EREjmIDJyJyFBs4EZGjeBx4wPzHhVMwSktLdfy73/3OmuvQoYOOM61VDxs2LO1cS2vc5rz/MsHr1q3Tsf/U7DVr1ujYvPsL5cf8/MJc8wbszy+qq6utOYfWvLPCPXAiIkexgRMROcqJJZRLLrnEGvfq1UvHf/vb38JOJ6PKysqWn0St9rnPfU7HgwcPTvs8/x1X5sz55M5h/sP2vvvd7+rYPPwzFXNpbPjw4dbcoUOHMr6WgpfpdHmT/+qDScM9cCIiR7GBExE5ig2ciMhRsV0DNy8RumrVKmvOvAxo7969rbkDBw4UNrEWZHtZWPPQM2rZrl27dOw/jNA8PfrOO++05sxLuLZta/+6V1RUZL39Bx98UMdc8w6feZcdALjvvvt07L/sgXnnpbfeeiuQ7Y8bN84ax2VtnXvgRESOYgMnInJUbJdQzD93zbPw/Myz8IDCL6H4c3nsscessXmIo3lDWwDYvn27jufPn1+A7JKrsbFRx7keqtmxY0drPGTIkKxfu3Tp0py2SYXRt29fHfsPI1y9erWOgzrzMi5LJn7cAyciclSLDVxEuovIShHZIiKbRGSa93gnEVkhItu97+l3kyl2WNfEase6Fo9s9sBPAZihlOoP4CoAd4pIOYAqAPVKqT4A6r0xuYN1TS7WtUi0uAaulGoC0OTFx0RkC4BuAMYAGO49bQGA1wDcW4gkM50qu2zZMmv8q1/9Sse1tbXWnP8qcqYuXbrouKyszJoz75bjP0zN/1wz1xMnTlhz3//+93X83nvvpc0lDHGoa5y9+uqr1thfyxg7qZR6C0hWXe+//35rbH6+5L/KY9wur1FIrfoQU0R6AhgIYC2Arl4TgFKqSUS6pHnNZACT88yTCoh1TSbWNfmybuAiUgKgFsB0pdRR/8Hz6SilqgFUe+8R23vsFSvWNZlY1+KQVQMXkXZo/mVYpJRa4j38voiUef+alwHYH2Ri5pl3/kN4zLOizKuSAcAzzzyj47vuusuaa2pqSrs986a2n/3sZ625XG9oe+2111pj/xmlUYuirq5YtGiRNXbpZgxJqat542L/zTjMevhvolJMN1XJ5igUATAPwBal1KPGVB2Ajw/IrQTAA2UdwromGutaJLLZAx8KoALAuyKy0XvsPgBzADwnIpMA7AHwtYJkSIXCuiZTCVjXopHNUShrAKRbQBsRbDoUFtY1sY4rpVjXIhHbU+mPHTum49tvv92aMw/BmzJlijV33nnn6di8omGqcRDMtXoA+OY3v6njN998M/DtUe5GjMi+fz377LMFzITyZR5GOG/ePGsuaTcuzoSn0hMROYoNnIjIUbFdQjGZyykAcM899+jYf8bcrFmzdDxo0KCctue/Me6TTz6pY//ZnVu3brXGR48ezWmbVHj+3yOTfymM4i3TYYTFhHvgRESOYgMnInIUGzgRkaMk19PEc9oYr60QGxmOFW41V+paUlJijbdt26bjhQsXWnMzZ84MJacC2KCUujKIN3KlrkUiZV25B05E5Cg2cCIiRzlxGCFREI4fP26Nu3XrFlEmRMHgHjgRkaPYwImIHMUGTkTkKDZwIiJHsYETETmKDZyIyFFs4EREjmIDJyJyFBs4EZGj2MCJiBwV9qn0BwHsBnChF8dBMebSI+D3Y10zCzOXIGvLumYWeV1DvZys3qjI+qAueZkv5hKcOOXPXIITp/yZi41LKEREjmIDJyJyVFQNvDqi7abCXIITp/yZS3DilD9zMUSyBk5ERPnjEgoRkaPYwImIHBVqAxeRUSKyTUR2iEhVmNv2tj9fRPaLSIPxWCcRWSEi273vpSHk0V1EVorIFhHZJCLTosolCKyrlUtiasu6WrnEsq6hNXARaQNgLoDrAZQDGC8i5WFt31MDYJTvsSoA9UqpPgDqvXGhnQIwQynVH8BVAO70fhZR5JIX1vUMiagt63qGeNZVKRXKF4D/A7DcGM8EMDOs7Rvb7QmgwRhvA1DmxWUAtkWQ01IAI+OQC+vK2rKu7tQ1zCWUbgD2GuNG77GodVVKNQGA971LmBsXkZ4ABgJYG3UuOWJd03C8tqxrGnGqa5gNXFI8VtTHMIpICYBaANOVUkejzidHrGsKCagt65pC3OoaZgNvBNDdGF8KYF+I20/nfREpAwDv+/4wNioi7dD8i7BIKbUkylzyxLr6JKS2rKtPHOsaZgNfB6CPiPQSkbMB3AqgLsTtp1MHoNKLK9G8tlVQIiIA5gHYopR6NMpcAsC6GhJUW9bVENu6hrzwPxrAPwHsBHB/BB88LAbQBOAkmvcwJgHojOZPj7d73zuFkMcwNP85+g8AG72v0VHkwrqytqyru3XlqfRERI7imZhERI5iAycichQbOBGRo9jAiYgcxQZOROQoNnAiIkexgRMROer/AV0cR+Eq5ur9AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import matplotlib.pyplot as plt\n",
    "import sys\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "writer = SummaryWriter(\"runs\\\\mnist\")\n",
    "\n",
    "# device config\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# hyper parameters\n",
    "input_size = 784 # 28x28\n",
    "hidden_size = 100\n",
    "num_classes = 10\n",
    "num_epochs = 2\n",
    "batch_size = 100\n",
    "learning_rate = 0.001\n",
    "\n",
    "# import the dataset\n",
    "train_dataset = torchvision.datasets.MNIST(root = \"./data/mnist/MNIST/\", train=True, transform=transforms.ToTensor(), download=True)\n",
    "test_dataset = torchvision.datasets.MNIST(root = \"./data/mnist/MNIST/\", train=False, transform=transforms.ToTensor())\n",
    "\n",
    "# data loaders\n",
    "train_loader = torch.utils.data.DataLoader(dataset = train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset = test_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "examples = iter(train_loader)\n",
    "samples, labels = examples.next()\n",
    "print(samples.shape, labels.shape)\n",
    "\n",
    "for i in range(6):\n",
    "    plt.subplot(2, 3, i+1)\n",
    "    plt.imshow(samples[i][0], cmap=\"gray\")\n",
    "#plt.show()\n",
    "img_grid = torchvision.utils.make_grid(samples)\n",
    "writer.add_image(\"mnist_images\", img_grid)\n",
    "#sys.exit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1 / 2, step 100/600, loss = 0.4242\n",
      "epoch 1 / 2, step 200/600, loss = 0.3922\n",
      "epoch 1 / 2, step 300/600, loss = 0.2741\n",
      "epoch 1 / 2, step 400/600, loss = 0.3091\n",
      "epoch 1 / 2, step 500/600, loss = 0.1880\n",
      "epoch 1 / 2, step 600/600, loss = 0.1091\n",
      "epoch 2 / 2, step 100/600, loss = 0.1418\n",
      "epoch 2 / 2, step 200/600, loss = 0.1762\n",
      "epoch 2 / 2, step 300/600, loss = 0.2061\n",
      "epoch 2 / 2, step 400/600, loss = 0.0714\n",
      "epoch 2 / 2, step 500/600, loss = 0.1001\n",
      "epoch 2 / 2, step 600/600, loss = 0.1255\n"
     ]
    }
   ],
   "source": [
    "class NeuralNet(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, output_size):\n",
    "        super(NeuralNet, self).__init__()\n",
    "        self.linear1 = nn.Linear(input_size, hidden_size)\n",
    "        self.leaky_relu = nn.LeakyReLU()\n",
    "        self.linear2 = nn.Linear(hidden_size, hidden_size)\n",
    "        self.linear3 = nn.Linear(hidden_size, output_size)\n",
    "\n",
    "    def forward(self,x):\n",
    "        # because we are using crossentropy loss we dont need to define\n",
    "        # a last layer activation function because it is applied automatically\n",
    "        out = self.linear1(x)\n",
    "        out = self.leaky_relu(out)\n",
    "        out = self.linear2(out)\n",
    "        out = self.leaky_relu(out)\n",
    "        out = self.linear3(out)\n",
    "        return out\n",
    "\n",
    "model = NeuralNet(input_size, hidden_size, output_size=num_classes).to(device)\n",
    "\n",
    "# loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "writer.add_graph(model, samples.reshape(-1,28*28).to(device))\n",
    "\n",
    "# training loop\n",
    "\n",
    "n_total_steps = len(train_loader)\n",
    "running_loss = 0.0\n",
    "running_correct = 0\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        # 100 x 1 x 28 x 28 -> 100 x 784\n",
    "        images = images.reshape(-1, 28*28).to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        # forward\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "\n",
    "        # backward\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        running_correct += (predicted == labels).sum().item()\n",
    "\n",
    "        if (i+1) % 100 == 0:\n",
    "            print(f\"epoch {epoch+1} / {num_epochs}, step {i+1}/{n_total_steps}, loss = {loss.item():.4f}\")\n",
    "            writer.add_scalar(\"training loss\", running_loss / 100, epoch * n_total_steps + i)\n",
    "            writer.add_scalar(\"accuracy\", running_correct / 100, epoch * n_total_steps + i)\n",
    "            running_loss = 0.0\n",
    "            running_correct = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy = 96.35\n"
     ]
    }
   ],
   "source": [
    "# val\n",
    "labels = []\n",
    "preds = []\n",
    "with torch.no_grad():\n",
    "    n_correct = 0\n",
    "    n_samples = 0\n",
    "    for images, val_labels in test_loader:\n",
    "        images = images.reshape(-1, 28*28).to(device)\n",
    "        val_labels = val_labels.to(device)\n",
    "        outputs = model(images)\n",
    "\n",
    "        #value, index\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        n_samples += val_labels.shape[0]\n",
    "        n_correct += (predicted == val_labels).sum().item()\n",
    "        class_predictions = [F.softmax(output, dim=0) for output in outputs]\n",
    "\n",
    "        preds.append(class_predictions)\n",
    "        labels.append(predicted)\n",
    "    \n",
    "    preds = torch.cat([torch.stack(batch) for batch in preds])\n",
    "    labels = torch.cat(labels)\n",
    "\n",
    "    acc = 100.0 * n_correct / n_samples\n",
    "    print(f\"accuracy = {acc}\")\n",
    "\n",
    "    classes = range(10)\n",
    "    for i in classes:\n",
    "        labels_i = labels == i\n",
    "        preds_i = preds[:, i]\n",
    "        writer.add_pr_curve(str(i), labels_i, preds_i, global_step=0)\n",
    "        writer.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial 17 - Saving and Loading Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "PATH = \"models\\\\\"\n",
    "model = ConvNet().to(device)\n",
    "args = model.parameters()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save entire model, along with weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Complete model\n",
    "torch.save(model, PATH) # can save tensors, models or dicts (uses pickle to serialize objects)\n",
    "\n",
    "# model class must be defined somwhere\n",
    "model = torch.load(PATH)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save only model weights (recommended way of saving)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# State dict\n",
    "torch.save(model.state_dict(), PATH)\n",
    "\n",
    "#model must be created again with same architecture\n",
    "model = NeuralNet(*args)\n",
    "model.load_state_dict(torch.load(PATH))\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Running this will actually save the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "FILE= \"models/model.pth\"\n",
    "#################################\n",
    "# LAZY WAY\n",
    "# lazy_save(model, FILE)\n",
    "# model = lazy_load(FILE)\n",
    "# model.eval()\n",
    "#for param in model.parameters():\n",
    "#    print(param)\n",
    "#################################\n",
    "\n",
    "#################################\n",
    "# PROPER WAY\n",
    "torch.save(model.state_dict(), FILE)\n",
    "loaded_model = Model(n_input_features=16, n_hidden=8)\n",
    "loaded_model.load_state_dict(torch.load(FILE))\n",
    "loaded_model.eval()\n",
    "#################################\n",
    "for param in loaded_model.parameters():\n",
    "    print(param)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Practical example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'state': {}, 'param_groups': [{'lr': 0.01, 'momentum': 0, 'dampening': 0, 'weight_decay': 0, 'nesterov': False, 'maximize': False, 'params': [0, 1, 2, 3, 4, 5]}]}\n"
     ]
    }
   ],
   "source": [
    "class Model(nn.Module):\n",
    "    def __init__(self, n_input_features, n_hidden):\n",
    "        super(Model, self).__init__()\n",
    "        self.model = nn.Sequential(\n",
    "            nn.ReLU(nn.Linear(n_input_features, n_hidden)),\n",
    "            nn.ReLU(nn.Linear(n_hidden, n_hidden)),\n",
    "            nn.Linear(n_hidden, 1),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        return self.model(x)\n",
    "\n",
    "def lazy_save(model, FILE):\n",
    "    torch.save(model, FILE)\n",
    "\n",
    "def lazy_load(FILE):\n",
    "    torch.load(FILE)\n",
    "\n",
    "def proper_save(model, FILE):\n",
    "    torch.save(model.state_dict(), FILE)\n",
    "\n",
    "\n",
    "model = Model(n_input_features=16, n_hidden=8)\n",
    "\n",
    "# train model...\n",
    "learning_rate = 0.01\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n",
    "print(optimizer.state_dict())\n",
    "checkpoint_path = \"models\\\\checkpoints\\\\\"\n",
    "checkpoint = {\n",
    "    \"epoch\": 90,\n",
    "    \"model_state\": model.state_dict(),\n",
    "    \"optim_state\": optimizer.state_dict()\n",
    "}\n",
    "torch.save(checkpoint, os.path.join(checkpoint_path,\"checkpoint.pth\"))\n",
    "\n",
    "loaded_checkpoint = torch.load(os.path.join(checkpoint_path,\"checkpoint.pth\"))\n",
    "epoch = loaded_checkpoint[\"epoch\"]\n",
    "\n",
    "model = Model(n_input_features=16, n_hidden=8)\n",
    "optimizer = optim.SGD(model._parameters(), lr=learning_rate)\n",
    "\n",
    "model.load_state_dict(checkpoint[\"model_state\"])\n",
    "optimizer.load_state_dict(checkpoint[\"optim_state\"])\n",
    "\n",
    "print(optimizer.state_dict())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pytorch Tutorial 18: Learning rate Scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.6 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6 | packaged by conda-forge | (default, Jul 11 2021, 03:37:25) [MSC v.1916 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "d5b56a78e3e75a120fd5f35cf8efed775b2ff68e364aa87bad3970e34a65f885"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}