HitOrMiss_MNIST_Example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Necessary imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from torch import nn\n",
    "from time import time\n",
    "from capsules import *\n",
    "from torch.optim import Adam\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.autograd import Variable\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Definition of CapsNet model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HitNet4MNIST(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(HitNet4MNIST, self).__init__()\n",
    "        self.conv = nn.Sequential(\n",
    "            nn.Conv2d(\n",
    "                in_channels=1,\n",
    "                out_channels=256,\n",
    "                kernel_size=9,\n",
    "                stride=1\n",
    "            ),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(\n",
    "                in_channels=256,\n",
    "                out_channels=256,\n",
    "                kernel_size=9,\n",
    "                stride=2\n",
    "            ),\n",
    "            nn.ReLU(inplace=True)\n",
    "        )\n",
    "        self.hom = HitOrMissLayer()\n",
    "        self.decoder = RegularizingDecoder([32*10,512,1024,784])\n",
    "        \n",
    "    def forward(self, x):\n",
    "        preprocessed = self.conv(x)\n",
    "        hom = self.hom(preprocessed)\n",
    "        internal = 0.5-hom\n",
    "        lengths = (internal**2).sum(dim=-1)**0.5\n",
    "        _, max_caps_index = lengths.max(dim=-1)\n",
    "        masked_internal = mask_hom(hom, max_caps_index.type(torch.FloatTensor))\n",
    "        reconstructions = self.decoder(\n",
    "            masked_internal.reshape(masked_internal.shape[0], -1)\n",
    "        )\n",
    "        return(internal, lengths, reconstructions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cn = HitNet4MNIST().cuda()\n",
    "optimizer = Adam(cn.parameters())\n",
    "capsule_loss = CentripetalLoss(caps_dimension=32).cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "~/HDD/works/OpenSource/capsules/python-mnist/data ~/HDD/works/OpenSource/capsules/python-mnist\n",
      "~/HDD/works/OpenSource/capsules/python-mnist\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning into 'python-mnist'...\n",
      "--2019-10-10 12:33:11--  http://yann.lecun.com/exdb/mnist/\n",
      "Resolving yann.lecun.com (yann.lecun.com)... 216.165.22.6\n",
      "Connecting to yann.lecun.com (yann.lecun.com)|216.165.22.6|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 29170 (28K) [text/html]\n",
      "Saving to: ‘data/index.html.tmp’\n",
      "\n",
      "     0K .......... .......... ........                        100%  221K=0.1s\n",
      "\n",
      "2019-10-10 12:33:12 (221 KB/s) - ‘data/index.html.tmp’ saved [29170/29170]\n",
      "\n",
      "Loading robots.txt; please ignore errors.\n",
      "--2019-10-10 12:33:12--  http://yann.lecun.com/robots.txt\n",
      "Reusing existing connection to yann.lecun.com:80.\n",
      "HTTP request sent, awaiting response... 404 Not Found\n",
      "2019-10-10 12:33:12 ERROR 404: Not Found.\n",
      "\n",
      "Removing data/index.html.tmp since it should be rejected.\n",
      "\n",
      "--2019-10-10 12:33:12--  http://yann.lecun.com/\n",
      "Reusing existing connection to yann.lecun.com:80.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 38324 (37K) [text/html]\n",
      "Saving to: ‘data/index.html.tmp’\n",
      "\n",
      "     0K .......... .......... .......... .......              100% 5.19M=0.007s\n",
      "\n",
      "2019-10-10 12:33:12 (5.19 MB/s) - ‘data/index.html.tmp’ saved [38324/38324]\n",
      "\n",
      "Removing data/index.html.tmp since it should be rejected.\n",
      "\n",
      "--2019-10-10 12:33:12--  http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
      "Reusing existing connection to yann.lecun.com:80.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 9912422 (9.5M) [application/x-gzip]\n",
      "Saving to: ‘data/train-images-idx3-ubyte.gz’\n",
      "\n",
      "     0K .......... .......... .......... .......... ..........  0% 21.4M 0s\n",
      "    50K .......... .......... .......... .......... ..........  1%  241K 20s\n",
      "   100K .......... .......... .......... .......... ..........  1% 19.2M 13s\n",
      "   150K .......... .......... .......... .......... ..........  2%  242K 20s\n",
      "   200K .......... .......... .......... .......... ..........  2% 20.8M 16s\n",
      "   250K .......... .......... .......... .......... ..........  3%  401K 17s\n",
      "   300K .......... .......... .......... .......... ..........  3% 14.4M 15s\n",
      "   350K .......... .......... .......... .......... ..........  4%  407K 16s\n",
      "   400K .......... .......... .......... .......... ..........  4% 9.49M 14s\n",
      "   450K .......... .......... .......... .......... ..........  5%  425K 15s\n",
      "   500K .......... .......... .......... .......... ..........  5% 6.22M 13s\n",
      "   550K .......... .......... .......... .......... ..........  6% 4.52M 12s\n",
      "   600K .......... .......... .......... .......... ..........  6%  444K 13s\n",
      "   650K .......... .......... .......... .......... ..........  7% 9.03M 12s\n",
      "   700K .......... .......... .......... .......... ..........  7%  431K 12s\n",
      "   750K .......... .......... .......... .......... ..........  8% 7.02M 12s\n",
      "   800K .......... .......... .......... .......... ..........  8% 4.02M 11s\n",
      "   850K .......... .......... .......... .......... ..........  9%  457K 11s\n",
      "   900K .......... .......... .......... .......... ..........  9% 5.37M 11s\n",
      "   950K .......... .......... .......... .......... .......... 10% 5.46M 10s\n",
      "  1000K .......... .......... .......... .......... .......... 10%  460K 11s\n",
      "  1050K .......... .......... .......... .......... .......... 11% 5.16M 10s\n",
      "  1100K .......... .......... .......... .......... .......... 11% 1.47M 10s\n",
      "  1150K .......... .......... .......... .......... .......... 12%  563K 10s\n",
      "  1200K .......... .......... .......... .......... .......... 12% 9.82M 10s\n",
      "  1250K .......... .......... .......... .......... .......... 13% 6.30M 9s\n",
      "  1300K .......... .......... .......... .......... .......... 13%  394K 10s\n",
      "  1350K .......... .......... .......... .......... .......... 14% 27.9M 9s\n",
      "  1400K .......... .......... .......... .......... .......... 14% 18.2M 9s\n",
      "  1450K .......... .......... .......... .......... .......... 15%  457K 9s\n",
      "  1500K .......... .......... .......... .......... .......... 16% 2.79M 9s\n",
      "  1550K .......... .......... .......... .......... .......... 16% 21.0M 9s\n",
      "  1600K .......... .......... .......... .......... .......... 17%  506K 9s\n",
      "  1650K .......... .......... .......... .......... .......... 17% 1.41M 9s\n",
      "  1700K .......... .......... .......... .......... .......... 18% 38.6M 8s\n",
      "  1750K .......... .......... .......... .......... .......... 18% 4.46M 8s\n",
      "  1800K .......... .......... .......... .......... .......... 19%  554K 8s\n",
      "  1850K .......... .......... .......... .......... .......... 19% 1.84M 8s\n",
      "  1900K .......... .......... .......... .......... .......... 20% 7.48M 8s\n",
      "  1950K .......... .......... .......... .......... .......... 20%  562K 8s\n",
      "  2000K .......... .......... .......... .......... .......... 21% 1.78M 8s\n",
      "  2050K .......... .......... .......... .......... .......... 21% 11.6M 8s\n",
      "  2100K .......... .......... .......... .......... .......... 22% 5.96M 7s\n",
      "  2150K .......... .......... .......... .......... .......... 22%  571K 7s\n",
      "  2200K .......... .......... .......... .......... .......... 23% 1.92M 7s\n",
      "  2250K .......... .......... .......... .......... .......... 23% 10.5M 7s\n",
      "  2300K .......... .......... .......... .......... .......... 24% 5.26M 7s\n",
      "  2350K .......... .......... .......... .......... .......... 24%  580K 7s\n",
      "  2400K .......... .......... .......... .......... .......... 25% 1.95M 7s\n",
      "  2450K .......... .......... .......... .......... .......... 25% 8.40M 7s\n",
      "  2500K .......... .......... .......... .......... .......... 26% 5.37M 7s\n",
      "  2550K .......... .......... .......... .......... .......... 26% 1.40M 7s\n",
      "  2600K .......... .......... .......... .......... .......... 27%  924K 6s\n",
      "  2650K .......... .......... .......... .......... .......... 27% 1.96M 6s\n",
      "  2700K .......... .......... .......... .......... .......... 28% 6.12M 6s\n",
      "  2750K .......... .......... .......... .......... .......... 28% 7.93M 6s\n",
      "  2800K .......... .......... .......... .......... .......... 29%  585K 6s\n",
      "  2850K .......... .......... .......... .......... .......... 29% 2.14M 6s\n",
      "  2900K .......... .......... .......... .......... .......... 30% 6.21M 6s\n",
      "  2950K .......... .......... .......... .......... .......... 30% 8.13M 6s\n",
      "  3000K .......... .......... .......... .......... .......... 31% 4.45M 6s\n",
      "  3050K .......... .......... .......... .......... .......... 32%  626K 6s\n",
      "  3100K .......... .......... .......... .......... .......... 32% 2.01M 6s\n",
      "  3150K .......... .......... .......... .......... .......... 33% 7.57M 6s\n",
      "  3200K .......... .......... .......... .......... .......... 33% 7.07M 5s\n",
      "  3250K .......... .......... .......... .......... .......... 34% 6.49M 5s\n",
      "  3300K .......... .......... .......... .......... .......... 34%  616K 5s\n",
      "  3350K .......... .......... .......... .......... .......... 35% 1.99M 5s\n",
      "  3400K .......... .......... .......... .......... .......... 35% 7.54M 5s\n",
      "  3450K .......... .......... .......... .......... .......... 36% 12.7M 5s\n",
      "  3500K .......... .......... .......... .......... .......... 36% 7.15M 5s\n",
      "  3550K .......... .......... .......... .......... .......... 37%  622K 5s\n",
      "  3600K .......... .......... .......... .......... .......... 37% 2.14M 5s\n",
      "  3650K .......... .......... .......... .......... .......... 38% 5.09M 5s\n",
      "  3700K .......... .......... .......... .......... .......... 38% 6.50M 5s\n",
      "  3750K .......... .......... .......... .......... .......... 39% 13.0M 5s\n",
      "  3800K .......... .......... .......... .......... .......... 39%  626K 5s\n",
      "  3850K .......... .......... .......... .......... .......... 40% 6.92M 5s\n",
      "  3900K .......... .......... .......... .......... .......... 40% 1.91M 5s\n",
      "  3950K .......... .......... .......... .......... .......... 41% 7.93M 4s\n",
      "  4000K .......... .......... .......... .......... .......... 41% 9.50M 4s\n",
      "  4050K .......... .......... .......... .......... .......... 42% 7.99M 4s\n",
      "  4100K .......... .......... .......... .......... .......... 42%  646K 4s\n",
      "  4150K .......... .......... .......... .......... .......... 43% 1.71M 4s\n",
      "  4200K .......... .......... .......... .......... .......... 43% 12.8M 4s\n",
      "  4250K .......... .......... .......... .......... .......... 44% 11.2M 4s\n",
      "  4300K .......... .......... .......... .......... .......... 44% 5.44M 4s\n",
      "  4350K .......... .......... .......... .......... .......... 45% 16.5M 4s\n",
      "  4400K .......... .......... .......... .......... .......... 45%  652K 4s\n",
      "  4450K .......... .......... .......... .......... .......... 46% 1.86M 4s\n",
      "  4500K .......... .......... .......... .......... .......... 47% 7.80M 4s\n",
      "  4550K .......... .......... .......... .......... .......... 47% 9.51M 4s\n",
      "  4600K .......... .......... .......... .......... .......... 48% 4.35M 4s\n",
      "  4650K .......... .......... .......... .......... .......... 48% 30.6M 4s\n",
      "  4700K .......... .......... .......... .......... .......... 49%  729K 4s\n",
      "  4750K .......... .......... .......... .......... .......... 49% 3.37M 4s\n",
      "  4800K .......... .......... .......... .......... .......... 50%  408K 4s\n",
      "  4850K .......... .......... .......... .......... .......... 50% 51.2M 3s\n",
      "  4900K .......... .......... .......... .......... .......... 51% 83.3M 3s\n",
      "  4950K .......... .......... .......... .......... .......... 51% 28.8M 3s\n",
      "  5000K .......... .......... .......... .......... .......... 52% 25.7M 3s\n",
      "  5050K .......... .......... .......... .......... .......... 52% 20.6M 3s\n",
      "  5100K .......... .......... .......... .......... .......... 53%  255K 3s\n",
      "  5150K .......... .......... .......... .......... .......... 53% 22.9M 3s\n",
      "  5200K .......... .......... .......... .......... .......... 54% 23.9M 3s\n",
      "  5250K .......... .......... .......... .......... .......... 54%  247K 3s\n",
      "  5300K .......... .......... .......... .......... .......... 55% 33.0M 3s\n",
      "  5350K .......... .......... .......... .......... .......... 55% 61.6M 3s\n",
      "  5400K .......... .......... .......... .......... .......... 56%  243K 3s\n",
      "  5450K .......... .......... .......... .......... .......... 56% 26.2M 3s\n",
      "  5500K .......... .......... .......... .......... .......... 57% 51.8M 3s\n",
      "  5550K .......... .......... .......... .......... .......... 57% 63.5M 3s\n",
      "  5600K .......... .......... .......... .......... .......... 58%  353K 3s\n",
      "  5650K .......... .......... .......... .......... .......... 58%  443K 3s\n",
      "  5700K .......... .......... .......... .......... .......... 59% 1.67M 3s\n",
      "  5750K .......... .......... .......... .......... .......... 59%  107M 3s\n",
      "  5800K .......... .......... .......... .......... .......... 60%  421K 3s\n",
      "  5850K .......... .......... .......... .......... .......... 60% 10.1M 3s\n",
      "  5900K .......... .......... .......... .......... .......... 61% 16.3M 3s\n",
      "  5950K .......... .......... .......... .......... .......... 61% 3.68M 3s\n",
      "  6000K .......... .......... .......... .......... .......... 62%  466K 3s\n",
      "  6050K .......... .......... .......... .......... .......... 63% 8.28M 3s\n",
      "  6100K .......... .......... .......... .......... .......... 63% 3.76M 3s\n",
      "  6150K .......... .......... .......... .......... .......... 64%  409K 3s\n",
      "  6200K .......... .......... .......... .......... .......... 64% 8.67M 3s\n",
      "  6250K .......... .......... .......... .......... .......... 65% 33.0M 3s\n",
      "  6300K .......... .......... .......... .......... .......... 65%  389K 3s\n",
      "  6350K .......... .......... .......... .......... .......... 66% 32.1M 3s\n",
      "  6400K .......... .......... .......... .......... .......... 66% 37.3M 3s\n",
      "  6450K .......... .......... .......... .......... .......... 67%  213K 3s\n",
      "  6500K .......... .......... .......... .......... .......... 67% 19.1M 3s\n",
      "  6550K .......... .......... .......... .......... .......... 68% 19.4M 2s\n",
      "  6600K .......... .......... .......... .......... .......... 68% 19.7M 2s\n",
      "  6650K .......... .......... .......... .......... .......... 69%  194K 2s\n",
      "  6700K .......... .......... .......... .......... .......... 69% 83.3M 2s\n",
      "  6750K .......... .......... .......... .......... .......... 70% 96.1M 2s\n",
      "  6800K .......... .......... .......... .......... .......... 70%  113M 2s\n",
      "  6850K .......... .......... .......... .......... .......... 71% 5.85M 2s\n",
      "  6900K .......... .......... .......... .......... .......... 71%  462K 2s\n",
      "  6950K .......... .......... .......... .......... .......... 72% 4.58M 2s\n",
      "  7000K .......... .......... .......... .......... .......... 72% 5.91M 2s\n",
      "  7050K .......... .......... .......... .......... .......... 73%  461K 2s\n",
      "  7100K .......... .......... .......... .......... .......... 73% 4.90M 2s\n",
      "  7150K .......... .......... .......... .......... .......... 74% 5.07M 2s\n",
      "  7200K .......... .......... .......... .......... .......... 74%  451K 2s\n",
      "  7250K .......... .......... .......... .......... .......... 75% 4.46M 2s\n",
      "  7300K .......... .......... .......... .......... .......... 75%  481K 2s\n",
      "  7350K .......... .......... .......... .......... .......... 76% 3.59M 2s\n",
      "  7400K .......... .......... .......... .......... .......... 76% 5.97M 2s\n",
      "  7450K .......... .......... .......... .......... .......... 77%  463K 2s\n",
      "  7500K .......... .......... .......... .......... .......... 77% 5.39M 2s\n",
      "  7550K .......... .......... .......... .......... .......... 78% 6.02M 2s\n",
      "  7600K .......... .......... .......... .......... .......... 79%  462K 2s\n",
      "  7650K .......... .......... .......... .......... .......... 79% 3.93M 2s\n",
      "  7700K .......... .......... .......... .......... .......... 80% 6.54M 2s\n",
      "  7750K .......... .......... .......... .......... .......... 80%  470K 2s\n",
      "  7800K .......... .......... .......... .......... .......... 81% 4.27M 1s\n",
      "  7850K .......... .......... .......... .......... .......... 81% 5.30M 1s\n",
      "  7900K .......... .......... .......... .......... .......... 82%  478K 1s\n",
      "  7950K .......... .......... .......... .......... .......... 82% 4.36M 1s\n",
      "  8000K .......... .......... .......... .......... .......... 83% 1.99M 1s\n",
      "  8050K .......... .......... .......... .......... .......... 83%  557K 1s\n",
      "  8100K .......... .......... .......... .......... .......... 84% 4.55M 1s\n",
      "  8150K .......... .......... .......... .......... .......... 84% 1.85M 1s\n",
      "  8200K .......... .......... .......... .......... .......... 85%  567K 1s\n",
      "  8250K .......... .......... .......... .......... .......... 85% 4.78M 1s\n",
      "  8300K .......... .......... .......... .......... .......... 86% 1.88M 1s\n",
      "  8350K .......... .......... .......... .......... .......... 86%  559K 1s\n",
      "  8400K .......... .......... .......... .......... .......... 87% 6.97M 1s\n",
      "  8450K .......... .......... .......... .......... .......... 87% 1.73M 1s\n",
      "  8500K .......... .......... .......... .......... .......... 88%  557K 1s\n",
      "  8550K .......... .......... .......... .......... .......... 88% 7.39M 1s\n",
      "  8600K .......... .......... .......... .......... .......... 89% 1.74M 1s\n",
      "  8650K .......... .......... .......... .......... .......... 89% 1.03M 1s\n",
      "  8700K .......... .......... .......... .......... .......... 90%  996K 1s\n",
      "  8750K .......... .......... .......... .......... .......... 90% 1.75M 1s\n",
      "  8800K .......... .......... .......... .......... .......... 91% 1.17M 1s\n",
      "  8850K .......... .......... .......... .......... .......... 91%  878K 1s\n",
      "  8900K .......... .......... .......... .......... .......... 92% 1.87M 1s\n",
      "  8950K .......... .......... .......... .......... .......... 92% 3.13M 1s\n",
      "  9000K .......... .......... .......... .......... .......... 93%  617K 1s\n",
      "  9050K .......... .......... .......... .......... .......... 94% 1.80M 0s\n",
      "  9100K .......... .......... .......... .......... .......... 94% 2.76M 0s\n",
      "  9150K .......... .......... .......... .......... .......... 95%  621K 0s\n",
      "  9200K .......... .......... .......... .......... .......... 95% 1.76M 0s\n",
      "  9250K .......... .......... .......... .......... .......... 96% 3.32M 0s\n",
      "  9300K .......... .......... .......... .......... .......... 96%  605K 0s\n",
      "  9350K .......... .......... .......... .......... .......... 97% 5.29M 0s\n",
      "  9400K .......... .......... .......... .......... .......... 97% 1.56M 0s\n",
      "  9450K .......... .......... .......... .......... .......... 98% 1.43M 0s\n",
      "  9500K .......... .......... .......... .......... .......... 98%  828K 0s\n",
      "  9550K .......... .......... .......... .......... .......... 99% 2.13M 0s\n",
      "  9600K .......... .......... .......... .......... .......... 99% 1.18M 0s\n",
      "  9650K .......... .......... ..........                      100%  590K=7.9s\n",
      "\n",
      "2019-10-10 12:33:20 (1.20 MB/s) - ‘data/train-images-idx3-ubyte.gz’ saved [9912422/9912422]\n",
      "\n",
      "--2019-10-10 12:33:20--  http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
      "Reusing existing connection to yann.lecun.com:80.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 28881 (28K) [application/x-gzip]\n",
      "Saving to: ‘data/train-labels-idx1-ubyte.gz’\n",
      "\n",
      "     0K .......... .......... ........                        100% 23.9M=0.001s\n",
      "\n",
      "2019-10-10 12:33:21 (23.9 MB/s) - ‘data/train-labels-idx1-ubyte.gz’ saved [28881/28881]\n",
      "\n",
      "--2019-10-10 12:33:21--  http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
      "Reusing existing connection to yann.lecun.com:80.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 1648877 (1.6M) [application/x-gzip]\n",
      "Saving to: ‘data/t10k-images-idx3-ubyte.gz’\n",
      "\n",
      "     0K .......... .......... .......... .......... ..........  3% 20.0M 0s\n",
      "    50K .......... .......... .......... .......... ..........  6% 33.7M 0s\n",
      "   100K .......... .......... .......... .......... ..........  9% 14.4M 0s\n",
      "   150K .......... .......... .......... .......... .......... 12%  409K 1s\n",
      "   200K .......... .......... .......... .......... .......... 15% 17.6M 1s\n",
      "   250K .......... .......... .......... .......... .......... 18% 3.37M 1s\n",
      "   300K .......... .......... .......... .......... .......... 21%  454K 1s\n",
      "   350K .......... .......... .......... .......... .......... 24% 10.9M 1s\n",
      "   400K .......... .......... .......... .......... .......... 27% 4.70M 1s\n",
      "   450K .......... .......... .......... .......... .......... 31%  454K 1s\n",
      "   500K .......... .......... .......... .......... .......... 34% 8.91M 1s\n",
      "   550K .......... .......... .......... .......... .......... 37% 4.83M 1s\n",
      "   600K .......... .......... .......... .......... .......... 40% 1.85M 1s\n",
      "   650K .......... .......... .......... .......... .......... 43%  568K 1s\n",
      "   700K .......... .......... .......... .......... .......... 46% 3.28M 1s\n",
      "   750K .......... .......... .......... .......... .......... 49% 21.1M 1s\n",
      "   800K .......... .......... .......... .......... .......... 52%  464K 1s\n",
      "   850K .......... .......... .......... .......... .......... 55% 10.4M 1s\n",
      "   900K .......... .......... .......... .......... .......... 58% 2.38M 0s\n",
      "   950K .......... .......... .......... .......... .......... 62%  490K 0s\n",
      "  1000K .......... .......... .......... .......... .......... 65% 12.3M 0s\n",
      "  1050K .......... .......... .......... .......... .......... 68% 2.48M 0s\n",
      "  1100K .......... .......... .......... .......... .......... 71%  527K 0s\n",
      "  1150K .......... .......... .......... .......... .......... 74% 4.50M 0s\n",
      "  1200K .......... .......... .......... .......... .......... 77% 2.89M 0s\n",
      "  1250K .......... .......... .......... .......... .......... 80% 2.83M 0s\n",
      "  1300K .......... .......... .......... .......... .......... 83%  564K 0s\n",
      "  1350K .......... .......... .......... .......... .......... 86% 4.38M 0s\n",
      "  1400K .......... .......... .......... .......... .......... 90% 3.48M 0s\n",
      "  1450K .......... .......... .......... .......... .......... 93%  497K 0s\n",
      "  1500K .......... .......... .......... .......... .......... 96% 10.6M 0s\n",
      "  1550K .......... .......... .......... .......... .......... 99% 2.10M 0s\n",
      "  1600K ..........                                            100% 46.5M=1.2s\n",
      "\n",
      "2019-10-10 12:33:22 (1.34 MB/s) - ‘data/t10k-images-idx3-ubyte.gz’ saved [1648877/1648877]\n",
      "\n",
      "--2019-10-10 12:33:22--  http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
      "Reusing existing connection to yann.lecun.com:80.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 4542 (4.4K) [application/x-gzip]\n",
      "Saving to: ‘data/t10k-labels-idx1-ubyte.gz’\n",
      "\n",
      "     0K ....                                                  100%  831K=0.005s\n",
      "\n",
      "2019-10-10 12:33:22 (831 KB/s) - ‘data/t10k-labels-idx1-ubyte.gz’ saved [4542/4542]\n",
      "\n",
      "FINISHED --2019-10-10 12:33:22--\n",
      "Total wall clock time: 11s\n",
      "Downloaded: 6 files, 11M in 9.2s (1.21 MB/s)\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "git clone https://github.com/sorki/python-mnist.git\n",
    "cd python-mnist/\n",
    "./get_data.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mnist import MNIST\n",
    "mndata = MNIST('./python-mnist/data')\n",
    "images, labels = mndata.load_training()\n",
    "test_img, test_lab = mndata.load_testing()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "images = np.array(images)\n",
    "labels = np.array(labels)\n",
    "test_img = np.array(test_img)\n",
    "test_lab = np.array(test_lab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def iterate_minibatches(X, y, batchsize):\n",
    "    indices = np.random.permutation(np.arange(len(X)))\n",
    "    for start in range(0, len(indices), batchsize):\n",
    "        ix = indices[start: start + batchsize]\n",
    "        yield X[ix], y[ix]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch #0\n",
      "Minibatch 0 of 600\n",
      "Minibatch 100 of 600\n",
      "Minibatch 200 of 600\n",
      "Minibatch 300 of 600\n",
      "Minibatch 400 of 600\n",
      "Minibatch 500 of 600\n",
      "Training loss: 0.10137466\n",
      "Training accuracy: 0.9590333333333333\n",
      "Training in 0th epoch took 0.5446719328562418 minutes\n",
      "Testing minibatch 0 of 100\n",
      "Testing loss: 0.029154724\n",
      "Testing accuracy: 0.9901000000000001\n",
      "0th epoch took 0.5658048868179322 minutes\n",
      "Epoch #1\n",
      "Minibatch 0 of 600\n",
      "Minibatch 100 of 600\n",
      "Minibatch 200 of 600\n",
      "Minibatch 300 of 600\n",
      "Minibatch 400 of 600\n",
      "Minibatch 500 of 600\n",
      "Training loss: 0.0301739\n",
      "Training accuracy: 0.9917833333333335\n",
      "Training in 1th epoch took 0.5260341127713521 minutes\n",
      "Testing minibatch 0 of 100\n",
      "Testing loss: 0.017840901\n",
      "Testing accuracy: 0.992\n",
      "1th epoch took 0.5466638604799906 minutes\n",
      "Epoch #2\n",
      "Minibatch 0 of 600\n",
      "Minibatch 100 of 600\n",
      "Minibatch 200 of 600\n",
      "Minibatch 300 of 600\n",
      "Minibatch 400 of 600\n",
      "Minibatch 500 of 600\n",
      "Training loss: 0.022331133\n",
      "Training accuracy: 0.9952\n",
      "Training in 2th epoch took 0.5390448093414306 minutes\n",
      "Testing minibatch 0 of 100\n",
      "Testing loss: 0.013807792\n",
      "Testing accuracy: 0.9943000000000001\n",
      "2th epoch took 0.560029411315918 minutes\n",
      "Epoch #3\n",
      "Minibatch 0 of 600\n",
      "Minibatch 100 of 600\n",
      "Minibatch 200 of 600\n",
      "Minibatch 300 of 600\n",
      "Minibatch 400 of 600\n",
      "Minibatch 500 of 600\n",
      "Training loss: 0.017512131\n",
      "Training accuracy: 0.9970333333333333\n",
      "Training in 3th epoch took 0.5336638847986858 minutes\n",
      "Testing minibatch 0 of 100\n",
      "Testing loss: 0.012725604\n",
      "Testing accuracy: 0.9938000000000001\n",
      "3th epoch took 0.5549081524213155 minutes\n",
      "Epoch #4\n",
      "Minibatch 0 of 600\n",
      "Minibatch 100 of 600\n",
      "Minibatch 200 of 600\n",
      "Minibatch 300 of 600\n",
      "Minibatch 400 of 600\n",
      "Minibatch 500 of 600\n",
      "Training loss: 0.014651089\n",
      "Training accuracy: 0.9980166666666666\n",
      "Training in 4th epoch took 0.5395275592803955 minutes\n",
      "Testing minibatch 0 of 100\n",
      "Testing loss: 0.012087911\n",
      "Testing accuracy: 0.9941000000000001\n",
      "4th epoch took 0.5613043228785197 minutes\n",
      "Epoch #5\n",
      "Minibatch 0 of 600\n",
      "Minibatch 100 of 600\n",
      "Minibatch 200 of 600\n",
      "Minibatch 300 of 600\n",
      "Minibatch 400 of 600\n",
      "Minibatch 500 of 600\n",
      "Training loss: 0.011982471\n",
      "Training accuracy: 0.9989166666666666\n",
      "Training in 5th epoch took 0.536316963036855 minutes\n",
      "Testing minibatch 0 of 100\n",
      "Testing loss: 0.011031584\n",
      "Testing accuracy: 0.9936\n",
      "5th epoch took 0.5571611881256103 minutes\n",
      "Epoch #6\n",
      "Minibatch 0 of 600\n",
      "Minibatch 100 of 600\n",
      "Minibatch 200 of 600\n",
      "Minibatch 300 of 600\n",
      "Minibatch 400 of 600\n",
      "Minibatch 500 of 600\n",
      "Training loss: 0.010380201\n",
      "Training accuracy: 0.9994\n",
      "Training in 6th epoch took 0.5326623400052388 minutes\n",
      "Testing minibatch 0 of 100\n",
      "Testing loss: 0.012206419\n",
      "Testing accuracy: 0.9946\n",
      "6th epoch took 0.5540423393249512 minutes\n",
      "Epoch #7\n",
      "Minibatch 0 of 600\n",
      "Minibatch 100 of 600\n",
      "Minibatch 200 of 600\n",
      "Minibatch 300 of 600\n",
      "Minibatch 400 of 600\n",
      "Minibatch 500 of 600\n",
      "Training loss: 0.008874876\n",
      "Training accuracy: 0.9997500000000002\n",
      "Training in 7th epoch took 0.5533255696296692 minutes\n",
      "Testing minibatch 0 of 100\n",
      "Testing loss: 0.011004932\n",
      "Testing accuracy: 0.9942\n",
      "7th epoch took 0.5753164450327556 minutes\n",
      "Epoch #8\n",
      "Minibatch 0 of 600\n",
      "Minibatch 100 of 600\n",
      "Minibatch 200 of 600\n",
      "Minibatch 300 of 600\n",
      "Minibatch 400 of 600\n",
      "Minibatch 500 of 600\n",
      "Training loss: 0.007255888\n",
      "Training accuracy: 0.9998500000000001\n",
      "Training in 8th epoch took 0.5421809275945028 minutes\n",
      "Testing minibatch 0 of 100\n",
      "Testing loss: 0.0117318975\n",
      "Testing accuracy: 0.9943000000000001\n",
      "8th epoch took 0.5622164885203044 minutes\n",
      "Epoch #9\n",
      "Minibatch 0 of 600\n",
      "Minibatch 100 of 600\n",
      "Minibatch 200 of 600\n",
      "Minibatch 300 of 600\n",
      "Minibatch 400 of 600\n",
      "Minibatch 500 of 600\n",
      "Training loss: 0.006388204\n",
      "Training accuracy: 0.9999000000000001\n",
      "Training in 9th epoch took 0.5167104442914326 minutes\n",
      "Testing minibatch 0 of 100\n",
      "Testing loss: 0.01085101\n",
      "Testing accuracy: 0.9945\n",
      "9th epoch took 0.5368400096893311 minutes\n",
      "All epochs took 5.57437971830368 minutes\n"
     ]
    }
   ],
   "source": [
    "training_loss = []\n",
    "training_accuracy = []\n",
    "testing_loss = []\n",
    "testing_accuracy = []\n",
    "start_time = time()\n",
    "for epoch in range(n_epochs):\n",
    "    print(\"Epoch #\"+str(epoch))\n",
    "    cn.train()\n",
    "    running_loss = []\n",
    "    running_accuracy = []\n",
    "    i = 0\n",
    "    epoch_time = time()\n",
    "    for batch_X, batch_y in iterate_minibatches(images, labels, 100):\n",
    "        if i % 100 == 0:\n",
    "            print(\"Minibatch \"+str(i)+\" of \"+str(round(images.shape[0]/100)))\n",
    "        i += 1\n",
    "        optimizer.zero_grad()\n",
    "        inp = torch.from_numpy(\n",
    "            batch_X.reshape(batch_X.shape[0],1,28,28)/255.0\n",
    "        ).type(torch.FloatTensor).cuda()\n",
    "        real_class = make_y(\n",
    "            torch.from_numpy(batch_y).type(torch.LongTensor).cuda(), 10\n",
    "        )\n",
    "        _, lengths, reconstruction = cn(inp)\n",
    "        loss = capsule_loss(\n",
    "            lengths, real_class, inp, reconstruction\n",
    "        )\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        running_loss.append(loss.cpu().data.numpy())\n",
    "        running_accuracy.append(\n",
    "            accuracy_score(lengths.min(1)[1].cpu().data.numpy(), batch_y)\n",
    "        )\n",
    "    training_loss.append(np.mean(running_loss))\n",
    "    training_accuracy.append(np.mean(running_accuracy))\n",
    "    print(\"Training loss: \"+str(training_loss[-1]))\n",
    "    print(\"Training accuracy: \"+str(training_accuracy[-1]))\n",
    "    print(\"Training in \"+str(epoch)+\"th epoch took \"+str((time() - epoch_time)/60) + \" minutes\")\n",
    "    cn.eval()\n",
    "    running_test_loss = []\n",
    "    running_test_accuracy = []\n",
    "    i = 0\n",
    "    for batch_X, batch_y in iterate_minibatches(test_img, test_lab, 100):\n",
    "        if i % 100 == 0:\n",
    "            print(\"Testing minibatch \"+str(i)+\" of \"+str(round(test_img.shape[0]/100)))\n",
    "        i += 1\n",
    "        test_X = torch.from_numpy(\n",
    "            batch_X.reshape(batch_X.shape[0],1,28,28)/255.0\n",
    "        ).type(torch.FloatTensor).cuda()\n",
    "        test_Y = make_y(\n",
    "            torch.from_numpy(batch_y).type(torch.LongTensor).cuda(), 10\n",
    "        )\n",
    "        test_internal, test_lengths, test_reconstruction = cn(test_X)\n",
    "        running_test_loss.append(\n",
    "            capsule_loss(\n",
    "                test_lengths, test_Y, test_X, test_reconstruction\n",
    "            ).cpu().data.numpy()\n",
    "        )\n",
    "        running_test_accuracy.append(\n",
    "            accuracy_score(test_lengths.min(1)[1].cpu().data.numpy(), batch_y)\n",
    "        )\n",
    "    testing_loss.append(np.mean(running_test_loss))\n",
    "    testing_accuracy.append(np.mean(running_test_accuracy))\n",
    "    print(\"Testing loss: \"+str(testing_loss[-1]))\n",
    "    print(\"Testing accuracy: \"+str(testing_accuracy[-1]))\n",
    "    print(str(epoch)+\"th epoch took \"+str((time() - epoch_time)/60) + \" minutes\")\n",
    "print(\"All epochs took \"+str((time() - start_time)/60) + \" minutes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Accuracy')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOydeXxV1dX3vyu5GRgyQEbghskAgoKoRJxFHLCxzi3FqtVHW9u++NQOPk+1D9XW1qqtrbXFt32t4tBasI9tLVVBEQlOKIggKFOYkzAkEDKT6Wa9f5wTcrkkZLo34d6s7+dzPveevfc5e5/k3N/ZZ+291xJVxTAMw4hconq7AYZhGEZoMaE3DMOIcEzoDcMwIhwTesMwjAjHhN4wDCPCMaE3DMOIcEzoDcMwIhwT+hMMEdkpIpf2djsMoz1EJE9EDolIXG+3xTg+JvSGYXQaERkJXAAocHUP1uvpqboiCRP6MEFEviEiW0WkVEQWishQN11E5HERKRaRchFZJyKnunm5IrJBRCpFpEhE7undqzAiiK8BHwLPAbc2J4pIPxH5tYjscu/H90Skn5t3voh8ICJlIlIgIre56Xki8nW/c9wmIu/57auIzBaRfCDfTXvCPUeFiKwWkQv8ykeLyI9EZJt7768WkSwReVJEfu1/ESLybxH5bij+QCcSJvRhgIhMBx4GZgJDgF3AAjf7cuBCYCyQDHwFOOjmPQN8U1UTgFOBt3uw2UZk8zXgRXebISIZbvpjwJnAucBg4L+BJhEZDiwCfg+kAZOBtZ2o71pgKjDB3V/lnmMw8Ffgf0Uk3s37PnAjkAskArcDNcDzwI0iEgUgIqnAJcD8zlx4OGJCHx7cBMxT1U9UtQ64DzjHfX1uABKAkwFR1Y2qutc9rgGYICKJqnpIVT/phbYbEYaInA+MAP6mqquBbcBXXQG9HbhbVYtU1aeqH7j37E3AW6o6X1UbVPWgqnZG6B9W1VJVPQygqn9xz9Goqr8G4oBxbtmvA3NUdbM6fOqWXQmU44g7wCwgT1X3d/NPcsJjQh8eDMXpxQOgqlU4vfZhqvo2MBd4EtgvIk+JSKJb9AacXs0uEVkuIuf0cLuNyORW4E1VPeDu/9VNSwXicYQ/kKw20jtKgf+OiPxARDa65qEyIMmtv726ngdudr/fDPy5G20KG0zow4M9OD0oAERkAJACFAGo6u9U9UzgFBwTzn+56atU9RogHXgF+FsPt9uIMFx7+0zgIhHZJyL7gO8Bp+GYFWuBk1o5tKCNdIBqoL/ffmYrZY642XXt8T902zFIVZNxeurSgbr+AlwjIqcB43F+FxGPCf2JSYyIxDdvOAL9HyIy2Z3K9gvgI1XdKSI5IjJVRGJwfjC1gE9EYkXkJhFJUtUGoALw9doVGZHCtTj30QQcG/lkHMF8F8duPw/4jYgMdQdFz3Hv2ReBS0Vkpoh4RCRFRCa751wLXC8i/UUkG7ijnTYkAI1ACeARkftxbPHNPA38TETGuJMVJolICoCqFuLY9/8M/L3ZFBTpmNCfmLwOHPbbLgB+DPwd2IvTW5nllk0E/gQcwjHvHMQZEAO4BdgpIhXAt2h5ZTWMrnIr8Kyq7lbVfc0bjvnwJuBeYD2OmJYCjwJRqrobx4z4Azd9Lc5bAMDjQD2wH8e08mI7bXgDZ2B3C849X8vRpp3f4HSO3sTp4DwD9PPLfx6YSB8x24AzeNfbbTAMw+gxRORCHBPOSFVt6u329ATWozcMo8/gmjjvBp7uKyIPJvSGYfQRRGQ8UIYzaPzbXm5Oj2KmG8MwjAjHevSGYRgRzgnnICg1NVVHjhzZal51dTUDBgzo2Qb14boj9ZpXr159QFXTQnLy42D39olRb6TWfdz7WlVPqO3MM8/Utli2bFmbeaGmL9YdqdcMfKx2b/d63X3xmkNZ9/HuazPdGIZhRDgm9IZhGBGOCb1hGEaEY0JvGIYR4ZjQG4ZhRDgm9EbEc/vtt5Oens6pp57aar7r4fB3bqjGdSJyhl/erSKS727+IfPOFJH17jG/ExFp9eSGcQJgQm9EPLfddhuLFy8+XpEvAGPc7U7gDwAiMhh4ACeE3VnAAyIyyD3mD27Z5uOuCEnjDSMInHALptrig60H+Gd+PdOm9XZLjHDjwgsvZOfOnccrcg3wgjsX+UMRSRaRIcA0YImqlgKIyBLgChHJAxJVdYWb/gKOn/ZFIbsIIyxoalLqGpuoa/Q5nw1+3xt91DU0sb6kkaZN+2n0Kb4mpbHJ/7OJBt/R+41Nis/n7EdHCd+5ZEyn29UhoReRK4AngGgcr2+PBORfiOMkaBIwS1Vf9su7FZjj7v5cVZ/vdCuBj3cd4l/bGvhlo484T3RXTmEYbTGMo/2ZF7ppx0svbCX9GETkTpyePxkZGeTl5bXagKqqqjbzQk1v1X0iX3NDk3Ko1tlKa5XS2iZK3f2yOqWhCRp87meT81nvA19HXYet/rhL7Y6PhknRRZ0+rl2hF5FonHikl+Hc0KtEZKGqbvArthu4Dbgn4NjmV98pOKHAVrvHHupsQzMS4wAorqgja3D/dkobRqdozb6uXUg/NlH1KeApgClTpui0Nl5J8/LyaCsv1PRW3b1Vb31jE/98I4+UrInsqzjMnrJa9pXXsre8ln0Vh9lXXsuBqvpjjkuI9zAkqR9D0+LpFxtNfEw0cZ4od4smLsbvuyfK3W8pEx8TTawnis8+XUvOlDOJjhI80YInSoiOinI/xe8zimg3vzmtq0NBHenRnwVsVdXtACKyAOdV94jQq+pONy/Qv/MMWnn1BeZ3tqEZifEA7K+oNaE3gk0hTkDpZrw4cXoLccw3/ul5brq3lfLGCYKqUlJZx7aSarYfqGJ7STXbS6rYfqCagtIamhR498Mj5ZP6xTAkKZ7MpHgmDks+8n2Iu2Um9WNgXHAs3bW7ozktKzko5+ooHWl5a6+vUzt4/rZefY+iI6+3BZXOM+TtDz+hamfPDy2cyK+ZkVZvKOret28f1dXVbZ1zIXCX24mZCpSr6l4ReQP4hd8A7OXAfapaKiKVInI28BFOrNTfB62xRoc5XO/zE/JqdhxwxHx7STVVdY1HysXHRDEyZQCnDk3i6tOGUlO8m+lnn35E0PvHhs1wZZfoyNV1+DW1q8d25PX2UHU9P35/CSnek5h2/qgOVh887NU6fOu+8cYbycvL48CBA9x8880AqSLyLQBV/SNOjN5cYCtQA/yHm1cqIj/DiX8K8GDz2ynwbeA5nFiki7CB2JCjqmwrqWb5lhLeyy9h875K9pTXHlVmWHI/RqcN4IYzhjE6bSCj0wYwOm0gQxLjiYpqkaO8vL2cl53a05fQa3RE6Nt6re0Ibb36dprk/jF4oqC4orb9wobhx/z5R1sKReSAK/AAuLNtZrd2rKrOA+a1kv4x0PrEfCNoVNc18sG2g+RtLmb5lhIKDx0GYHTaAM4aNbhFzFMHMip1AP1ibaJGa3RE6FcBY0RkFFAEzAK+2sHzt/rq2+lWAiLCoDhhnwm9YUQsqsqW/VVHhH3VzlIafEr/2GjOPSmVb110EheNTbNxuk7SrtCraqOI3IUj2tHAPFX9XEQexPF/vFBEcoB/AoOAq0Tkp6p6Sjuvvp1mULyw34TeMCKKitoG3s8/wPItJSzfUsJe1xwzLiOB288bxUVj05gycjCxHlvf2VU6NAKhqq/j2DH90+73+76Ko2ch+Jdr9dW3KyTHCfsr6oJxKsMweglVZfP+Sl7dVs//3bSC1bsP4WtSEuI8nD8mlbsvSePCsWkMTe7X202NGMJqqHlQnPBZaS2q2uX5pIZh9DyNviY+3nWIJRv2s2TDfnaX1gAwYUgj37xwNBeNTeOMEYOIibZeeygIK6FPjo+ipr6eyrpGEuNjers5hmEch5r6Rt7ZcoAlG/bz9qb9HKppIDY6inOzU/jmRaPpf2gb111xQW83s08QVkI/KM7pxe8vrzWhN4wTkANVdSzduJ83P9/Pe1sPUNfYRGK8h0vGZ3DZhAwuHJt2ZOFRXt6OXm5t3yGshD453hX6ijrGZCT0cmsMwwDYXlLFm65J5pPdh1B15rPfeNZwLp+QQc6owWaS6WXCSuibe/Q2xdIwepfaBh/z3t/B31cXsq2kGoBThiby3UvGctmEDMYPSbBxtBOIsBL6lh69Cb1h9AaqyqLP9vHQaxspKjvMOaNT+No5I7l0QgbDbJbMCUtYCX1ctJAY7zGhN4xe4PM95fz03xtYuaOUkzMTmP+NsznnpJTebpbRAcJK6AEyk+JN6A2jBzlQVcev39zCglW7GdQ/ll9cN5Gv5GQRHWWmmXAh7IQ+IzGefbZoyjBCTn1jEy+s2MkTb+VzuMHH7eeN4juXjCGpn814CzfCUujz9x/o7WYYRsSiqizbXMzPX93I9gPVXDwujf+5cgLZ6QN7u2lGFwlDoY+jpKoOnxs/0TCM4LG1uJKfvbqR5VtKGJ02gGdvy+Hik9N7u1lGNwk7oc9MjMfXpBysqiPdjTplGEb3KK9p4LdLt/DnFbvoFxvNnCvH87VzRpojsQgh7IS+JaSgCb3RMRYvXszdd9+Nz+fj61//+jH5IjICx/FeGlAK3KyqhW7eo8CVbtGfqepLbvpzwEVAuZt3m6quDemFhIBGXxNv727ge+8so/xwA7POGs4PLhtLysC43m6aEUTCVuj3VdQykaRebo1xouPz+Zg9ezZLlizB6/WSk5MDENhDeAx4QVWfF5HpwMPALSJyJXAGMBmIA5aLyCJVrXCP+y9VfbmHLiXo7Cuv5T+eW8XGvfVMHTWYB646hQlDE3u7WUYICLv3ssykliDhhtEeK1euJDs7m9GjRxMbG8usWbMAAiMzTwCWut+XAdf4pS9X1UZVrQY+xQluH/aUVtdz8zMfUVBaw+zJcSy482wT+Qgm7Hr0KQNiiRITeqNjFBUVkZXVEgnT6/UCxAYU+xS4AXgCuA5IEJEUN/0BEfkN0B+4GNjgd9xDInI/zkPiXlU9Zt5vRwLfQ88GYj/cqPxyVS0FlU3cMyUeb+xhli9f3iN1+xNJwedP9LrDTug90VGkJcSxr9yE3mgfJxzssckB+/cAc0XkNuAdnJCZjar6phs97QOgBFgBNLrH3Afsw3loPAX8EHiwlfrbDXwPPReIvbbBx+3PrWJ35WH+381TuHRCRt8JPl+xFza/Bpteo2rfNgYOGw/JwwO2EdBvEITQT0+HrruhFmrL3a2s5XtjHZx+U6frDDuhB8dOv7/SFk0Z7eP1eikoKDiyX1hYCNDgX0ZV9wDXA4jIQOAGVS138x4CHnLz/grku+l73cPrRORZnIfFCU2jr4nvzF/DB9sO8vhXTuPScYPhs38w+OB2KB3hCF1UhAXXPrAVNv0bNr4KRR87aYNPojY+nYHlRbDrA6irOPqY2ISjxX/QiKP3+7khsJuawFcHjbWOAHfos5as3Z/CW3l+Ql4Oh8uO3ve1oW+e+L4l9LsP1vR2M4wwICcnh/z8fHbs2MGwYcNYsGABQJl/GRFJBUpVtQmnpz7PTY8GklX1oIhMAiYBb7p5Q1R1rzguGq8FPuu5q+o8TU3KD/++njc37OcnV03guhENMG8GFK1mEsD6nzkikpINqWMgdWzLlpINsd0Mxt3kg+oDULX/yDas8FPIb3TqS8qCqCAMGarCnk9g02uOuB/Y7KQPmQzT58DJV0HaOD5bvrylV324DMp2Q9ku99PdDu2Cne9CfdXRdXj6gfrAV9+lJp4EsNMD8ckQn9SyJXmdz37+6QFl4pOca+zkG0eYCn0cq3Z2Oca40YfweDzMnTuXGTNm4PP5uP3221m3bl2tf3B7YBrwsIgojulmtnt4DPCu6263AmfaZbPp5kURSQMEWAt8qwcvq1OoKj97bQN//6SQ7106ltsSVsEfvw8SBdc/zSc7DnBG1gAo2QwH8mHPWtjwL9CmlpMkDXcEOW3c0Q8CTxxUFfsJuPu9MmC/5sDR5wPGAGx9ytnx9IPU7KMfMKljIeUkiGnHK6avAXa97wj75tehoggkGkaeBzl3wMlXOiLaFv2SnW3IpNb+eHD4kN8DYBdU7oPoGOfB6Ilr5zMgLTqOd1au5cLpl4fUPBRIWAp9ZmI8ZTUN1Db4iI+JsFdNI+jk5uaSm5t7ZH/OnDmBwe1fBo6ZJqmqtTgzb45BVaeHoKkh4fdvb+XZ93fyrbPT+U7lY/DeAsiaCjc8DcnDqSjNgzOmHX1QQy2UboMDW6Bki/N5YItj6mg8fPwKozwwMAMGpjsCO+yMlv2BGc6WkMEHKz/h3JMzj66j8GP47B+0DKOIYy5JG+eKv/uQGTQSilY74r5lsWPH9vSD7Etg+o9h7AzoP7j7fzwR5zz9B8PQyd0/H9AUHdejIg9hKvTNc+mLK+oYntLNV0rDiGCe/2Anv1myhbvHV/Hd3XOQQzvhoh/Chf8N0cf5+cfEQ8YpzuZPUxNUFLrCn+/0phMyjxbx+OQOmWHq43bCiHOdzZ+Gw3Bwa0sdzQ+CHe84dm5/4pNh3Bfg5C/CSdO7b2KKUMJa6PdV1JrQG0YbvLKmiJ8sXM+vhr7Dl3bNQwZmwm2vHSusnSEqqmVQMvvS4DXWn5h+kDnR2fxpaoLyAkf8S7c7vfwR5zpmFOO4hKXQ26Ipwzg+b23YzyP/m8e/kp5mUuknMP5quPp3LTNGwpGoKGcGzKARvd2SsCMshT4jwYTeMNriw+0HeWn+M7wR9wcSfQ1w1RNwxq09bhc2ThzCUugT+3mIj4myRVOGEcBnu4rJf/5u/hT9Oo2ppyBfngfpJ/d2s4xeJiyFXkRs0ZRhBLBr0xpiFtzGLbKT6slfZ8CVDzmDqkafJyyFHtzVsdajNwxQ5dB7T5Ox9EccJp79Vz5PRs61vd0q4wQirIV+XWFZ+wUNI5KpKaXule8waMu/WcFEUm5+lrHZY3q7VcYJRti5KW4mM9FxbNaG0yrDiGyamuCTF2j6/RSit7zOr5q+Ssytr5jIG60StkKfkRhPXWMTFYcb2y9sGJHEnjXwzGWw8D/ZG5PF1fU/J+emnzJlVGpvt8w4QQlr0w04i6aS+tuCCaMPUFMKSx+E1c/BgDS47v/x+OaTOVR3kGnjLIC30TZhK/TNi6b2VdQyLjOhl1tjGCGkyQefvABLfwq1FXD2t2HavRCfRPHqlaQnWHxX4/h0yHQjIleIyGYR2Soi97aSHyciL7n5H4nISDc9RkSeF5H1IrJRRO4LVsNt0ZTRJyhcDU9fAq9+F9InwLfehSsedtzVAsUVtaSZ0Bvt0K7Quz65nwS+gOPJ70YRCfTodwdwSFWzgceBR930LwNxqjoROBP4ZvNDoLukJzo3t02xNCKS6oOw8D8dka/YC9c/7fipCXAyVlJZR1qCzZU3jk9HTDdnAVtVdTuAiCzACZ7sHzvzGuAn7veXccKyCY6v0QEi4gH6AfU4fr27TXxMNMn9Y9hfaUJvRBBNPlj9LCz9GdRVwjmzHW+T8ccG7m7wNXGwut5MN0a7dETohwEFfvuFwNS2yqhqo4iUAyk4on8NsBcnuPL3VPWYiCFdDaA8MKqRz7cXkZd3sAOX0T36WjDh3qw32HWvXLmSuXPn4vP5uPLKK4/JF5EROFGl0oBSnAAjhW7eo0DzQT9T1Zfc9FHAAmAw8Alwi6p2LeRQMwWr4PUfwN5PYeQFkPsrSB/fZvEDVc7K8Oa3W8Noi44IfWuekAInr7dV5izABwwFBuFE63mr+e3gSMEuBlAevX0lZTX1TJt2fgcuo3v0VgDl3qw7Eq7Z5/Nxxx13kJeXh9frJScnByDQ1vEY8IKqPi8i04GHgVtE5ErgDGAyEAcsF5FFqlqBY558XFUXiMgfccyXf+hSI6tKGLfpd5C3FBKGwA3PwKk3tOuErMR1AZJuphujHToyGFsIZPnte4E9bZVxzTRJOD2jrwKLVbVBVYuB94Ep3W10M82LpgyjLVauXEl2djajR48mNjaWWbNmASQHFJsALHW/L8N5C21OX66qjapaDXwKXOGaJafTEpXqeZy4sZ2n4TD88Twy9ufBud+Bu1bBxC91yNNkcUWz0FuP3jg+HenRrwLGuK+qRcAsHAH3ZyFwK7AC+BLwtqqqiOwGpovIX3BMN2cDvw1W4zMT4zlQVUejrwlPdNiu/TJCSFFREVlZLf0Ur9cLEBtQ7FPgBuAJ4DogQURS3PQHROQ3OPfvxThjUylAmV/82EIc8+UxdMQsmeH9Kvujh0DsOFixusPX9l5BAwBbP/uEQ9u6fv+baTDy625X6F2b+13AG0A0ME9VPw8IrvwM8GcR2YrTk5/lHv4k8CzwGY5551lVXResxqcnxtOkcKCq/si8esPwpw0XGYGJ9+BMILgNJzh4EdCoqm+KSA7wAVCC05FppGPmzOb6O2CWnMb+Lpiq1r61BT7P56rLphHr6brQm2kw8uvu0IIpVX0deD0gzT+4ci3OVMrA46paSw8WmX6rY03ojdbwer0UFLTMJSgsLARo8C+jqnuA6wFEZCBwg6qWu3kPAQ+5eX8F8oEDQLKIeNxefWvmzJBTXFnH4AGx3RJ5o28Q1ndIsxsEWzRltEVOTg75+fns2LGD+vp6FixYAHCU21MRSRWR5t/CfTgzcBCRaNeEg4hMAiYBb6rzmrAMx0wJjtnyX6G/mqMprqgz+7zRIcJb6JPcRVMm9EYbeDwe5s6dy4wZMxg/fjwzZ84EqBWRB0XkarfYNGCziGwBMnB78EAMzkyxDTjml5v97PI/BL7vmitTcMyXPUpJVZ2tijU6RNj6ugFIGRBHdJSY0BvHJTc3l9zc3CP7c+bMCTQ9vkzLDBr80mtxZt4cgztF+Kzgt7bjlFTUkp1mHiuN9gnrHn10lJCeEMe+cgspaPQtVJWSqjpbLGV0iLAWenBm3hSbGwSjj3GopoEGn5I20ITeaJ+wF3pbNGX0RZo7N9ajNzpCBAh9PPvMRm/0MVpWxdq0YqN9wl7o0xPjqaxtpKbeQgoafYcWPzfWozfaJ+yFPvPIXHobkDX6DsWV5rnS6DhhL/S2aMroixRX1jIwzkP/2LCeIW30EGEv9Jm2aMrogxRX2mIpo+OEvdCnW4/e6IOUVJjQGx0n7IU+Ic5D/9hoWzRl9CmKK2ttINboMGEv9CJCRmK8xY41+hQllXU2tdLoMGEv9AAZiXHst0VTRh+huq6R6nqfzbgxOkxECL0tmjL6EsU2h97oJBEh9BmJ8RRX1LUVTcgwIopit1Njg7FGR4kYoa/3NXGopqH9woYR5rT06M1Gb3SMiBF6sCmWRussXryYcePGkZ2dzSOPPHJMvoiMEJGlIrJORPJExOuX90sR+VxENorI70RE3PQ8EdksImvdLb2nrsdMN0ZniYhldc2LpvZV1DJ+SGIvt8Y4kfD5fMyePZslS5bg9XrJyckBCOwKPwa8oKrPi8h04GHgFhE5FzgPJ4QgwHvARUCeu3+Tqn4c8osIoLiyltjoKJL7x/R01UaYEhE9+uZX2GLr0RsBrFy5kuzsbEaPHk1sbCyzZs0CSA4oNgFY6n5fBlzjflech0IsEIcTWnB/6Ft9fErcVbHuy4VhtEtE9OibTTe2aMoIpKioiKysrCP7Xq8XHOH251PgBuAJ4DogQURSVHWFiCwD9gICzFXVjX7HPSsiPuDvwM+1ldkAInIncCdARkYGeXl5rbazqqqqzbxANu86TJzS4fLt0Zm6g0lv1dsX644IoY/1RJEyINamWBrH0MZMrMDEe4C5InIb8A5QBDSKSDYwHmi22S8RkQtV9R0cs02RiCTgCP0twAut1P8UTmBxpkyZotOmTWu1nXl5ebSVF8jDa97hpIz+TJs2pUPl26MzdQeT3qq3L9YdEaYbcEMKmtAbAXi9XgoKCo7sFxYWAhw1PUtV96jq9ap6OvA/blo5Tu/+Q1WtUtUqYBFwtptf5H5WAn+lBwOFm/sDo7NEjNBnJsZZj944hpycHPLz89mxYwf19fUsWLAAoMy/jIikikjzb+E+YJ77fTdwkYh4RCQGZyB2o7uf6h4bA3wR+Kwnrqe+0ZlGbFMrjc4QMUKfkRhvwUeMY/B4PMydO5cZM2Ywfvx4Zs6cCVArIg+KyNVusWnAZhHZAmQAD7npLwPbgPU4dvxPVfXfOAOzb4jIOmAtjqnnTz1xPSVVFnDE6DwRYaMHR+gPVtfR4GsiJjpinl9GEMjNzSU3N/fI/pw5c1DV+5v3VfVlHFE/ClX1Ad9sJb0aODM0rT0+FkLQ6AoRo4gZifGotvwQDCMSaR6HMtON0RkiRuj9F00ZRqTSvCrW/NwYnSFihN4WTRl9geLKOkQgdWDgUgDDaJuIEfrMpOZFUyb0RuRSUllLyoBYPDYOZXSCiLlbBvePJSZa2Gczb4wIpriijjSzzxudpENCLyJXuJ76torIva3kx4nIS27+RyIy0i9vkoiscD0ArheRkNylUVFCeoItmjIim5KqOptxY3SadoVeRKKBJ4Ev4Dh/ulFEJgQUuwM4pKrZwOPAo+6xHuAvwLdU9RSc+cohcxqfYYumjAinuMKE3ug8HenRnwVsVdXtqloPLKDFu18z1wDPu99fBi5x/XZfDqxT1U8BVPWgOzc5JDiLpkzojcikqUk5UFVni6WMTtORBVPDgAK//UJgaltlVLVRRMqBFGAsoCLyBpAGLFDVXwZWECwPf42VdRSVNobEM1xf83bXm/X2dt0nKqU19TQ2KWkDTeiNztERoW/N6XWg97+2yniA84EcoAZYKiKrVXXpUQWD5OFvI9tYsmsTU845n4FxwV3029e83fVmvb1d94lKcUWz+wMbjDU6R0dMN4VAlt++F9jTVhnXLp8ElLrpy1X1gKrWAK8DZ3S30W3RvGjKzDdGJFJc2bwq1nr0RufoiNCvAsaIyCgRiQVmAQsDyiwEbnW/fwl42w3C8AYwSUT6uw+Ai4ANwWn6sWS4087221x6IwIpsaDgRhdp177h2tzvwhHtaGCeqn4uIg8CH6vqQuAZ4M8ishWnJz/LPfaQiPwG52GhwOuq+lqIroUMd9HU/koTeiPyOBIU3AZjjU7SIUO2qr6OY3bxT/P3/lcLfLmNY/+CM58bIOEAACAASURBVMUy5FhIQSOSKamsIyHeQ3xMdG83xQgzImZlLMDAOA8D4zxmozcikuLKWnNmZnSJiBJ6cBZNmdAb/ixevJhx48aRnZ3NI488cky+iIwQkaUisk5E8kTE65f3S3dV90YR+Z27PgQROdNd6b3VPz2U2GIpo6tEoNDboimjBZ/Px+zZs1m0aBEbNmxg/vz5AIGjmY8BL6jqJOBB4GEAETkXOA+YBJyKM034IveYP+Cs/RjjbleE+lqKK+tsINboEhEn9JkWUtDwY+XKlWRnZzN69GhiY2OZNWsWQHJAsQlA89qOZbSs/Fach0IsTvjAGGC/iAwBElV1hTu77AXg2lBeh6pSUmk9eqNrRJzQpyfGU1xZS1NT4Jouoy9SVFREVlbLMhCv1wuOcPvzKXCD+/06IEFEUlR1BY7w73W3N1R1I85K8EK/4wvdtJBRVdfI4QafzbgxukTExIxtJjMxjgafUlpTT6otFe/zOB3uY5MD9u8B5orIbcA7OMG+G0UkGxiPs0gQYImIXAgc7sA5geC599hb1QTAgcId5OUVtFmuK5iLjcivO+KEvmWKZa0JvYHX66WgoEUYCwsLIcCDqqruAa4HEJGBwA2qWu6K9IeqWuXmLQLOBv5Mi/hD66vFm88dFPceK7YdhPc+5KKzJnNedmrbF9wFzMVG5Ncdcaab5kVTxbZoygBycnLIz89nx44d1NfXs2DBAoAy/zIikioizb+F+4B57vfdwEUi4hGRGJyB2I2quheoFJGz3dk2XwP+FcrrMPcHRneIOKHPtEVThh8ej4e5c+cyY8YMxo8fz8yZMwFqReRBEbnaLTYN2CwiW4AM4CE3/WVgG7Aex47/qar+2837NvA0sNUtsyiU12HuD4zuEHGmm7SEOETMsZnRQm5uLrm5uUf258yZE7iy+2UcUT8KN3bCN1s7p6p+jDPlskcoqawj1hNFYr+I+8kaPUDE9ehjoqNIGWCLpozIotidWtkD67KMCCTihB5sdawReRRX1pp93ugyESn0mYnx7LNFU0YEUVxRZ35ujC4TkUKfbm4QjAjD3B8Y3SEihT4zMZ7S6nrqGkMWh9wweozaBh/lhxvMdGN0mYgU+gx3mXixmW+MCOBAlQUcMbpHZAq9LZoyIohim0NvdJOIFHpbNGVEEs1vpjYYa3SViBT6Zn83NiBrRAIl5v7A6CYRKfSD+scQGx1lQm9EBMWVdUQJpJiTPqOLRKTQiwjpiXHsM6E3IoDiijpSBsYRHWWrYo2uEZFCD82RpkzojfDHVsUa3SVihT7DQgoaEUJJlYUQNLpHhAt9bVsRhgwjbCiusFWxRveIYKGPo6beR2VdY283xTC6jK9JOVBlfm6M7hGxQp/ZvGjK7PRGGHOwuo4mtVWxRveIWKHPsEVThsvixYsZN24c2dnZPPLII8fki8gIEVkqIutEJE9EvG76xSKy1m+rFZFr3bznRGSHX97kULS9ebGU2eiN7hCx4Wps0ZQB4PP5mD17NkuWLMHr9ZKTkwMQaPB+DHhBVZ8XkenAw8AtqroMmAwgIoNxwga+6Xfcf7nRqUJGcwjBNLPRG90ggnv0Tg/I5tL3bVauXEl2djajR48mNjaWWbNmASQHFJsALHW/LwOuaeVUXwIWqWpN6Fp7LC2xYq1Hb3SdiO3R94/1kBDvsR59H6eoqIisrKwj+16vFyA2oNinwA3AE8B1QIKIpKjqQb8ys4DfBBz3kIjcj/OQuFdVj7ETisidwJ0AGRkZ5OXltdrOqqqqVvM+3FYPwMY1H7EtOjQLptqqO9T0Vr19se6IFXqwRVMGbU2vDUy8B5grIrcB7wBFwJHpWiIyBJgIvOF3zH3APpyHxlPAD4EHW6n/KTefKVOm6LRp01ptZ15eHq3lvV3+GUmFe7j8kotbPS4YtFV3qOmtevti3REt9BkWUrDP4/V6KSgoOLJfWFgI0OBfRlX3ANcDiMhA4AZVLfcrMhP4p6o2+B2z1/1aJyLP4jwsgo6FEDSCQcTa6MERepte2bfJyckhPz+fHTt2UF9fz4IFCwDK/MuISKqINP8W7gPmBZzmRmB+wDFD3E8BrgU+C0X7zf2BEQw6JPQicoWIbBaRrSJybyv5cSLykpv/kYiMDMgfLiJVIhKSXk9bZCbFUVxZh6/JVsf2VTweD3PnzmXGjBmMHz+emTNnAtSKyIMicrVbbBqwWUS2ABnAQ83Hu/dyFrA84NQvish6YD2QCvw8FO13YsWa0Bvdo13TjYhEA08ClwGFwCoRWaiqG/yK3QEcUtVsEZkFPAp8xS//cWBR8JrdMTIS4/E1KQerbQl5XyY3N5fc3Nwj+3PmzEFV72/ed6dItjpNUlV3AsNaSZ8e/JYeUwcllXWkJ9q9a3SPjvTozwK2qup2Va0HFnDs9LNrgOfd7y8Dl7ivtLgLTLYDnwenyR3nyFx6WzRlhCEVtY3UNTZZj97oNh0ZjB0GFPjtFwJT2yqjqo0iUg6kiMhhnNkIl3GcwaruTkFri6IyHwBvfbCKg+ndG3fua9OxerPe3q77RKE5spQNxhrdpSPq19rk3UCjd1tlfgo8rqpVbge/Vbo7Ba0tTi6v5cEPl5I2fAzTzh7R4eOCUXcwselvfROLFWsEi44IfSHOYFQzXmBPG2UKRcQDJAGlOD3/L4nIL3FWIzaJSK2qzu12yztA6sBYosTcIBjhSfGRVbFmoze6R0eEfhUwRkRG4SwkmQV8NaDMQuBWYAXOUvG31VmpckFzARH5CVDVUyIP4ImOInVgnAm9EZYUNwcFN8+VRjdpV+hdm/tdOKsCo4F5qvq5iDwIfKyqC4FngD+LyFacnvysUDa6M9iiKSNcKamsIz4mioS4iF7XaPQAHbqDVPV14PWANP/pabXAl9s5x0+60L5uk5EYT+GhHvVDZRhBwZlDH8/xxrcMoyNE9MpYcBZNmQdLIxxxQgia2cboPhEv9BkJ8ZTVNFDb4OvtphhGpyiurLUZN0ZQiHyhPxJS0Oz0Rnhh7g+MYBH5Qt8cUtDMN0YYUdvgo7K20dwfGEEh4oU+00IKGmFISwhB69Eb3Sfihb45pKAJvRFOHJlDb0JvBIGIF/qkfjHEeaJM6I2wonlMyVbFGsEg4oVeRGzRlBF2FJvpxggiES/0YLFj+zqLFy9m3LhxZGdn88gjjxyTLyIjRGSpiKwTkTwR8brpF4vIWr+t1nW7jYiMcoPs5LtBdwIDjneL4spaoqOElAFBPa3RR+kTQp+RZELfV/H5fMyePZtFixaxYcMG5s+fDxBoD3kMeEFVJ+EE+H4YQFWXqepkVZ0MTAdqgDfdYx7F8cw6BjiEE3wnaBRX1DlO+aJsVazRffqG0CfEsa+8FsfPmtGXWLlyJdnZ2YwePZrY2FhmzZoFjidVfyYAS93vyzg2sA44zvoWqWqNG1RnOi1RqZ7HiRsbNEqqLCqaETz6hLekzKR46hqbqDjcSFL/mN5ujtGDFBUVkZXV4mXb6/UCBNpDPgVuAJ4ArgMSRCRFVQ/6lZkF/Mb9ngKUqWqju19IK+EGoetBdbbvOcygeOmR4CsW2Cby6+4TQp/ut2jKhL5v0cZbXGDiPcBcEbkNeAfHHXeziCMiQ4CJOB5coWPBeJrr71JQnZr33uK8UelMmzap1fLBxALbRH7dfcJ0Y4um+i5er5eCgpZImIWFhQAN/mVUdY+qXq+qpwP/46aV+xWZCfxTVZuPOwAku0F2oPVgPF2m0dfEweo60gbajBsjOPQJoR+R0h8RmPf+Duobm3q7OUYPkpOTQ35+Pjt27KC+vp4FCxYAlPmXEZFUEWn+LdwHzAs4zY3A/OYdN6jOMhy7PThBd/4VrDYfrK5HFdLM/YERJMJH6BsOk1T2WZcOzUiM5+fXnkre5hK+97e1+JpsULav4PF4mDt3LjNmzGD8+PHMnDkToFZEHhSRq91i04DNIrIFyAAeaj5eREbihMlcHnDqHwLfd4PtpOAE3wkKLYulrEdvBIfwsdEveYBJ656F86+A1OxOH37T1BFU1Tby8KJNJMR5ePj6iRbQoY+Qm5tLbm7ukf05c+YEBs55mZYZNEehqjtpZaBVVbcDZwW9sZj7AyP4hE+P/oLv0xQVAwvvgqaumV++edFJ3HVxNgtWFfDQaxttuqVxQtLs0Mw8VxrBInyEPiGTbSfdDrtXwMddf0v+weVjue3ckTz93g5+t3RrEBtoGMHhiPsDG4w1gkT4mG6AfZmXcHLj57DkARhzOQwa0elziAj3f3EClbWNPP7WFhLiPdx+/qgQtNYwukZxZS2D+scQ6wmffphxYhNed5IIXPWE8/nvu6GLppeoKOHRGyZyxSmZPPjqBv62qqD9gwyjhyiuqDNnZkZQCS+hB0geDpf+BLYvg7Uvdvk0nugonrhxMheMSeXef6zjtXV7g9ZEw+gOTghBs88bwSP8hB5gyh0w/Fx440dQua/Lp4nzRPP/bjmTM4YP4rsvrSFvc3EQG2kYXaPEYsUaQSY8hT4qCq7+PTTWwWs/6LIJB6B/rIdnbsthTHoC3/rLalbuKA1iQw2jc6gqJZV1pCWa0BvBIzyFHpy59NPug02vwuf/7NapkvrF8Oc7zmJYcj9uf24V6wrL2j/IMEJA+eEG6n1NZroxgkr4Cj3AOXfBkMnw+n9B9cH2yx+HlIFx/OXrU0nqF8Ot81aSv78ySI00jI7TPLXSTDdGMAlvoY/2wDVPQm0ZLL6326cbktSPF78+FU90FDc9/RG7D9YEoZGG0XGa3R/YrBsjmIS30ANkngoX3APr/wabF3f7dCNTB/CXO6ZS72vipmc+ZF+5ebw0eg5zf2CEgvAXeoALfgDpE+DV70Ftefvl22FcZgLP/8dZlFbVc/MzH1FaXR+ERhpG+xSb+wMjBESG0Hti4Zq5ULUP3vxxUE55WlYyT9+aQ0FpDbfOW0lNg/nFMUJPSWUd/WOjGRgXVovWjROcyBB6gGFnwjmz4ZPnYXugR9mucc5JKfzh5jPYuLeCh1fW8llR998WDON4FNsceiMERI7QA0z7EQweDQv/E+qrg3LK6Sdn8KevTaGiXrnmyff51RubqG3wBeXchhFIcUWtTa00gk6HhF5ErhCRzSKyVUSOmd4iInEi8pKb/5EbrAERuUxEVovIevdzenCbH0Bsf7h6LpTtgrd/HrTTXnxyOr84vx/XnT6MJ5dt48rfvcvqXYeCdn4jtCxevJhx48aRnZ3NI488cky+iIwQkaUisk5E8kTE65c3XETeFJGNIrLB795+TkR2iMhad5scjLaWVJqfGyP4tCv0IhINPAl8AZgA3CgiEwKK3QEcUtVs4HHgUTf9AHCVqk7ECbf252A1vE1Gnue4SPjwD1CwMminHRAjPPbl03j+9rOobWjiS3/8gJ/++3Nq6hvbP9joNXw+H7Nnz2bRokVs2LCB+fPnAwR2mR8DXlDVScCDwMN+eS8Av1LV8TiBRvz9ZPyXqk52t7XBaG+xCb0RAjrSoz8L2Kqq21W1HlgAXBNQ5hrgeff7y8AlIiKqukZVm4Mmfw7Ei0jo7+LLfgpJXvjXbGgI7vTIi8am8cb3LuSWs0fw7Ps7mfHbd3h/64Gg1mEEj5UrV5Kdnc3o0aOJjY1l1qxZAMkBxSYAS93vy3Dvb7dD41HVJQCqWqWqIVtcUVPfSFVdI+nm/sAIMh0Z2h8G+PvxLQSmtlVGVRtFpBwnjqa/At4ArFHVuq43t4PEJcBVv4W/3ADv/BIuub/9YzrBwDgPD15zKl+cNJQf/n0dNz39EbNysvjRleNJjI8Jal1G9ygqKiIrK+vIvtfrBYgNKPYpzv35BHAdkCAiKcBYoExE/gGMAt4C7lXV5kGah0TkfpyHxL2t3dsicidwJ0BGRgZ5eXmttrOqqopX33oHgNKiHeTlFXblcrtEVVVVm+2KxHr7Yt0dEfrWAqsGzjU8bhkROQXHnHN5qxV04sfQ8T+Qh5MzppPx7uOsrvFSlTC6g8e1Tlt133e68srWGF5aVcDidYXcekosp6cHd2qc/RC7zmeffcbevXuPnGvjxo1w7P17DzBXRG4D3gGKgEac38cFwOnAbuAl4DacQOD3AftwHhpP4QQLfzCwflV9ys1nypQpOm3atFbbmZeXR8bISfDOCi7MmcyFY9O6eMWdJy8vj7baFYn19sW6O6JIhUCW374X2NNGmUIR8QBJQCmAO7D1T+BrqrqttQo682Po1B/orEnw5FSmFM2DbyyD6K73to9X9+WXwLrCMv775XU88Ukl10xO4YGrTmHwgMCOY/DrDiWR8GOIi4tjxYoVR861YsUKgAb/Mq558XoAERkI3KCq5SJSiPMWut3NewU4G3hGVZsDGNSJyLM4D4tu0ez+wEw3RrDpiI1+FTBGREaJSCwwC1gYUGYhzmArwJeAt1VVRSQZeA24T1XfD1ajO0z/wXDlr2Hfenj/iZBWNcmbzMK7zud7l47l9fV7ufQ3y1n46R4LQN7L5OTkkJ+fz44dO6ivr2fBggUAR7knFZFUEWn+LdwHzHO/rwIGiUhz93o6sME9Zoj7KcC1wGfdbWuz+wOLFWsEm3aFXlUbgbuAN4CNwN9U9XMReVBErnaLPQOkiMhW4PtA8xTMu4Bs4Md+09DSg34Vx2PC1TDhGlj+KGxe1C3f9e0R64ni7kvH8Op/XkDWoH58Z/4avvHCavZXmL+c3sLj8TB37lxmzJjB+PHjmTlzJkBtwP07DdgsIluADOAhANcWfw+wVETW45go/+Qe86Kbth5IBbo9n7e4sg5PlDCof3DeBA2jmQ4Zk1X1deD1gLT7/b7XAl9u5bifE4QfQLfJfQz2fw7zZ8HIC+CyB2HYGSGrblxmAv/4P+cx770dPPbmZqY/lsfVk4fxlZwsTvMm4XQCjZ4iNzeX3NzcI/tz5swJvH9fxpktdgzujJtJraQHfU1Ic6zYqCi7P4zgElkrY9tiYDp8ewV84VdQvAH+dDG8fDuU7ghZldFRwjcuHM0b372QK04dwj/XFHLtk+/zhSfeZd57OzhkjtKMAEqqzP2BERr6htCD4/hs6p3wnbWOW+NNr8PcHFh8H9SELnzgyNQB/Hrmaaz8n0t56LpTifNE8eCrG5j6i6XM/usnvLOlhKYms+MbjvuDNHN/YISAvuciLz4RLvkx5NwBeQ/DR3+ENS/C+d+Fs78NMf1CUm1ifAw3TR3BTVNHsGlfBS+tKuCfa4p4bd1ehiX340tnevnyFC/eQf1DUr9x4lNSWccZIwb1djOMCKTv9OgDSRzqBBj/9gcw4hxY+lP4/ZmO6DeF1mnZyZmJPHDVKXz0o0uY+9XTGZ02gN+9nc8Fv1zGLc98xKvr9lDXaI7T+hKNTcrB6nqbcWOEhL7Xow8kfTx89SXY+Z7jy/5f/wdWPOkM2GZfAiEcOI3zRPPFSUP54qShFB6q4X8/LuTl1YXc9dc1DOofw7WnOwO4RuRTUe+Y72wOvREK+m6PPpCR58M33oYvPQsN1fDiDfDCNbAnKL6q2sU7qD/fu2ws7/z3xbxw+1mcm53Kix/u5orfvssDHxzmmfd2HJlnbUQeZXWu0JuN3ggB1qP3RwROvR5O/iKsftaZe//URTDxyyR6cuDwZOgX6A8ruERHCReOTePCsWkcqq7nlbVFvLB8Ez97dQMPvbaBC8akcd3pw7j8lAz6x9q/L1IoPyL01qM3go8pRWt4YmHqN+G0Wc6K2hX/lzMa/xfW/Df0T4WUkyAl2wlykpLt7A8eDbEDgtqMQQNi+Y/zRjGqYRfeCWfyypo9/HNNEd99aS39Y6O54pRMrjtjGOeelEq0zb0Oa44IvZlujBBgQn884pMcz5dTv8X6xc8ycUg8HNzmbNvehrUvHl0+Yaj7EGh+ELifg0Y6D49ukJ2ewD0zxvH9y8by8a5D/HNNIa+u28s/1hSRnhDHNZOHcu3pw5gwJNEWZIUhZXWKCKTaYKwRAkzoO8LAdA6mToXzph2dXlcFpdvh4FYo3dbyENiwEA4HzM3vNwgGpLlbahvf3f345DYHgaOihLNGDeasUYN54KpTWLapmH+sKeK5D3byp3d3MDZjINed7uWayUMZmhyaqaJG8CmrUwb3jyUm2obNjOBjQt8d4gbCkEnOFsjhQ3DQfQgc2gnVJe52AIo3QfW7xz4MmomKcR8AqTAgnbE1HohZ12IiGjQKYuKJj4nmCxOH8IWJQzhUXc+r6/fyypoiHl28iV++sYmzR6XwhYmZnDF8EOMyE9oWkcZ654F1YDMc2AIlW+DAZs49sBO2nARpJ0PaWPdzHCSPgKjoYP0VDRzTjUWWMkKFCX2o6DcIvGc6W1v4GqDm4NEPgSPf3f2q/aSVbIUlb/gdKE4ErcGjHPPQ4NEMSjmJW0afxC1nnsGuCp9rzy/k/n99DkB8TBRTh8QwLbWcM/oXc5IUMbBiuyPspdtB/ebtJ2VB6lgOpqQxJK4Bti+DT//aku+Jh5Qxjug3b6njnIdQeyYqXyNUF0PlPqja38rnXqjcz3mHK2HTSKctSVnO9SZntewPSIOoyOn9ltUp3nQTeiM0mND3JtExkJDpbMfh/bw8pk09zTUTbXfMRKXbXTPRvwLeDIQRSV7uHjya75w8mqrGKGr3bCCubCuJxSVHIp7WazTbo4ZS1n8kDL+QpOGnMiz7NOIzxzlvKsDmvDyGNPuEP1zm9vY3Q8km57NwJXzm5wssyuM8eNLGOsIPULUPKve3fFaXcGzcD5xB7oRMGJgB6RMoLj7IsAHqvA3teBfqKwP+dnGQNKxF+JPdh0GS32c3x0V6kvI65QybWmmECBP6cKHfIBh2prMFUlPqOGgrPfohIBteIcHXSELqGJhwKaSOpWFwNvlNQ1lZlsgnhdWsLShj9+Ya2AzRS/dzcmYNk7OSOX34IOqrmmhqUsebYr9kyDrL2fypr24x9zQ/APZvgE2vOfkD0iEhwxmoHno6JAxxxDwhEwZmOnkD0o8R5fy8PIb5Bx45XAblhVBe4HyW7W7Z37bUeSPwf4Bc9xSc9pWg/OlDjapSXqc248YIGSb0kUD/wc52PDORSwxOJOwJODHxAA5W1bG2oIw1u8tYW1DGwrV7ePGj3QD8YtWbnDoskUneZCYOS+I0bzJZg/u1zOyJHeAI+NDTj66osd6x4wfLlt8v2dkyT209v7EeKopaHgTDzw5OvT3AoZoGfGpz6I3QYUJvkDIwjkvGZ3DJ+AwAmpqUbSVVzF/yIQ0Dh7CuqJzn3t9Jva8JgKR+MUzyJjFxWJLz6U1maFL80dM6e9ps4ol1xyxGHZO1ePFi7r77bnw+H1//+tePyReREThRpdJwQmDerKqFbt5w4GmcUJkK5KrqThEZBSwABgOfALeoapd8TzeveLZVsUaoMKE3jiEqShiTkcCF3himTXN60PWNTWzZX8m6wnLWF5WxrrCcp97ZTqPrYjl1YCwThzmiP8l9AKQn9r5w+Xw+Zs+ezZIlS/B6veTk5AAENuwx4AVVfV5EpgMPA7e4eS8AD6nqEjeebJOb/ijwuKouEJE/AncAf+hKG5tjxdqsGyNUmNAbHSLWE8Wpw5I4dVgSMByA2gYfG/dWsL6o3HkAFJazfEs+ze71hybFc/rwQZw+PJnThydzytAk4mN6dlrmypUryc7OZvTo0QDMmjWLdevWBfqxmAB8z/2+DHgFQEQmAB43yhSqWuWmC0782K+6xzwP/ISuCn2lGxTchN4IESb0RpeJj4l2hbzFh3pNfSMb9lTwaWG5a/c/xGvr9wIQEy1MGJJ4RPzPGD4I76B+IV3JW1RURFZWiwdQr9cLEGhX+hS4AXgCuA5IEJEUYCxQJiL/AEYBb+HEQx4ElLnxlAEKgWGt1S8idwJ3AmRkZJCXl3dMmQ+3OxafzWtXstPT86uaq6qqWm1XpNbbF+s2oTeCSv9YD1NGDmbKyMFH0oora1m7u4w1BWV8susQL60q4LkPdgKOyWdyVkuv/zRvMgPigndbauvB4AMT7wHmishtwDtAEdCI8/u4ADgd2A28hDOGvbAD52yu/yngKYApU6boNP+ZRC7LKz8nfttOZlx6cXuXExLy8vJorV2RWm9frNuE3gg56QnxXH5KJpef4qwXaPQ1sXl/JWt2OzN91hQc4q2N+wGIEhibkUBqdB1rG7cwNLkfw5L7MTS5H0OS4jtt+vF6vRQUFBzZLywsBGjwL6Oqe4DrAVw7/A2qWi4ihcAaVd3u5r0CnI0zcJssIh63V+8F9nT+L+NQXFlHcpz5JzJChwm90eN4oqM4ZWgSpwxN4uazRwBQVlN/ZIrnmoIy1u2q4r238o85NnVg7FHi73yPP/I9ZUDsUaagnJwc8vPz2bFjB8OGDWPBggUAZf7nFJFUoFRVm4D7cIQcYBUwSETSVLUExy7/saqqiCwDvoQz8+ZW4F9d/XuUVNSRZEJvhBATeuOEILl/LNPGpTNtXDrgvN6ec/4F7C+vo6jsMHuat/LDFJXVsmV/JXmbSzjccHTIxVhPFMOS+/HjL45n+skZeDwe5s6dy4wZM/D5fNx+++2sW7euVkQexBHthcA04GERURzTzWwAVfWJyD3AUncAdjXwJ7eqHwILROTnwBrgma5ee3FlLekm9EYIMaE3TljiPNEMT+nP8JTWA6arKmU1DQEPglqKyg4zqH/LeGtubi65ublH9ufMmYOq3u93npeBl2kFd8bNMV7rXHPOWcce0XnOzU7FU7kvGKcyjFYxoTfCFhFh0IBYBg2Idad9hie/uG4ieXkHe7sZRgQTOe7/DMMwjFYxoTcMw4hwTOgNwzAiHBN6wzCMCMeE3jAMI8IxoTcMw4hwTOgNwzAiHBN6wzCMCEfa8O7Xa4hICbCrjexU4EAPNqev1x2p1zxCVdNCdO42sXv7hKk3Uutu874+4YT+eIjIx6o6xeqO7Hp7u+7eoC/+nZNmpAAAAxFJREFUrfviNfdW3Wa6MQzDiHBM6A3DMCKccBP6p6zuPlFvb9fdG/TFv3VfvOZeqTusbPSGYRhG5wm3Hr1hGIbRSUzoDcMwIpywEXoRuUJENovIVhG5twfrzRKRZSKyUUQ+F5G7e6put/5oEVkjIq/2cL3JIvKyiGxyr/2cHqz7e+7f+jMRmS8i8T1Vd0/TV+9rtw196t7uzfv6/7d3765RBFAUh3+HRMFEBFsTIRFEDYJELIIBC2MjirYKWlj7BEHQv0FEC7EJ2hiwiCksRC3sgxgF0TQSxaxGTOMDmxg8FjMLKVIp3mFn7lftTnNmlzOXeTHTEYNeUhdwEzgIDAHHJQ0FxS8DF23vAEaA04HZAOeB2cC8thvAI9vbgV1R6yCpDzgH7LG9E+gCjkVkR2t4r6FB3a661x0x6CnezfnW9pztJeAecDQi2PaC7Zny8w+KUvRFZEvqBw4B4xF5K3I3APsoX3hte8n218BV6AbWSeoGeoBPgdmRGtlraGy3K+t1pwz6PmB+xfcWgaVskzQADAPTQZHXgUvA76C8ti3AInCnPLQel9QbEWz7I3AV+AAsAN9sP4nIrkBTew0N63bVve6UQa9VloXeFyppPXAfuGD7e0DeYeCL7ef/O2sV3cBu4JbtYeAnEHL+WNJGir3aQWAT0CvpRER2BRrX6zKzcd2uutedMuhbwOYV3/sJPOyRtIZiY5iwPRUUOwockfSe4pB+v6S7QdktoGW7vYc3SbFxRDgAvLO9aPsXMAXsDcqO1sReQzO7XWmvO2XQPwO2ShqUtJbiIsaDiGBJojifN2v7WkQmgO3LtvttD1D83qe2Q/YAbH8G5iVtKxeNAW8isikObUck9ZT//RjVXLCL0LheQ2O7XWmvu6OC/oXtZUlngMcUV6tv234dFD8KnAReSXpZLrti+2FQflXOAhPlAJoDTkWE2p6WNAnMUNwZ8oKaPg4he12Z8G5X3et8BEJKKdVcp5y6SSml9Jdy0KeUUs3loE8ppZrLQZ9SSjWXgz6llGouB31KKdVcDvqUUqq5PyAyF3/azL4pAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(1,2,1)\n",
    "plt.plot(training_loss)\n",
    "plt.plot(testing_loss)\n",
    "plt.grid(True)\n",
    "plt.title(\"Loss\")\n",
    "plt.subplot(1,2,2)\n",
    "plt.plot(training_accuracy)\n",
    "plt.plot(testing_accuracy)\n",
    "plt.grid(True)\n",
    "plt.title(\"Accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(cn, \"mnist_hitnet.ptch\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "cn.eval()\n",
    "random_sample = test_img[np.random.choice(range(test_img.shape[0]), 2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f4cfc719828>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAC4CAYAAAD61bdSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAP40lEQVR4nO3df5BV9XnH8c/DsqCiVomKCCSggvFnoG6RxtJojRlitGobU2lHqbFB02i1Y50S/qhJZtIYW39NdGIgIJhYNY0aSQc1zMYpof6Ii/E3/iCEKmFlg9BKfojs8vSPvXZWvt/r3r2/n7vv1wyz9z77Pfc8h3149nDO95xj7i4AQDwjGp0AAKA8NHAACIoGDgBB0cABICgaOAAERQMHgKAqauBmNsfMXjaz9Wa2oFpJAY1GbSMCK3ceuJm1SXpF0umSNkl6UtJcd3+x2DKjbLTvpTFlrQ8YzNv6jd7xnVbp51DbaDbFantkBZ85U9J6d98gSWZ2t6SzJRUt8r00RifZaRWsEijuCe+s1kdR22gqxWq7kkMoEyS9PuD9pkLsPcxsvpl1mVnXLu2sYHVA3VDbCKGSBp77r2pyPMbdF7l7h7t3tGt0BasD6obaRgiVNPBNkiYNeD9R0ubK0gGaArWNECpp4E9KmmpmU8xslKTzJa2oTlpAQ1HbCKHsk5ju3mtml0l6WFKbpKXu/kLVMgMahNpGFJXMQpG7r5S0skq5AE2D2kYEXIkJAEHRwAEgqIoOoQDAQOu/MyOJ/fy025PYtP+cl11+ytxnqp5TK2MPHACCooEDQFA0cAAIigYOAEFxEhNA1Xz8qJeS2C7vS2JnTMtfF7Wu6hm1NvbAASAoGjgABEUDB4CgaOAAEBQNHACCYhYKgKEb0ZYNt1l5D0lHedgDB4CgaOAAEBQNHACCooEDQFAVncQ0s42Sdkjqk9Tr7h3VSApoNGr7/fX98Uey8ZsPW1zS8q+8dUiR72wuM6PhqRqzUE51961V+Byg2VDbaGocQgGAoCpt4C7pR2a21szmVyMhoElQ22h6lR5COdndN5vZIZJWmdlL7r564IBC8c+XpL20T4WrA+qG2kbTq2gP3N03F772SLpf0szMmEXu3uHuHe0aXcnqgLqhthFB2XvgZjZG0gh331F4/QlJX6laZi1gxH77ZeM9c49LYttmpDe9//Kp92WXv3D/9LzatNUXZsceec1vk1jfKz/PjpVzGbREbe/J2kclse7Ld5a8/K93p2N/fdPE7Ni9mYUyJJUcQhkn6X4ze/dz/s3dH6pKVkBjUdsIoewG7u4bJOUngwKBUduIgmmEABAUDRwAguJ+4FXSduxRSezI5RuyY1eMv6Wide3KnGt8Yfbt+cE/TkMd112eHXrozY9WkBVa1Ygxeyexp2fdUfLyn9v4p0ls7wd+WlFO6MceOAAERQMHgKBo4AAQFA0cAIKigQNAUMxCGaK3z0puiSFJuvPWG5LYuLb07L0krfjNgUnsH3/4l0nsgBet5Ly2H5u/DP76s76bxFZffX127Jyev09i+9/1eMk5oDVtPeeYTDQzvamIn609MokdKW6zXg3sgQNAUDRwAAiKBg4AQdHAASAoTmK+j5ETJySxSV98JTs2d8Lyzh3js2Nvve7Pk9gRSx8bYnbv9YEi8UXfmpPEPt6ZntiUpLempL/P968kKYTSNu2IbPz8f3i4os89/PtvV7Q8imMPHACCooEDQFA0cAAIigYOAEHRwAEgqEFnoZjZUklnSupx9+MKsbGS7pE0WdJGSZ9x9+21S7Mxts2elMR+8KFbS17+nk//STY+9vnKZpwMRd/L65PYTdum1239zWw413ZOz+xDsvHLD3y15M+Y9dTcJHbQo8+VnRPeXyl74Msk7TkXbYGkTnefKqmz8B6IZpmobQQ2aAN399WStu0RPlvS8sLr5ZLOqXJeQM1R24iu3GPg49y9W5IKX/P/95JkZvPNrMvMunZpZ5mrA+qG2kYYNT+J6e6L3L3D3TvaNbrWqwPqhtpGo5V7Kf0WMxvv7t1mNl5STzWTahW2qbvRKWSt2Zq/ZPorf51eYv/VbX+VHXvwbfU7EVtnw7a2t59a+iXv3X2/y8bHXjcmDe7uKzclDKLcPfAVkuYVXs+T9EB10gEajtpGGIM2cDO7S9Jjko4ys01mdrGkayWdbmavSjq98B4IhdpGdIMeQnH3dGJnv9OqnAtQV9Q2ouNKTAAIigYOAEHxQIf38Xv/3pXE/nnB8dmxCw9KLxe+6ZmV2bHz//bKJLZ3Z2WXG7/z0WPz31j4qyT00FH3l/y5ay/5r3z8Nn73RzbiI0cnsQdn31Jk9F5JZMn2k/Kf+5OfVZKWRpzw4SQ2cfHr2bH7j0xnwjz4i2OyY/d5aL8k9oFvx59Jxb9CAAiKBg4AQdHAASAoGjgABMVJzPfhvb1J7LG/OTE79kd3bUhin0gfVC9JWrXotoryysufbByK2c/8RRI74J/SE1j9nq94fWicnQfvk8SmjCz2s059/+6PZeMT9WgaHNGWhDZ+eWZ2+c55/5LExrUV+YeUce2hT2bjv/iD9DYBn9/wd9mxI3+8tuT1NRp74AAQFA0cAIKigQNAUDRwAAiKk5hD5E/mr5j8xtzzktgXZ6dXf0nSZy/OX6FZqsmjtiaxT+3zvyUv//U381dt5k5YehcnK1vRaxftrmj5tiE8gKjn0vSqzec/+40io0s/YTkUuRO013x7SXbstZ/8dBLLPRy8GbAHDgBB0cABICgaOAAERQMHgKBo4AAQ1KCzUMxsqaQzJfW4+3GF2JckfU7SuzebXujulU2tCC43O2V8/qpePXjDARWta9O9JyexT826Izu283fpJdNrTtw/O9Z3Da8ZJ9R2abZmnkA/6Z6N2bHpzSckm/NmdROqkj8c3ZeN944dk8Ss1smUqZQ98GWS5mTiN7r79MKfYV3gCGuZqG0ENmgDd/fVkrbVIRegrqhtRFfJMfDLzOxZM1tqZgcWG2Rm882sy8y6dmkIs/+BxqG2EUK5Dfybko6QNF1St6Triw1090Xu3uHuHe0aXebqgLqhthFGWZfSu/uWd1+b2WJJ/1G1jPD/Rk44LBu/9MNrSv6Mzz9yYRKbtqvI2VVQ2xm7MrHeX27Ojt09e0YSWzE9d9l8bS6ZH27K2gM3s/ED3p4r7u6PFkFtI5JSphHeJekUSQeZ2SZJ10g6xcymS3JJGyVdUsMcgZqgthHdoA3c3edmwvnbeAGBUNuIjisxASAoGjgABMUDHZrYxgsnZ+OXHvDDkj9j2hLmJ6MyY0ekbeKNKz6aHdv3sfTBIkN5qjyGhj1wAAiKBg4AQdHAASAoGjgABMVJzCZ26GmbSh4766nclGbp4K4Xk5iXnRGGo9HWnsQeuuq67Nj8Hbab8yTmCd+6PBuf/OwzSWx3rZMpE3vgABAUDRwAgqKBA0BQNHAACIoGDgBBMQulWcw8PgmtPHppkcHp793fvp1/Ioz35p4TjuFu97ZRSSz39HlJOihzKXwu1gyufuOkbPyZhdOT2AdXPZEdu3t3fi5NM2IPHACCooEDQFA0cAAIigYOAEGV8kzMSZLukHSo+q8oXeTuN5vZWEn3SJqs/mcHfsbdt9cu1RY3wtIQv19rajjX9tTL0xN4p26/Ojv2uYtvqXU6Zfn6m8cmsa6vnZgdO+bh/AnL6ErpEL2SrnL3oyXNkvQFMztG0gJJne4+VVJn4T0QCbWN0AZt4O7e7e5PFV7vkLRO0gRJZ0taXhi2XNI5tUoSqAVqG9EN6f/oZjZZ0gxJT0ga5+7dUv8/BEmHFFlmvpl1mVnXLvF4LzQnahsRldzAzWxfSfdKutLd3yp1OXdf5O4d7t7RrvzFJkAjUduIqqQGbmbt6i/wO939vkJ4i5mNL3x/vKSe2qQI1A61jchKmYVikpZIWufuNwz41gpJ8yRdW/j6QE0yBGqE2n6vw298KRs/c+VFSeyVS9KHPBRz1vHPJrE/O3BtduxFq9N1TV2cvx3EyJdfT2Jj3mzN2SbFlHIvlJMlXSDpOTN7uhBbqP7i/p6ZXSzpNUnn1SZFoGaobYQ2aAN39zWS0knK/U6rbjpA/VDbiI4rRQAgKBo4AATF/cABSJL6the5W8DjaXza46V/7suZ2Nd0QnbsNOVPbubEuWt37bAHDgBB0cABICgaOAAERQMHgKBo4AAQFA0cAIKigQNAUDRwAAiKBg4AQdHAASAoLqVvFS/s1+gMANQZe+AAEBQNHACCooEDQFA0cAAIatAGbmaTzOwRM1tnZi+Y2RWF+JfM7Jdm9nThzxm1TxeoHmob0ZUyC6VX0lXu/pSZ7SdprZmtKnzvRnf/19qlN3yMfON/ktjv//SCkpc//Pb0Cd1S/w8PRVHbCK2Uhxp3S+ouvN5hZuskTah1YkCtUduIbkjHwM1ssqQZkp4ohC4zs2fNbKmZHVhkmflm1mVmXbu0s6JkgVqhthFRyQ3czPaVdK+kK939LUnflHSEpOnq34u5Precuy9y9w5372jX6CqkDFQXtY2oSmrgZtau/gK/093vkyR33+Lufe6+W9JiSTNrlyZQG9Q2Ihv0GLiZmaQlkta5+w0D4uMLxxAl6VxJz9cmxeGhd+NrSeywc4ewfBVzGS6obURXyiyUkyVdIOk5M3u6EFsoaa6ZTZfkkjZKuqQmGQK1Q20jtFJmoayRZJlvrax+OkD9UNuIjisxASAoGjgABEUDB4CgaOAAEBQNHACCooEDQFA0cAAIigYOAEGZu9dvZWa/kvTfhbcHSdpat5XXD9vVOB9y94MbseIBtR3h76lcrbptEbYrW9t1beDvWbFZl7t3NGTlNcR2DW+t/PfUqtsWebs4hAIAQdHAASCoRjbwRQ1cdy2xXcNbK/89teq2hd2uhh0DBwBUhkMoABAUDRwAgqp7AzezOWb2spmtN7MF9V5/NRWeWN5jZs8PiI01s1Vm9mrha/aJ5s3MzCaZ2SNmts7MXjCzKwrx8NtWS61S29R1nG2rawM3szZJt0r6pKRj1P/oqmPqmUOVLZM0Z4/YAkmd7j5VUmfhfTS9kq5y96MlzZL0hcLPqRW2rSZarLaXiboOod574DMlrXf3De7+jqS7JZ1d5xyqxt1XS9q2R/hsScsLr5dLOqeuSVWBu3e7+1OF1zskrZM0QS2wbTXUMrVNXcfZtno38AmSXh/wflMh1krGvftE88LXQxqcT0XMbLKkGZKeUIttW5W1em231M++Veq63g089wBZ5jE2KTPbV9K9kq5097canU+To7aDaKW6rncD3yRp0oD3EyVtrnMOtbbFzMZLUuFrT4PzKYuZtau/yO909/sK4ZbYthpp9dpuiZ99q9V1vRv4k5KmmtkUMxsl6XxJK+qcQ62tkDSv8HqepAcamEtZzMwkLZG0zt1vGPCt8NtWQ61e2+F/9q1Y13W/EtPMzpB0k6Q2SUvd/at1TaCKzOwuSaeo/3aUWyRdI+kHkr4n6YOSXpN0nrvveUKoqZnZH0n6iaTnJO0uhBeq/3hh6G2rpVapbeo6zrZxKT0ABMWVmAAQFA0cAIKigQNAUDRwAAiKBg4AQdHAASAoGjgABPV/YRDZ2/z5FJwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(1,2,1)\n",
    "plt.imshow(random_sample[0].reshape(28,28))\n",
    "plt.subplot(1,2,2)\n",
    "plt.imshow(random_sample[1].reshape(28,28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "_,cl,rec, = cn(Variable(torch.from_numpy(random_sample.reshape(2,1,28,28)/255).type(torch.FloatTensor)).cuda())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 10])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cl.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "rec = rec.cpu().data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f4ceb59b4e0>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAC4CAYAAAD61bdSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAXsklEQVR4nO3da4yc5XUH8P+Z2Zld73ptr732rr22sQ0bbHBak2wIKUElpQSSonKpoPGHiA8ojqKQgoIqWfRDUKtKUVugaVVFcoRlR3K4qFyFUAJxgwxNS1juYBvbGGOvb+vb3ndn53L6YQfVcM7Lzu5cn+H/kyKvj5+Zed7ZZw5v5jwXUVUQEVF4YtXuABERzQ4TOBFRoJjAiYgCxQRORBQoJnAiokAxgRMRBaqoBC4i14vI+yJyQEQ2l6pTRNXGsU0hkNnOAxeROIB9AK4F0AfgVQAbVXV31GOS0qhNaJnV6xFNZwKjmNSUFPs8HNtUa6LGdkMRz3k5gAOqehAAROQRADcCiBzkTWjBV+WaIl6SKNorurNUT8WxTTUlamwX8xVKF4Aj5/29Lx/7BBHZJCK9ItKbRqqIlyOqGI5tCkIxCdz7v6rm+xhV3aKqParak0BjES9HVDEc2xSEYr5C6QOw4ry/LwdwrLjuENUEju3ZkgJLEBJx76i54l7/c7a3UzF34K8C6BaR1SKSBPAdAM+UpltEVcWxTUGY9R24qmZE5E4AvwEQB7BVVd8rWc+IqoRjm0JRzFcoUNXnADxXor4Q1QyObQoBV2ISEQWKCZyIKFBFfYVCRIHyZotEzAyJJRO2aTLpP21Lsw02OGkmk/H7FY/7cU/CPq/GI+5Jzw2aUG5oxG2qmXThfajyrBfegRMRBYoJnIgoUEzgRESBYgInIgoUi5hE9WIGhUmJ2bbS6O/nIs22MClznWIlAJ1jnyPXbAue2Tm2MAoAmRabkjJz/GtIzbPxlhN+AXJOn+1DLOYXTHV42May/hJ/TU+68UrhHTgRUaCYwImIAsUETkQUKCZwIqJAMYETEQWKs1CIQhN1aIIz40Sc5eZTT+HMQmmKmIXizDhJL2tz24512udIN9t+ZZvchyM13/Yr4094QdJOFkHjcMTMkgbbh1hDxLL9uc7h1KnCZ5tELsUvw7J73oETEQWKCZyIKFBM4EREgWICJyIKVFFFTBE5BGAYQBZARlV7StGpzyVnWa+33BkANOcUQ6JO8/6cndJdKjUztmeyPN7ZS1u8vbjh7+ctrXPdttlFrSY2vsQveI4ss32YaLft0mvG3ccvXmQrk2cG/H7lPphjYq2H3abIOsv5Y3P8Sqr7qYvcv9z5XWhEITWbdYLFfT5LMQvlG6p6ugTPQ1RrOLappvErFCKiQBWbwBXA8yLymohsKkWHiGoExzbVvGK/QrlSVY+JyBIAL4jIXlXddX6D/ODfBABNiJiRT1R7OLap5hV1B66qx/J/9gN4EsDlTpstqtqjqj0J+IUPolrDsU0hmPUduIi0AIip6nD+528C+PuS9Sw0ziySeNt8t2n/LRebWOttx0zsr7t63cc3xexS3f888WW37cGdq01s9Q77WgCQOXTEBnNO5bzOVWVsz2R5fMTsJHiHNEQtpXdOj/dmmwDAeKdtO77Iv/cbXW5nVTRfPGBiG1e/5T6+NT5hYo+oP7aHs841JP33ZrLNmYUy4SyZBxBzZpZEvOP+QQ8RM0u853BnpnzGc3xaMV+hdAB4Mr+nQgOAX6nqr4t4PqJawbFNQZh1AlfVgwD+uIR9IaoJHNsUCk4jJCIKFBM4EVGguB/4TEUUmxpWdpnY7r/rcNs++uf/ZmLrE4UvqR3M2b2JL7ngqP9aN33VxJ6TK9y2q56wS4tzu/f7nfgcFjfLaganx5fieeEUN3NJPx2k5tvnGO3y+9XYPWhiV3Z9aGLpiOXmvz211sTOvL/IbTv/hP3MxDKFf47Sbf5S+qRTQJQJfz9wb1/1qBxRjk0teAdORBQoJnAiokAxgRMRBYoJnIgoUEzgRESB4iyUGWpYudyNH/xnu2z+2a/Y2SYAsNx513sn7eb0O05/zX38869/0QYb/VkhK5adNbGJTn9z+vGVdil1c1/EJv+DQzbIwyMqYyYHOjTaJeQAkGux4y21yN/PZXSp83rrnSPhAdy05m37vDk74J8/amebAP6Mk3kH/OttPm3HfMNoxMEmjmyj/7y5pH0fYzOYDaQRnwP/IJbiPjO8AyciChQTOBFRoJjAiYgCxQRORBQoFjE/Q6zVFvUOfM8vYv7ssq0m1hrzC4s/7rvexPb906UmNu+NE+7j1w1/YGKZ7oji6i2dJiZNfqFneIUt3jS3L3TbysioiWnUyd30SVF7fxf6cO8kdADS6BQhm22xEgAyC+wy8tGl/vL20dV2HP/lmj1uW2+v+icOOBs7vjHPffzS953C5Jh9TsBfNq8Rb20uYd+ziNX8UO/99fb9xmfs5+3wtkXQwmuuLt6BExEFigmciChQTOBERIFiAiciChQTOBFRoKadhSIiWwHcAKBfVdfnYwsBPApgFYBDAG5T1XPl62aZRcwKkGX2QIZrrnvDbdsZt0vL//30VW7bw3/bbWJzX7WndGejZnU4S6kTfWfcpvM+sCd3D6z3n3bwC7aq37GrficqBTG2neXxbgyAODNOcnP9QwvS8xImlmrzPweJ9nETOzjS7rZ9aWiNiWX32tlc7fv82RuJETstI2q2yGSr/QeN+9eQabLxprN+H2IZZ2rITGabzORAh6gZSQUusS/kDnwbgE/Pe9sMYKeqdgPYmf87UWi2gWObAjZtAlfVXQA+vSPSjQC253/eDuCmEveLqOw4til0s/0OvENVjwNA/s8lUQ1FZJOI9IpIbxqpWb4cUcVwbFMwyl7EVNUtqtqjqj0J+NtVEoWIY5uqbbYVqpMislRVj4vIUgD9pexUrRhb02ZiV7S+6LadUPtWPv7OZW7bdYfs2+WtqJWkv5ezt2Q62273IweAsaW2SNK8bMRtO/GhLTYh7RdSZ7KEODDlH9tugarwNdXevt8AgKQtTOaa/TE02mnHa6rNL5w1Ndql7KNp/3nPfWQ/M4v32XYN4/5rpebba5uc6xf6so02nov472jWiWcb/fcxOWzfm3jUex7zlt37n41a2g/8GQC353++HcDTRfWCqHZwbFMwpk3gIvIwgP8BcLGI9InIHQB+CuBaEdkP4Nr834mCwrFNoZv2KxRV3RjxT9eUuC9EFcWxTaHjSkwiokAxgRMRBap+10nPREQlODkwaWJvja50217aeMzErl3nb3r/Xz/aYGJtu1fYbkWssh3rtP+QavdnMdxw9R9MLBfxxC+/0GP7cPK03wmeQF89USekO7+TXMRMi8n5zgyOlXbJPABctNCOgZNjzowlAI399vUkZ8fmxEK/X6kFzthe4DZFZq693myHPx8/nrB9mDzkH3aRHLJTVhoG/VleMjFhYlEztNwDHaI+5CVcSk9ERDWICZyIKFBM4EREgWICJyIKFIuYn6Fh72ETe/JdW4AEgDuv3mVi93W+4Lb9wa0vmtihzCITe2PsAvfx7w0tNbGbl/j7lP/pnI9MbNuALVYCwMK9tgCk435hi2qPOkvpU202BgDDq2xRb/3y427bL863Bfq+4Uv8PjTY4ltqvr1PnPQPpcfYSrt1Q1vXoNv2yx19Jrauxb+GtLOp+K8X+9dw9lSXiSWHW9y2zeN26wCM28ImAHcff2jEnv8F4h04EVGgmMCJiALFBE5EFCgmcCKiQLGI+RmyAwMmtvYf/PNt/yx3l4lt3PCq27arsbAzcj8cs4VNAFg2xxZ1bo4o3uScX/FjB/19yrt699vHRx2sTJXh7SHtFcMAaLNdQTi22F/x2OjsCX/VIvv7n+qCfb3hMf+wZMnZlYXDq+w1xJaPuY//5oV28/A7FtsJAgBweaMt0I7k/ALiyxN2JeV/Jy902/Y7l5aZE/GeJ+znS5wYAOik3Ve9WLwDJyIKFBM4EVGgmMCJiALFBE5EFCgmcCKiQE07C0VEtgK4AUC/qq7Px+4D8D0Ap/LN7lXV58rVyapx9uTNHvjQbfqFO+yy+zea5rpt31zebWJjFy00sYGL/GXQ9/3olybWHPNPCfeq8i2P+Xsb50b2uvF6FezYbvA/ttkWOwZSbf5+001JOyMiIf4+1ueydlrGxIA/CyW2wC7R71zbb2LfWrbbffzN8+yWEGsT/lHzxzN2Js1vxta4bfeNd5rY0SF/PX9qkbO3fsS23Wgo8h5Yyr8f+DYA1zvxB1V1Q/5/tTXAiQqzDRzbFLBpE7iq7gJwtgJ9Iaoojm0KXTH3/3eKyNsislVEnC25pojIJhHpFZHeNPzjjohqDMc2BWG2CfznAC4EsAHAcQD3RzVU1S2q2qOqPQn432UR1RCObQrGrJbSq+rJj38WkV8AeLZkPQpVzhaAcmP+cmHs+8CEmg7afbuX9l3kPrz7x6ecqH9A64G0LZIseOptt22OBxXX3NiWuL3HEmffbwCITTpjMOITfsECu51DKuc/76tnnX3p4/5YmbNy2MS+stgW+L/eYpfMA8CJrC38/+/4Yrft0/12b/6hlF9cbWqwRdvJjP/mxCa9wmLEZyNjC56a9Q8Ydw87LvIzN6s7cBE5/0SBmwG8W1QviGoExzaFpJBphA8DuBpAu4j0AfgJgKtFZAOm/rN0CMD3y9hHorLg2KbQTZvAVXWjE36oDH0hqiiObQodV2ISEQWKCZyIKFA80KFGeBXqkW5/yftFzobxWfUr37c8+zcm1j32ygx7R2UXcUgDEs7MkEZ/24RMi22bnufPchhO22mPExGzUC5qtbOeTi72t4lobZw0sTjs2Hxq4Evu4/vGFpjY8VF/yfuJU/bzEU/42wGsaLeHs0xO+ukvPmFnoTSM+58vmbDz/zXnt/WfoPxL6YmIqAYxgRMRBYoJnIgoUEzgRESBYhGzVjhFrD/a/JbbtFFssWkwN+62XXe/Pa2e58zXHon5xSxxilzqLK8HgEyzPYHeOVAeADA0YZectyfsMngAWD+nz8Qubj7hth1x9g7vn2w1sX1DS9zHn5uwW0K0NfljO7vIXlx786jbNia2KJg+7W8/sfiQbdt00u+DjNp4LuN/wrzfsfo114LxDpyIKFBM4EREgWICJyIKFBM4EVGgmMCJiALFWSg1It5mlwVv7ngmorVdxrxlYL3bMtt3rJhuUbV5s1MillnHMjbecsSf3XI2udDEds5b67b9q47XTezalr3+8zqzUH6rl5rYX3S84z7+XKbFxBLiT9U4mrLL7s9O2scDwO/3XWhi83fbWTsAsGCfPe0+NugfzqJpe1BEJfEOnIgoUEzgRESBYgInIgoUEzgRUaAKORNzBYBfAugEkAOwRVV/JiILATwKYBWmzg68TVXtUddUkKFvdJtYe8zf9zmltnCybcd1btvlmd8X17E6FsTYdrZYkLS/VLth1MbnHfHv0WIZW8B7U2yhDwCG19vC5NlOfz/wjoTdd/trLftNbFncX7a/P91uYkfSi9y2/Sm7RP8PB1a5bVvfsvufzz/kFyBjI3ZPcwzZwiYAaMppG0FzTvG5AqfSZwDco6rrAFwB4IcicgmAzQB2qmo3gJ35vxOFhGObgjZtAlfV46r6ev7nYQB7AHQBuBHA9nyz7QBuKlcnicqBY5tCN6PvwEVkFYDLALwCoENVjwNTHwQA7vZiIrJJRHpFpDcNe/wQUS3g2KYQFZzARWQugMcB3K2qQ4U+TlW3qGqPqvYkYL+HIqo2jm0KVUEJXEQSmBrgO1T1iXz4pIgszf/7UgD95ekiUflwbFPICpmFIgAeArBHVR8475+eAXA7gJ/m/3y6LD2sR84m/XN+YJe8J8Rf6ns4YzeRX/H8oNu2uBp3fava2PYOachG7OzvxTN+24YBOy4STf4YanUmssTSftu+MytM7ME1/oEMyzrtZJ1LF9pDRRYn/VkdB0YXm9juUx1u25Gj9rT6Be/696TzD9rZIo1n/a+9YoO2b5qK+IrMW0rvzTYBAJ3BafUFKmQvlCsBfBfAOyLyZj52L6YG92MicgeAwwBuLXnviMqLY5uCNm0CV9WXAfg74gDXlLY7RJXDsU2h40pMIqJAMYETEQWK+4FXQbzVLgG+ruM9E/OWzAPAQ2f/xMRiH/mnhBd56DWVg7d8OuYXEHNO8SwWdYJ9zN6PJeN+20TSfvSTgwm3bctJGx8/7G/zMNzWaWIvtdiYRmSehDOJMzHqFwW7TtmiYHOfXxyND9j9vL0T5QFAh50i5qS/ZF6ztg+aidgjvMhl8x7egRMRBYoJnIgoUEzgRESBYgInIgoUEzgRUaA4C6UKxq662MTWNNqTv49FLK9++LXLTWxt6v3iO0bVE7XM2hkDufEJt2nMWcIdizo1PWFnlsSa/JklDUN2o66G0Wa3bctxZ9aLs3WARCw3bxix/ZWM/97EB+0sEok6eMGZRRK1fYHf1u9D5BYIFcI7cCKiQDGBExEFigmciChQTOBERIFiEbOcIpZHn+u2BaSJnI29OGZPqgeAee85xaaoYhWFIWKZtVcki9o+0StuSsQScMTt2BQnBgCStGOz6Yw9qX7qeZ17wqj9sT0Zu1G5Rux/jrS9tlxEsXEmnw91+hDd2Hm9MiyZj8I7cCKiQDGBExEFigmciChQTOBERIGaNoGLyAoR+Z2I7BGR90Tkrnz8PhE5KiJv5v/37fJ3l6h0OLYpdIXMQskAuEdVXxeRVgCvicgL+X97UFX/pXzdC1vMqd4DQKrNxp449SUT23/GntANAEt67RJib+N/mlbtj21nRkP08m0b1xlMqIiahYKUM2MlYjk/nEMlxJuZIv69o7uMPWpWiDe7JeKwC69t5PtY5ZklM1HIocbHARzP/zwsInsAdJW7Y0TlxrFNoZvRd+AisgrAZQBeyYfuFJG3RWSriDj3lYCIbBKRXhHpTYN3iVSbOLYpRAUncBGZC+BxAHer6hCAnwO4EMAGTN3F3O89TlW3qGqPqvYkYHc1I6o2jm0KVUEJXEQSmBrgO1T1CQBQ1ZOqmlXVHIBfALB7nBLVOI5tCtm034GLiAB4CMAeVX3gvPjS/HeIAHAzgHfL08VwRe0hfOEvPjKx4Sc7TGzZuWH38dmjB+xr1WiRpZYFO7bL9LvWiCXv4hRHc2m/sChOEVG94mjEa7mFxai90t0niHhvnD3JZ/wcNaiQWShXAvgugHdE5M187F4AG0VkAwAFcAjA98vSQ6Ly4dimoBUyC+Vl+PvnPFf67hBVDsc2hY4rMYmIAsUETkQUKCZwIqJA8UCHMlJnw3kAyPQdtUEnNoPaO1HxchGntM9kEojX1lsKX+lZIQHNLJkJ3oETEQWKCZyIKFBM4EREgWICJyIKlFRyCbaInALw8TrydgCnK/bilcPrqp4LVNXfRL3MzhvbIbxPs1Wv1xbCdblju6IJ/BMvLNKrqj1VefEy4nV9vtXz+1Sv1xbydfErFCKiQDGBExEFqpoJfEsVX7uceF2fb/X8PtXrtQV7XVX7DpyIiIrDr1CIiALFBE5EFKiKJ3ARuV5E3heRAyKyudKvX0r5E8v7ReTd82ILReQFEdmf/9M90byWicgKEfmdiOwRkfdE5K58PPhrK6d6Gdsc1+FcW0UTuIjEAfwHgG8BuARTR1ddUsk+lNg2ANd/KrYZwE5V7QawM//30GQA3KOq6wBcAeCH+d9TPVxbWdTZ2N4GjusgVPoO/HIAB1T1oKpOAngEwI0V7kPJqOouAGc/Fb4RwPb8z9sB3FTRTpWAqh5X1dfzPw8D2AOgC3VwbWVUN2Ob4zqca6t0Au8CcOS8v/flY/Wk4+MTzfN/Lqlyf4oiIqsAXAbgFdTZtZVYvY/tuvrd18u4rnQC93Zx5zzGGiUicwE8DuBuVR2qdn9qHMd2IOppXFc6gfcBWHHe35cDOFbhPpTbSRFZCgD5P/ur3J9ZEZEEpgb5DlV9Ih+ui2srk3of23Xxu6+3cV3pBP4qgG4RWS0iSQDfAfBMhftQbs8AuD3/8+0Anq5iX2ZFRATAQwD2qOoD5/1T8NdWRvU+toP/3dfjuK74SkwR+TaAfwUQB7BVVf+xoh0oIRF5GMDVmNqO8iSAnwB4CsBjAFYCOAzgVlX9dEGoponI1wG8BOAd/P/RnPdi6vvCoK+tnOplbHNch3NtXEpPRBQorsQkIgoUEzgRUaCYwImIAsUETkQUKCZwIqJAMYETEQWKCZyIKFD/B1DwnzmZFu4sAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(1,2,1)\n",
    "plt.imshow(rec[0].reshape(28,28))\n",
    "plt.subplot(1,2,2)\n",
    "plt.imshow(rec[1].reshape(28,28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.return_types.min(\n",
       "values=tensor([0.0422, 0.0452], device='cuda:0', grad_fn=<MinBackward0>),\n",
       "indices=tensor([9, 6], device='cuda:0'))"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cl.min(1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}