11a_transfer_learning_jcat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from exp.nb_11 import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Train ImageWoof model (10 dogbreeds) from scratch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jump_to lesson 12 video](https://course.fast.ai/videos/?lesson=12&t=2920)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 Get the ImageWoof data, and preprocess it "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = datasets.untar_data(datasets.URLs.IMAGEWOOF_160)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "size = 128\n",
    "bs = 64\n",
    "# using all code we built in this class, not fastai v1.0\n",
    "tfms = [make_rgb, RandomResizedCrop(size, scale=(0.35,1)), np_to_float, PilRandomFlip()]\n",
    "val_tfms = [make_rgb, CenterCrop(size), np_to_float]\n",
    "il = ImageList.from_files(path, tfms=tfms)\n",
    "sd = SplitData.split_by_func(il, partial(grandparent_splitter, valid_name='val'))\n",
    "ll = label_by_func(sd, parent_labeler, proc_y=CategoryProcessor())\n",
    "ll.valid.x.tfms = val_tfms\n",
    "data = ll.to_databunch(bs, c_in=3, c_out=10, num_workers=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12954"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(il)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 Create a Learner object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_func = LabelSmoothingCrossEntropy()\n",
    "opt_func = adam_opt(mom=0.9, mom_sqr=0.99, eps=1e-6, wd=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = cnn_learner(xresnet18, data, loss_func, opt_func, norm=norm_imagenette)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3 Create a learning schedule, and train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sched_1cycle(lr, pct_start=0.3, mom_start=0.95, mom_mid=0.85, mom_end=0.95):\n",
    "    phases = create_phases(pct_start)\n",
    "    sched_lr  = combine_scheds(phases, cos_1cycle_anneal(lr/10., lr, lr/1e5))\n",
    "    sched_mom = combine_scheds(phases, cos_1cycle_anneal(mom_start, mom_mid, mom_end))\n",
    "    return [ParamScheduler('lr', sched_lr),\n",
    "            ParamScheduler('mom', sched_mom)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 3e-3\n",
    "pct_start = 0.5\n",
    "cbsched = sched_1cycle(lr, pct_start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# note: on Jeremy's 1080ti, each epoch takes 10 sec, compared to 90 sec on my RTX-2070 on Windows 10\n",
    "#      got 84% accuracy\n",
    "learn.fit(40, cbsched)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.4 Save the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# state dictionary is an ordered dictionary with all the parameter names and values\n",
    "st = learn.model.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(st)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "', '.join(st.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "st['10.bias']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "mdl_path = path/'models'\n",
    "mdl_path.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It's also possible to save the whole model, including the architecture, but it gets quite fiddly and we don't recommend it. Instead, just save the parameters, and recreate the model directly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(st, mdl_path/'iw5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Train a Pets model from scratch (37 dog and cat breeds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jump_to lesson 12 video](https://course.fast.ai/videos/?lesson=12&t=3127)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Get the Pets dataset, make an ImageList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "pets = datasets.untar_data(datasets.URLs.PETS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/annotations'),\n",
       " WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images')]"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pets.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "pets_path = pets/'images'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "il = ImageList.from_files(pets_path, tfms=tfms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ImageList (7390 items)\n",
       "[WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_1.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_10.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_100.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_101.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_102.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_103.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_104.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_105.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_106.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_107.jpg')...]\n",
       "Path: C:\\Users\\cross-entropy\\.fastai\\data\\oxford-iiit-pet\\images"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "il"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 Split data into training and validation sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def random_splitter(fn, p_valid): return random.random() < p_valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "sd = SplitData.split_by_func(il, partial(random_splitter, p_valid=0.1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SplitData\n",
       "Train: ImageList (6667 items)\n",
       "[WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_1.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_100.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_101.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_102.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_103.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_104.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_106.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_108.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_109.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_110.jpg')...]\n",
       "Path: C:\\Users\\cross-entropy\\.fastai\\data\\oxford-iiit-pet\\images\n",
       "Valid: ImageList (723 items)\n",
       "[WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_10.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_105.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_107.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_11.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_116.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_122.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_123.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_136.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_139.jpg'), WindowsPath('C:/Users/cross-entropy/.fastai/data/oxford-iiit-pet/images/Abyssinian_160.jpg')...]\n",
       "Path: C:\\Users\\cross-entropy\\.fastai\\data\\oxford-iiit-pet\\images"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Abyssinian_1.jpg\n"
     ]
    }
   ],
   "source": [
    "n = il.items[0].name\n",
    "print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Abyssinian'"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'^(.*)_\\d+.jpg$', n)[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3 Use a regular expression to identify the part of the filename associated with the dog or cat class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pet_labeler(fn): return re.findall(r'^(.*)_\\d+.jpg$', fn.name)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "proc = CategoryProcessor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll = label_by_func(sd, pet_labeler, proc_y=proc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Abyssinian, american_bulldog, american_pit_bull_terrier, basset_hound, beagle, Bengal, Birman, Bombay, boxer, British_Shorthair, chihuahua, Egyptian_Mau, english_cocker_spaniel, english_setter, german_shorthaired, great_pyrenees, havanese, japanese_chin, keeshond, leonberger, Maine_Coon, miniature_pinscher, newfoundland, Persian, pomeranian, pug, Ragdoll, Russian_Blue, saint_bernard, samoyed, scottish_terrier, shiba_inu, Siamese, Sphynx, staffordshire_bull_terrier, wheaten_terrier, yorkshire_terrier'"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "', '.join(proc.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll.valid.x.tfms = val_tfms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "c_out = len(proc.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = ll.to_databunch(bs, c_in=3, c_out=c_out, num_workers=8) #why did it work?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.4 Train the pets model from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "# why are we using imagenette norm ?????\n",
    "learn = cnn_learner(xresnet18, data, loss_func, opt_func, norm=norm_imagenette)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>3.474966</td>\n",
       "      <td>0.088346</td>\n",
       "      <td>3.429850</td>\n",
       "      <td>0.094053</td>\n",
       "      <td>01:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.268964</td>\n",
       "      <td>0.140393</td>\n",
       "      <td>3.901183</td>\n",
       "      <td>0.106501</td>\n",
       "      <td>01:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.039603</td>\n",
       "      <td>0.201740</td>\n",
       "      <td>3.093634</td>\n",
       "      <td>0.200553</td>\n",
       "      <td>01:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2.706843</td>\n",
       "      <td>0.299835</td>\n",
       "      <td>2.690228</td>\n",
       "      <td>0.322268</td>\n",
       "      <td>01:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2.396690</td>\n",
       "      <td>0.401530</td>\n",
       "      <td>2.415632</td>\n",
       "      <td>0.402490</td>\n",
       "      <td>01:19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 40% accuracy -- not a great result!\n",
    "learn.fit(5, cbsched)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Apply transfer learning from ImageWoof model to Pets model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jump_to lesson 12 video](https://course.fast.ai/videos/?lesson=12&t=3265)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 Make a Learner object for the Pets data, then load pretrained weights from the ImageWoof model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a learner for the pets data set, with 10 outputs like the imagewoof model \n",
    "learn = cnn_learner(xresnet18, data, loss_func, opt_func, c_out=10, norm=norm_imagenette)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the pretrained imagewoof model\n",
    "st = torch.load(mdl_path/'iw5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = learn.model\n",
    "m.load_state_dict(st)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 Remove the head from the body"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove the final layer, and create a new model from the body\n",
    "#      that includes everything before the AdaptiveAvgPool2d layer\n",
    "cut = next(i for i,o in enumerate(m.children()) if isinstance(o,nn.AdaptiveAvgPool2d))\n",
    "m_cut = m[:cut]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  3.3 Determine how many outputs from the body"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "xb,yb = get_batch(data.valid_dl, learn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = m_cut(xb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([128, 512, 4, 4])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "512"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ni = pred.shape[1]\n",
    "ni"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.4 Create the penultimate pooling layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class AdaptiveConcatPool2d(nn.Module):\n",
    "    def __init__(self, sz=1):\n",
    "        super().__init__()\n",
    "        self.output_size = sz\n",
    "        self.ap = nn.AdaptiveAvgPool2d(sz)\n",
    "        self.mp = nn.AdaptiveMaxPool2d(sz)\n",
    "    def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5 Create the transfer learning model \n",
    "by combining the body with a new head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nh = 40 ????? nh does not seem to be used anywhere in this code ?????\n",
    "# new model = body + adaptive pooling layers + flatten + new head\n",
    "m_new = nn.Sequential(\n",
    "    m_cut, AdaptiveConcatPool2d(), Flatten(),\n",
    "    nn.Linear(ni*2, data.c_out))\n",
    "learn.model = m_new"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.6 Train the transfer learning model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.900010</td>\n",
       "      <td>0.283936</td>\n",
       "      <td>2.179667</td>\n",
       "      <td>0.486860</td>\n",
       "      <td>01:21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.969682</td>\n",
       "      <td>0.545523</td>\n",
       "      <td>2.208702</td>\n",
       "      <td>0.473029</td>\n",
       "      <td>01:21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.726557</td>\n",
       "      <td>0.634918</td>\n",
       "      <td>1.854407</td>\n",
       "      <td>0.593361</td>\n",
       "      <td>01:22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.485310</td>\n",
       "      <td>0.721914</td>\n",
       "      <td>1.536633</td>\n",
       "      <td>0.706777</td>\n",
       "      <td>01:22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.303791</td>\n",
       "      <td>0.795860</td>\n",
       "      <td>1.471868</td>\n",
       "      <td>0.734440</td>\n",
       "      <td>01:24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# transfer learning result #1: 73.4% error\n",
    "learn.fit(5, cbsched)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Putting it together: a method to take a Learner, and adapt it to another model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jump_to lesson 12 video](https://course.fast.ai/videos/?lesson=12&t=3483)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def adapt_model(learn, data):\n",
    "    cut = next(i for i,o in enumerate(learn.model.children())\n",
    "               if isinstance(o,nn.AdaptiveAvgPool2d))\n",
    "    m_cut = learn.model[:cut]\n",
    "    xb,yb = get_batch(data.valid_dl, learn)\n",
    "    pred = m_cut(xb)\n",
    "    ni = pred.shape[1]\n",
    "    m_new = nn.Sequential(\n",
    "        m_cut, AdaptiveConcatPool2d(), Flatten(),\n",
    "        nn.Linear(ni*2, data.c_out))\n",
    "    learn.model = m_new"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set up the transfer learning process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn = cnn_learner(xresnet18, data, loss_func, opt_func, c_out=10, norm=norm_imagenette)\n",
    "learn.model.load_state_dict(torch.load(mdl_path/'iw5'))\n",
    "adapt_model(learn, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# freeze body parameters, i.e. except the head\n",
    "for p in learn.model[0].parameters(): p.requires_grad_(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.774446</td>\n",
       "      <td>0.308085</td>\n",
       "      <td>2.573833</td>\n",
       "      <td>0.373444</td>\n",
       "      <td>01:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.428881</td>\n",
       "      <td>0.417729</td>\n",
       "      <td>2.267052</td>\n",
       "      <td>0.459198</td>\n",
       "      <td>01:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2.047926</td>\n",
       "      <td>0.530523</td>\n",
       "      <td>2.087088</td>\n",
       "      <td>0.506224</td>\n",
       "      <td>01:15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# train just the head\n",
    "learn.fit(3, sched_1cycle(1e-2, 0.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# unfreeze body parameters\n",
    "for p in learn.model[0].parameters(): p.requires_grad_(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.842971</td>\n",
       "      <td>0.594870</td>\n",
       "      <td>1.884222</td>\n",
       "      <td>0.590595</td>\n",
       "      <td>01:21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.727079</td>\n",
       "      <td>0.637618</td>\n",
       "      <td>1.871323</td>\n",
       "      <td>0.586445</td>\n",
       "      <td>01:23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.634446</td>\n",
       "      <td>0.672566</td>\n",
       "      <td>1.764705</td>\n",
       "      <td>0.625173</td>\n",
       "      <td>01:25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.453012</td>\n",
       "      <td>0.747113</td>\n",
       "      <td>1.588173</td>\n",
       "      <td>0.680498</td>\n",
       "      <td>01:22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.260233</td>\n",
       "      <td>0.820609</td>\n",
       "      <td>1.501467</td>\n",
       "      <td>0.724758</td>\n",
       "      <td>01:21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# transfer learning result #2: 72.5%, slightly worse than result #1\n",
    "#      reset_opt = True zeros all the gradients\n",
    "learn.fit(5, cbsched, reset_opt=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## what happened here? \n",
    "Jeremy: \"Anytime something weird happens in your neural net, it's almost certainly because of batchnorm, because batchnorm makes everything weird\".\n",
    "Result is better than training from scratch, but worse than naive fine-tuning where we didn't freeze/unfreeze.\n",
    "Frozen part of model was designed for image woof, tuned for mean & std dev, but pets data set has different means & std devs. inside the model. Different set of batch norm statistics. Would be interesting to see what's really going on using histograms as we did earlier in the course.\n",
    "Good news is it's easily fixed."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Batch norm transfer\n",
    "The trick is: don't freeze all of the body parameters, but freeze only the body parameters that are not in the batchnorm layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jump_to lesson 12 video](https://course.fast.ai/videos/?lesson=12&t=3567)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set up the transfer learning process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn = cnn_learner(xresnet18, data, loss_func, opt_func, c_out=10, norm=norm_imagenette)\n",
    "learn.model.load_state_dict(torch.load(mdl_path/'iw5'))\n",
    "adapt_model(learn, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_mod(m, f):\n",
    "    f(m)\n",
    "    for l in m.children(): apply_mod(l, f)\n",
    "\n",
    "def set_grad(m, b):\n",
    "    if isinstance(m, (nn.Linear,nn.BatchNorm2d)): return\n",
    "    if hasattr(m, 'weight'):\n",
    "        for p in m.parameters(): p.requires_grad_(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# freeze only the body parameters that are *not* in linear or batchnorm layers\n",
    "apply_mod(learn.model, partial(set_grad, b=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.644171</td>\n",
       "      <td>0.335383</td>\n",
       "      <td>2.253329</td>\n",
       "      <td>0.460581</td>\n",
       "      <td>01:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.065932</td>\n",
       "      <td>0.513124</td>\n",
       "      <td>2.003434</td>\n",
       "      <td>0.567082</td>\n",
       "      <td>01:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.821127</td>\n",
       "      <td>0.605820</td>\n",
       "      <td>1.806843</td>\n",
       "      <td>0.607192</td>\n",
       "      <td>01:19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit(3, sched_1cycle(1e-2, 0.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# unfreeze body parameters that were just frozen\n",
    "apply_mod(learn.model, partial(set_grad, b=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.697546</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>1.794863</td>\n",
       "      <td>0.625173</td>\n",
       "      <td>01:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.653100</td>\n",
       "      <td>0.664917</td>\n",
       "      <td>1.944319</td>\n",
       "      <td>0.572614</td>\n",
       "      <td>01:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.605481</td>\n",
       "      <td>0.682316</td>\n",
       "      <td>1.769455</td>\n",
       "      <td>0.615491</td>\n",
       "      <td>01:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.439478</td>\n",
       "      <td>0.750262</td>\n",
       "      <td>1.538387</td>\n",
       "      <td>0.716459</td>\n",
       "      <td>01:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.270474</td>\n",
       "      <td>0.819109</td>\n",
       "      <td>1.477680</td>\n",
       "      <td>0.737206</td>\n",
       "      <td>01:20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# transfer learning result #3: 73.7%\n",
    "#      not much better than result #1\n",
    "learn.fit(5, cbsched, reset_opt=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pytorch already has an `apply` method we can use to accomplish the same task as our own method\n",
    "\n",
    "`apply_mod(learn.model, partial(set_grad, b=False))`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.model.apply(partial(set_grad, b=False));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Discriminative LR and param groups\n",
    "can selectively set learning rate to zero for some layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jump_to lesson 12 video](https://course.fast.ai/videos/?lesson=12&t=3799)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set up the transfer learning process: \n",
    "create a pretrained model for the Pets data set  using the weights from the trained ImageWoof model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn = cnn_learner(xresnet18, data, loss_func, opt_func, c_out=10, norm=norm_imagenette)\n",
    "learn.model.load_state_dict(torch.load(mdl_path/'iw5'))\n",
    "adapt_model(learn, data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Method to create two groups of parameters\n",
    "first group has body layers, second group has batchnorm layers plus everything after the head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def bn_splitter(m):\n",
    "    def _bn_splitter(l, g1, g2):\n",
    "        if isinstance(l, nn.BatchNorm2d): g2 += l.parameters()\n",
    "        elif hasattr(l, 'weight'): g1 += l.parameters()\n",
    "        for ll in l.children(): _bn_splitter(ll, g1, g2)\n",
    "        \n",
    "    g1,g2 = [],[]\n",
    "    _bn_splitter(m[0], g1, g2)\n",
    "    \n",
    "    g2 += m[1:].parameters()\n",
    "    return g1,g2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "a,b = bn_splitter(learn.model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_eq(len(a)+len(b), len(list(m.parameters())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22 46\n"
     ]
    }
   ],
   "source": [
    "print(len(a),len(b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Callback dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['begin_validate',\n",
       " 'after_epoch',\n",
       " 'after_cancel_batch',\n",
       " 'after_cancel_train',\n",
       " 'after_fit',\n",
       " 'after_batch',\n",
       " 'after_pred',\n",
       " 'begin_fit',\n",
       " 'begin_epoch',\n",
       " 'after_loss',\n",
       " 'after_backward',\n",
       " 'begin_batch',\n",
       " 'after_step',\n",
       " 'after_cancel_epoch']"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[o for o in Learner.ALL_CBS]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from types import SimpleNamespace\n",
    "cb_types = SimpleNamespace(**{o:o for o in Learner.ALL_CBS})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "namespace(after_backward='after_backward', after_batch='after_batch', after_cancel_batch='after_cancel_batch', after_cancel_epoch='after_cancel_epoch', after_cancel_train='after_cancel_train', after_epoch='after_epoch', after_fit='after_fit', after_loss='after_loss', after_pred='after_pred', after_step='after_step', begin_batch='begin_batch', begin_epoch='begin_epoch', begin_fit='begin_fit', begin_validate='begin_validate')"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cb_types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'after_backward'"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cb_types.after_backward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A method to create schedule callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def sched_1cycle(lrs, pct_start=0.3, mom_start=0.95, mom_mid=0.85, mom_end=0.95):\n",
    "    phases = create_phases(pct_start)\n",
    "    sched_lr  = [combine_scheds(phases, cos_1cycle_anneal(lr/10., lr, lr/1e5))\n",
    "                 for lr in lrs]\n",
    "    sched_mom = combine_scheds(phases, cos_1cycle_anneal(mom_start, mom_mid, mom_end))\n",
    "    return [ParamScheduler('lr', sched_lr),\n",
    "            ParamScheduler('mom', sched_mom)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A method to debug callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class DebugCallback(Callback):\n",
    "    _order = 999\n",
    "    def __init__(self, cb_name, f=None): self.cb_name,self.f = cb_name,f\n",
    "    def __call__(self, cb_name):\n",
    "        if cb_name==self.cb_name:\n",
    "            if self.f: self.f(self.run)\n",
    "            else:      set_trace() # Q: Don't you have to call import pdb; pdb.set_trace() ?????"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set up the transfer learning process again, this time with discriminative learning rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn = cnn_learner(xresnet18, data, loss_func, opt_func,\n",
    "                    c_out=10, norm=norm_imagenette, splitter=bn_splitter)\n",
    "learn.model.load_state_dict(torch.load(mdl_path/'iw5'))\n",
    "adapt_model(learn, data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make the discriminative learning schedule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<exp.nb_09.ParamScheduler object at 0x0000021A21006F28>, <exp.nb_09.ParamScheduler object at 0x0000021A21006E80>]\n"
     ]
    }
   ],
   "source": [
    "disc_lr_sched = sched_1cycle([0,3e-2], 0.5) # (lrs, pct_start)???\n",
    "print(disc_lr_sched)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use the callback debug method to view the parameter groups with the discriminative learning model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 [{'mom': 0.9499999999999997, 'mom_sqr': 0.99, 'eps': 1e-06, 'wd': 0.01, 'lr': 0.0, 'sqr_mom': 0.99}, {'mom': 0.9499999999999997, 'mom_sqr': 0.99, 'eps': 1e-06, 'wd': 0.01, 'lr': 0.0030000000000000512, 'sqr_mom': 0.99}]\n"
     ]
    }
   ],
   "source": [
    "def _print_det(o): \n",
    "    print (len(o.opt.param_groups), o.opt.hypers)\n",
    "    raise CancelTrainException()\n",
    "\n",
    "# Q: I thought lrs = [0.,3.e-2], but they are [0.,3.e-3] ?????\n",
    "learn.fit(1, disc_lr_sched + [DebugCallback(cb_types.after_batch, _print_det)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.514204</td>\n",
       "      <td>0.372731</td>\n",
       "      <td>2.259981</td>\n",
       "      <td>0.450899</td>\n",
       "      <td>01:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.208817</td>\n",
       "      <td>0.468427</td>\n",
       "      <td>2.318595</td>\n",
       "      <td>0.434302</td>\n",
       "      <td>01:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.958383</td>\n",
       "      <td>0.549123</td>\n",
       "      <td>1.882161</td>\n",
       "      <td>0.571231</td>\n",
       "      <td>01:21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit(3, disc_lr_sched)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adjust the schedule, then train some more"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "disc_lr_sched = sched_1cycle([1e-3,1e-2], 0.3) # (lrs, pct_start)???"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.794170</td>\n",
       "      <td>0.609720</td>\n",
       "      <td>1.995908</td>\n",
       "      <td>0.546335</td>\n",
       "      <td>01:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.789848</td>\n",
       "      <td>0.609120</td>\n",
       "      <td>1.888118</td>\n",
       "      <td>0.589212</td>\n",
       "      <td>01:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.658342</td>\n",
       "      <td>0.657867</td>\n",
       "      <td>1.705317</td>\n",
       "      <td>0.647303</td>\n",
       "      <td>01:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.523009</td>\n",
       "      <td>0.710664</td>\n",
       "      <td>1.635593</td>\n",
       "      <td>0.679115</td>\n",
       "      <td>01:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.408390</td>\n",
       "      <td>0.757012</td>\n",
       "      <td>1.591546</td>\n",
       "      <td>0.694329</td>\n",
       "      <td>01:19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# transfer learning result #4: 69.4%, much worse than result #3 which should be comparable\n",
    "# jeremy's 1080ti only takes 5 seconds per epoch -- 16x faster!!!!!\n",
    "learn.fit(5, disc_lr_sched)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted 11a_transfer_learning.ipynb to exp\\nb_11a.py\n"
     ]
    }
   ],
   "source": [
    "# !./notebook2script.py 11a_transfer_learning.ipynb\n",
    "!python notebook2script.py 11a_transfer_learning.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}