diff --git a/.floydignore b/.floydignore index 0b293f1..02f9be9 100644 --- a/.floydignore +++ b/.floydignore @@ -12,4 +12,3 @@ var *.pyc *.swp .DS_Store -models diff --git a/experiment6.ipynb b/experiment6.ipynb index 10f6fda..043c502 100644 --- a/experiment6.ipynb +++ b/experiment6.ipynb @@ -255,7 +255,7 @@ "metadata": {}, "outputs": [], "source": [ - "handler = EarlyStopping(patience=50, score_function=lambda engine: engine.state.metrics['accuracy'], trainer=trainer)\n", + "handler = EarlyStopping(patience=30, score_function=lambda engine: engine.state.metrics['accuracy'], trainer=trainer)\n", "val_evaluator.add_event_handler(Events.COMPLETED, handler)" ] }, @@ -272,7 +272,7 @@ "metadata": {}, "outputs": [], "source": [ - "scheduler = CosineAnnealingScheduler(optimizer, 'lr', 3e-4, 1e-7, len(loader))\n", + "scheduler = CosineAnnealingScheduler(optimizer, 'lr', 2e-4, 1e-7, len(loader))\n", "trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)\n", "\n", "@trainer.on(Events.ITERATION_COMPLETED)\n", @@ -439,7 +439,7 @@ ], "source": [ "print('Training started\\n')\n", - "trainer.run(loader, max_epochs=50)" + "trainer.run(loader, max_epochs=120)" ] }, { diff --git a/experiment6.py b/experiment6.py index cc8a056..71d5528 100644 --- a/experiment6.py +++ b/experiment6.py @@ -188,7 +188,7 @@ def forward(self, x1, x2): # In[8]: -handler = EarlyStopping(patience=50, score_function=lambda engine: engine.state.metrics['accuracy'], trainer=trainer) +handler = EarlyStopping(patience=30, score_function=lambda engine: engine.state.metrics['accuracy'], trainer=trainer) val_evaluator.add_event_handler(Events.COMPLETED, handler) @@ -197,7 +197,7 @@ def forward(self, x1, x2): # In[9]: -scheduler = CosineAnnealingScheduler(optimizer, 'lr', 3e-4, 1e-7, len(loader)) +scheduler = CosineAnnealingScheduler(optimizer, 'lr', 2e-4, 1e-7, len(loader)) trainer.add_event_handler(Events.ITERATION_STARTED, scheduler) @trainer.on(Events.ITERATION_COMPLETED) @@ -280,7 +280,7 @@ def save_best_epoch_only(engine): print('Training started\n') -trainer.run(loader, max_epochs=50) +trainer.run(loader, max_epochs=120) # #### Evaluate diff --git a/experiment7.ipynb b/experiment7.ipynb new file mode 100644 index 0000000..805cd82 --- /dev/null +++ b/experiment7.ipynb @@ -0,0 +1,736 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[31mmenpo 0.8.1 has requirement matplotlib<2.0,>=1.4, but you'll have matplotlib 3.0.2 which is incompatible.\u001b[0m\n", + "\u001b[31mmenpo 0.8.1 has requirement pillow<5.0,>=3.0, but you'll have pillow 5.4.1 which is incompatible.\u001b[0m\n", + "\u001b[31mmenpo 0.8.1 has requirement scipy<1.0,>=0.16, but you'll have scipy 1.2.0 which is incompatible.\u001b[0m\n", + "\u001b[33mYou are using pip version 10.0.1, however version 19.2.2 is available.\n", + "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n" + ] + } + ], + "source": [ + "!pip install -q -r requirements.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", + "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5" + }, + "outputs": [], + "source": [ + "import sys\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from PIL import Image\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.utils.data as D\n", + "from torch.optim.lr_scheduler import ExponentialLR\n", + "import torch.nn.functional as F\n", + "from torch.autograd import Variable\n", + "\n", + "from torchvision import transforms\n", + "\n", + "from ignite.engine import Events\n", + "from scripts.ignite import create_supervised_evaluator, create_supervised_trainer\n", + "from ignite.metrics import Loss, Accuracy\n", + "from ignite.contrib.handlers.tqdm_logger import ProgressBar\n", + "from ignite.handlers import EarlyStopping, ModelCheckpoint\n", + "from ignite.contrib.handlers import LinearCyclicalScheduler, CosineAnnealingScheduler\n", + "\n", + "from tqdm import tqdm_notebook\n", + "\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "from efficientnet_pytorch import EfficientNet\n", + "\n", + "from scripts.evaluate import eval_model \n", + "\n", + "import warnings\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define dataset and model" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0", + "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a" + }, + "outputs": [], + "source": [ + "img_dir = '../input/rxrxairgb512'\n", + "path_data = '../input/rxrxaicsv'\n", + "device = 'cuda'\n", + "batch_size = 32\n", + "torch.manual_seed(0)\n", + "model_name = 'efficientnet-b3'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "jitter = (0.6, 1.4)\n", + "class ImagesDS(D.Dataset):\n", + " # taken textbook from https://arxiv.org/pdf/1812.01187.pdf\n", + " transform_train = transforms.Compose([\n", + " transforms.RandomResizedCrop(224),\n", + " transforms.ColorJitter(brightness=jitter, contrast=jitter, saturation=jitter, hue=.1),\n", + " transforms.RandomHorizontalFlip(p=0.5),\n", + " # PCA Noise should go here,\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(mean=(123.68, 116.779, 103.939), std=(58.393, 57.12, 57.375))\n", + " ])\n", + " \n", + " transform_validation = transforms.Compose([\n", + " transforms.CenterCrop(224),\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(mean=(123.68, 116.779, 103.939), std=(58.393, 57.12, 57.375))\n", + " ])\n", + "\n", + " def __init__(self, df, img_dir=img_dir, mode='train', validation=False, site=1):\n", + " self.records = df.to_records(index=False)\n", + " self.site = site\n", + " self.mode = mode\n", + " self.img_dir = img_dir\n", + " self.len = df.shape[0]\n", + " self.validation = validation\n", + " \n", + " @staticmethod\n", + " def _load_img_as_tensor(file_name, validation):\n", + " with Image.open(file_name) as img:\n", + " if not validation:\n", + " return ImagesDS.transform_train(img)\n", + " else:\n", + " return ImagesDS.transform_validation(img)\n", + "\n", + " def _get_img_path(self, index, site=1):\n", + " experiment, well, plate = self.records[index].experiment, self.records[index].well, self.records[index].plate\n", + " return f'{self.img_dir}/{self.mode}/{experiment}_{plate}_{well}_s{site}.jpeg'\n", + " \n", + " def __getitem__(self, index):\n", + " img1, img2 = [self._load_img_as_tensor(self._get_img_path(index, site), self.validation) for site in [1,2]]\n", + " if self.mode == 'train':\n", + " return img1, img2, int(self.records[index].sirna)\n", + " else:\n", + " return img1, img2, self.records[index].id_code\n", + "\n", + " def __len__(self):\n", + " return self.len" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# dataframes for training, cross-validation, and testing\n", + "df = pd.read_csv(path_data+'/train.csv')\n", + "df_train, df_val = train_test_split(df, test_size = 0.1, random_state=42)\n", + "df_test = pd.read_csv(path_data+'/test.csv')\n", + "\n", + "# pytorch training dataset & loader\n", + "ds = ImagesDS(df_train, mode='train', validation=False)\n", + "loader = D.DataLoader(ds, batch_size=batch_size, shuffle=True, num_workers=4)\n", + "\n", + "# pytorch cross-validation dataset & loader\n", + "ds_val = ImagesDS(df_val, mode='train', validation=True)\n", + "val_loader = D.DataLoader(ds_val, batch_size=batch_size, shuffle=True, num_workers=4)\n", + "\n", + "# pytorch test dataset & loader\n", + "ds_test = ImagesDS(df_test, mode='test', validation=True)\n", + "tloader = D.DataLoader(ds_test, batch_size=batch_size, shuffle=False, num_workers=4)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded pretrained weights for efficientnet-b3\n" + ] + } + ], + "source": [ + "class EfficientNetTwoInputs(nn.Module):\n", + " def __init__(self):\n", + " super(EfficientNetTwoInputs, self).__init__()\n", + " self.classes = 1108\n", + " \n", + " model = model = EfficientNet.from_pretrained(model_name, num_classes=1108) \n", + " num_ftrs = model._fc.in_features\n", + " model._fc = nn.Identity()\n", + " \n", + " self.resnet = model\n", + " self.fc = nn.Linear(num_ftrs * 2, self.classes)\n", + "\n", + " def forward(self, x1, x2):\n", + " x1_out = self.resnet(x1)\n", + " x2_out = self.resnet(x2)\n", + " \n", + " N, _, _, _ = x1.size()\n", + " x1_out = x1_out.view(N, -1)\n", + " x2_out = x2_out.view(N, -1)\n", + " \n", + " out = torch.cat((x1_out, x2_out), 1)\n", + " out = self.fc(out)\n", + "\n", + " return out \n", + " \n", + "model = EfficientNetTwoInputs()\n", + "model.load_state_dict(torch.load('models/Model_efficientnet-b3_49.pth'))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "criterion = nn.CrossEntropyLoss()\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=2e-4)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "metrics = {\n", + " 'loss': Loss(criterion),\n", + " 'accuracy': Accuracy(),\n", + "}\n", + "\n", + "trainer = create_supervised_trainer(model, optimizer, criterion, device=device)\n", + "val_evaluator = create_supervised_evaluator(model, metrics=metrics, device=device)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### EarlyStopping" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "handler = EarlyStopping(patience=30, score_function=lambda engine: engine.state.metrics['accuracy'], trainer=trainer)\n", + "val_evaluator.add_event_handler(Events.COMPLETED, handler)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### LR Scheduler" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "scheduler = CosineAnnealingScheduler(optimizer, 'lr', 2e-4, 1e-7, len(loader))\n", + "trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)\n", + "\n", + "@trainer.on(Events.ITERATION_COMPLETED)\n", + "def print_lr(engine):\n", + " epoch = engine.state.epoch\n", + " iteration = engine.state.iteration\n", + " \n", + " if epoch < 2 and iteration % 100 == 0:\n", + " print(f'Iteration {iteration} | LR {optimizer.param_groups[0][\"lr\"]}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Compute and display metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "@trainer.on(Events.EPOCH_COMPLETED)\n", + "def compute_and_display_val_metrics(engine):\n", + " epoch = engine.state.epoch\n", + " metrics = val_evaluator.run(val_loader).metrics\n", + " print(\"Validation Results - Epoch: {} | Average Loss: {:.4f} | Accuracy: {:.4f} \"\n", + " .format(engine.state.epoch, metrics['loss'], metrics['accuracy']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Save best epoch only" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "!mkdir -p models" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def get_saved_model_path(epoch):\n", + " return f'models/Model_{model_name}_{epoch + 49}.pth'\n", + "\n", + "best_acc = 0.\n", + "best_epoch = 1\n", + "best_epoch_file = ''\n", + "\n", + "@trainer.on(Events.EPOCH_COMPLETED)\n", + "def save_best_epoch_only(engine):\n", + " epoch = engine.state.epoch\n", + "\n", + " global best_acc\n", + " global best_epoch\n", + " global best_epoch_file\n", + " best_acc = 0. if epoch == 1 else best_acc\n", + " best_epoch = 1 if epoch == 1 else best_epoch\n", + " best_epoch_file = '' if epoch == 1 else best_epoch_file\n", + "\n", + " metrics = val_evaluator.run(val_loader).metrics\n", + "\n", + " if metrics['accuracy'] > best_acc:\n", + " prev_best_epoch_file = get_saved_model_path(best_epoch)\n", + " if os.path.exists(prev_best_epoch_file):\n", + " os.remove(prev_best_epoch_file)\n", + " \n", + " best_acc = metrics['accuracy']\n", + " best_epoch = epoch\n", + " best_epoch_file = get_saved_model_path(best_epoch)\n", + " print(f'\\nEpoch: {best_epoch} - New best accuracy! Accuracy: {best_acc}\\n\\n\\n')\n", + " torch.save(model.state_dict(), best_epoch_file)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Progress bar - uncomment when testing in notebook" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "# pbar = ProgressBar(bar_format='')\n", + "# pbar.attach(trainer, output_transform=lambda x: {'loss': x})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Train" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training started\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "94c4e7cd554e488ba9278d4098093725", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=1027), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Training started\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, data, max_epochs)\u001b[0m\n\u001b[1;32m 357\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 358\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Engine run is terminating due to exception: %s.\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 359\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 360\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 361\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36m_handle_exception\u001b[0;34m(self, e)\u001b[0m\n\u001b[1;32m 322\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEXCEPTION_RAISED\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 323\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 325\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 326\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, data, max_epochs)\u001b[0m\n\u001b[1;32m 344\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 345\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEPOCH_STARTED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m \u001b[0mhours\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msecs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_once_on_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 347\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Epoch[%s] Complete. Time taken: %02d:%02d:%02d\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhours\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msecs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 348\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_terminate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36m_run_once_on_dataset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 311\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Current run is terminating due to exception: %s.\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 313\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 314\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 315\u001b[0m \u001b[0mtime_taken\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36m_handle_exception\u001b[0;34m(self, e)\u001b[0m\n\u001b[1;32m 322\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEXCEPTION_RAISED\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 323\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 325\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 326\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36m_run_once_on_dataset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 303\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miteration\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 304\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mITERATION_STARTED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 305\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_process_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 306\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mITERATION_COMPLETED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 307\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_terminate\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_terminate_single_epoch\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/floyd/home/scripts/ignite.py\u001b[0m in \u001b[0;36m_update\u001b[0;34m(engine, batch)\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[0mx1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprepare_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_blocking\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnon_blocking\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 51\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 491\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 492\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 494\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 495\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x1, x2)\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0mx1_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 15\u001b[0m \u001b[0mx2_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 491\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 492\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 494\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 495\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/efficientnet_pytorch/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 177\u001b[0m \u001b[0;31m# Convolution layers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 178\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 179\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 180\u001b[0m \u001b[0;31m# Pooling and final linear layer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/efficientnet_pytorch/model.py\u001b[0m in \u001b[0;36mextract_features\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 165\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdrop_connect_rate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 166\u001b[0m \u001b[0mdrop_connect_rate\u001b[0m \u001b[0;34m*=\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_blocks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 167\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mblock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdrop_connect_rate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdrop_connect_rate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 168\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 169\u001b[0m \u001b[0;31m# Head\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 491\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 492\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 494\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 495\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/efficientnet_pytorch/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, inputs, drop_connect_rate)\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mid_skip\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_block_args\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstride\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0minput_filters\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0moutput_filters\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdrop_connect_rate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdrop_connect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdrop_connect_rate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 91\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0minputs\u001b[0m \u001b[0;31m# skip connection\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 92\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/efficientnet_pytorch/utils.py\u001b[0m in \u001b[0;36mdrop_connect\u001b[0;34m(inputs, p, training)\u001b[0m\n\u001b[1;32m 73\u001b[0m \u001b[0mrandom_tensor\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 74\u001b[0m \u001b[0mbinary_tensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrandom_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 75\u001b[0;31m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minputs\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mkeep_prob\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mbinary_tensor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 76\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 77\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "print('Training started\\n')\n", + "trainer.run(loader, max_epochs=50)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Evaluate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eval_model(model, tloader, best_epoch_file, path_data)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "2b62ae829edc4d60acf1d9a9e1d598d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.4.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.4.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.1.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7740dfb227e54da8b1510dac2d094406": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.1.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.1.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.1.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "921a9c670b6e4a2db86c75a7ff5d9ee6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.1.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.1.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.1.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9dfcb7497f8842af817750eec565b8b9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.4.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.4.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.4.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_921a9c670b6e4a2db86c75a7ff5d9ee6", + "placeholder": "​", + "style": "IPY_MODEL_2b62ae829edc4d60acf1d9a9e1d598d8", + "value": " 94% 2151/2283 [22:45<01:23, 1.58it/s]" + } + }, + "d2df0eb5abab4e3895ec792681cfa8d2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.4.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.4.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.1.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "initial" + } + }, + "e3ff3ae302394523bb5b28ee009842d5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.4.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.4.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.4.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ff74a4321a59419cb24e116db9dd1e3e", + "IPY_MODEL_9dfcb7497f8842af817750eec565b8b9" + ], + "layout": "IPY_MODEL_7740dfb227e54da8b1510dac2d094406" + } + }, + "fad7703039454db7af5d7fb4bce65003": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.1.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.1.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.1.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ff74a4321a59419cb24e116db9dd1e3e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.4.0", + "model_name": "IntProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.4.0", + "_model_name": "IntProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.4.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "Loss: 128.54232788085938", + "description_tooltip": null, + "layout": "IPY_MODEL_fad7703039454db7af5d7fb4bce65003", + "max": 2283, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d2df0eb5abab4e3895ec792681cfa8d2", + "value": 2151 + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/experiment7.py b/experiment7.py new file mode 100644 index 0000000..0e3ff8a --- /dev/null +++ b/experiment7.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python +# coding: utf-8 + +# ## Load libraries + +# In[1]: + + +get_ipython().system('pip install -q -r requirements.txt') + + +# In[1]: + + +import sys +import os +import numpy as np +import pandas as pd + +from PIL import Image + +import torch +import torch.nn as nn +import torch.utils.data as D +from torch.optim.lr_scheduler import ExponentialLR +import torch.nn.functional as F +from torch.autograd import Variable + +from torchvision import transforms + +from ignite.engine import Events +from scripts.ignite import create_supervised_evaluator, create_supervised_trainer +from ignite.metrics import Loss, Accuracy +from ignite.contrib.handlers.tqdm_logger import ProgressBar +from ignite.handlers import EarlyStopping, ModelCheckpoint +from ignite.contrib.handlers import LinearCyclicalScheduler, CosineAnnealingScheduler + +from tqdm import tqdm_notebook + +from sklearn.model_selection import train_test_split + +from efficientnet_pytorch import EfficientNet + +from scripts.evaluate import eval_model + +import warnings +warnings.filterwarnings('ignore') + + +# ## Define dataset and model + +# In[2]: + + +img_dir = '../input/rxrxairgb512' +path_data = '../input/rxrxaicsv' +device = 'cuda' +batch_size = 32 +torch.manual_seed(0) +model_name = 'efficientnet-b3' + + +# In[3]: + + +jitter = (0.6, 1.4) +class ImagesDS(D.Dataset): + # taken textbook from https://arxiv.org/pdf/1812.01187.pdf + transform_train = transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.ColorJitter(brightness=jitter, contrast=jitter, saturation=jitter, hue=.1), + transforms.RandomHorizontalFlip(p=0.5), + # PCA Noise should go here, + transforms.ToTensor(), + transforms.Normalize(mean=(123.68, 116.779, 103.939), std=(58.393, 57.12, 57.375)) + ]) + + transform_validation = transforms.Compose([ + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=(123.68, 116.779, 103.939), std=(58.393, 57.12, 57.375)) + ]) + + def __init__(self, df, img_dir=img_dir, mode='train', validation=False, site=1): + self.records = df.to_records(index=False) + self.site = site + self.mode = mode + self.img_dir = img_dir + self.len = df.shape[0] + self.validation = validation + + @staticmethod + def _load_img_as_tensor(file_name, validation): + with Image.open(file_name) as img: + if not validation: + return ImagesDS.transform_train(img) + else: + return ImagesDS.transform_validation(img) + + def _get_img_path(self, index, site=1): + experiment, well, plate = self.records[index].experiment, self.records[index].well, self.records[index].plate + return f'{self.img_dir}/{self.mode}/{experiment}_{plate}_{well}_s{site}.jpeg' + + def __getitem__(self, index): + img1, img2 = [self._load_img_as_tensor(self._get_img_path(index, site), self.validation) for site in [1,2]] + if self.mode == 'train': + return img1, img2, int(self.records[index].sirna) + else: + return img1, img2, self.records[index].id_code + + def __len__(self): + return self.len + + +# In[4]: + + +# dataframes for training, cross-validation, and testing +df = pd.read_csv(path_data+'/train.csv') +df_train, df_val = train_test_split(df, test_size = 0.1, random_state=42) +df_test = pd.read_csv(path_data+'/test.csv') + +# pytorch training dataset & loader +ds = ImagesDS(df_train, mode='train', validation=False) +loader = D.DataLoader(ds, batch_size=batch_size, shuffle=True, num_workers=4) + +# pytorch cross-validation dataset & loader +ds_val = ImagesDS(df_val, mode='train', validation=True) +val_loader = D.DataLoader(ds_val, batch_size=batch_size, shuffle=True, num_workers=4) + +# pytorch test dataset & loader +ds_test = ImagesDS(df_test, mode='test', validation=True) +tloader = D.DataLoader(ds_test, batch_size=batch_size, shuffle=False, num_workers=4) + + +# In[5]: + + +class EfficientNetTwoInputs(nn.Module): + def __init__(self): + super(EfficientNetTwoInputs, self).__init__() + self.classes = 1108 + + model = model = EfficientNet.from_pretrained(model_name, num_classes=1108) + num_ftrs = model._fc.in_features + model._fc = nn.Identity() + + self.resnet = model + self.fc = nn.Linear(num_ftrs * 2, self.classes) + + def forward(self, x1, x2): + x1_out = self.resnet(x1) + x2_out = self.resnet(x2) + + N, _, _, _ = x1.size() + x1_out = x1_out.view(N, -1) + x2_out = x2_out.view(N, -1) + + out = torch.cat((x1_out, x2_out), 1) + out = self.fc(out) + + return out + +model = EfficientNetTwoInputs() +model.load_state_dict(torch.load('models/Model_efficientnet-b3_49.pth')) + + +# In[6]: + + +criterion = nn.CrossEntropyLoss() +optimizer = torch.optim.Adam(model.parameters(), lr=2e-4) + + +# In[7]: + + +metrics = { + 'loss': Loss(criterion), + 'accuracy': Accuracy(), +} + +trainer = create_supervised_trainer(model, optimizer, criterion, device=device) +val_evaluator = create_supervised_evaluator(model, metrics=metrics, device=device) + + +# #### EarlyStopping + +# In[8]: + + +handler = EarlyStopping(patience=30, score_function=lambda engine: engine.state.metrics['accuracy'], trainer=trainer) +val_evaluator.add_event_handler(Events.COMPLETED, handler) + + +# #### LR Scheduler + +# In[9]: + + +scheduler = CosineAnnealingScheduler(optimizer, 'lr', 2e-4, 1e-7, len(loader)) +trainer.add_event_handler(Events.ITERATION_STARTED, scheduler) + +@trainer.on(Events.ITERATION_COMPLETED) +def print_lr(engine): + epoch = engine.state.epoch + iteration = engine.state.iteration + + if epoch < 2 and iteration % 100 == 0: + print(f'Iteration {iteration} | LR {optimizer.param_groups[0]["lr"]}') + + +# #### Compute and display metrics + +# In[10]: + + +@trainer.on(Events.EPOCH_COMPLETED) +def compute_and_display_val_metrics(engine): + epoch = engine.state.epoch + metrics = val_evaluator.run(val_loader).metrics + print("Validation Results - Epoch: {} | Average Loss: {:.4f} | Accuracy: {:.4f} " + .format(engine.state.epoch, metrics['loss'], metrics['accuracy'])) + + +# #### Save best epoch only + +# In[11]: + + +get_ipython().system('mkdir -p models') + + +# In[12]: + + +def get_saved_model_path(epoch): + return f'models/Model_{model_name}_{epoch + 49}.pth' + +best_acc = 0. +best_epoch = 1 +best_epoch_file = '' + +@trainer.on(Events.EPOCH_COMPLETED) +def save_best_epoch_only(engine): + epoch = engine.state.epoch + + global best_acc + global best_epoch + global best_epoch_file + best_acc = 0. if epoch == 1 else best_acc + best_epoch = 1 if epoch == 1 else best_epoch + best_epoch_file = '' if epoch == 1 else best_epoch_file + + metrics = val_evaluator.run(val_loader).metrics + + if metrics['accuracy'] > best_acc: + prev_best_epoch_file = get_saved_model_path(best_epoch) + if os.path.exists(prev_best_epoch_file): + os.remove(prev_best_epoch_file) + + best_acc = metrics['accuracy'] + best_epoch = epoch + best_epoch_file = get_saved_model_path(best_epoch) + print(f'\nEpoch: {best_epoch} - New best accuracy! Accuracy: {best_acc}\n\n\n') + torch.save(model.state_dict(), best_epoch_file) + + +# #### Progress bar - uncomment when testing in notebook + +# In[13]: + + +# pbar = ProgressBar(bar_format='') +# pbar.attach(trainer, output_transform=lambda x: {'loss': x}) + + +# #### Train + +# In[14]: + + +print('Training started\n') +trainer.run(loader, max_epochs=50) + + +# #### Evaluate + +# In[ ]: + + +eval_model(model, tloader, best_epoch_file, path_data) + diff --git a/experiment8.ipynb b/experiment8.ipynb new file mode 100644 index 0000000..4b8fa63 --- /dev/null +++ b/experiment8.ipynb @@ -0,0 +1,798 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[31mmenpo 0.8.1 has requirement matplotlib<2.0,>=1.4, but you'll have matplotlib 3.0.2 which is incompatible.\u001b[0m\n", + "\u001b[31mmenpo 0.8.1 has requirement pillow<5.0,>=3.0, but you'll have pillow 5.4.1 which is incompatible.\u001b[0m\n", + "\u001b[31mmenpo 0.8.1 has requirement scipy<1.0,>=0.16, but you'll have scipy 1.2.0 which is incompatible.\u001b[0m\n", + "\u001b[33mYou are using pip version 10.0.1, however version 19.2.2 is available.\n", + "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n" + ] + } + ], + "source": [ + "!pip install -q -r requirements.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", + "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5" + }, + "outputs": [], + "source": [ + "import sys\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from PIL import Image\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.utils.data as D\n", + "from torch.optim.lr_scheduler import ExponentialLR\n", + "import torch.nn.functional as F\n", + "from torch.autograd import Variable\n", + "\n", + "from torchvision import transforms\n", + "\n", + "from ignite.engine import Events\n", + "from scripts.ignite import create_supervised_evaluator, create_supervised_trainer\n", + "from ignite.metrics import Loss, Accuracy\n", + "from ignite.contrib.handlers.tqdm_logger import ProgressBar\n", + "from ignite.handlers import EarlyStopping, ModelCheckpoint\n", + "from ignite.contrib.handlers import LinearCyclicalScheduler, CosineAnnealingScheduler\n", + "\n", + "from tqdm import tqdm_notebook\n", + "\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "from efficientnet_pytorch import EfficientNet\n", + "\n", + "from scripts.evaluate import eval_model \n", + "\n", + "import warnings\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define dataset and model" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0", + "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a" + }, + "outputs": [], + "source": [ + "img_dir = '../input/rxrxairgb'\n", + "path_data = '../input/rxrxaicsv'\n", + "device = 'cuda'\n", + "batch_size = 2\n", + "torch.manual_seed(0)\n", + "model_name = 'efficientnet-b6'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "jitter = (0.6, 1.4)\n", + "class ImagesDS(D.Dataset):\n", + " # taken textbook from https://arxiv.org/pdf/1812.01187.pdf\n", + " transform_train = transforms.Compose([\n", + " transforms.RandomResizedCrop(224),\n", + " transforms.ColorJitter(brightness=jitter, contrast=jitter, saturation=jitter, hue=.1),\n", + " transforms.RandomHorizontalFlip(p=0.5),\n", + " # PCA Noise should go here,\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(mean=(123.68, 116.779, 103.939), std=(58.393, 57.12, 57.375))\n", + " ])\n", + " \n", + " transform_validation = transforms.Compose([\n", + " transforms.CenterCrop(224),\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(mean=(123.68, 116.779, 103.939), std=(58.393, 57.12, 57.375))\n", + " ])\n", + "\n", + " def __init__(self, df, img_dir=img_dir, mode='train', validation=False, site=1):\n", + " self.records = df.to_records(index=False)\n", + " self.site = site\n", + " self.mode = mode\n", + " self.img_dir = img_dir\n", + " self.len = df.shape[0]\n", + " self.validation = validation\n", + " \n", + " @staticmethod\n", + " def _load_img_as_tensor(file_name, validation):\n", + " with Image.open(file_name) as img:\n", + " if not validation:\n", + " return ImagesDS.transform_train(img)\n", + " else:\n", + " return ImagesDS.transform_validation(img)\n", + "\n", + " def _get_img_path(self, index, site=1):\n", + " experiment, well, plate = self.records[index].experiment, self.records[index].well, self.records[index].plate\n", + " return f'{self.img_dir}/{self.mode}/{experiment}_{plate}_{well}_s{site}.jpeg'\n", + " \n", + " def __getitem__(self, index):\n", + " img1, img2 = [self._load_img_as_tensor(self._get_img_path(index, site), self.validation) for site in [1,2]]\n", + " if self.mode == 'train':\n", + " return img1, img2, int(self.records[index].sirna)\n", + " else:\n", + " return img1, img2, self.records[index].id_code\n", + "\n", + " def __len__(self):\n", + " return self.len" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# dataframes for training, cross-validation, and testing\n", + "df = pd.read_csv(path_data+'/train.csv')\n", + "df_train, df_val = train_test_split(df, test_size = 0.1, random_state=42)\n", + "df_test = pd.read_csv(path_data+'/test.csv')\n", + "\n", + "# pytorch training dataset & loader\n", + "ds = ImagesDS(df_train, mode='train', validation=False)\n", + "loader = D.DataLoader(ds, batch_size=batch_size, shuffle=True, num_workers=4)\n", + "\n", + "# pytorch cross-validation dataset & loader\n", + "ds_val = ImagesDS(df_val, mode='train', validation=True)\n", + "val_loader = D.DataLoader(ds_val, batch_size=batch_size, shuffle=True, num_workers=4)\n", + "\n", + "# pytorch test dataset & loader\n", + "ds_test = ImagesDS(df_test, mode='test', validation=True)\n", + "tloader = D.DataLoader(ds_test, batch_size=batch_size, shuffle=False, num_workers=4)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded pretrained weights for efficientnet-b3\n" + ] + } + ], + "source": [ + "class EfficientNetTwoInputs(nn.Module):\n", + " def __init__(self):\n", + " super(EfficientNetTwoInputs, self).__init__()\n", + " self.classes = 1108\n", + " \n", + " model = model = EfficientNet.from_pretrained(model_name, num_classes=1108) \n", + " num_ftrs = model._fc.in_features\n", + " model._fc = nn.Identity()\n", + " \n", + " self.resnet = model\n", + " self.fc = nn.Linear(num_ftrs * 2, self.classes)\n", + "\n", + " def forward(self, x1, x2):\n", + " x1_out = self.resnet(x1)\n", + " x2_out = self.resnet(x2)\n", + " \n", + " N, _, _, _ = x1.size()\n", + " x1_out = x1_out.view(N, -1)\n", + " x2_out = x2_out.view(N, -1)\n", + " \n", + " out = torch.cat((x1_out, x2_out), 1)\n", + " out = self.fc(out)\n", + "\n", + " return out \n", + " \n", + "model = EfficientNetTwoInputs()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "criterion = nn.CrossEntropyLoss()\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "metrics = {\n", + " 'loss': Loss(criterion),\n", + " 'accuracy': Accuracy(),\n", + "}\n", + "\n", + "trainer = create_supervised_trainer(model, optimizer, criterion, device=device)\n", + "val_evaluator = create_supervised_evaluator(model, metrics=metrics, device=device)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### EarlyStopping" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "handler = EarlyStopping(patience=10, score_function=lambda engine: engine.state.metrics['accuracy'], trainer=trainer)\n", + "val_evaluator.add_event_handler(Events.COMPLETED, handler)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### LR Scheduler" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "scheduler = CosineAnnealingScheduler(optimizer, 'lr', 3e-4, 1e-7, len(loader))\n", + "trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)\n", + "\n", + "# @trainer.on(Events.ITERATION_COMPLETED)\n", + "# def print_lr(engine):\n", + "# epoch = engine.state.epoch\n", + "# iteration = engine.state.iteration\n", + " \n", + "# if epoch < 2 and iteration % 10 == 0:\n", + "# print(f'Iteration {iteration} | LR {optimizer.param_groups[0][\"lr\"]}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Compute and display metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "@trainer.on(Events.EPOCH_COMPLETED)\n", + "def compute_and_display_val_metrics(engine):\n", + " epoch = engine.state.epoch\n", + " metrics = val_evaluator.run(val_loader).metrics\n", + " print(\"Validation Results - Epoch: {} | Average Loss: {:.4f} | Accuracy: {:.4f} \"\n", + " .format(engine.state.epoch, metrics['loss'], metrics['accuracy']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Save best epoch only" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "!mkdir -p models" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def get_saved_model_path(epoch):\n", + " return f'models/Model_{model_name}_{epoch}.pth'\n", + "\n", + "best_acc = 0.\n", + "best_epoch = 1\n", + "best_epoch_file = ''\n", + "\n", + "@trainer.on(Events.EPOCH_COMPLETED)\n", + "def save_best_epoch_only(engine):\n", + " epoch = engine.state.epoch\n", + "\n", + " global best_acc\n", + " global best_epoch\n", + " global best_epoch_file\n", + " best_acc = 0. if epoch == 1 else best_acc\n", + " best_epoch = 1 if epoch == 1 else best_epoch\n", + " best_epoch_file = '' if epoch == 1 else best_epoch_file\n", + "\n", + " metrics = val_evaluator.run(val_loader).metrics\n", + "\n", + " if metrics['accuracy'] > best_acc:\n", + " prev_best_epoch_file = get_saved_model_path(best_epoch)\n", + " if os.path.exists(prev_best_epoch_file):\n", + " os.remove(prev_best_epoch_file)\n", + " \n", + " best_acc = metrics['accuracy']\n", + " best_epoch = epoch\n", + " best_epoch_file = get_saved_model_path(best_epoch)\n", + " print(f'\\nEpoch: {best_epoch} - New best accuracy! Accuracy: {best_acc}\\n\\n\\n')\n", + " torch.save(model.state_dict(), best_epoch_file)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Progress bar - uncomment when testing in notebook" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "# pbar = ProgressBar(bar_format='')\n", + "# pbar.attach(trainer, output_transform=lambda x: {'loss': x})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Train" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training started\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3ae3ea27298d489c9804b4498df966f0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=1027), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iteration 10 | LR 0.00029994317585367494\n", + "Iteration 20 | LR 0.0002997468019973758\n", + "Iteration 30 | LR 0.0002994103605967601\n", + "Iteration 40 | LR 0.0002989341664516109\n", + "Iteration 50 | LR 0.0002983186651248503\n", + "Iteration 60 | LR 0.00029756443252563743\n", + "Iteration 70 | LR 0.0002966721743705047\n", + "Iteration 80 | LR 0.0002956427255230365\n", + "Iteration 90 | LR 0.0002944770492127082\n", + "Iteration 100 | LR 0.00029317623613361676\n", + "Iteration 110 | LR 0.0002917415034239452\n", + "Iteration 120 | LR 0.0002901741935271173\n", + "Iteration 130 | LR 0.00028847577293570676\n", + "Iteration 140 | LR 0.00028664783081927683\n", + "Iteration 150 | LR 0.0002846920775374343\n", + "Iteration 160 | LR 0.0002826103430394889\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Traceback (most recent call last):\n", + " File \"/usr/local/lib/python3.6/multiprocessing/queues.py\", line 240, in _feed\n", + " send_bytes(obj)\n", + " File \"/usr/local/lib/python3.6/multiprocessing/connection.py\", line 200, in send_bytes\n", + " self._send_bytes(m[offset:offset + size])\n", + " File \"/usr/local/lib/python3.6/multiprocessing/connection.py\", line 404, in _send_bytes\n", + " self._send(header + buf)\n", + " File \"/usr/local/lib/python3.6/multiprocessing/connection.py\", line 368, in _send\n", + " n = write(self._handle, buf)\n", + "BrokenPipeError: [Errno 32] Broken pipe\n", + "Traceback (most recent call last):\n", + " File \"/usr/local/lib/python3.6/multiprocessing/queues.py\", line 240, in _feed\n", + " send_bytes(obj)\n", + " File \"/usr/local/lib/python3.6/multiprocessing/connection.py\", line 200, in send_bytes\n", + " self._send_bytes(m[offset:offset + size])\n", + " File \"/usr/local/lib/python3.6/multiprocessing/connection.py\", line 404, in _send_bytes\n", + " self._send(header + buf)\n", + " File \"/usr/local/lib/python3.6/multiprocessing/connection.py\", line 368, in _send\n", + " n = write(self._handle, buf)\n", + "BrokenPipeError: [Errno 32] Broken pipe\n", + "Traceback (most recent call last):\n", + " File \"/usr/local/lib/python3.6/multiprocessing/queues.py\", line 240, in _feed\n", + " send_bytes(obj)\n", + " File \"/usr/local/lib/python3.6/multiprocessing/connection.py\", line 200, in send_bytes\n", + " self._send_bytes(m[offset:offset + size])\n", + " File \"/usr/local/lib/python3.6/multiprocessing/connection.py\", line 404, in _send_bytes\n", + " self._send(header + buf)\n", + " File \"/usr/local/lib/python3.6/multiprocessing/connection.py\", line 368, in _send\n", + " n = write(self._handle, buf)\n", + "BrokenPipeError: [Errno 32] Broken pipe\n", + "Traceback (most recent call last):\n", + " File \"/usr/local/lib/python3.6/multiprocessing/queues.py\", line 240, in _feed\n", + " send_bytes(obj)\n", + " File \"/usr/local/lib/python3.6/multiprocessing/connection.py\", line 200, in send_bytes\n", + " self._send_bytes(m[offset:offset + size])\n", + " File \"/usr/local/lib/python3.6/multiprocessing/connection.py\", line 404, in _send_bytes\n", + " self._send(header + buf)\n", + " File \"/usr/local/lib/python3.6/multiprocessing/connection.py\", line 368, in _send\n", + " n = write(self._handle, buf)\n", + "BrokenPipeError: [Errno 32] Broken pipe\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Training started\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, data, max_epochs)\u001b[0m\n\u001b[1;32m 357\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 358\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Engine run is terminating due to exception: %s.\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 359\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 360\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 361\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36m_handle_exception\u001b[0;34m(self, e)\u001b[0m\n\u001b[1;32m 322\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEXCEPTION_RAISED\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 323\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 325\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 326\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, data, max_epochs)\u001b[0m\n\u001b[1;32m 344\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 345\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEPOCH_STARTED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m \u001b[0mhours\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msecs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_once_on_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 347\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Epoch[%s] Complete. Time taken: %02d:%02d:%02d\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhours\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msecs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 348\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_terminate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36m_run_once_on_dataset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 311\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Current run is terminating due to exception: %s.\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 313\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 314\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 315\u001b[0m \u001b[0mtime_taken\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36m_handle_exception\u001b[0;34m(self, e)\u001b[0m\n\u001b[1;32m 322\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEXCEPTION_RAISED\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 323\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 325\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 326\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36m_run_once_on_dataset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 303\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miteration\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 304\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mITERATION_STARTED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 305\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_process_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 306\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mITERATION_COMPLETED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 307\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_terminate\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_terminate_single_epoch\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/floyd/home/scripts/ignite.py\u001b[0m in \u001b[0;36m_update\u001b[0;34m(engine, batch)\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 51\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 53\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0moutput_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m 105\u001b[0m \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 106\u001b[0m \"\"\"\n\u001b[0;32m--> 107\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m 91\u001b[0m Variable._execution_engine.run_backward(\n\u001b[1;32m 92\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m allow_unreachable=True) # allow_unreachable flag\n\u001b[0m\u001b[1;32m 94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "print('Training started\\n')\n", + "trainer.run(loader, max_epochs=60)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Evaluate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model.cuda()\n", + "eval_model(model, tloader, best_epoch_file, path_data)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "2b62ae829edc4d60acf1d9a9e1d598d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.4.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.4.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.1.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7740dfb227e54da8b1510dac2d094406": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.1.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.1.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.1.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "921a9c670b6e4a2db86c75a7ff5d9ee6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.1.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.1.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.1.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9dfcb7497f8842af817750eec565b8b9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.4.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.4.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.4.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_921a9c670b6e4a2db86c75a7ff5d9ee6", + "placeholder": "​", + "style": "IPY_MODEL_2b62ae829edc4d60acf1d9a9e1d598d8", + "value": " 94% 2151/2283 [22:45<01:23, 1.58it/s]" + } + }, + "d2df0eb5abab4e3895ec792681cfa8d2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.4.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.4.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.1.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "initial" + } + }, + "e3ff3ae302394523bb5b28ee009842d5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.4.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.4.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.4.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ff74a4321a59419cb24e116db9dd1e3e", + "IPY_MODEL_9dfcb7497f8842af817750eec565b8b9" + ], + "layout": "IPY_MODEL_7740dfb227e54da8b1510dac2d094406" + } + }, + "fad7703039454db7af5d7fb4bce65003": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.1.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.1.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.1.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ff74a4321a59419cb24e116db9dd1e3e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.4.0", + "model_name": "IntProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.4.0", + "_model_name": "IntProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.4.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "Loss: 128.54232788085938", + "description_tooltip": null, + "layout": "IPY_MODEL_fad7703039454db7af5d7fb4bce65003", + "max": 2283, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d2df0eb5abab4e3895ec792681cfa8d2", + "value": 2151 + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/experiment8.py b/experiment8.py new file mode 100644 index 0000000..6a5552b --- /dev/null +++ b/experiment8.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python +# coding: utf-8 + +# ## Load libraries + +# In[1]: + + +get_ipython().system('pip install -q -r requirements.txt') + + +# In[1]: + + +import sys +import os +import numpy as np +import pandas as pd + +from PIL import Image + +import torch +import torch.nn as nn +import torch.utils.data as D +from torch.optim.lr_scheduler import ExponentialLR +import torch.nn.functional as F +from torch.autograd import Variable + +from torchvision import transforms + +from ignite.engine import Events +from scripts.ignite import create_supervised_evaluator, create_supervised_trainer +from ignite.metrics import Loss, Accuracy +from ignite.contrib.handlers.tqdm_logger import ProgressBar +from ignite.handlers import EarlyStopping, ModelCheckpoint +from ignite.contrib.handlers import LinearCyclicalScheduler, CosineAnnealingScheduler + +from tqdm import tqdm_notebook + +from sklearn.model_selection import train_test_split + +from efficientnet_pytorch import EfficientNet + +from scripts.evaluate import eval_model + +import warnings +warnings.filterwarnings('ignore') + + +# ## Define dataset and model + +# In[2]: + + +img_dir = '../input/rxrxairgb' +path_data = '../input/rxrxaicsv' +device = 'cuda' +batch_size = 2 +torch.manual_seed(0) +model_name = 'efficientnet-b6' + + +# In[3]: + + +jitter = (0.6, 1.4) +class ImagesDS(D.Dataset): + # taken textbook from https://arxiv.org/pdf/1812.01187.pdf + transform_train = transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.ColorJitter(brightness=jitter, contrast=jitter, saturation=jitter, hue=.1), + transforms.RandomHorizontalFlip(p=0.5), + # PCA Noise should go here, + transforms.ToTensor(), + transforms.Normalize(mean=(123.68, 116.779, 103.939), std=(58.393, 57.12, 57.375)) + ]) + + transform_validation = transforms.Compose([ + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=(123.68, 116.779, 103.939), std=(58.393, 57.12, 57.375)) + ]) + + def __init__(self, df, img_dir=img_dir, mode='train', validation=False, site=1): + self.records = df.to_records(index=False) + self.site = site + self.mode = mode + self.img_dir = img_dir + self.len = df.shape[0] + self.validation = validation + + @staticmethod + def _load_img_as_tensor(file_name, validation): + with Image.open(file_name) as img: + if not validation: + return ImagesDS.transform_train(img) + else: + return ImagesDS.transform_validation(img) + + def _get_img_path(self, index, site=1): + experiment, well, plate = self.records[index].experiment, self.records[index].well, self.records[index].plate + return f'{self.img_dir}/{self.mode}/{experiment}_{plate}_{well}_s{site}.jpeg' + + def __getitem__(self, index): + img1, img2 = [self._load_img_as_tensor(self._get_img_path(index, site), self.validation) for site in [1,2]] + if self.mode == 'train': + return img1, img2, int(self.records[index].sirna) + else: + return img1, img2, self.records[index].id_code + + def __len__(self): + return self.len + + +# In[4]: + + +# dataframes for training, cross-validation, and testing +df = pd.read_csv(path_data+'/train.csv') +df_train, df_val = train_test_split(df, test_size = 0.1, random_state=42) +df_test = pd.read_csv(path_data+'/test.csv') + +# pytorch training dataset & loader +ds = ImagesDS(df_train, mode='train', validation=False) +loader = D.DataLoader(ds, batch_size=batch_size, shuffle=True, num_workers=4) + +# pytorch cross-validation dataset & loader +ds_val = ImagesDS(df_val, mode='train', validation=True) +val_loader = D.DataLoader(ds_val, batch_size=batch_size, shuffle=True, num_workers=4) + +# pytorch test dataset & loader +ds_test = ImagesDS(df_test, mode='test', validation=True) +tloader = D.DataLoader(ds_test, batch_size=batch_size, shuffle=False, num_workers=4) + + +# In[5]: + + +class EfficientNetTwoInputs(nn.Module): + def __init__(self): + super(EfficientNetTwoInputs, self).__init__() + self.classes = 1108 + + model = model = EfficientNet.from_pretrained(model_name, num_classes=1108) + num_ftrs = model._fc.in_features + model._fc = nn.Identity() + + self.resnet = model + self.fc = nn.Linear(num_ftrs * 2, self.classes) + + def forward(self, x1, x2): + x1_out = self.resnet(x1) + x2_out = self.resnet(x2) + + N, _, _, _ = x1.size() + x1_out = x1_out.view(N, -1) + x2_out = x2_out.view(N, -1) + + out = torch.cat((x1_out, x2_out), 1) + out = self.fc(out) + + return out + +model = EfficientNetTwoInputs() + + +# In[6]: + + +criterion = nn.CrossEntropyLoss() +optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) + + +# In[7]: + + +metrics = { + 'loss': Loss(criterion), + 'accuracy': Accuracy(), +} + +trainer = create_supervised_trainer(model, optimizer, criterion, device=device) +val_evaluator = create_supervised_evaluator(model, metrics=metrics, device=device) + + +# #### EarlyStopping + +# In[8]: + + +handler = EarlyStopping(patience=10, score_function=lambda engine: engine.state.metrics['accuracy'], trainer=trainer) +val_evaluator.add_event_handler(Events.COMPLETED, handler) + + +# #### LR Scheduler + +# In[9]: + + +scheduler = CosineAnnealingScheduler(optimizer, 'lr', 3e-4, 1e-7, len(loader)) +trainer.add_event_handler(Events.ITERATION_STARTED, scheduler) + +# @trainer.on(Events.ITERATION_COMPLETED) +# def print_lr(engine): +# epoch = engine.state.epoch +# iteration = engine.state.iteration + +# if epoch < 2 and iteration % 10 == 0: +# print(f'Iteration {iteration} | LR {optimizer.param_groups[0]["lr"]}') + + +# #### Compute and display metrics + +# In[10]: + + +@trainer.on(Events.EPOCH_COMPLETED) +def compute_and_display_val_metrics(engine): + epoch = engine.state.epoch + metrics = val_evaluator.run(val_loader).metrics + print("Validation Results - Epoch: {} | Average Loss: {:.4f} | Accuracy: {:.4f} " + .format(engine.state.epoch, metrics['loss'], metrics['accuracy'])) + + +# #### Save best epoch only + +# In[11]: + + +get_ipython().system('mkdir -p models') + + +# In[12]: + + +def get_saved_model_path(epoch): + return f'models/Model_{model_name}_{epoch}.pth' + +best_acc = 0. +best_epoch = 1 +best_epoch_file = '' + +@trainer.on(Events.EPOCH_COMPLETED) +def save_best_epoch_only(engine): + epoch = engine.state.epoch + + global best_acc + global best_epoch + global best_epoch_file + best_acc = 0. if epoch == 1 else best_acc + best_epoch = 1 if epoch == 1 else best_epoch + best_epoch_file = '' if epoch == 1 else best_epoch_file + + metrics = val_evaluator.run(val_loader).metrics + + if metrics['accuracy'] > best_acc: + prev_best_epoch_file = get_saved_model_path(best_epoch) + if os.path.exists(prev_best_epoch_file): + os.remove(prev_best_epoch_file) + + best_acc = metrics['accuracy'] + best_epoch = epoch + best_epoch_file = get_saved_model_path(best_epoch) + print(f'\nEpoch: {best_epoch} - New best accuracy! Accuracy: {best_acc}\n\n\n') + torch.save(model.state_dict(), best_epoch_file) + + +# #### Progress bar - uncomment when testing in notebook + +# In[13]: + + +# pbar = ProgressBar(bar_format='') +# pbar.attach(trainer, output_transform=lambda x: {'loss': x}) + + +# #### Train + +# In[14]: + + +print('Training started\n') +trainer.run(loader, max_epochs=60) + + +# #### Evaluate + +# In[ ]: + + +model.cuda() +eval_model(model, tloader, best_epoch_file, path_data) + diff --git a/job_template.txt b/job_template.txt index eca9652..370d4f2 100644 --- a/job_template.txt +++ b/job_template.txt @@ -1 +1 @@ -floyd run --gpu2 --data michelml/datasets/rxrxaicsv/1:rxrxaicsv --data michelml/datasets/rxrxairgb512/1:rxrxairgb512 --env pytorch-1.0 "ipython experiment6.py" --message "experiment 6 - same as experiment 5 but w/ 512x512 images, 50 epochs only" \ No newline at end of file +floyd run --gpu2 --data michelml/datasets/rxrxaicsv/1:rxrxaicsv --data michelml/datasets/rxrxairgb512/1:rxrxairgb --env pytorch-1.0 "ipython experiment8.py" --message "experiment 8 - same as experiment 5 but w/ efficientnet b6, 60 epochs" \ No newline at end of file