experiment13-cellline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: toolz in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 1)) (0.10.0)\n",
      "Requirement already satisfied: scikit-image in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 2)) (0.15.0)\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 3)) (0.23.4)\n",
      "Requirement already satisfied: tqdm==4.31.1 in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 4)) (4.31.1)\n",
      "Requirement already satisfied: pytorch-ignite==0.2.0 in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 5)) (0.2.0)\n",
      "Requirement already satisfied: pytorch-pretrained-bert==0.6.2 in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 6)) (0.6.2)\n",
      "Requirement already satisfied: torch==1.1.0 in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 7)) (1.1.0)\n",
      "Requirement already satisfied: torchvision==0.3.0 in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 8)) (0.3.0)\n",
      "Requirement already satisfied: Pillow==5.4.1 in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 9)) (5.4.1)\n",
      "Requirement already satisfied: efficientnet-pytorch==0.4.0 in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 10)) (0.4.0)\n",
      "Requirement already satisfied: PyWavelets>=0.4.0 in /usr/local/lib/python3.6/site-packages (from scikit-image->-r requirements.txt (line 2)) (1.0.3)\n",
      "Requirement already satisfied: matplotlib!=3.0.0,>=2.0.0 in /usr/local/lib/python3.6/site-packages (from scikit-image->-r requirements.txt (line 2)) (3.0.2)\n",
      "Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.6/site-packages (from scikit-image->-r requirements.txt (line 2)) (2.3)\n",
      "Requirement already satisfied: imageio>=2.0.1 in /usr/local/lib/python3.6/site-packages (from scikit-image->-r requirements.txt (line 2)) (2.5.0)\n",
      "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/site-packages (from scikit-image->-r requirements.txt (line 2)) (1.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/lib/python3.6/site-packages (from pandas->-r requirements.txt (line 3)) (2.7.5)\n",
      "Requirement already satisfied: pytz>=2011k in /usr/local/lib/python3.6/site-packages (from pandas->-r requirements.txt (line 3)) (2018.7)\n",
      "Requirement already satisfied: numpy>=1.9.0 in /usr/local/lib/python3.6/site-packages (from pandas->-r requirements.txt (line 3)) (1.15.4)\n",
      "Requirement already satisfied: regex in /usr/local/lib/python3.6/site-packages (from pytorch-pretrained-bert==0.6.2->-r requirements.txt (line 6)) (2019.8.19)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/site-packages (from pytorch-pretrained-bert==0.6.2->-r requirements.txt (line 6)) (2.22.0)\n",
      "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/site-packages (from pytorch-pretrained-bert==0.6.2->-r requirements.txt (line 6)) (1.9.231)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/site-packages (from torchvision==0.3.0->-r requirements.txt (line 8)) (1.12.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->-r requirements.txt (line 2)) (0.10.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->-r requirements.txt (line 2)) (1.0.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->-r requirements.txt (line 2)) (2.3.0)\n",
      "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.6/site-packages (from networkx>=2.0->scikit-image->-r requirements.txt (line 2)) (4.4.0)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/site-packages (from requests->pytorch-pretrained-bert==0.6.2->-r requirements.txt (line 6)) (1.24.3)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/site-packages (from requests->pytorch-pretrained-bert==0.6.2->-r requirements.txt (line 6)) (3.0.4)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/site-packages (from requests->pytorch-pretrained-bert==0.6.2->-r requirements.txt (line 6)) (2.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/site-packages (from requests->pytorch-pretrained-bert==0.6.2->-r requirements.txt (line 6)) (2019.6.16)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/site-packages (from boto3->pytorch-pretrained-bert==0.6.2->-r requirements.txt (line 6)) (0.9.4)\n",
      "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/site-packages (from boto3->pytorch-pretrained-bert==0.6.2->-r requirements.txt (line 6)) (0.2.1)\n",
      "Requirement already satisfied: botocore<1.13.0,>=1.12.231 in /usr/local/lib/python3.6/site-packages (from boto3->pytorch-pretrained-bert==0.6.2->-r requirements.txt (line 6)) (1.12.231)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/site-packages (from kiwisolver>=1.0.1->matplotlib!=3.0.0,>=2.0.0->scikit-image->-r requirements.txt (line 2)) (41.2.0)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.231->boto3->pytorch-pretrained-bert==0.6.2->-r requirements.txt (line 6)) (0.15.2)\n",
      "\u001b[31mfloyd-cli 0.11.17 has requirement click<7,>=6.7, but you'll have click 7.0 which is incompatible.\u001b[0m\n",
      "\u001b[33mYou are using pip version 10.0.1, however version 19.2.3 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils.data as D\n",
    "from torch.optim.lr_scheduler import ExponentialLR\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "\n",
    "from torchvision import transforms\n",
    "\n",
    "from ignite.engine import Events\n",
    "from scripts.ignite import create_supervised_evaluator, create_supervised_trainer\n",
    "from ignite.metrics import Loss, Accuracy\n",
    "from ignite.contrib.handlers.tqdm_logger import ProgressBar\n",
    "from ignite.handlers import  EarlyStopping, ModelCheckpoint\n",
    "from ignite.contrib.handlers import LinearCyclicalScheduler, CosineAnnealingScheduler\n",
    "\n",
    "import random\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from efficientnet_pytorch import EfficientNet, utils as enet_utils\n",
    "\n",
    "from scripts.evaluate import eval_model_per_cell\n",
    "from scripts.transforms import gen_transform_train, gen_transform_validation\n",
    "from scripts.plates_leak import apply_plates_leak\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define dataset and model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "img_dir = '/storage/rxrxai'\n",
    "path_data = '/storage/rxrxai'\n",
    "device = 'cuda'\n",
    "batch_size = 4\n",
    "torch.manual_seed(0)\n",
    "model_name = 'efficientnet-b4'\n",
    "init_lr = 3e-4\n",
    "end_lr = 1e-7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ImagesDS(D.Dataset):\n",
    "    transform_validation = gen_transform_validation(crop_size=448)\n",
    "    \n",
    "    def __init__(self, df, img_dir=img_dir, mode='train', validation=False, channels=[1,2,3,4,5,6]):\n",
    "        self.records = df.to_records(index=False)\n",
    "        self.mode = mode\n",
    "        self.img_dir = img_dir\n",
    "        self.len = df.shape[0]\n",
    "        self.validation = validation\n",
    "        self.channels = channels\n",
    "\n",
    "    def _get_img_path(self, index, channel, site):\n",
    "        experiment, well, plate = self.records[index].experiment, self.records[index].well, self.records[index].plate\n",
    "        return '/'.join([self.img_dir,self.mode,experiment,f'Plate{plate}',f'{well}_s{site}_w{channel}.png'])\n",
    "        \n",
    "    @staticmethod\n",
    "    def _load_img_as_tensor(file_name, transform):\n",
    "        with Image.open(file_name) as img:\n",
    "            return transform(img)\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        transform1 = ImagesDS.transform_validation if self.validation else gen_transform_train()\n",
    "        transform2 = ImagesDS.transform_validation if self.validation else gen_transform_train()\n",
    "        \n",
    "        paths1 = [self._get_img_path(index, ch, 1) for ch in self.channels]\n",
    "        paths2 = [self._get_img_path(index, ch, 2) for ch in self.channels]\n",
    "        \n",
    "        img1 = torch.cat([self._load_img_as_tensor(img_path, transform1) for img_path in paths1])\n",
    "        img2 = torch.cat([self._load_img_as_tensor(img_path, transform2) for img_path in paths2])\n",
    "        \n",
    "        if random.random() > 0.5 and not self.validation:\n",
    "            img1, img2 = img2, img1\n",
    "        \n",
    "        if self.mode == 'train':\n",
    "            return img1, img2, int(self.records[index].sirna)\n",
    "        else:\n",
    "            return img1, img2, self.records[index].id_code\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataframes for training, cross-validation, and testing\n",
    "df = pd.read_csv(path_data+'/train.csv')\n",
    "df['category'] = df['experiment'].apply(lambda x: x.split('-')[0])\n",
    "df_test = pd.read_csv(path_data+'/test.csv')\n",
    "df_test['category'] = df_test['experiment'].apply(lambda x: x.split('-')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EfficientNetTwoInputs(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(EfficientNetTwoInputs, self).__init__()\n",
    "        self.classes = 1108\n",
    "        \n",
    "        model = EfficientNet.from_pretrained(model_name, num_classes=1108) \n",
    "        num_ftrs = model._fc.in_features\n",
    "        model._fc = nn.Identity()\n",
    "        \n",
    "        # accept 6 channels\n",
    "        trained_kernel = model._conv_stem.weight\n",
    "        new_conv = enet_utils.Conv2dStaticSamePadding(6, 48, kernel_size=(3, 3), stride=(2, 2), bias=False, image_size=512)\n",
    "        with torch.no_grad():\n",
    "            new_conv.weight[:,:] = torch.stack([torch.mean(trained_kernel, 1)]*6, dim=1)\n",
    "        model._conv_stem = new_conv\n",
    "        \n",
    "        self.resnet = model\n",
    "        self.fc = nn.Linear(num_ftrs * 2, self.classes)\n",
    "\n",
    "    def forward(self, x1, x2):\n",
    "        x1_out = self.resnet(x1)\n",
    "        x2_out = self.resnet(x2)\n",
    "   \n",
    "        N, _, _, _ = x1.size()\n",
    "        x1_out = x1_out.view(N, -1)\n",
    "        x2_out = x2_out.view(N, -1)\n",
    "        \n",
    "        out = torch.cat((x1_out, x2_out), 1)\n",
    "        out = self.fc(out)\n",
    "\n",
    "        return out "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'HEPG2': 25, 'HUVEC': 15, 'RPE': 25, 'U2OS': 40}\n"
     ]
    }
   ],
   "source": [
    "cells = df['category'].unique()\n",
    "epochs_per_cell = {\n",
    "    cells[0]: 25,\n",
    "    cells[1]: 15,\n",
    "    cells[2]: 25,\n",
    "    cells[3]: 40,\n",
    "}\n",
    "print(epochs_per_cell)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# utilities to save best epoch\n",
    "def get_saved_model_path(epoch, cell=''):\n",
    "    return f'/artifacts/Model_{model_name}{cell}_{epoch}.pth'\n",
    "\n",
    "best_acc = 0.\n",
    "best_epoch = 1\n",
    "best_epoch_file = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p /artifacts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "========================================\n",
      "CURRENT CATEGORY: HEPG2\n",
      "----------------------------------------\n",
      "Loaded pretrained weights for efficientnet-b4\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-b7b86ce01a26>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0;31m# model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mEfficientNetTwoInputs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/storage/rxrxmodels/Model_efficientnet-b4_57.pth'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[1;32m    385\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 387\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_load\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmap_location\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mpickle_load_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    388\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    389\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mnew_fd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_load\u001b[0;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[1;32m    572\u001b[0m     \u001b[0munpickler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mUnpickler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mpickle_load_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    573\u001b[0m     \u001b[0munpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpersistent_load\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpersistent_load\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 574\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0munpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    575\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    576\u001b[0m     \u001b[0mdeserialized_storage_keys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mpickle_load_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36mpersistent_load\u001b[0;34m(saved_id)\u001b[0m\n\u001b[1;32m    535\u001b[0m                 \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    536\u001b[0m                 \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_torch_load_uninitialized\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 537\u001b[0;31m                 \u001b[0mdeserialized_objects\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot_key\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrestore_location\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    538\u001b[0m             \u001b[0mstorage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdeserialized_objects\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot_key\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    539\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mview_metadata\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36mdefault_restore_location\u001b[0;34m(storage, location)\u001b[0m\n\u001b[1;32m    117\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdefault_restore_location\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstorage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0m_package_registry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstorage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    120\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_cuda_deserialize\u001b[0;34m(obj, location)\u001b[0m\n\u001b[1;32m     97\u001b[0m             \u001b[0mstorage_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mstorage_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    100\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/torch/cuda/__init__.py\u001b[0m in \u001b[0;36m_lazy_new\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m    597\u001b[0m     \u001b[0;31m# We need this method only for lazy init, so we can remove it\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m     \u001b[0;32mdel\u001b[0m \u001b[0m_CudaBase\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__new__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 599\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_CudaBase\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__new__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    600\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    601\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "all_preds = []\n",
    "\n",
    "for cell in cells:\n",
    "    category_df = df[df['category'] == cell].copy()\n",
    "    cat_test_df = df_test[df_test['category'] == cell].copy()\n",
    "\n",
    "    print('\\n' + '=' * 40)\n",
    "    print(\"CURRENT CATEGORY:\", cell)\n",
    "    print('-' * 40)\n",
    "\n",
    "    cat_train_df, cat_val_df = train_test_split(\n",
    "        category_df, \n",
    "        random_state=2019,\n",
    "        test_size=0.05\n",
    "    )\n",
    "    \n",
    "    # pytorch training dataset & loader\n",
    "    cat_train_ds = ImagesDS(cat_train_df, mode='train', validation=False)\n",
    "    cat_train_loader = D.DataLoader(cat_train_ds, batch_size=batch_size, shuffle=True, num_workers=8)\n",
    "\n",
    "    # pytorch cross-validation dataset & loader\n",
    "    cat_val_ds = ImagesDS(cat_val_df, mode='train', validation=True)\n",
    "    cat_val_loader = D.DataLoader(cat_val_ds, batch_size=batch_size, shuffle=True, num_workers=8)\n",
    "\n",
    "    # model\n",
    "    model = EfficientNetTwoInputs()\n",
    "    model.load_state_dict(torch.load('/storage/rxrxmodels/Model_efficientnet-b4_57.pth'))\n",
    "    model.train()\n",
    "    \n",
    "    # metrics\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=init_lr)\n",
    "    \n",
    "    # multi-gpus training\n",
    "    if torch.cuda.device_count() > 1:\n",
    "        print(f\"Using {torch.cuda.device_count()} GPUs to train on {cell}\")\n",
    "        model = nn.DataParallel(model)    \n",
    "    \n",
    "    metrics = {\n",
    "        'loss': Loss(criterion),\n",
    "        'accuracy': Accuracy(),\n",
    "    }\n",
    "\n",
    "    trainer = create_supervised_trainer(model, optimizer, criterion, device=device)\n",
    "    val_evaluator = create_supervised_evaluator(model, metrics=metrics, device=device)\n",
    "    \n",
    "    # LR Scheduler\n",
    "    scheduler = CosineAnnealingScheduler(optimizer, 'lr', init_lr, end_lr, len(cat_train_loader))\n",
    "    trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)\n",
    "\n",
    "    # print lr\n",
    "    def print_lr(engine):\n",
    "        epoch = engine.state.epoch\n",
    "        iteration = engine.state.iteration\n",
    "    \n",
    "        if epoch < 2 and iteration % 100 == 0:\n",
    "            print(f'Iteration {iteration} | LR {optimizer.param_groups[0][\"lr\"]}')\n",
    "    trainer.add_event_handler(Events.ITERATION_COMPLETED, print_lr)\n",
    "\n",
    "        \n",
    "    # save best epoch only\n",
    "    def save_best_epoch_only(engine):\n",
    "        epoch = engine.state.epoch\n",
    "\n",
    "        global best_acc\n",
    "        global best_epoch\n",
    "        global best_epoch_file\n",
    "        best_acc = 0. if epoch == 1 else best_acc\n",
    "        best_epoch = 1 if epoch == 1 else best_epoch\n",
    "        best_epoch_file = '' if epoch == 1 else best_epoch_file\n",
    "\n",
    "        metrics = val_evaluator.run(cat_val_loader).metrics\n",
    "        print(\"Validation Results - Epoch: {} | Average Loss: {:.4f} | Accuracy: {:.4f} \"\n",
    "              .format(engine.state.epoch, metrics['loss'], metrics['accuracy']))\n",
    "\n",
    "        if metrics['accuracy'] > best_acc:\n",
    "            prev_best_epoch_file = get_saved_model_path(best_epoch, cell)\n",
    "            if os.path.exists(prev_best_epoch_file):\n",
    "                os.remove(prev_best_epoch_file)\n",
    "\n",
    "            best_acc = metrics['accuracy']\n",
    "            best_epoch = epoch\n",
    "            best_epoch_file = get_saved_model_path(best_epoch, cell)\n",
    "            print(f'\\nEpoch: {best_epoch} - New best accuracy! Accuracy: {best_acc}\\n\\n\\n')\n",
    "            torch.save(model.state_dict(), best_epoch_file)\n",
    "    trainer.add_event_handler(Events.EPOCH_COMPLETED, save_best_epoch_only)\n",
    "\n",
    "\n",
    "    # progress bar\n",
    "#     pbar = ProgressBar(bar_format='')\n",
    "#     pbar.attach(trainer, output_transform=lambda x: {'loss': x})\n",
    "                  \n",
    "    print('Training started\\n')\n",
    "    trainer.run(cat_train_loader, max_epochs=epochs_per_cell[cell])\n",
    "     \n",
    "    cat_test_ds = ImagesDS(cat_test_df, mode='test', validation=True)\n",
    "    cat_test_loader = D.DataLoader(cat_test_ds, batch_size=1, shuffle=False, num_workers=8)\n",
    "    cell_preds, _ = eval_model_per_cell(model, cat_test_loader, best_epoch_file, path_data, cat_test_df.copy().drop(['category'], axis=1), sub_file=f'/artifacts/submission_{cell}.csv')\n",
    "    all_preds += cell_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# aggregate submission files\n",
    "submissions = []\n",
    "for cell in cells:\n",
    "    submissions += [pd.read_csv(f'/artifacts/submission_{cell}.csv')]\n",
    "\n",
    "submissions = pd.concat(submissions)\n",
    "submissions.to_csv(f'/artifacts/submission.csv', index=False, columns=['id_code','sirna'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### apply plates leak"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_plates_leak(all_preds)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "2b62ae829edc4d60acf1d9a9e1d598d8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "7740dfb227e54da8b1510dac2d094406": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "921a9c670b6e4a2db86c75a7ff5d9ee6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9dfcb7497f8842af817750eec565b8b9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_921a9c670b6e4a2db86c75a7ff5d9ee6",
       "placeholder": "​",
       "style": "IPY_MODEL_2b62ae829edc4d60acf1d9a9e1d598d8",
       "value": " 94% 2151/2283 [22:45&lt;01:23,  1.58it/s]"
      }
     },
     "d2df0eb5abab4e3895ec792681cfa8d2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "e3ff3ae302394523bb5b28ee009842d5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_ff74a4321a59419cb24e116db9dd1e3e",
        "IPY_MODEL_9dfcb7497f8842af817750eec565b8b9"
       ],
       "layout": "IPY_MODEL_7740dfb227e54da8b1510dac2d094406"
      }
     },
     "fad7703039454db7af5d7fb4bce65003": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ff74a4321a59419cb24e116db9dd1e3e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "Loss: 128.54232788085938",
       "description_tooltip": null,
       "layout": "IPY_MODEL_fad7703039454db7af5d7fb4bce65003",
       "max": 2283,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_d2df0eb5abab4e3895ec792681cfa8d2",
       "value": 2151
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}