EfficientNet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting keras-rectified-adam\r\n",
      "  Downloading https://files.pythonhosted.org/packages/21/79/9521f66b92186702cb58a214c1b923b416266381cd824e15a1733f6a5b06/keras-rectified-adam-0.17.0.tar.gz\r\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from keras-rectified-adam) (1.16.4)\r\n",
      "Requirement already satisfied: Keras in /opt/conda/lib/python3.6/site-packages (from keras-rectified-adam) (2.3.0)\r\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /opt/conda/lib/python3.6/site-packages (from Keras->keras-rectified-adam) (1.1.0)\r\n",
      "Requirement already satisfied: scipy>=0.14 in /opt/conda/lib/python3.6/site-packages (from Keras->keras-rectified-adam) (1.2.1)\r\n",
      "Requirement already satisfied: keras-applications>=1.0.6 in /opt/conda/lib/python3.6/site-packages (from Keras->keras-rectified-adam) (1.0.8)\r\n",
      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.6/site-packages (from Keras->keras-rectified-adam) (5.1.2)\r\n",
      "Requirement already satisfied: six>=1.9.0 in /opt/conda/lib/python3.6/site-packages (from Keras->keras-rectified-adam) (1.12.0)\r\n",
      "Requirement already satisfied: h5py in /opt/conda/lib/python3.6/site-packages (from Keras->keras-rectified-adam) (2.9.0)\r\n",
      "Building wheels for collected packages: keras-rectified-adam\r\n",
      "  Building wheel for keras-rectified-adam (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n",
      "\u001b[?25h  Created wheel for keras-rectified-adam: filename=keras_rectified_adam-0.17.0-cp36-none-any.whl size=14781 sha256=cd29489ece42e2ae7c94a6fae912fb32a726b9b39fb9e02598c5b7d89e8c24ca\r\n",
      "  Stored in directory: /root/.cache/pip/wheels/7b/01/27/3a934e1a5644f5b93c720422a6ef97034ea78a21ba71cfb549\r\n",
      "Successfully built keras-rectified-adam\r\n",
      "Installing collected packages: keras-rectified-adam\r\n",
      "Successfully installed keras-rectified-adam-0.17.0\r\n",
      "Collecting git+https://github.com/qubvel/efficientnet\r\n",
      "  Cloning https://github.com/qubvel/efficientnet to /tmp/pip-req-build-0z9skdth\r\n",
      "  Running command git clone -q https://github.com/qubvel/efficientnet /tmp/pip-req-build-0z9skdth\r\n",
      "Requirement already satisfied, skipping upgrade: keras_applications<=1.0.8,>=1.0.7 in /opt/conda/lib/python3.6/site-packages (from efficientnet==1.1.0) (1.0.8)\r\n",
      "Requirement already satisfied, skipping upgrade: scikit-image in /opt/conda/lib/python3.6/site-packages (from efficientnet==1.1.0) (0.15.0)\r\n",
      "Requirement already satisfied, skipping upgrade: numpy>=1.9.1 in /opt/conda/lib/python3.6/site-packages (from keras_applications<=1.0.8,>=1.0.7->efficientnet==1.1.0) (1.16.4)\r\n",
      "Requirement already satisfied, skipping upgrade: h5py in /opt/conda/lib/python3.6/site-packages (from keras_applications<=1.0.8,>=1.0.7->efficientnet==1.1.0) (2.9.0)\r\n",
      "Requirement already satisfied, skipping upgrade: PyWavelets>=0.4.0 in /opt/conda/lib/python3.6/site-packages (from scikit-image->efficientnet==1.1.0) (1.0.3)\r\n",
      "Requirement already satisfied, skipping upgrade: pillow>=4.3.0 in /opt/conda/lib/python3.6/site-packages (from scikit-image->efficientnet==1.1.0) (5.4.1)\r\n",
      "Requirement already satisfied, skipping upgrade: imageio>=2.0.1 in /opt/conda/lib/python3.6/site-packages (from scikit-image->efficientnet==1.1.0) (2.5.0)\r\n",
      "Requirement already satisfied, skipping upgrade: matplotlib!=3.0.0,>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from scikit-image->efficientnet==1.1.0) (3.0.3)\r\n",
      "Requirement already satisfied, skipping upgrade: networkx>=2.0 in /opt/conda/lib/python3.6/site-packages (from scikit-image->efficientnet==1.1.0) (2.3)\r\n",
      "Requirement already satisfied, skipping upgrade: six in /opt/conda/lib/python3.6/site-packages (from h5py->keras_applications<=1.0.8,>=1.0.7->efficientnet==1.1.0) (1.12.0)\r\n",
      "Requirement already satisfied, skipping upgrade: cycler>=0.10 in /opt/conda/lib/python3.6/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet==1.1.0) (0.10.0)\r\n",
      "Requirement already satisfied, skipping upgrade: kiwisolver>=1.0.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet==1.1.0) (1.1.0)\r\n",
      "Requirement already satisfied, skipping upgrade: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet==1.1.0) (2.4.2)\r\n",
      "Requirement already satisfied, skipping upgrade: python-dateutil>=2.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet==1.1.0) (2.8.0)\r\n",
      "Requirement already satisfied, skipping upgrade: decorator>=4.3.0 in /opt/conda/lib/python3.6/site-packages (from networkx>=2.0->scikit-image->efficientnet==1.1.0) (4.4.0)\r\n",
      "Requirement already satisfied, skipping upgrade: setuptools in /opt/conda/lib/python3.6/site-packages (from kiwisolver>=1.0.1->matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet==1.1.0) (41.2.0)\r\n",
      "Building wheels for collected packages: efficientnet\r\n",
      "  Building wheel for efficientnet (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n",
      "\u001b[?25h  Created wheel for efficientnet: filename=efficientnet-1.1.0-cp36-none-any.whl size=18328 sha256=d2e17636a8e27fd6677d21881a23d81673ca6265007e780ace9375e4a814c0b9\r\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-xccgnjqg/wheels/64/60/2e/30ebaa76ed1626e86bfb0cc0579b737fdb7d9ff8cb9522663a\r\n",
      "Successfully built efficientnet\r\n",
      "Installing collected packages: efficientnet\r\n",
      "Successfully installed efficientnet-1.1.0\r\n"
     ]
    }
   ],
   "source": [
    "!pip install keras-rectified-adam\n",
    "!pip install -U git+https://github.com/qubvel/efficientnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os, glob, random, cv2\n",
    "import keras\n",
    "import multiprocessing\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import keras.backend as K\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from copy import deepcopy\n",
    "\n",
    "from sklearn.metrics import precision_recall_curve, auc\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import Callback\n",
    "from keras.applications.densenet import DenseNet201\n",
    "from keras.layers import Dense, Flatten\n",
    "from keras.models import Model, load_model\n",
    "from keras.utils import Sequence\n",
    "from keras_radam import RAdam\n",
    "\n",
    "from albumentations import Compose, VerticalFlip, HorizontalFlip, Rotate, GridDistortion\n",
    "from IPython.display import Image\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from numpy.random import seed\n",
    "from tensorflow import set_random_seed\n",
    "\n",
    "seed(10)\n",
    "set_random_seed(10)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "test_imgs_dir = '../input/understanding_cloud_organization/test_images/'\n",
    "train_imgs_dir = '../input/understanding_cloud_organization/train_images/'\n",
    "num_cores = multiprocessing.cpu_count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Generators"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One hot encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Image_Label</th>\n",
       "      <th>EncodedPixels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0011165.jpg_Fish</td>\n",
       "      <td>264918 937 266318 937 267718 937 269118 937 27...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0011165.jpg_Flower</td>\n",
       "      <td>1355565 1002 1356965 1002 1358365 1002 1359765...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0011165.jpg_Gravel</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0011165.jpg_Sugar</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>002be4f.jpg_Fish</td>\n",
       "      <td>233813 878 235213 878 236613 878 238010 881 23...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Image_Label                                      EncodedPixels\n",
       "0    0011165.jpg_Fish  264918 937 266318 937 267718 937 269118 937 27...\n",
       "1  0011165.jpg_Flower  1355565 1002 1356965 1002 1358365 1002 1359765...\n",
       "2  0011165.jpg_Gravel                                                NaN\n",
       "3   0011165.jpg_Sugar                                                NaN\n",
       "4    002be4f.jpg_Fish  233813 878 235213 878 236613 878 238010 881 23..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv('../input/understanding_cloud_organization/train.csv')\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_kg_hide-input": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Image</th>\n",
       "      <th>Class</th>\n",
       "      <th>Fish</th>\n",
       "      <th>Flower</th>\n",
       "      <th>Sugar</th>\n",
       "      <th>Gravel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0011165.jpg</td>\n",
       "      <td>{Fish, Flower}</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>002be4f.jpg</td>\n",
       "      <td>{Fish, Sugar, Flower}</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0031ae9.jpg</td>\n",
       "      <td>{Fish, Sugar, Flower}</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0035239.jpg</td>\n",
       "      <td>{Gravel, Flower}</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>003994e.jpg</td>\n",
       "      <td>{Fish, Sugar, Gravel}</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Image                  Class  Fish  Flower  Sugar  Gravel\n",
       "0  0011165.jpg         {Fish, Flower}     1       1      0       0\n",
       "1  002be4f.jpg  {Fish, Sugar, Flower}     1       1      1       0\n",
       "2  0031ae9.jpg  {Fish, Sugar, Flower}     1       1      1       0\n",
       "3  0035239.jpg       {Gravel, Flower}     0       1      0       1\n",
       "4  003994e.jpg  {Fish, Sugar, Gravel}     1       0      1       1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = train_df[~train_df['EncodedPixels'].isnull()]\n",
    "train_df['Image'] = train_df['Image_Label'].map(lambda x: x.split('_')[0])\n",
    "train_df['Class'] = train_df['Image_Label'].map(lambda x: x.split('_')[1])\n",
    "\n",
    "classes = train_df['Class'].unique()\n",
    "\n",
    "train_df = train_df.groupby('Image')['Class'].agg(set).reset_index()\n",
    "\n",
    "for class_name in classes:\n",
    "    train_df[class_name] = train_df['Class'].map(lambda x: 1 if class_name in x else 0)\n",
    "\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dictionary for fast access\n",
    "img_2_ohe_vector = {img:vec for img, vec in zip(train_df['Image'], train_df.iloc[:, 2:].values)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train-val split\n",
    "train_imgs, val_imgs = train_test_split(train_df['Image'].values, \n",
    "                                        test_size = 0.2, \n",
    "                                        stratify = train_df['Class'].map(lambda x: str(sorted(list(x)))), \n",
    "                                        random_state = 2019)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataGenenerator(Sequence):\n",
    "    def __init__(self, images_list = None, folder_imgs = train_imgs_dir, \n",
    "                 batch_size = 32, shuffle = True, augmentation = None,\n",
    "                 resized_height = 260, resized_width = 260, num_channels = 3):\n",
    "        self.batch_size = batch_size\n",
    "        self.shuffle = shuffle\n",
    "        self.augmentation = augmentation\n",
    "        \n",
    "        if images_list is None:\n",
    "            self.images_list = os.listdir(folder_imgs)\n",
    "        else:\n",
    "            self.images_list = deepcopy(images_list)\n",
    "        \n",
    "        \n",
    "        self.folder_imgs = folder_imgs\n",
    "        self.len = len(self.images_list) // self.batch_size\n",
    "        self.resized_height = resized_height\n",
    "        self.resized_width = resized_width\n",
    "        self.num_channels = num_channels\n",
    "        self.num_classes = 4\n",
    "        self.is_test = not 'train' in folder_imgs\n",
    "        \n",
    "        if not shuffle and not self.is_test:\n",
    "            self.labels = [img_2_ohe_vector[img] for img in self.images_list[:self.len * self.batch_size]]\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.len\n",
    "    \n",
    "    def on_epoch_start(self):\n",
    "        if self.shuffle:\n",
    "            random.shuffle(self.images_list)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        current_batch = self.images_list[idx * self.batch_size: (idx + 1) * self.batch_size]\n",
    "        X = np.empty((self.batch_size, self.resized_height, self.resized_width, self.num_channels))\n",
    "        y = np.empty((self.batch_size, self.num_classes))\n",
    "\n",
    "        for i, image_name in enumerate(current_batch):\n",
    "            path = os.path.join(self.folder_imgs, image_name)\n",
    "            img = cv2.resize(cv2.imread(path), (self.resized_height, self.resized_width)).astype(np.float32)\n",
    "            \n",
    "            if not self.augmentation is None:\n",
    "                augmented = self.augmentation(image=img)\n",
    "                img = augmented['image']\n",
    "            \n",
    "            X[i, :, :, :] = img/255.0\n",
    "            \n",
    "            if not self.is_test:\n",
    "                y[i, :] = img_2_ohe_vector[image_name]\n",
    "        \n",
    "        return X, y\n",
    "\n",
    "    def get_labels(self):\n",
    "        if self.shuffle:\n",
    "            images_current = self.images_list[:self.len * self.batch_size]\n",
    "            labels = [img_2_ohe_vector[img] for img in images_current]\n",
    "        else:\n",
    "            labels = self.labels\n",
    "        return np.array(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "albumentations_train = Compose([ VerticalFlip(), HorizontalFlip(), Rotate(limit=20), GridDistortion() ], p = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generator instances\n",
    "data_generator_train = DataGenenerator(train_imgs, augmentation = albumentations_train)\n",
    "data_generator_train_eval = DataGenenerator(train_imgs, shuffle = False)\n",
    "data_generator_val = DataGenenerator(val_imgs, shuffle = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PrAucCallback(Callback):\n",
    "    def __init__(self, data_generator, num_workers = num_cores, \n",
    "                 early_stopping_patience = 5, \n",
    "                 plateau_patience = 3, reduction_rate = 0.5,\n",
    "                 stage = 'train', checkpoints_path = 'checkpoints/'):\n",
    "        super(Callback, self).__init__()\n",
    "        self.data_generator = data_generator\n",
    "        self.num_workers = num_workers\n",
    "        self.class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']\n",
    "        self.history = [[] for _ in range(len(self.class_names) + 1)] # to store per each class and also mean PR AUC\n",
    "        self.early_stopping_patience = early_stopping_patience\n",
    "        self.plateau_patience = plateau_patience\n",
    "        self.reduction_rate = reduction_rate\n",
    "        self.stage = stage\n",
    "        self.best_pr_auc = -float('inf')\n",
    "        \n",
    "        if not os.path.exists(checkpoints_path):\n",
    "            os.makedirs(checkpoints_path)\n",
    "        \n",
    "        self.checkpoints_path = checkpoints_path\n",
    "        \n",
    "    def compute_pr_auc(self, y_true, y_pred):\n",
    "        pr_auc_mean = 0\n",
    "        for class_i in range(len(self.class_names)):\n",
    "            precision, recall, _ = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])\n",
    "            pr_auc = auc(recall, precision)\n",
    "            pr_auc_mean += pr_auc/len(self.class_names)\n",
    "            print(f\"PR-AUC {self.class_names[class_i]}, {self.stage}: {pr_auc:.3f}\")\n",
    "            self.history[class_i].append(pr_auc)        \n",
    "        print(f\"\\nmean, {self.stage}: {pr_auc_mean:.3f}\")\n",
    "        self.history[-1].append(pr_auc_mean)\n",
    "        return pr_auc_mean\n",
    "              \n",
    "    def is_patience_lost(self, patience):\n",
    "        if len(self.history[-1]) > patience:\n",
    "            best_performance = max(self.history[-1][-(patience + 1):-1])\n",
    "            return best_performance == self.history[-1][-(patience + 1)] and best_performance >= self.history[-1][-1]    \n",
    "              \n",
    "    def early_stopping_check(self, pr_auc_mean):\n",
    "        if self.is_patience_lost(self.early_stopping_patience):\n",
    "            self.model.stop_training = True    \n",
    "              \n",
    "    def model_checkpoint(self, pr_auc_mean, epoch):\n",
    "        if pr_auc_mean > self.best_pr_auc:\n",
    "            \n",
    "            # remove previous checkpoints to save space\n",
    "            for checkpoint in glob.glob(os.path.join(self.checkpoints_path, 'classifier_epoch_*')):\n",
    "                os.remove(checkpoint)\n",
    "            self.best_pr_auc = pr_auc_mean\n",
    "            self.model.save(os.path.join(self.checkpoints_path, f'classifier_epoch_{epoch}_val_pr_auc_{pr_auc_mean}.h5'))              \n",
    "            print(f\"\\nSaved new checkpoint\")\n",
    "              \n",
    "    def reduce_lr_on_plateau(self):\n",
    "        if self.is_patience_lost(self.plateau_patience):\n",
    "            new_lr = float(keras.backend.get_value(self.model.optimizer.lr)) * self.reduction_rate\n",
    "            keras.backend.set_value(self.model.optimizer.lr, new_lr)\n",
    "            print(f\"\\nReduced learning rate to {new_lr}.\")\n",
    "        \n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        y_pred = self.model.predict_generator(self.data_generator, workers=self.num_workers)\n",
    "        y_true = self.data_generator.get_labels()\n",
    "        \n",
    "        # estimate AUC under precision recall curve for each class\n",
    "        pr_auc_mean = self.compute_pr_auc(y_true, y_pred)\n",
    "              \n",
    "        if self.stage == 'val':\n",
    "            self.early_stopping_check(pr_auc_mean)\n",
    "            self.model_checkpoint(pr_auc_mean, epoch)\n",
    "            self.reduce_lr_on_plateau()            \n",
    "        \n",
    "    def get_pr_auc_history(self):\n",
    "        return self.history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_metric_callback = PrAucCallback(data_generator_train_eval)\n",
    "val_callback = PrAucCallback(data_generator_val, stage='val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CLASSIFIER\n",
    "\n",
    "from keras.losses import binary_crossentropy\n",
    "def dice_coef(y_true, y_pred, smooth=1):\n",
    "    y_true_f = K.flatten(y_true)\n",
    "    y_pred_f = K.flatten(y_pred)\n",
    "    intersection = K.sum(y_true_f * y_pred_f)\n",
    "    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n",
    "\n",
    "def dice_loss(y_true, y_pred):\n",
    "    smooth = 1.\n",
    "    y_true_f = K.flatten(y_true)\n",
    "    y_pred_f = K.flatten(y_pred)\n",
    "    intersection = y_true_f * y_pred_f\n",
    "    score = (2. * K.sum(intersection) + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n",
    "    return 1. - score\n",
    "\n",
    "def bce_dice_loss(y_true, y_pred):\n",
    "    return binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://github.com/Callidior/keras-applications/releases/download/efficientnet/efficientnet-b0_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5\n",
      "16809984/16804768 [==============================] - 0s 0us/step\n"
     ]
    }
   ],
   "source": [
    "import efficientnet.keras as efn \n",
    "def get_model():\n",
    "    K.clear_session()\n",
    "    base_model =  efn.EfficientNetB0(weights='imagenet', include_top=False, pooling='avg', input_shape=(260, 260, 3))\n",
    "#     base_model =  efn.EfficientNetB1(weights='imagenet', include_top=False, pooling='avg', input_shape=(260, 260, 3))\n",
    "#     base_model =  efn.EfficientNetB1(weights='imagenet', include_top=False, pooling='avg', input_shape=(260, 260, 3))\n",
    "#     base_model =  efn.EfficientNetB2(weights='imagenet', include_top=False, pooling='avg', input_shape=(260, 260, 3))\n",
    "#     base_model =  efn.EfficientNetB3(weights='imagenet', include_top=False, pooling='avg', input_shape=(260, 260, 3))\n",
    "#     base_model =  efn.EfficientNetB4(weights='imagenet', include_top=False, pooling='avg', input_shape=(260, 260, 3))\n",
    "#     base_model =  efn.EfficientNetB5(weights='imagenet', include_top=False, pooling='avg', input_shape=(260, 260, 3))\n",
    "    x = base_model.output\n",
    "    y_pred = Dense(4, activation='sigmoid')(x)\n",
    "    return Model(inputs = base_model.input, outputs = y_pred)\n",
    "\n",
    "model = get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initial tuning of the added fully-connected layer\n",
    "for base_layer in model.layers[:-3]:\n",
    "    base_layer.trainable = False\n",
    "    \n",
    "model.compile(optimizer = RAdam(warmup_proportion = 0.1, min_lr = 1e-5),  \n",
    "                              loss = 'categorical_crossentropy', \n",
    "                              metrics = ['accuracy'])\n",
    "\n",
    "history_0 = model.fit_generator(generator = data_generator_train,\n",
    "                              validation_data = data_generator_val,\n",
    "                              epochs = 20,\n",
    "                              callbacks = [train_metric_callback, val_callback],\n",
    "                              workers = num_cores,\n",
    "                              verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fine-tuning the whole model\n",
    "for base_layer in model.layers[:-3]:\n",
    "    base_layer.trainable = True\n",
    "    \n",
    "model.compile(optimizer = RAdam(warmup_proportion = 0.1, min_lr = 1e-5),  \n",
    "                              loss = 'categorical_crossentropy', \n",
    "                              metrics = ['accuracy'])\n",
    "history_1 = model.fit_generator(generator = data_generator_train,\n",
    "                              validation_data = data_generator_val,\n",
    "                              epochs = 20,\n",
    "                              callbacks = [train_metric_callback, val_callback],\n",
    "                              workers = num_cores,\n",
    "                              verbose = 1,\n",
    "                              initial_epoch = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fs = 10\n",
    "fnt1 = 12\n",
    "fnt2 = 15\n",
    "pr_auc_history_train = train_metric_callback.get_pr_auc_history()\n",
    "pr_auc_history_val = val_callback.get_pr_auc_history()\n",
    "\n",
    "plt.figure(figsize = (fs, fs))\n",
    "plot(plt, pr_auc_history_train[-1])\n",
    "plot(plt, pr_auc_history_val[-1])\n",
    "\n",
    "plt.xlabel('Epoch', fontsize = fnt1)\n",
    "plt.ylabel('Mean PR-AUC', fontsize = fnt1)\n",
    "plt.legend(['Train', 'Validation'])\n",
    "plt.title('Training & validation PR-AUC', fontsize = fnt2)\n",
    "plt.savefig('pr_auc_hist.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize = (fs, fs))\n",
    "plot_with_dots(plt, history_0.history['loss']+history_1.history['loss'])\n",
    "plot_with_dots(plt, history_0.history['val_loss']+history_1.history['val_loss'])\n",
    "\n",
    "plt.xlabel('Epoch', fontsize = fnt1)\n",
    "plt.ylabel('Binary Crossentropy', fontsize = fnt1)\n",
    "plt.legend(['Train', 'Validation'])\n",
    "plt.title('Training & Validation Loss', fontsize = fnt2)\n",
    "plt.savefig('loss_hist.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select  post processing threshold\n",
    "class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']\n",
    "def get_threshold_for_recall(y_true, y_pred, class_i, recall_threshold = 0.94, precision_threshold = 0.90, plot = False):\n",
    "    precision, recall, thresholds = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])\n",
    "    i = len(thresholds) - 1\n",
    "    best_recall_threshold = None\n",
    "    while best_recall_threshold is None:\n",
    "        next_threshold = thresholds[i]\n",
    "        next_recall = recall[i]\n",
    "        if next_recall >= recall_threshold:\n",
    "            best_recall_threshold = next_threshold\n",
    "        i -= 1\n",
    "        \n",
    "    # consice, even though unnecessary passing through all the values\n",
    "    best_precision_threshold = [thres for prec, thres in zip(precision, thresholds) if prec >= precision_threshold][0]\n",
    "    \n",
    "    if plot:\n",
    "        plt.figure(figsize = (fs, fs))\n",
    "        plt.step(recall, precision, color='g', alpha=0.3, where='post')\n",
    "        plt.fill_between(recall, precision, alpha=0.3, color='b')\n",
    "        plt.axhline(y=precision[i + 1])\n",
    "        recall_for_prec_thres = [rec for rec, thres in zip(recall, thresholds) if thres == best_precision_threshold][0]\n",
    "        plt.axvline(x=recall_for_prec_thres, color='r')\n",
    "        plt.xlabel('Recall')\n",
    "        plt.ylabel('Precision')\n",
    "        plt.ylim([0.0, 1.05])\n",
    "        plt.xlim([0.0, 1.0])\n",
    "        plt.legend(['PR curve', \n",
    "                    f'Precision {precision[i + 1]: .2f} corresponding to selected recall threshold',\n",
    "                    f'Recall {recall_for_prec_thres: .2f} corresponding to selected precision threshold'])\n",
    "        plt.title(f'Precision-Recall curve for Class {class_names[class_i]}')\n",
    "    return best_recall_threshold, best_precision_threshold\n",
    "\n",
    "y_pred = model.predict_generator(data_generator_val, workers=num_cores)\n",
    "y_true = data_generator_val.get_labels()\n",
    "recall_thresholds = dict()\n",
    "precision_thresholds = dict()\n",
    "for i, class_name in tqdm(enumerate(class_names)):\n",
    "    recall_thresholds[class_name], precision_thresholds[class_name] = get_threshold_for_recall(y_true, y_pred, i, plot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SUBMISSION\n",
    "# prediction of cloud classes\n",
    "data_generator_test = DataGenenerator(folder_imgs=test_imgs_dir, shuffle=False)\n",
    "y_pred_test = model.predict_generator(data_generator_test, workers=num_cores)\n",
    "\n",
    "# Estimating set of images without masks\n",
    "image_labels_empty = set()\n",
    "for i, (img, predictions) in enumerate(zip(os.listdir(test_imgs_dir), y_pred_test)):\n",
    "    for class_i, class_name in enumerate(class_names):\n",
    "        if predictions[class_i] < recall_thresholds[class_name]:\n",
    "            image_labels_empty.add(f'{img}_{class_name}')\n",
    "\n",
    "# Segmentation results:            \n",
    "submission = pd.read_csv('../input/densenet201cloudy/densenet201.csv')\n",
    "submission.head()\n",
    "\n",
    "predictions_nonempty = set(submission.loc[~submission['EncodedPixels'].isnull(), 'Image_Label'].values)\n",
    "\n",
    "print(f'{len(image_labels_empty.intersection(predictions_nonempty))} masks would be removed')\n",
    "\n",
    "#removing masks\n",
    "submission.loc[submission['Image_Label'].isin(image_labels_empty), 'EncodedPixels'] = np.nan\n",
    "submission.to_csv('submission.csv', index=None)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "007f09756431484e959f31e49feb5061": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_101c6e687955408aa21203427ae57c0e",
       "max": 1,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_2372eecd16d0445d8e490bf17fdfcf87",
       "value": 1
      }
     },
     "101c6e687955408aa21203427ae57c0e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "2372eecd16d0445d8e490bf17fdfcf87": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "4003f5b770e841d193aca3ac46f6f4af": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "52a297c38a334d039d9c5a733daa403e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_007f09756431484e959f31e49feb5061",
        "IPY_MODEL_9ce268a1465940298e231f34e9f1dd81"
       ],
       "layout": "IPY_MODEL_6ac97c65b428480b8fc11d92fb6f25ab"
      }
     },
     "6ac97c65b428480b8fc11d92fb6f25ab": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "89173705a8d24d258f13f8c208525ae3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9ce268a1465940298e231f34e9f1dd81": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_89173705a8d24d258f13f8c208525ae3",
       "placeholder": "​",
       "style": "IPY_MODEL_4003f5b770e841d193aca3ac46f6f4af",
       "value": " 4/? [00:00&lt;00:00, 32.20it/s]"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}