13-dropout.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Activation, BatchNormalization, Dropout, Input\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "import keras\n",
    "from datetime import datetime\n",
    "from sklearn import preprocessing\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import plot_model\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.utils import shuffle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_dataframe = pd.read_csv('dataset/1. istanbul/train_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualize some training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject ID</th>\n",
       "      <th>Jitter (local)</th>\n",
       "      <th>Jitter (local, absolute)</th>\n",
       "      <th>Jitter (rap)</th>\n",
       "      <th>Jitter (ppq5)</th>\n",
       "      <th>Jitter (ddp)</th>\n",
       "      <th>Shimmer (local)</th>\n",
       "      <th>Shimmer (local, dB)</th>\n",
       "      <th>Shimmer (apq3)</th>\n",
       "      <th>Shimmer (apq5)</th>\n",
       "      <th>...</th>\n",
       "      <th>Maximum pitch</th>\n",
       "      <th>Number of pulses</th>\n",
       "      <th>Number of periods</th>\n",
       "      <th>Mean period</th>\n",
       "      <th>Standard deviation of period</th>\n",
       "      <th>Fraction of locally unvoiced frames</th>\n",
       "      <th>Number of voice breaks</th>\n",
       "      <th>Degree of voice breaks</th>\n",
       "      <th>UPDRS</th>\n",
       "      <th>Class information</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.488</td>\n",
       "      <td>0.000090</td>\n",
       "      <td>0.900</td>\n",
       "      <td>0.794</td>\n",
       "      <td>2.699</td>\n",
       "      <td>8.334</td>\n",
       "      <td>0.779</td>\n",
       "      <td>4.517</td>\n",
       "      <td>4.609</td>\n",
       "      <td>...</td>\n",
       "      <td>187.576</td>\n",
       "      <td>160</td>\n",
       "      <td>159</td>\n",
       "      <td>0.006065</td>\n",
       "      <td>0.000416</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.728</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>0.353</td>\n",
       "      <td>0.376</td>\n",
       "      <td>1.059</td>\n",
       "      <td>5.864</td>\n",
       "      <td>0.642</td>\n",
       "      <td>2.058</td>\n",
       "      <td>3.180</td>\n",
       "      <td>...</td>\n",
       "      <td>234.505</td>\n",
       "      <td>170</td>\n",
       "      <td>169</td>\n",
       "      <td>0.005181</td>\n",
       "      <td>0.000403</td>\n",
       "      <td>2.247</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1.220</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.732</td>\n",
       "      <td>0.670</td>\n",
       "      <td>2.196</td>\n",
       "      <td>8.719</td>\n",
       "      <td>0.875</td>\n",
       "      <td>4.347</td>\n",
       "      <td>5.166</td>\n",
       "      <td>...</td>\n",
       "      <td>211.442</td>\n",
       "      <td>1431</td>\n",
       "      <td>1427</td>\n",
       "      <td>0.006071</td>\n",
       "      <td>0.000474</td>\n",
       "      <td>10.656</td>\n",
       "      <td>1</td>\n",
       "      <td>0.178</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2.502</td>\n",
       "      <td>0.000123</td>\n",
       "      <td>1.156</td>\n",
       "      <td>1.634</td>\n",
       "      <td>3.469</td>\n",
       "      <td>13.513</td>\n",
       "      <td>1.273</td>\n",
       "      <td>5.263</td>\n",
       "      <td>8.771</td>\n",
       "      <td>...</td>\n",
       "      <td>220.230</td>\n",
       "      <td>94</td>\n",
       "      <td>92</td>\n",
       "      <td>0.004910</td>\n",
       "      <td>0.000320</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3.509</td>\n",
       "      <td>0.000167</td>\n",
       "      <td>1.715</td>\n",
       "      <td>1.539</td>\n",
       "      <td>5.145</td>\n",
       "      <td>9.112</td>\n",
       "      <td>1.040</td>\n",
       "      <td>3.102</td>\n",
       "      <td>4.927</td>\n",
       "      <td>...</td>\n",
       "      <td>225.162</td>\n",
       "      <td>117</td>\n",
       "      <td>114</td>\n",
       "      <td>0.004757</td>\n",
       "      <td>0.000380</td>\n",
       "      <td>18.182</td>\n",
       "      <td>1</td>\n",
       "      <td>13.318</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Subject ID  Jitter (local)  Jitter (local, absolute)  Jitter (rap)  \\\n",
       "0           1           1.488                  0.000090         0.900   \n",
       "1           1           0.728                  0.000038         0.353   \n",
       "2           1           1.220                  0.000074         0.732   \n",
       "3           1           2.502                  0.000123         1.156   \n",
       "4           1           3.509                  0.000167         1.715   \n",
       "\n",
       "   Jitter (ppq5)  Jitter (ddp)  Shimmer (local)  Shimmer (local, dB)  \\\n",
       "0          0.794         2.699            8.334                0.779   \n",
       "1          0.376         1.059            5.864                0.642   \n",
       "2          0.670         2.196            8.719                0.875   \n",
       "3          1.634         3.469           13.513                1.273   \n",
       "4          1.539         5.145            9.112                1.040   \n",
       "\n",
       "   Shimmer (apq3)  Shimmer (apq5)        ...          Maximum pitch  \\\n",
       "0           4.517           4.609        ...                187.576   \n",
       "1           2.058           3.180        ...                234.505   \n",
       "2           4.347           5.166        ...                211.442   \n",
       "3           5.263           8.771        ...                220.230   \n",
       "4           3.102           4.927        ...                225.162   \n",
       "\n",
       "   Number of pulses  Number of periods  Mean period  \\\n",
       "0               160                159     0.006065   \n",
       "1               170                169     0.005181   \n",
       "2              1431               1427     0.006071   \n",
       "3                94                 92     0.004910   \n",
       "4               117                114     0.004757   \n",
       "\n",
       "   Standard deviation of period  Fraction of locally unvoiced frames  \\\n",
       "0                      0.000416                                0.000   \n",
       "1                      0.000403                                2.247   \n",
       "2                      0.000474                               10.656   \n",
       "3                      0.000320                                0.000   \n",
       "4                      0.000380                               18.182   \n",
       "\n",
       "   Number of voice breaks  Degree of voice breaks  UPDRS  Class information  \n",
       "0                       0                   0.000     23                  1  \n",
       "1                       0                   0.000     23                  1  \n",
       "2                       1                   0.178     23                  1  \n",
       "3                       0                   0.000     23                  1  \n",
       "4                       1                  13.318     23                  1  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataframe.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1040, 26)\n"
     ]
    }
   ],
   "source": [
    "train_x = train_dataframe.drop(['Subject ID', 'Class information', 'UPDRS'], axis=1)\n",
    "train_x = train_x.as_matrix()\n",
    "print(train_x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare training labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1040,)\n",
      "(1040, 2)\n"
     ]
    }
   ],
   "source": [
    "train_y = train_dataframe['Class information']\n",
    "train_y = train_y.as_matrix()\n",
    "print(train_y.shape)\n",
    "train_y = np_utils.to_categorical(train_y)\n",
    "print(train_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(168, 26)\n"
     ]
    }
   ],
   "source": [
    "test_dataframe = pd.read_csv('dataset/1. istanbul/test_data.csv')\n",
    "test_x = test_dataframe.drop(['Subject ID', 'Class information'], axis=1)\n",
    "test_x = test_x.as_matrix()\n",
    "print(test_x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare test labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(168,)\n",
      "(168, 2)\n"
     ]
    }
   ],
   "source": [
    "test_y = test_dataframe['Class information']\n",
    "test_y = test_y.as_matrix()\n",
    "print(test_y.shape)\n",
    "test_y = np_utils.to_categorical(test_y)\n",
    "print(test_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine train and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1208, 26)\n",
      "(1208, 2)\n"
     ]
    }
   ],
   "source": [
    "train_x = np.vstack([train_x, test_x])\n",
    "train_y = np.vstack([train_y, test_y])\n",
    "print(train_x.shape)\n",
    "print(train_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalize data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "      <th>25</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.096023</td>\n",
       "      <td>0.110551</td>\n",
       "      <td>0.106928</td>\n",
       "      <td>0.054303</td>\n",
       "      <td>0.106923</td>\n",
       "      <td>0.186514</td>\n",
       "      <td>0.266893</td>\n",
       "      <td>0.162585</td>\n",
       "      <td>0.056981</td>\n",
       "      <td>0.142044</td>\n",
       "      <td>...</td>\n",
       "      <td>0.034343</td>\n",
       "      <td>0.193353</td>\n",
       "      <td>0.204351</td>\n",
       "      <td>0.107383</td>\n",
       "      <td>0.106783</td>\n",
       "      <td>0.378729</td>\n",
       "      <td>0.062423</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.042716</td>\n",
       "      <td>0.042501</td>\n",
       "      <td>0.038278</td>\n",
       "      <td>0.023294</td>\n",
       "      <td>0.038318</td>\n",
       "      <td>0.125260</td>\n",
       "      <td>0.215176</td>\n",
       "      <td>0.065922</td>\n",
       "      <td>0.037237</td>\n",
       "      <td>0.150903</td>\n",
       "      <td>...</td>\n",
       "      <td>0.049169</td>\n",
       "      <td>0.238354</td>\n",
       "      <td>0.295334</td>\n",
       "      <td>0.114094</td>\n",
       "      <td>0.113499</td>\n",
       "      <td>0.295624</td>\n",
       "      <td>0.060411</td>\n",
       "      <td>0.025488</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.077225</td>\n",
       "      <td>0.089595</td>\n",
       "      <td>0.085843</td>\n",
       "      <td>0.045104</td>\n",
       "      <td>0.085882</td>\n",
       "      <td>0.196062</td>\n",
       "      <td>0.303133</td>\n",
       "      <td>0.155902</td>\n",
       "      <td>0.064677</td>\n",
       "      <td>0.158903</td>\n",
       "      <td>...</td>\n",
       "      <td>0.043064</td>\n",
       "      <td>0.204329</td>\n",
       "      <td>0.250621</td>\n",
       "      <td>0.960403</td>\n",
       "      <td>0.958361</td>\n",
       "      <td>0.379296</td>\n",
       "      <td>0.071557</td>\n",
       "      <td>0.120874</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.002575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.167146</td>\n",
       "      <td>0.152809</td>\n",
       "      <td>0.139056</td>\n",
       "      <td>0.116617</td>\n",
       "      <td>0.139134</td>\n",
       "      <td>0.314949</td>\n",
       "      <td>0.453379</td>\n",
       "      <td>0.191910</td>\n",
       "      <td>0.114487</td>\n",
       "      <td>0.367528</td>\n",
       "      <td>...</td>\n",
       "      <td>0.035815</td>\n",
       "      <td>0.298746</td>\n",
       "      <td>0.267658</td>\n",
       "      <td>0.063087</td>\n",
       "      <td>0.061786</td>\n",
       "      <td>0.270109</td>\n",
       "      <td>0.047233</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.237778</td>\n",
       "      <td>0.209959</td>\n",
       "      <td>0.209212</td>\n",
       "      <td>0.109570</td>\n",
       "      <td>0.209245</td>\n",
       "      <td>0.205808</td>\n",
       "      <td>0.365421</td>\n",
       "      <td>0.106962</td>\n",
       "      <td>0.061375</td>\n",
       "      <td>0.278121</td>\n",
       "      <td>...</td>\n",
       "      <td>0.038016</td>\n",
       "      <td>0.299027</td>\n",
       "      <td>0.277220</td>\n",
       "      <td>0.078523</td>\n",
       "      <td>0.076561</td>\n",
       "      <td>0.255738</td>\n",
       "      <td>0.056711</td>\n",
       "      <td>0.206243</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.192688</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0         1         2         3         4         5         6   \\\n",
       "0  0.096023  0.110551  0.106928  0.054303  0.106923  0.186514  0.266893   \n",
       "1  0.042716  0.042501  0.038278  0.023294  0.038318  0.125260  0.215176   \n",
       "2  0.077225  0.089595  0.085843  0.045104  0.085882  0.196062  0.303133   \n",
       "3  0.167146  0.152809  0.139056  0.116617  0.139134  0.314949  0.453379   \n",
       "4  0.237778  0.209959  0.209212  0.109570  0.209245  0.205808  0.365421   \n",
       "\n",
       "         7         8         9     ...           16        17        18  \\\n",
       "0  0.162585  0.056981  0.142044    ...     0.034343  0.193353  0.204351   \n",
       "1  0.065922  0.037237  0.150903    ...     0.049169  0.238354  0.295334   \n",
       "2  0.155902  0.064677  0.158903    ...     0.043064  0.204329  0.250621   \n",
       "3  0.191910  0.114487  0.367528    ...     0.035815  0.298746  0.267658   \n",
       "4  0.106962  0.061375  0.278121    ...     0.038016  0.299027  0.277220   \n",
       "\n",
       "         19        20        21        22        23        24        25  \n",
       "0  0.107383  0.106783  0.378729  0.062423  0.000000  0.000000  0.000000  \n",
       "1  0.114094  0.113499  0.295624  0.060411  0.025488  0.000000  0.000000  \n",
       "2  0.960403  0.958361  0.379296  0.071557  0.120874  0.083333  0.002575  \n",
       "3  0.063087  0.061786  0.270109  0.047233  0.000000  0.000000  0.000000  \n",
       "4  0.078523  0.076561  0.255738  0.056711  0.206243  0.083333  0.192688  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mins = np.min(train_x, axis=0)\n",
    "maxs = np.max(train_x, axis=0)\n",
    "rng = maxs - mins\n",
    "train_x = 1.0 - (((1.0 - 0.0) * (maxs - train_x)) / rng)\n",
    "test_x = 1.0 - (((1.0 - 0.0) * (maxs - test_x)) / rng)\n",
    "pd.DataFrame(train_x).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shuffle data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1\n",
       "0  0.0  1.0\n",
       "1  0.0  1.0\n",
       "2  1.0  0.0\n",
       "3  0.0  1.0\n",
       "4  1.0  0.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x, train_y = shuffle(train_x, train_y)\n",
    "pd.DataFrame(train_y).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tuning parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_split = 0.172;\n",
    "epochs = 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "root_directory = 'results/results-dropout/'\n",
    "model_directory = root_directory + \"model/\"\n",
    "tensorboard_directory = root_directory + \"tensorboard/\"\n",
    "def create_model(param, dropout):\n",
    "    global date\n",
    "    x = Input(shape=(train_x.shape[1],))\n",
    "\n",
    "    y = Dense(units = param['layer_1'], activation='relu')(x)\n",
    "    y = Dropout(dropout)(y)\n",
    "\n",
    "    y = Dense(units = param['layer_2'], activation='relu')(y)\n",
    "    y = Dropout(dropout)(y)\n",
    "    \n",
    "    y = Dense(units = param['layer_3'], activation='relu')(y)\n",
    "    y = Dropout(dropout)(y)\n",
    "    \n",
    "    if (param['layer_4'] > 0):\n",
    "        y = Dense(units = param['layer_4'], activation='relu')(y)\n",
    "        y = Dropout(dropout)(y)\n",
    "\n",
    "    y = Dense(units = train_y.shape[1], activation='softmax')(y)\n",
    "    model = Model(x, y)\n",
    "    \n",
    "    # Create directory\n",
    "    directory = model_directory + date + '/'\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "    \n",
    "    # Write model hyper-parameters\n",
    "    file = open(directory + \"params.txt\", \"a\")\n",
    "    file.write(\"optimizer: %s, layer 1: %d, layer 2: %d, layer 3: %d, layer 4: %d, dropout: %f\" % (param['optimizer_text'], param['layer_1'], param['layer_2'], param['layer_3'], param['layer_4'], dropout))\n",
    "    file.close()\n",
    "    \n",
    "    # Write model summary\n",
    "    file2 = open(directory + \"summary.txt\", \"a\")\n",
    "    model.summary(print_fn=lambda x: file2.write(x + '\\n'))\n",
    "    file2.close()\n",
    "\n",
    "    # Write model diagram\n",
    "    plot_model(model, to_file=directory + 'model.png', show_shapes=True, show_layer_names=False)\n",
    "    \n",
    "    # Compile the model\n",
    "    model.compile(optimizer=param['optimizer'], loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class KerasClassifierTensorBoard(KerasClassifier):\n",
    "    def fit(self, x, y, **kwargs):\n",
    "        global date\n",
    "        date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "        tensorboard_callback = keras.callbacks.TensorBoard(log_dir=tensorboard_directory + date + '/')\n",
    "        csv_logger = keras.callbacks.CSVLogger(model_directory + date + '/epochs.csv')\n",
    "        callbacks = [tensorboard_callback, csv_logger]\n",
    "        return super(KerasClassifierTensorBoard, self).fit(x, y, callbacks=callbacks, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grid search hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 1 folds for each of 18 candidates, totalling 18 fits\n",
      "[CV] param={'layer_2': 8, 'optimizer': <keras.optimizers.Adam object at 0x000002387A4958D0>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.5 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7128 - acc: 0.5110 - val_loss: 0.6974 - val_acc: 0.4808\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7072 - acc: 0.5030 - val_loss: 0.6965 - val_acc: 0.4952\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7237 - acc: 0.4900 - val_loss: 0.6957 - val_acc: 0.5192\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7090 - acc: 0.4940 - val_loss: 0.6951 - val_acc: 0.5000\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7012 - acc: 0.5220 - val_loss: 0.6947 - val_acc: 0.4952\n",
      "1000/1208 [=======================>......] - ETA: 0s[CV]  param={'layer_2': 8, 'optimizer': <keras.optimizers.Adam object at 0x000002387A4958D0>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.5, total=   4.5s\n",
      "[CV] param={'layer_2': 8, 'optimizer': <keras.optimizers.Adam object at 0x000002387A50CC50>, 'optimizer_text': 'Adam, lr=0.000001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    4.6s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7625 - acc: 0.4860 - val_loss: 0.7089 - val_acc: 0.4760\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7598 - acc: 0.5220 - val_loss: 0.7089 - val_acc: 0.4760\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7587 - acc: 0.5080 - val_loss: 0.7088 - val_acc: 0.4760\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7606 - acc: 0.5170 - val_loss: 0.7088 - val_acc: 0.4760\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7594 - acc: 0.5370 - val_loss: 0.7087 - val_acc: 0.4760\n",
      " 820/1208 [===================>..........] - ETA: 0s[CV]  param={'layer_2': 8, 'optimizer': <keras.optimizers.Adam object at 0x000002387A50CC50>, 'optimizer_text': 'Adam, lr=0.000001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.5, total=   4.2s\n",
      "[CV] param={'layer_2': 8, 'optimizer': <keras.optimizers.RMSprop object at 0x000002387A52AC18>, 'optimizer_text': 'RMSprop, lr=0.0001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.5 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6911 - acc: 0.5390 - val_loss: 0.7048 - val_acc: 0.4038\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7022 - acc: 0.5650 - val_loss: 0.7038 - val_acc: 0.5000\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7094 - acc: 0.5450 - val_loss: 0.7026 - val_acc: 0.5048\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6976 - acc: 0.5550 - val_loss: 0.7018 - val_acc: 0.5144\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7046 - acc: 0.5510 - val_loss: 0.7009 - val_acc: 0.5192\n",
      "1200/1208 [============================>.] - ETA: 0s[CV]  param={'layer_2': 8, 'optimizer': <keras.optimizers.RMSprop object at 0x000002387A52AC18>, 'optimizer_text': 'RMSprop, lr=0.0001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.5, total=   4.5s\n",
      "[CV] param={'layer_2': 64, 'optimizer': <keras.optimizers.Adam object at 0x000002387A543828>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 8, 'layer_4': 0, 'layer_1': 4}, dropout=0.5 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7016 - acc: 0.5630 - val_loss: 0.6939 - val_acc: 0.5240\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7035 - acc: 0.5830 - val_loss: 0.6935 - val_acc: 0.5240\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6906 - acc: 0.5810 - val_loss: 0.6931 - val_acc: 0.5240\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6940 - acc: 0.5730 - val_loss: 0.6928 - val_acc: 0.5240\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6965 - acc: 0.5740 - val_loss: 0.6925 - val_acc: 0.5240\n",
      " 840/1208 [===================>..........] - ETA: 0s[CV]  param={'layer_2': 64, 'optimizer': <keras.optimizers.Adam object at 0x000002387A543828>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 8, 'layer_4': 0, 'layer_1': 4}, dropout=0.5, total=   5.6s\n",
      "[CV] param={'layer_2': 6, 'optimizer': <keras.optimizers.Adam object at 0x000002387A4E9588>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 32, 'layer_4': 0, 'layer_1': 6}, dropout=0.5 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7198 - acc: 0.4830 - val_loss: 0.6993 - val_acc: 0.4087\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7127 - acc: 0.5070 - val_loss: 0.6966 - val_acc: 0.4183\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7036 - acc: 0.5240 - val_loss: 0.6949 - val_acc: 0.4760\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7035 - acc: 0.5170 - val_loss: 0.6934 - val_acc: 0.5192\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6970 - acc: 0.5290 - val_loss: 0.6922 - val_acc: 0.5192\n",
      " 760/1208 [=================>............] - ETA: 0s[CV]  param={'layer_2': 6, 'optimizer': <keras.optimizers.Adam object at 0x000002387A4E9588>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 32, 'layer_4': 0, 'layer_1': 6}, dropout=0.5, total=   6.3s\n",
      "[CV] param={'layer_2': 6, 'optimizer': <keras.optimizers.Adam object at 0x000002387A59CAC8>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 32, 'layer_4': 8, 'layer_1': 6}, dropout=0.5 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7325 - acc: 0.4960 - val_loss: 0.6901 - val_acc: 0.5240\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7129 - acc: 0.5190 - val_loss: 0.6889 - val_acc: 0.5481\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7207 - acc: 0.5230 - val_loss: 0.6881 - val_acc: 0.5529\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7093 - acc: 0.4970 - val_loss: 0.6876 - val_acc: 0.5433\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7069 - acc: 0.5040 - val_loss: 0.6874 - val_acc: 0.5577\n",
      " 680/1208 [===============>..............] - ETA: 0s[CV]  param={'layer_2': 6, 'optimizer': <keras.optimizers.Adam object at 0x000002387A59CAC8>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 32, 'layer_4': 8, 'layer_1': 6}, dropout=0.5, total=   7.6s\n",
      "[CV] param={'layer_2': 8, 'optimizer': <keras.optimizers.Adam object at 0x000002387A4958D0>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.4 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6944 - acc: 0.5680 - val_loss: 0.7039 - val_acc: 0.5240\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6957 - acc: 0.5720 - val_loss: 0.7011 - val_acc: 0.5240\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7026 - acc: 0.5630 - val_loss: 0.6984 - val_acc: 0.5240\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7054 - acc: 0.5680 - val_loss: 0.6964 - val_acc: 0.5240\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6962 - acc: 0.5750 - val_loss: 0.6950 - val_acc: 0.5240\n",
      " 740/1208 [=================>............] - ETA: 0s[CV]  param={'layer_2': 8, 'optimizer': <keras.optimizers.Adam object at 0x000002387A4958D0>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.4, total=   8.1s\n",
      "[CV] param={'layer_2': 8, 'optimizer': <keras.optimizers.Adam object at 0x000002387A50CC50>, 'optimizer_text': 'Adam, lr=0.000001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.4 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7013 - acc: 0.5200 - val_loss: 0.6894 - val_acc: 0.5481\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7070 - acc: 0.5200 - val_loss: 0.6894 - val_acc: 0.5481\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7072 - acc: 0.5210 - val_loss: 0.6894 - val_acc: 0.5481\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7111 - acc: 0.4800 - val_loss: 0.6894 - val_acc: 0.5529\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7056 - acc: 0.5280 - val_loss: 0.6894 - val_acc: 0.55290.52\n",
      " 760/1208 [=================>............] - ETA: 0s[CV]  param={'layer_2': 8, 'optimizer': <keras.optimizers.Adam object at 0x000002387A50CC50>, 'optimizer_text': 'Adam, lr=0.000001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.4, total=   9.0s\n",
      "[CV] param={'layer_2': 8, 'optimizer': <keras.optimizers.RMSprop object at 0x000002387A52AC18>, 'optimizer_text': 'RMSprop, lr=0.0001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.4 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6883 - acc: 0.5500 - val_loss: 0.6857 - val_acc: 0.5240\n",
      "Epoch 2/5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000/1000 [==============================] - 0s - loss: 0.6890 - acc: 0.5280 - val_loss: 0.6855 - val_acc: 0.5240\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6856 - acc: 0.5390 - val_loss: 0.6853 - val_acc: 0.5240\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6807 - acc: 0.5530 - val_loss: 0.6852 - val_acc: 0.5240\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6737 - acc: 0.5720 - val_loss: 0.6853 - val_acc: 0.5240\n",
      " 760/1208 [=================>............] - ETA: 0s[CV]  param={'layer_2': 8, 'optimizer': <keras.optimizers.RMSprop object at 0x000002387A52AC18>, 'optimizer_text': 'RMSprop, lr=0.0001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.4, total=   9.0s\n",
      "[CV] param={'layer_2': 64, 'optimizer': <keras.optimizers.Adam object at 0x000002387A543828>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 8, 'layer_4': 0, 'layer_1': 4}, dropout=0.4 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6997 - acc: 0.5200 - val_loss: 0.6903 - val_acc: 0.5240\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6944 - acc: 0.5780 - val_loss: 0.6892 - val_acc: 0.5240\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6856 - acc: 0.5780 - val_loss: 0.6884 - val_acc: 0.5240\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6831 - acc: 0.5770 - val_loss: 0.6880 - val_acc: 0.5240\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6889 - acc: 0.5790 - val_loss: 0.6875 - val_acc: 0.5240\n",
      " 900/1208 [=====================>........] - ETA: 0s[CV]  param={'layer_2': 64, 'optimizer': <keras.optimizers.Adam object at 0x000002387A543828>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 8, 'layer_4': 0, 'layer_1': 4}, dropout=0.4, total=  10.4s\n",
      "[CV] param={'layer_2': 6, 'optimizer': <keras.optimizers.Adam object at 0x000002387A4E9588>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 32, 'layer_4': 0, 'layer_1': 6}, dropout=0.4 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7436 - acc: 0.4490 - val_loss: 0.7043 - val_acc: 0.4615\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7322 - acc: 0.4320 - val_loss: 0.6984 - val_acc: 0.4375\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7139 - acc: 0.4690 - val_loss: 0.6952 - val_acc: 0.4760\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7038 - acc: 0.4980 - val_loss: 0.6937 - val_acc: 0.4952\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7030 - acc: 0.4960 - val_loss: 0.6923 - val_acc: 0.5096\n",
      " 880/1208 [====================>.........] - ETA: 0s[CV]  param={'layer_2': 6, 'optimizer': <keras.optimizers.Adam object at 0x000002387A4E9588>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 32, 'layer_4': 0, 'layer_1': 6}, dropout=0.4, total=  11.2s\n",
      "[CV] param={'layer_2': 6, 'optimizer': <keras.optimizers.Adam object at 0x000002387A59CAC8>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 32, 'layer_4': 8, 'layer_1': 6}, dropout=0.4 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7141 - acc: 0.4970 - val_loss: 0.6919 - val_acc: 0.5240\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6935 - acc: 0.5430 - val_loss: 0.6918 - val_acc: 0.5240\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6921 - acc: 0.5630 - val_loss: 0.6920 - val_acc: 0.5240\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6912 - acc: 0.5570 - val_loss: 0.6924 - val_acc: 0.5240\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6893 - acc: 0.5610 - val_loss: 0.6926 - val_acc: 0.5240\n",
      " 820/1208 [===================>..........] - ETA: 0s[CV]  param={'layer_2': 6, 'optimizer': <keras.optimizers.Adam object at 0x000002387A59CAC8>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 32, 'layer_4': 8, 'layer_1': 6}, dropout=0.4, total=  12.1s\n",
      "[CV] param={'layer_2': 8, 'optimizer': <keras.optimizers.Adam object at 0x000002387A4958D0>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.6 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7890 - acc: 0.4960 - val_loss: 0.7053 - val_acc: 0.4808\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7323 - acc: 0.5090 - val_loss: 0.6993 - val_acc: 0.4952\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7207 - acc: 0.5380 - val_loss: 0.6962 - val_acc: 0.4760\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7213 - acc: 0.5490 - val_loss: 0.6939 - val_acc: 0.4904\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7051 - acc: 0.5610 - val_loss: 0.6927 - val_acc: 0.5096\n",
      " 740/1208 [=================>............] - ETA: 0s[CV]  param={'layer_2': 8, 'optimizer': <keras.optimizers.Adam object at 0x000002387A4958D0>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.6, total=  12.4s\n",
      "[CV] param={'layer_2': 8, 'optimizer': <keras.optimizers.Adam object at 0x000002387A50CC50>, 'optimizer_text': 'Adam, lr=0.000001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.6 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7689 - acc: 0.5460 - val_loss: 0.6924 - val_acc: 0.5721\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7696 - acc: 0.5730 - val_loss: 0.6924 - val_acc: 0.5721\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7823 - acc: 0.5510 - val_loss: 0.6924 - val_acc: 0.5721\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7660 - acc: 0.5740 - val_loss: 0.6924 - val_acc: 0.5721\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7816 - acc: 0.5590 - val_loss: 0.6924 - val_acc: 0.5721\n",
      " 780/1208 [==================>...........] - ETA: 0s[CV]  param={'layer_2': 8, 'optimizer': <keras.optimizers.Adam object at 0x000002387A50CC50>, 'optimizer_text': 'Adam, lr=0.000001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.6, total=  13.5s\n",
      "[CV] param={'layer_2': 8, 'optimizer': <keras.optimizers.RMSprop object at 0x000002387A52AC18>, 'optimizer_text': 'RMSprop, lr=0.0001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.6 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7612 - acc: 0.5170 - val_loss: 0.6978 - val_acc: 0.4856\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7568 - acc: 0.4900 - val_loss: 0.6970 - val_acc: 0.4471\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7428 - acc: 0.5290 - val_loss: 0.6964 - val_acc: 0.4615\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7385 - acc: 0.5290 - val_loss: 0.6955 - val_acc: 0.4615\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7346 - acc: 0.5370 - val_loss: 0.6950 - val_acc: 0.4760\n",
      " 780/1208 [==================>...........] - ETA: 0s[CV]  param={'layer_2': 8, 'optimizer': <keras.optimizers.RMSprop object at 0x000002387A52AC18>, 'optimizer_text': 'RMSprop, lr=0.0001, decay=0.0001', 'layer_3': 16, 'layer_4': 0, 'layer_1': 4}, dropout=0.6, total=  13.9s\n",
      "[CV] param={'layer_2': 64, 'optimizer': <keras.optimizers.Adam object at 0x000002387A543828>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 8, 'layer_4': 0, 'layer_1': 4}, dropout=0.6 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6966 - acc: 0.5510 - val_loss: 0.6918 - val_acc: 0.5288\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6919 - acc: 0.5700 - val_loss: 0.6916 - val_acc: 0.5240\n",
      "Epoch 3/5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000/1000 [==============================] - 0s - loss: 0.6937 - acc: 0.5590 - val_loss: 0.6916 - val_acc: 0.5240\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6904 - acc: 0.5810 - val_loss: 0.6916 - val_acc: 0.5240\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6843 - acc: 0.5930 - val_loss: 0.6915 - val_acc: 0.5240\n",
      " 760/1208 [=================>............] - ETA: 0s[CV]  param={'layer_2': 64, 'optimizer': <keras.optimizers.Adam object at 0x000002387A543828>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 8, 'layer_4': 0, 'layer_1': 4}, dropout=0.6, total=  15.0s\n",
      "[CV] param={'layer_2': 6, 'optimizer': <keras.optimizers.Adam object at 0x000002387A4E9588>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 32, 'layer_4': 0, 'layer_1': 6}, dropout=0.6 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6914 - acc: 0.5520 - val_loss: 0.6913 - val_acc: 0.5288\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6899 - acc: 0.5710 - val_loss: 0.6908 - val_acc: 0.5240\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6875 - acc: 0.5700 - val_loss: 0.6907 - val_acc: 0.5240\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6853 - acc: 0.5710 - val_loss: 0.6907 - val_acc: 0.5240\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6829 - acc: 0.5730 - val_loss: 0.6907 - val_acc: 0.5240\n",
      " 760/1208 [=================>............] - ETA: 0s[CV]  param={'layer_2': 6, 'optimizer': <keras.optimizers.Adam object at 0x000002387A4E9588>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 32, 'layer_4': 0, 'layer_1': 6}, dropout=0.6, total=  15.8s\n",
      "[CV] param={'layer_2': 6, 'optimizer': <keras.optimizers.Adam object at 0x000002387A59CAC8>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 32, 'layer_4': 8, 'layer_1': 6}, dropout=0.6 \n",
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7040 - acc: 0.5470 - val_loss: 0.6928 - val_acc: 0.5144\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.7031 - acc: 0.5460 - val_loss: 0.6926 - val_acc: 0.5240\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6971 - acc: 0.5670 - val_loss: 0.6925 - val_acc: 0.5240\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6937 - acc: 0.5870 - val_loss: 0.6924 - val_acc: 0.5240\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6960 - acc: 0.5820 - val_loss: 0.6922 - val_acc: 0.5240\n",
      " 740/1208 [=================>............] - ETA: 0s[CV]  param={'layer_2': 6, 'optimizer': <keras.optimizers.Adam object at 0x000002387A59CAC8>, 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_3': 32, 'layer_4': 8, 'layer_1': 6}, dropout=0.6, total=  16.6s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:  3.0min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1000 samples, validate on 208 samples\n",
      "Epoch 1/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6938 - acc: 0.5750 - val_loss: 0.6950 - val_acc: 0.5240\n",
      "Epoch 2/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6948 - acc: 0.5890 - val_loss: 0.6950 - val_acc: 0.5240\n",
      "Epoch 3/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6971 - acc: 0.5890 - val_loss: 0.6950 - val_acc: 0.5240\n",
      "Epoch 4/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6957 - acc: 0.5760 - val_loss: 0.6950 - val_acc: 0.5240\n",
      "Epoch 5/5\n",
      "1000/1000 [==============================] - 0s - loss: 0.6975 - acc: 0.5780 - val_loss: 0.6950 - val_acc: 0.5240\n"
     ]
    }
   ],
   "source": [
    "model = KerasClassifierTensorBoard(build_fn=create_model, epochs=epochs, batch_size=20)\n",
    "\n",
    "param = [\n",
    "    {'optimizer': keras.optimizers.Adam(lr=0.0001, decay=0.0001), 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_1': 4, 'layer_2': 8, 'layer_3': 16, 'layer_4': 0},\n",
    "    {'optimizer': keras.optimizers.Adam(lr=0.000001, decay=0.0001), 'optimizer_text': 'Adam, lr=0.000001, decay=0.0001', 'layer_1': 4, 'layer_2': 8, 'layer_3': 16, 'layer_4': 0},\n",
    "    {'optimizer': keras.optimizers.RMSprop(lr=0.0001, decay=0.0001), 'optimizer_text': 'RMSprop, lr=0.0001, decay=0.0001', 'layer_1': 4, 'layer_2': 8, 'layer_3': 16, 'layer_4': 0},\n",
    "    {'optimizer': keras.optimizers.Adam(lr=0.0001, decay=0.0001), 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_1': 4, 'layer_2': 64, 'layer_3': 8, 'layer_4': 0},\n",
    "    {'optimizer': keras.optimizers.Adam(lr=0.0001, decay=0.0001), 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_1': 6, 'layer_2': 6, 'layer_3': 32, 'layer_4': 0},\n",
    "    {'optimizer': keras.optimizers.Adam(lr=0.0001, decay=0.0001), 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_1': 6, 'layer_2': 6, 'layer_3': 32, 'layer_4': 8},\n",
    "]\n",
    "dropouts = [0.5, 0.4, 0.6]\n",
    "param_grid = dict(\n",
    "    param=param, \n",
    "    dropout = dropouts\n",
    ")\n",
    "\n",
    "grid = GridSearchCV(estimator = model, param_grid = param_grid, error_score=0, verbose=2, cv=[(slice(None), slice(None))], n_jobs=1, fit_params=dict(validation_split=validation_split))\n",
    "grid_result = grid.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Print results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"Mean %f, Std %f with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The End"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}