31-fine-tune-binary-50.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras.models import Sequential, Model, load_model\n",
    "from keras.layers import Dense, Activation, BatchNormalization, Dropout, Input\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "import keras\n",
    "from datetime import datetime\n",
    "from sklearn import preprocessing\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import plot_model\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.utils import shuffle\n",
    "import pickle\n",
    "import glob\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load processed dataset\n",
    "The data we load here are already shuffled and normalized."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train: (604, 26)\n",
      "y_train: (604, 2)\n",
      "x_test: (604, 26)\n",
      "y_test: (604, 2)\n"
     ]
    }
   ],
   "source": [
    "data_directory = 'dataset/4. normalized-50/'\n",
    "x_train = pickle.load(open(data_directory + \"x_train.p\", \"rb\"))\n",
    "y_train = pickle.load(open(data_directory + \"y_train.p\", \"rb\"))\n",
    "x_test = pickle.load(open(data_directory + \"x_test.p\", \"rb\"))\n",
    "y_test = pickle.load(open(data_directory + \"y_test.p\", \"rb\"))\n",
    "print('x_train: ' + str(x_train.shape))\n",
    "print('y_train: ' + str(y_train.shape))\n",
    "print('x_test: ' + str(x_test.shape))\n",
    "print('y_test: ' + str(y_test.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert back from one hot vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(604,)\n",
      "(604,)\n"
     ]
    }
   ],
   "source": [
    "y_train = y_train.argmax(axis=1)\n",
    "y_test = y_test.argmax(axis=1)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tunable parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 5000\n",
    "batch_size = 20\n",
    "learning_rate = 0.0001\n",
    "decay = 0.00001"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load auto encoders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 26)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                270       \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 8)                 88        \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 8)                 0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 26)                234       \n",
      "=================================================================\n",
      "Total params: 592\n",
      "Trainable params: 592\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "autoencoder_path = 'results/30-stacked-autoencoder-50-test-size-01/model/2018-03-08-06-43-23/weights.hdf5'\n",
    "weights = load_model(autoencoder_path)\n",
    "weights.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set up constants for weight transfer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "autoencoder_layer_1_units = 10\n",
    "autoencoder_layer_2_units = 8\n",
    "autoencoder_layer_1_name = 'dense_4'\n",
    "autoencoder_layer_2_name = 'dense_5'\n",
    "autoencoder_layer_3_name = 'dense_6'\n",
    "autoencoder_layer_1_dropout = 0.3*0.75\n",
    "autoencoder_layer_2_dropout = 0.3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model for classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "root_directory = 'results/31-fine-tune-binary-50-01/'\n",
    "model_directory = root_directory + \"model/\"\n",
    "tensorboard_directory = root_directory + \"tensorboard/\"\n",
    "\n",
    "def create_model(layers, dropout):\n",
    "    global date\n",
    "    global autoencoder_path\n",
    "    x = Input(shape=(x_train.shape[1],))\n",
    "    \n",
    "    y = Dense(units = autoencoder_layer_1_units, activation='relu')(x)\n",
    "    y = Dropout(autoencoder_layer_1_dropout)(y)\n",
    "    \n",
    "    y = Dense(units = autoencoder_layer_2_units, activation='relu')(y)\n",
    "    y = Dropout(autoencoder_layer_2_dropout)(y)\n",
    "    \n",
    "    y = Dense(units = x_train.shape[1], activation='relu')(y)\n",
    "\n",
    "    y = Dense(units = layers['layer_1'], activation='relu')(y)\n",
    "    y = Dropout(dropout * 0.5)(y)\n",
    "    \n",
    "    if (layers['layer_2'] > 0):\n",
    "        y = Dense(units = layers['layer_2'], activation='relu')(y)\n",
    "        y = Dropout(dropout * 0.75)(y)\n",
    "    \n",
    "    if (layers['layer_3'] > 0):\n",
    "        y = Dense(units = layers['layer_3'], activation='relu')(y)\n",
    "        y = Dropout(dropout)(y)\n",
    "    \n",
    "    if (layers['layer_4'] > 0):\n",
    "        y = Dense(units = layers['layer_4'], activation='relu')(y)\n",
    "        y = Dropout(dropout)(y)\n",
    "    \n",
    "    y = Dense(units = 1, activation='sigmoid')(y)\n",
    "    model = Model(x, y)\n",
    "    \n",
    "    # Create directory\n",
    "    directory = model_directory + date + '/'\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "    \n",
    "    # Write model hyper-parameters\n",
    "    df = pd.DataFrame({\n",
    "        'Layer 4': layers['layer_1'],\n",
    "        'Layer 5': layers['layer_2'],\n",
    "        'Layer 6': layers['layer_3'],\n",
    "        'Layer 8': layers['layer_4'],\n",
    "        'Dropout 4': dropout * 0.5,\n",
    "        'Dropout 5': dropout * 0.75,\n",
    "        'Dropout 6': dropout,\n",
    "        'Dropout 8': dropout,\n",
    "        'Learning Rate': learning_rate,\n",
    "        'Decay': decay,\n",
    "        'Batch size': batch_size,\n",
    "        'Autoencoder': autoencoder_path,\n",
    "        'Epochs': epochs,\n",
    "        'Fine tune': True,\n",
    "        'Loss': 'binary_crossentropy'\n",
    "    }, index=[0])\n",
    "    df.to_csv(model_directory + date + '/params.csv')\n",
    "    \n",
    "    # Write model summary\n",
    "    file2 = open(directory + \"summary.txt\", \"a\")\n",
    "    model.summary(print_fn=lambda line: file2.write(line + '\\n'))\n",
    "    file2.close()\n",
    "\n",
    "    # Write model diagram\n",
    "    plot_model(model, to_file=directory + 'model.png', show_shapes=True, show_layer_names=False)\n",
    "    \n",
    "    # Compile the model\n",
    "    model.compile(optimizer=keras.optimizers.Adam(lr=learning_rate, decay=decay), loss=keras.losses.binary_crossentropy, metrics=['accuracy'])\n",
    "    \n",
    "    # Set layer weights to pre trained autoencoder weights\n",
    "    model.layers[1].set_weights(weights.get_layer(autoencoder_layer_1_name).get_weights())\n",
    "    model.layers[3].set_weights(weights.get_layer(autoencoder_layer_2_name).get_weights())\n",
    "    model.layers[5].set_weights(weights.get_layer(autoencoder_layer_3_name).get_weights())\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class KerasClassifierTensorBoard(KerasClassifier):\n",
    "    def fit(self, x, y, **kwargs):\n",
    "        global date\n",
    "        date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "        tensorboard_callback = keras.callbacks.TensorBoard(log_dir=tensorboard_directory + date + '/')\n",
    "        csv_logger = keras.callbacks.CSVLogger(model_directory + date + '/epochs.csv')\n",
    "        model_checkpoint = keras.callbacks.ModelCheckpoint(model_directory + date + '/weights.hdf5', save_best_only=True)\n",
    "        callbacks = [tensorboard_callback, csv_logger, model_checkpoint]\n",
    "        return super(KerasClassifierTensorBoard, self).fit(x, y, callbacks=callbacks, verbose=0, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 1 folds for each of 24 candidates, totalling 24 fits\n",
      "[CV] layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 32, 'layer_3': 8}, dropout=0.3 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 32, 'layer_3': 8}, dropout=0.3, total=17.5min\n",
      "[CV] layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 24, 'layer_3': 8}, dropout=0.3 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed: 17.5min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 24, 'layer_3': 8}, dropout=0.3, total=18.5min\n",
      "[CV] layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 24, 'layer_3': 0}, dropout=0.3 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 24, 'layer_3': 0}, dropout=0.3, total=16.8min\n",
      "[CV] layers={'layer_2': 32, 'layer_4': 0, 'layer_1': 64, 'layer_3': 16}, dropout=0.3 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 32, 'layer_4': 0, 'layer_1': 64, 'layer_3': 16}, dropout=0.3, total=17.6min\n",
      "[CV] layers={'layer_2': 32, 'layer_4': 8, 'layer_1': 64, 'layer_3': 16}, dropout=0.3 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 32, 'layer_4': 8, 'layer_1': 64, 'layer_3': 16}, dropout=0.3, total=19.6min\n",
      "[CV] layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 64, 'layer_3': 0}, dropout=0.3 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 64, 'layer_3': 0}, dropout=0.3, total=16.9min\n",
      "[CV] layers={'layer_2': 8, 'layer_4': 0, 'layer_1': 16, 'layer_3': 4}, dropout=0.3 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 8, 'layer_4': 0, 'layer_1': 16, 'layer_3': 4}, dropout=0.3, total=18.0min\n",
      "[CV] layers={'layer_2': 4, 'layer_4': 0, 'layer_1': 8, 'layer_3': 0}, dropout=0.3 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 4, 'layer_4': 0, 'layer_1': 8, 'layer_3': 0}, dropout=0.3, total=17.0min\n",
      "[CV] layers={'layer_2': 0, 'layer_4': 0, 'layer_1': 16, 'layer_3': 0}, dropout=0.3 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 0, 'layer_4': 0, 'layer_1': 16, 'layer_3': 0}, dropout=0.3, total=16.1min\n",
      "[CV] layers={'layer_2': 12, 'layer_4': 0, 'layer_1': 8, 'layer_3': 16}, dropout=0.3 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 12, 'layer_4': 0, 'layer_1': 8, 'layer_3': 16}, dropout=0.3, total=18.6min\n",
      "[CV] layers={'layer_2': 12, 'layer_4': 32, 'layer_1': 8, 'layer_3': 16}, dropout=0.3 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 12, 'layer_4': 32, 'layer_1': 8, 'layer_3': 16}, dropout=0.3, total=20.0min\n",
      "[CV] layers={'layer_2': 8, 'layer_4': 0, 'layer_1': 16, 'layer_3': 16}, dropout=0.3 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 8, 'layer_4': 0, 'layer_1': 16, 'layer_3': 16}, dropout=0.3, total=19.0min\n",
      "[CV] layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 32, 'layer_3': 8}, dropout=0.5 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 32, 'layer_3': 8}, dropout=0.5, total=18.9min\n",
      "[CV] layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 24, 'layer_3': 8}, dropout=0.5 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 24, 'layer_3': 8}, dropout=0.5, total=18.9min\n",
      "[CV] layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 24, 'layer_3': 0}, dropout=0.5 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 24, 'layer_3': 0}, dropout=0.5, total=18.2min\n",
      "[CV] layers={'layer_2': 32, 'layer_4': 0, 'layer_1': 64, 'layer_3': 16}, dropout=0.5 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 32, 'layer_4': 0, 'layer_1': 64, 'layer_3': 16}, dropout=0.5, total=19.0min\n",
      "[CV] layers={'layer_2': 32, 'layer_4': 8, 'layer_1': 64, 'layer_3': 16}, dropout=0.5 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 32, 'layer_4': 8, 'layer_1': 64, 'layer_3': 16}, dropout=0.5, total=20.6min\n",
      "[CV] layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 64, 'layer_3': 0}, dropout=0.5 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 16, 'layer_4': 0, 'layer_1': 64, 'layer_3': 0}, dropout=0.5, total=18.4min\n",
      "[CV] layers={'layer_2': 8, 'layer_4': 0, 'layer_1': 16, 'layer_3': 4}, dropout=0.5 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 8, 'layer_4': 0, 'layer_1': 16, 'layer_3': 4}, dropout=0.5, total=19.8min\n",
      "[CV] layers={'layer_2': 4, 'layer_4': 0, 'layer_1': 8, 'layer_3': 0}, dropout=0.5 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 4, 'layer_4': 0, 'layer_1': 8, 'layer_3': 0}, dropout=0.5, total=18.6min\n",
      "[CV] layers={'layer_2': 0, 'layer_4': 0, 'layer_1': 16, 'layer_3': 0}, dropout=0.5 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 0, 'layer_4': 0, 'layer_1': 16, 'layer_3': 0}, dropout=0.5, total=17.2min\n",
      "[CV] layers={'layer_2': 12, 'layer_4': 0, 'layer_1': 8, 'layer_3': 16}, dropout=0.5 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 12, 'layer_4': 0, 'layer_1': 8, 'layer_3': 16}, dropout=0.5, total=20.9min\n",
      "[CV] layers={'layer_2': 12, 'layer_4': 32, 'layer_1': 8, 'layer_3': 16}, dropout=0.5 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 12, 'layer_4': 32, 'layer_1': 8, 'layer_3': 16}, dropout=0.5, total=22.0min\n",
      "[CV] layers={'layer_2': 8, 'layer_4': 0, 'layer_1': 16, 'layer_3': 16}, dropout=0.5 \n",
      " 20/604 [..............................] - ETA: 0s[CV]  layers={'layer_2': 8, 'layer_4': 0, 'layer_1': 16, 'layer_3': 16}, dropout=0.5, total=20.9min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed: 449.0min finished\n"
     ]
    }
   ],
   "source": [
    "model = KerasClassifierTensorBoard(build_fn=create_model, epochs=epochs, batch_size=batch_size)\n",
    "\n",
    "layers = [\n",
    "    {'layer_1': 32, 'layer_2': 16, 'layer_3': 8, 'layer_4': 0},\n",
    "    {'layer_1': 24, 'layer_2': 16, 'layer_3': 8, 'layer_4': 0},\n",
    "    {'layer_1': 24, 'layer_2': 16, 'layer_3': 0, 'layer_4': 0},\n",
    "    {'layer_1': 64, 'layer_2': 32, 'layer_3': 16, 'layer_4': 0},\n",
    "    {'layer_1': 64, 'layer_2': 32, 'layer_3': 16, 'layer_4': 8},\n",
    "    {'layer_1': 64, 'layer_2': 16, 'layer_3': 0, 'layer_4': 0},\n",
    "    {'layer_1': 16, 'layer_2': 8, 'layer_3': 4, 'layer_4': 0},\n",
    "    {'layer_1': 8, 'layer_2': 4, 'layer_3': 0, 'layer_4': 0},\n",
    "    {'layer_1': 16, 'layer_2': 0, 'layer_3': 0, 'layer_4': 0},\n",
    "    {'layer_1': 8, 'layer_2': 12, 'layer_3': 16, 'layer_4': 0},\n",
    "    {'layer_1': 8, 'layer_2': 12, 'layer_3': 16, 'layer_4': 32},\n",
    "    {'layer_1': 16, 'layer_2': 8, 'layer_3': 16, 'layer_4': 0}\n",
    "]\n",
    "param_grid = dict(\n",
    "    layers=layers,\n",
    "    dropout=[0.3, 0.5]\n",
    ")\n",
    "    \n",
    "grid = GridSearchCV(estimator = model, param_grid = param_grid, error_score=0, verbose=2, cv=[(slice(None), slice(None))], n_jobs=1, fit_params=dict(validation_data=(x_test, y_test)))\n",
    "grid_result = grid.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}