BayesianOptimization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bayesian Optimization of an Attention Direction Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make sure you don't hog all the video memory\n",
    "import tensorflow as tf\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)\n",
    "from keras import backend as K\n",
    "K.set_session(sess)\n",
    "###################################\n",
    "\n",
    "import GPy, GPyOpt\n",
    "from keras.layers import Input, Dense, Conv1D, MaxPooling1D, Dropout, BatchNormalization, Flatten, ELU, Permute\n",
    "from keras.models import Model\n",
    "from keras.models import load_model\n",
    "from keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint\n",
    "from keras import optimizers\n",
    "from keras.initializers import Orthogonal as orth\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "import scipy.io as sio\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# EEG class\n",
    "class EEG():\n",
    "    def __init__(self,\n",
    "                 SHUFFLE = True, BATCH_SIZE = 20, TEST_TRAIN = 0.25,\n",
    "                 KER0=100, KER1=50, KER2=25, KER3=2, KER4=2,\n",
    "                 N_KERNELS = (10,20,40,80,160),\n",
    "                 POOL_SIZE = (5,2,5,2),\n",
    "                 DO0 = 0.1, DO1 = 0.2, DO2 = 0.25, DO3 = 0.25, DO4 = 0.8, DO5 = 0.8,\n",
    "                 DENSE_LEN = 10):\n",
    "        self.__SHUFFLE = SHUFFLE\n",
    "        self.__KER0 = KER0\n",
    "        self.__KER1 = KER1\n",
    "        self.__KER2 = KER2\n",
    "        self.__KER3 = KER3\n",
    "        self.__KER4 = KER4\n",
    "        self.__BATCH_SIZE = BATCH_SIZE\n",
    "        self.__TEST_TRAIN = TEST_TRAIN\n",
    "        self.__N_KERNELS = N_KERNELS\n",
    "        self.__POOL_SIZE = POOL_SIZE\n",
    "        self.__DROPOUT = (DO0, DO1, DO2, DO3, DO4, DO5)\n",
    "        self.__DENSE_LEN = DENSE_LEN\n",
    "        self.__x_train, self.__x_test, self.__y_train, self.__y_test = self.load_data()\n",
    "        self.__model = self.CNN_model()\n",
    "        \n",
    "    # load dataset:\n",
    "    def load_data(self):\n",
    "        \n",
    "#         mat_contents = sio.loadmat('/home/amplifier/home/DATASETS/ksenia2_long_chunked.mat')\n",
    "#         mat_contents = sio.loadmat('/home/amplifier/home/DATASETS/Merged456->2-30Hz,Env=1_TD1_94_.mat')\n",
    "        mat_contents = sio.loadmat(\n",
    "            '/home/amplifier/home/DATASETS/Merged123, DS2=80Hz, FIR=1-9Hz, centnorm=1, Env=1, TD, 143-202.mat')\n",
    "        X = mat_contents['X']\n",
    "        Y = mat_contents['Z']\n",
    "\n",
    "        if X.shape[1]<X.shape[2]:\n",
    "            X = np.transpose(X,[0,2,1])\n",
    "\n",
    "        if Y.shape[1] > Y.shape[0]:\n",
    "            Y = Y.T\n",
    "\n",
    "#         print('Original data shape:', X.shape)\n",
    "#         print('Original labels shape:', Y.shape)\n",
    "\n",
    "        x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=self.__TEST_TRAIN, shuffle=self.__SHUFFLE)\n",
    "#         print('Original data type:', x_train.dtype)\n",
    "\n",
    "        # convert to float64 for numerical stability:\n",
    "        x_train = x_train.astype('float64')\n",
    "        y_train = y_train.astype('float64')\n",
    "        x_test = x_test.astype('float64')\n",
    "        y_test = y_test.astype('float64')\n",
    "\n",
    "        # one hot encode the labels:\n",
    "        onehot_encoder = preprocessing.OneHotEncoder(sparse=False)\n",
    "        y_train = onehot_encoder.fit_transform(y_train)\n",
    "        y_test = onehot_encoder.fit_transform(y_test)\n",
    "\n",
    "        # convert to float16 to save space:\n",
    "        x_train = x_train.astype('float16')\n",
    "        y_train = y_train.astype('float16')\n",
    "        x_test = x_test.astype('float16')\n",
    "        y_test = y_test.astype('float16')\n",
    "#         print('Normalized data type:', x_train.dtype)\n",
    "\n",
    "        self.leng = X.shape[1] # if you work in the FD, this is the height of the sample time-frequency image, othewise EEG channels\n",
    "        self.chan = X.shape[2] # if you work in the FD, this is the width of the sample time-frequency image, othewise time samples of EEG signal\n",
    "\n",
    "        if len(X.shape)==3:\n",
    "            streams = 1 # this is EEG channels if you work with frequency domain, in the TD streams = 1\n",
    "        if len(X.shape)==4:\n",
    "            streams = X.shape[3] # this is EEG channels if you work with frequency domain, in the TD streams = 1\n",
    "\n",
    "#         print('test input shape', x_test.shape, \"Nomralized MEAN:\", np.mean(x_test), \"min\", np.min(x_test),\"max\", np.max(x_test))\n",
    "#         print('train input shape', x_train.shape, \"Nomralized MEAN:\", np.mean(x_train), \"min\", np.min(x_train),\"max\", np.max(x_train))\n",
    "\n",
    "#         print('test labels shape', y_test.shape, \"Nomralized MEAN:\", np.mean(y_test), \"min\", np.min(y_test),\"max\", np.max(y_test))\n",
    "#         print('train labels shape', y_train.shape, \"Nomralized MEAN:\", np.mean(y_train), \"min\", np.min(y_train),\"max\", np.max(y_train))\n",
    "\n",
    "        return x_train, x_test, y_train, y_test\n",
    "    \n",
    "    def activation(self):\n",
    "        return ELU(alpha=1.0) # Activation('relu') #\n",
    "    \n",
    "    # define model\n",
    "    def CNN_model(self):\n",
    "        N_KERNELS = self.__N_KERNELS\n",
    "        POOL_SIZE = self.__POOL_SIZE\n",
    "        KER0 = self.__KER0,\n",
    "        KER1 = self.__KER1,\n",
    "        KER2= self.__KER2,\n",
    "        KER3 = self.__KER3,\n",
    "        KER4 = self.__KER4,\n",
    "        DROPOUT = self.__DROPOUT\n",
    "        DENSE_LEN = self.__DENSE_LEN\n",
    "        \n",
    "        input_img = Input(shape=(self.leng, self.chan))  # adapt this if using `channels_first` image\n",
    "        input_norm = BatchNormalization()(input_img)\n",
    "\n",
    "        e2 = Conv1D(N_KERNELS[0], KER0, padding='same',\n",
    "                                        kernel_initializer = orth(gain=1.0, seed=None))(input_norm)\n",
    "        e2act = self.activation()(e2)\n",
    "        e2p = MaxPooling1D(pool_size=POOL_SIZE[0], strides=None, padding='same')(e2act)\n",
    "        e2b = BatchNormalization()(e2p)\n",
    "        e2a = Dropout(DROPOUT[0])(e2b)\n",
    "\n",
    "\n",
    "        e3 = Conv1D(N_KERNELS[1], KER1, padding='same',\n",
    "                                        kernel_initializer = orth(gain=1.0, seed=None))(e2a)\n",
    "        e3act = self.activation()(e3)\n",
    "        e4 = MaxPooling1D(pool_size=POOL_SIZE[1], strides=None, padding='same')(e3act)\n",
    "        e4b = BatchNormalization()(e4)\n",
    "        e4a = Dropout(DROPOUT[1])(e4b)\n",
    "\n",
    "\n",
    "        e5 = Conv1D(N_KERNELS[2], KER2, padding='same',\n",
    "                                        kernel_initializer = orth(gain=1.0, seed=None))(e4a)\n",
    "        e5act = self.activation()(e5)\n",
    "        e6 = MaxPooling1D(pool_size=POOL_SIZE[2], strides=None, padding='same')(e5act)\n",
    "        e6b = BatchNormalization()(e6)\n",
    "        e6a = Dropout(DROPOUT[2])(e6b)\n",
    "\n",
    "\n",
    "        e7 = Conv1D(N_KERNELS[3], KER3, padding='same',\n",
    "                                        kernel_initializer = orth(gain=1.0, seed=None))(e6a)\n",
    "        e7act = self.activation()(e7)\n",
    "        e8 = MaxPooling1D(pool_size=POOL_SIZE[3], strides=None, padding='same')(e7act)\n",
    "        e8b = BatchNormalization()(e8)\n",
    "        e9a = Dropout(DROPOUT[3])(e8b)\n",
    "\n",
    "\n",
    "        e10 = Conv1D(N_KERNELS[4], KER4, padding='same',\n",
    "                                        kernel_initializer = orth(gain=1.0, seed=None))(e9a)\n",
    "        e10act = self.activation()(e10)\n",
    "        e10b = BatchNormalization()(e10act)\n",
    "        e10a = Dropout(DROPOUT[4])(e10b)\n",
    "\n",
    "\n",
    "        e11 = Flatten()(e10a)\n",
    "\n",
    "        e12 = Dense(DENSE_LEN, activation='sigmoid',\n",
    "                                         kernel_initializer = orth(gain=1.0, seed=None))(e11)\n",
    "        e12b = BatchNormalization()(e12)\n",
    "        e12a = Dropout(DROPOUT[5])(e12b)\n",
    "\n",
    "        e13 = Dense(2, activation='softmax')(e12a)\n",
    "        \n",
    "        model = Model(input_img, e13)\n",
    "        model.compile(loss='binary_crossentropy', optimizer='adadelta', metrics=['accuracy'])\n",
    "#         model.summary()\n",
    "        return model\n",
    "    \n",
    "    # fit the model\n",
    "    def CNN_fit(self):\n",
    "        early_stopping = EarlyStopping(monitor='val_loss', patience=50, mode='min')\n",
    "        \n",
    "        checkpointer = ModelCheckpoint(\n",
    "            filepath='/home/amplifier/home/NEW_DL/weights/EEGnet_weights_TD_1D.h5',\n",
    "            verbose=0,\n",
    "            monitor='val_acc',\n",
    "            save_best_only=True)\n",
    "        \n",
    "        train_history = self.__model.fit(self.__x_train, self.__y_train,\n",
    "                epochs=1000,\n",
    "                batch_size=self.__BATCH_SIZE,\n",
    "                verbose=0,\n",
    "                shuffle=True,\n",
    "                validation_data=(self.__x_test, self.__y_test),\n",
    "                callbacks=[checkpointer, early_stopping])\n",
    "    \n",
    "    # evaluate the model\n",
    "    def CNN_evaluate(self):\n",
    "        self.CNN_fit()\n",
    "        evaluation = self.__model.evaluate(self.__x_test, self.__y_test, \n",
    "                                           batch_size=self.__BATCH_SIZE, verbose=0)\n",
    "        return evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to define model, read in data:\n",
    "def run_CNN(SHUFFLE = True, BATCH_SIZE = 20, TEST_TRAIN = 0.25,\n",
    "            \n",
    "            KER0=100,\n",
    "            KER1=50,\n",
    "            KER2=25,\n",
    "            KER3=2,\n",
    "            KER4=2,\n",
    "            N_KERNELS = (10,20,40,80,160),\n",
    "            POOL_SIZE = (5,2,5,2),\n",
    "            DO0 = 0.1, DO1 = 0.2, DO2 = 0.25, DO3 = 0.25, DO4 = 0.8, DO5 = 0.8,\n",
    "            DENSE_LEN = 10):\n",
    "    \n",
    "    _eeg = EEG(SHUFFLE = SHUFFLE, BATCH_SIZE = BATCH_SIZE, TEST_TRAIN = TEST_TRAIN,\n",
    "               KER0 = KER0, KER1 = KER1, KER2 = KER2, KER3 = KER3, KER4 = KER4,\n",
    "               N_KERNELS = N_KERNELS,\n",
    "               POOL_SIZE = POOL_SIZE, \n",
    "               DO0 = DO0, DO1 = DO1, DO2 = DO2, DO3 = DO3, DO4 = DO4, DO5 = DO5,\n",
    "               DENSE_LEN = DENSE_LEN)\n",
    "    CNN_evaluation = _eeg.CNN_evaluate()\n",
    "    return CNN_evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run_CNN()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bounds for hyper-parameters in CNN model\n",
    "# the bounds dict should be in order of continuous type and then discrete type\n",
    "bounds = [{'name': 'KER0',             'type': 'discrete',    'domain': (30, 50, 70, 100)},\n",
    "          {'name': 'KER1',             'type': 'discrete',    'domain': (30, 50, 70)},\n",
    "          {'name': 'KER2',             'type': 'discrete',    'domain': (10, 20, 30, 40)},\n",
    "          {'name': 'KER3',             'type': 'discrete',    'domain': (4, 6, 8)},\n",
    "          {'name': 'DO3',              'type': 'discrete',    'domain': (0.4, 0.7, 0.9)},\n",
    "          {'name': 'DO4',              'type': 'discrete',    'domain': (0.4, 0.7, 0.9)},\n",
    "          {'name': 'DO5',              'type': 'discrete',    'domain': (0.5, 0.7, 0.9)},\n",
    "          {'name': 'DENSE_LEN',        'type': 'discrete',    'domain': (10, 20)}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to optimize the CNN model\n",
    "def f(x):\n",
    "    print(x)\n",
    "    evaluation = run_CNN(\n",
    "        KER0 = int(x[:,0]), \n",
    "        KER1 = int(x[:,1]),\n",
    "        KER2 = int(x[:,2]), \n",
    "        KER3 = int(x[:,3]), \n",
    "        DO3 = int(x[:,4]),\n",
    "        DO4 = int(x[:,5]),\n",
    "        DO5 = int(x[:,6]),\n",
    "        DENSE_LEN = int(x[:,7]))\n",
    "    print(\"LOSS:\\t{0} \\t ACCURACY:\\t{1}\".format(evaluation[0], evaluation[1]))\n",
    "    return evaluation[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 100.    30.    40.     4.     0.7    0.4    0.5   20. ]]\n",
      "LOSS:\t1.9087007309483215 \t ACCURACY:\t0.5398230046297596\n",
      "[[ 50.   50.   30.    8.    0.9   0.7   0.9  20. ]]\n",
      "LOSS:\t1.803819335667433 \t ACCURACY:\t0.5752212510699719\n",
      "[[ 50.   50.   20.    4.    0.7   0.4   0.9  20. ]]\n",
      "LOSS:\t1.8839498695019072 \t ACCURACY:\t0.5663716782510808\n",
      "[[ 30.   70.   20.    4.    0.4   0.9   0.7  20. ]]\n",
      "LOSS:\t2.2141897150900514 \t ACCURACY:\t0.5044247837720719\n",
      "[[ 100.    30.    20.     8.     0.7    0.9    0.7   10. ]]\n",
      "LOSS:\t1.8676535998825479 \t ACCURACY:\t0.5044247784973246\n"
     ]
    }
   ],
   "source": [
    "# run Bayesian optimization of the hyperparameters\n",
    "opt_hparams = GPyOpt.methods.BayesianOptimization(f=f, domain=bounds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 50.   50.   30.    8.    0.9   0.7   0.9  20. ]]\n",
      "LOSS:\t2.4080165339782176 \t ACCURACY:\t0.46017699431529085\n",
      "[[ 100.    30.    20.     8.     0.7    0.9    0.9   10. ]]\n",
      "LOSS:\t1.9100385186946498 \t ACCURACY:\t0.5398230204540017\n",
      "[[ 100.    30.    20.     8.     0.4    0.9    0.9   10. ]]\n",
      "LOSS:\t1.717380706187898 \t ACCURACY:\t0.5486725800860245\n",
      "[[ 100.    30.    20.     8.     0.7    0.9    0.9   10. ]]\n",
      "LOSS:\t1.8664990515835518 \t ACCURACY:\t0.5132743441953068\n",
      "[[ 100.    30.    20.     8.     0.7    0.7    0.9   10. ]]\n",
      "LOSS:\t1.9729630451286788 \t ACCURACY:\t0.5132743460414684\n",
      "[[ 100.    30.    20.     8.     0.7    0.4    0.7   10. ]]\n",
      "LOSS:\t2.3170954716944063 \t ACCURACY:\t0.46902655130994003\n",
      "[[ 100.    30.    20.     8.     0.4    0.9    0.7   10. ]]\n",
      "LOSS:\t1.8183828792740813 \t ACCURACY:\t0.566371682734616\n",
      "[[ 100.    30.    20.     8.     0.9    0.9    0.9   10. ]]\n",
      "LOSS:\t1.693705149456463 \t ACCURACY:\t0.5840707967239144\n",
      "[[ 100.    30.    20.     8.     0.7    0.9    0.9   10. ]]\n",
      "LOSS:\t2.0727604152881995 \t ACCURACY:\t0.5486725800860245\n",
      "[[ 100.    30.    20.     8.     0.4    0.7    0.9   10. ]]\n",
      "LOSS:\t1.8823045692612639 \t ACCURACY:\t0.4955752215026754\n",
      "[[ 100.    30.    20.     8.     0.4    0.9    0.9   10. ]]\n",
      "LOSS:\t2.1434733498412952 \t ACCURACY:\t0.49557522493126116\n",
      "[[ 100.    30.    20.     8.     0.9    0.9    0.9   10. ]]\n",
      "LOSS:\t1.852985889510771 \t ACCURACY:\t0.49557522493126116\n",
      "[[ 100.    30.    20.     8.     0.4    0.7    0.9   10. ]]\n",
      "LOSS:\t1.775037244357894 \t ACCURACY:\t0.5398230012011739\n",
      "[[ 100.    30.    20.     8.     0.4    0.9    0.7   10. ]]\n",
      "LOSS:\t2.1360089610108233 \t ACCURACY:\t0.5309734502724842\n",
      "[[ 100.    30.    20.     8.     0.7    0.9    0.9   10. ]]\n",
      "LOSS:\t2.0863218687276923 \t ACCURACY:\t0.5398230196627896\n",
      "[[ 100.    30.    40.     4.     0.4    0.4    0.7   20. ]]\n",
      "LOSS:\t2.0535831767900854 \t ACCURACY:\t0.5840707948777528\n",
      "[[ 50.   50.   20.    4.    0.4   0.4   0.7  20. ]]\n",
      "LOSS:\t1.8059495181108998 \t ACCURACY:\t0.5752212484325983\n",
      "[[ 100.    30.    20.     8.     0.9    0.7    0.7   10. ]]\n",
      "LOSS:\t1.9206648210508634 \t ACCURACY:\t0.5486725721739035\n",
      "[[ 50.   50.   20.    4.    0.7   0.4   0.9  20. ]]\n",
      "LOSS:\t2.0914726341720176 \t ACCURACY:\t0.4690265426066069\n",
      "[[ 50.   50.   20.    4.    0.4   0.4   0.5  20. ]]\n",
      "LOSS:\t1.9847639472083707 \t ACCURACY:\t0.5221238951239966\n",
      "[[ 100.    30.    40.     4.     0.7    0.7    0.5   20. ]]\n",
      "LOSS:\t2.0736677520043028 \t ACCURACY:\t0.5840707975151265\n",
      "[[ 100.    30.    20.     8.     0.4    0.4    0.9   10. ]]\n",
      "LOSS:\t1.9760493683604012 \t ACCURACY:\t0.5486725774486508\n",
      "[[ 100.    30.    20.     8.     0.9    0.7    0.7   10. ]]\n",
      "LOSS:\t1.8567697157902001 \t ACCURACY:\t0.5575221212564316\n",
      "[[ 100.    30.    20.     8.     0.9    0.9    0.9   10. ]]\n",
      "LOSS:\t1.4933080198490514 \t ACCURACY:\t0.5575221265311789\n",
      "[[ 30.   70.   20.    8.    0.9   0.7   0.9  10. ]]\n",
      "LOSS:\t1.6987359281134817 \t ACCURACY:\t0.5929203439602809\n",
      "[[ 70.   30.   10.    8.    0.9   0.9   0.5  20. ]]\n",
      "LOSS:\t1.9678893616769166 \t ACCURACY:\t0.4867256618706526\n",
      "[[ 30.   70.   20.    8.    0.9   0.7   0.9  10. ]]\n",
      "LOSS:\t2.1304547755064163 \t ACCURACY:\t0.47787610223862975\n",
      "[[ 50.   50.   20.    4.    0.4   0.4   0.7  20. ]]\n",
      "LOSS:\t2.0998015002866763 \t ACCURACY:\t0.5132743460414684\n",
      "[[ 100.    30.    20.     8.     0.9    0.9    0.9   10. ]]\n",
      "LOSS:\t2.487901385906523 \t ACCURACY:\t0.4690265618594347\n",
      "[[ 100.    30.    20.     8.     0.7    0.7    0.9   10. ]]\n",
      "LOSS:\t1.9321264503276454 \t ACCURACY:\t0.48672567242014725\n",
      "[[ 100.    30.    20.     8.     0.4    0.9    0.7   10. ]]\n",
      "LOSS:\t2.0548829409928446 \t ACCURACY:\t0.5309734555472315\n",
      "[[ 100.    30.    20.     8.     0.7    0.9    0.9   10. ]]\n",
      "LOSS:\t1.7167906444684593 \t ACCURACY:\t0.5575221283773405\n",
      "[[ 100.    30.    20.     8.     0.9    0.7    0.7   10. ]]\n",
      "LOSS:\t1.7485272251399218 \t ACCURACY:\t0.6106194632243266\n",
      "[[ 100.    30.    20.     8.     0.4    0.7    0.7   10. ]]\n",
      "LOSS:\t1.7297204954434284 \t ACCURACY:\t0.6017699107132127\n",
      "[[ 100.    30.    20.     8.     0.7    0.9    0.9   10. ]]\n",
      "LOSS:\t2.4738517113491496 \t ACCURACY:\t0.46017699431529085\n",
      "[[ 100.    30.    20.     8.     0.7    0.9    0.9   10. ]]\n",
      "LOSS:\t1.6377168165898957 \t ACCURACY:\t0.5663716782510808\n"
     ]
    }
   ],
   "source": [
    "opt_hparams.run_optimization(max_iter=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Optimized Parameters:\n",
      "\tKER0:\t\t100.0\n",
      "\tKER1:\t\t30.0\n",
      "\tKER2:\t\t20.0\n",
      "\tKER3:\t\t8.0\n",
      "\tDO3:\t\t0.9\n",
      "\tDO4:\t\t0.9\n",
      "\tDO5:\t\t0.9\n",
      "\tDENSE_LEN:\t10.0\n",
      "\n",
      "optimized loss: [ 1.49330802]\n"
     ]
    }
   ],
   "source": [
    "# print optimized mnist model\n",
    "print(\"\"\"\n",
    "Optimized Parameters:\n",
    "\\t{0}:\\t\\t{1}\n",
    "\\t{2}:\\t\\t{3}\n",
    "\\t{4}:\\t\\t{5}\n",
    "\\t{6}:\\t\\t{7}\n",
    "\\t{8}:\\t\\t{9}\n",
    "\\t{10}:\\t\\t{11}\n",
    "\\t{12}:\\t\\t{13}\n",
    "\\t{14}:\\t{15}\n",
    "\"\"\".format(bounds[0][\"name\"],opt_hparams.x_opt[0],\n",
    "           bounds[1][\"name\"],opt_hparams.x_opt[1],\n",
    "           bounds[2][\"name\"],opt_hparams.x_opt[2],\n",
    "           bounds[3][\"name\"],opt_hparams.x_opt[3],\n",
    "           bounds[4][\"name\"],opt_hparams.x_opt[4],\n",
    "          bounds[5][\"name\"],opt_hparams.x_opt[5],\n",
    "          bounds[6][\"name\"],opt_hparams.x_opt[6],\n",
    "          bounds[7][\"name\"],opt_hparams.x_opt[7]))\n",
    "print(\"optimized loss: {0}\".format(opt_hparams.fx_opt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}