Siamese-L1-EEGnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make sure you don't hog all the video memory\n",
    "import tensorflow as tf\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)\n",
    "from keras import backend as K\n",
    "K.set_session(sess)\n",
    "###################################\n",
    "\n",
    "from keras.layers import Input, Lambda, merge, Dense, DepthwiseConv2D, Activation, AveragePooling2D\n",
    "from keras.layers import Flatten,Conv2D, MaxPooling2D, Dropout, BatchNormalization, SeparableConv2D\n",
    "from keras.constraints import max_norm\n",
    "from keras.models import Model, Sequential\n",
    "from keras.regularizers import l2\n",
    "from keras.initializers import RandomUniform\n",
    "from keras.optimizers import SGD,Adam\n",
    "from keras.losses import binary_crossentropy\n",
    "import numpy.random as rng\n",
    "import numpy as np\n",
    "import os\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "# import seaborn as sns\n",
    "from sklearn.utils import shuffle\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SHUFFLE = False\n",
    "Seed = 4\n",
    "SCRAMBLE_FOR_TEST = False\n",
    "AUGMENT = True\n",
    "\n",
    "file = 'Merged456-197-289_ICA(-eyes)+AUDpreproc.mat, DS2=64Hz, FIR=2-30Hz, centnorm=1, step=2, win=2, TD, 1-93.mat' #0.75(vanilla EEGnet), 0. (SiameseL1)\n",
    "# file = 'Merged456-1-94_ICA(-2,3ICs)+AUDpreproc.mat, DS2=64Hz, FIR=2-30Hz, centnorm=1, step=2, win=2, TD, 1-95.mat' #0.61(vanilla EEGnet)\n",
    "# get the Dataset:\n",
    "import scipy.io as sio\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "\n",
    "path = '/home/amplifier/home/DATASETS/' + file\n",
    "mat_contents = sio.loadmat(path)\n",
    "X = mat_contents['X']\n",
    "Y = mat_contents['Z']\n",
    "\n",
    "if X.shape[1]<X.shape[2]:\n",
    "    X = np.transpose(X,[0,2,1])\n",
    "\n",
    "if Y.shape[1] > Y.shape[0]:\n",
    "    Y = Y.T\n",
    "\n",
    "    \n",
    "# # one hot encode the labels:\n",
    "# onehot_encoder = preprocessing.OneHotEncoder(sparse=False)\n",
    "# Y = onehot_encoder.fit_transform(Y)\n",
    "\n",
    "X = X.transpose(0,2,1)\n",
    "\n",
    "print('Original data shape:', X.shape)\n",
    "print('Original labels shape:', Y.shape)\n",
    "\n",
    "\n",
    "# verify that the model REALLY finds a mapping between the input and the labels. If we get\n",
    "# our accuracy by chance, then we should get the same accuracy on a permuted dataset:\n",
    "if SCRAMBLE_FOR_TEST==True:\n",
    "    Y = np.random.permutation(Y)\n",
    "Y = Y.flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import product\n",
    "from sys import getsizeof\n",
    "\n",
    "    \n",
    "def get_pairs(X,Y):\n",
    "    print('X', X.shape)\n",
    "    print('Y', Y.shape)\n",
    "    \n",
    "    # get the full permutation (very LARGE DATASET!)\n",
    "\n",
    "    temp = []\n",
    "    for x in product(range(Y.shape[0]), repeat=2):\n",
    "        temp.append(np.array(x))\n",
    "    \n",
    "    n_perm = len(temp)\n",
    "    temp = np.array(temp)\n",
    "    print('temp', temp.shape)\n",
    "\n",
    "    XX = np.zeros([n_perm,2,60,128])\n",
    "    XX[:,0,:,:] = X[[temp[i,0] for i in range(n_perm)],:,:]\n",
    "    XX[:,1,:,:] = X[[temp[i,1] for i in range(n_perm)],:,:]\n",
    "\n",
    "    print('Data size in memory (GB):', np.round(getsizeof(XX)/1024/1024/1024, 2))\n",
    "\n",
    "    YY = np.zeros([n_perm, 2])\n",
    "    YY[:,0] = Y[[temp[i,0] for i in range(n_perm)]].flatten()\n",
    "    YY[:,1] = Y[[temp[i,1] for i in range(n_perm)]].flatten()\n",
    "\n",
    "    YYY = []\n",
    "    for i in range(len(YY)):\n",
    "        YYY.append(0 if YY[i,0]==YY[i,1] else 1)\n",
    "    YYY = np.array(YYY).flatten()\n",
    "    \n",
    "    # diag:\n",
    "#     print (XX.shape, YYY.shape)\n",
    "#     for i in range(XX.shape[0]):\n",
    "#         print(temp[i,0], YY[i,0], temp[i,1], YY[i,1], YYY[i])\n",
    "\n",
    "    print('Left Col', np.mean(YY[:,0]))\n",
    "    print('Right Col', np.mean(YY[:,1]))\n",
    "    print('Labels', np.mean(YYY), '\\n')\n",
    "    \n",
    "    print(\"\\nPairs\", XX.shape)\n",
    "    print(\"Labels\", YYY.shape)\n",
    "    \n",
    "    return XX, YYY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test, = train_test_split(X, Y, test_size=0.1, shuffle=True)\n",
    "\n",
    "print(x_train.shape)\n",
    "print(x_test.shape)\n",
    "\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test, Y_test = get_pairs(x_test, y_test)\n",
    "X_train, Y_train = get_pairs(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(X.shape)\n",
    "# val_set = np.random.choice(range(len(Y)), 50, replace=False)\n",
    "# print(val_set.shape)\n",
    "# X_val = X[val_set,:,:]\n",
    "# Y_val = Y[val_set]\n",
    "# print('Validation Set')\n",
    "# print('X_val:', X_val.shape)\n",
    "# print('Y_val:', Y_val.shape)\n",
    "\n",
    "# X = np.delete(X,val_set,0)\n",
    "# Y = np.delete(Y,val_set,0)\n",
    "# print('Train & Test')\n",
    "# print(X.shape)\n",
    "# print(Y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SZ = 195\n",
    "# idx_a = []\n",
    "# idx_b = []\n",
    "# idy_a = []\n",
    "# idy_b = []\n",
    "# YYY = []\n",
    "# for i in range(2):\n",
    "#     addr = np.array(np.where(Y==i)).flatten()\n",
    "#     a = np.random.choice(addr, size=SZ, replace=False).tolist()\n",
    "#     b = np.random.choice(addr, size=SZ, replace=False).tolist()\n",
    "#     idx_a = idx_a + a\n",
    "#     idx_b = idx_b + b\n",
    "#     idy_a = idy_a + a\n",
    "#     idy_b = idy_b + b\n",
    "\n",
    "# for i in range(2):\n",
    "#     addr_a = np.array(np.where(Y==i)).flatten()\n",
    "#     addr_b = np.array(np.where(Y!=i)).flatten()\n",
    "#     a = np.random.choice(addr_a, size=SZ, replace=False).tolist()\n",
    "#     b = np.random.choice(addr_b, size=SZ, replace=False).tolist()\n",
    "#     idx_a = idx_a + a\n",
    "#     idx_b = idx_b + b\n",
    "#     idy_a = idy_a + a\n",
    "#     idy_b = idy_b + b\n",
    "    \n",
    "# XX = [X[idx_a], X[idx_b]]\n",
    "# YY = [Y[idy_a], Y[idy_b]]\n",
    "\n",
    "# for i in range(len(idx_a)):\n",
    "#     YYY.append(0 if YY[0][i]==YY[1][i] else 1)\n",
    "# YYY = np.array(YYY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(len(idy_a)):\n",
    "#     print(i, idy_a[i], idy_b[i], YY[0][i], YY[1][i], YYY[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rng = np.arange(len(idx_a))\n",
    "# train_test = 0.8\n",
    "\n",
    "# # sample ordinal numbers of \n",
    "# train_id = np.random.choice(rng ,np.round(len(rng)*train_test).astype('int64'), replace=False)\n",
    "# test_id = rng[np.isin(rng,train_id, invert=True)]\n",
    "\n",
    "# x_train = [X[np.array(idx_a)[train_id]], X[np.array(idx_b)[train_id]]]\n",
    "# x_test = [X[np.array(idx_a)[test_id]], X[np.array(idx_b)[test_id]]]\n",
    "# y_train = YYY[train_id]\n",
    "# y_test = YYY[test_id]\n",
    "\n",
    "# print(len(idx_a))\n",
    "# print(len(y_train))\n",
    "# print(len(y_test))\n",
    "\n",
    "# print(np.mean(y_train))\n",
    "# print(np.mean(y_test))\n",
    "\n",
    "# x_train = np.array(x_train).transpose(1,0,2,3)\n",
    "# x_test = np.array(x_test).transpose(1,0,2,3)\n",
    "# y_train = np.array(y_train).flatten()\n",
    "# y_test = np.array(y_test).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # get the full permutation (very LARGE DATASET!)\n",
    "# from itertools import product\n",
    "# from sys import getsizeof\n",
    "\n",
    "# temp = []\n",
    "# for x in product(range(Y.shape[0]), repeat=2):\n",
    "#     temp.append(np.array(x))\n",
    "\n",
    "# XX = np.zeros([len(temp), 2, 60, 128])\n",
    "# XX[:,0,:,:] = X[[temp[i][0] for i in range(len(temp))],:,:]\n",
    "# XX[:,1,:,:] = X[[temp[i][1] for i in range(len(temp))],:,:]\n",
    "\n",
    "\n",
    "# print('Data size in memory (GB):', np.round(getsizeof(XX)/1024/1024/1024, 2))\n",
    "\n",
    "# YY = np.zeros([len(temp), 2])\n",
    "# YY[:,0] = Y[[temp[i][0] for i in range(len(temp))]].flatten()\n",
    "# YY[:,1] = Y[[temp[i][1] for i in range(len(temp))]].flatten()\n",
    "\n",
    "# YYY = []\n",
    "# for i in range(len(YY)):\n",
    "#     YYY.append(1 if YY[i,0]==YY[i,1] else 0)\n",
    "# YYY = np.array(YYY).flatten()\n",
    "\n",
    "# x_train, x_test, y_train, y_test = train_test_split(XX,YYY, test_size=0.1, shuffle=True, random_state=10)\n",
    "# print (x_train.shape, y_train.shape, x_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def EEGNet_my(input1, nb_classes, Chans = 64, Samples = 128, \n",
    "             dropoutRate = 0.25, kernLength = 64, F1 = 4, \n",
    "             D = 2, F2 = 8, dropoutType = 'Dropout'):\n",
    "    \n",
    "    \"\"\" Keras Implementation of EEGNet (https://arxiv.org/abs/1611.08024)\n",
    "\n",
    "    Inputs:\n",
    "        \n",
    "      nb_classes      : int, number of classes to classify\n",
    "      Chans, Samples  : number of channels and time points in the EEG data\n",
    "      dropoutRate     : dropout fraction\n",
    "      kernLength      : length of temporal convolution in first layer. We found\n",
    "                        that setting this to be half the sampling rate worked\n",
    "                        well in practice. For the SMR dataset in particular\n",
    "                        since the data was high-passed at 4Hz we used a kernel\n",
    "                        length of 32.     \n",
    "      F1, F2          : number of temporal filters (F1) and number of pointwise\n",
    "                        filters (F2) to learn. Default: F1 = 4, F2 = F1 * D. \n",
    "      D               : number of spatial filters to learn within each temporal\n",
    "                        convolution. Default: D = 2\n",
    "      dropoutType     : Either SpatialDropout2D or Dropout, passed as a string.\n",
    "\n",
    "    \"\"\"\n",
    "    \n",
    "    if dropoutType == 'SpatialDropout2D':\n",
    "        dropoutType = SpatialDropout2D\n",
    "    elif dropoutType == 'Dropout':\n",
    "        dropoutType = Dropout\n",
    "    else:\n",
    "        raise ValueError('dropoutType must be one of SpatialDropout2D '\n",
    "                         'or Dropout, passed as a string.')\n",
    "    \n",
    "    init = RandomUniform(minval=-0.1, maxval=0.1, seed=29)\n",
    "    net = Sequential()\n",
    "    net.add (Conv2D(F1, (1, kernLength), padding = 'same',\n",
    "                                   input_shape = (Chans, Samples,1),\n",
    "                                   use_bias = False, bias_initializer=init, kernel_initializer=init))\n",
    "    net.add (BatchNormalization())\n",
    "    net.add (DepthwiseConv2D((Chans, 1), use_bias = False, \n",
    "                                   depth_multiplier = D,\n",
    "                                   depthwise_constraint = max_norm(1.), bias_initializer=init, kernel_initializer=init))\n",
    "    net.add (BatchNormalization())\n",
    "    net.add (Activation('elu'))\n",
    "    net.add (AveragePooling2D((1, 4)))\n",
    "    net.add (dropoutType(dropoutRate))\n",
    "    net.add (SeparableConv2D(F2, (1, 16), use_bias = False, padding = 'same', bias_initializer=init, kernel_initializer=init))\n",
    "    net.add (BatchNormalization())\n",
    "    net.add (Activation('elu'))\n",
    "    net.add (AveragePooling2D((1, 8)))\n",
    "    net.add (dropoutType(dropoutRate))\n",
    "    net.add (Flatten())\n",
    "    net.add (Dense(nb_classes, kernel_constraint = max_norm(0.25), bias_initializer=init, kernel_initializer=init))\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.engine.topology import Layer\n",
    "from keras.layers import Concatenate\n",
    "\n",
    "# def triplet_loss(y_true, y_pred):\n",
    "#     norm1 = K.sqrt(K.sum(K.square(y_pred[0] - y_pred[1]), axis=-1, keepdims=True))\n",
    "#     norm2 = K.sqrt(K.sum(K.square(y_pred[0] - y_pred[2]), axis=-1, keepdims=True))\n",
    "#     loss = norm1 - norm2 + 0.2\n",
    "#     return loss\n",
    "\n",
    "input_shape = X_train[-1,-1,:,:].shape + (1,)\n",
    "a_input = Input(input_shape)\n",
    "r_input = Input(input_shape)\n",
    "\n",
    "#call the convnet Sequential model on each of the input tensors so params will be shared\n",
    "# encoded NOT as Sequential (stack of layers), but as a Tensor!!!! if you add an argument, a Tensor is returned\n",
    "encoded_a = EEGNet_my(input_shape, 10, Chans=input_shape[0])(a_input)\n",
    "encoded_r = EEGNet_my(input_shape, 10, Chans=input_shape[0])(r_input)\n",
    "\n",
    "L1_distance = Lambda(lambda tensors: (K.abs(tensors[0] - tensors[1])))([encoded_a, encoded_r])\n",
    "L1_distance = Dense(1,activation='sigmoid', use_bias=True)(L1_distance)\n",
    "\n",
    "# siamese_net = Model(inputs=[a_input, r_input], outputs=[encoded_a, encoded_r, L1_distance])\n",
    "siamese_net = Model(inputs=[a_input, r_input], outputs=[L1_distance])\n",
    "optimizer = Adam(0.00006)\n",
    "# optimizer = 'adam'\n",
    "siamese_net.compile(loss=['binary_crossentropy'], optimizer=optimizer, metrics=['accuracy'])\n",
    "siamese_net.summary()\n",
    "\n",
    "a1 = np.linalg.norm(siamese_net.layers[2].layers[13].get_weights()[0])\n",
    "a2 = np.linalg.norm(siamese_net.layers[3].layers[13].get_weights()[0])\n",
    "print(a1, a2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "early_stopping = EarlyStopping(monitor='val_acc', patience=20, mode='min')\n",
    "checkpointer = ModelCheckpoint(filepath='/home/amplifier/home/NEW_DL/weights/Siam3EEGnet_wts.h5',\n",
    "                               verbose=1,\n",
    "                               monitor='val_acc',\n",
    "                               save_best_only=True)\n",
    "\n",
    "train_history = siamese_net.fit([X_train[:,0,:,:,None], X_train[:,1,:,:,None]], Y_train,\n",
    "                epochs=100,\n",
    "                batch_size=20,\n",
    "                verbose=1,\n",
    "                shuffle=True,\n",
    "                validation_data=([X_test[:,0,:,:,None], X_test[:,1,:,:,None]], Y_test),\n",
    "                callbacks=[checkpointer, early_stopping])\n",
    "\n",
    "# model.save('/home/amplifier/home/NEW_DL/models/Siam3.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "siamese_net.load_weights('/home/amplifier/home/NEW_DL/weights/Siam3_wts.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(np.array(history.losses))\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "loss = train_history.history['loss']\n",
    "val_loss = train_history.history['val_loss']\n",
    "plt.plot(loss)\n",
    "plt.plot(val_loss)\n",
    "plt.legend(['loss', 'val_loss'])\n",
    "plt.title('AdaDelta')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_acc = []\n",
    "rng = np.random.choice(y_train.shape[0], 195, replace=False)\n",
    "prediction = []\n",
    "for i in rng:\n",
    "    pred = siamese_net.predict([x_train[None,i,0,:,:,None], x_train[None,i,1,:,:,None]])\n",
    "    prediction.append(pred)\n",
    "    acc = 'correct' if np.round(pred)==y_train[i] else \"\"\n",
    "    train_acc.append(1 if np.round(pred)==y_train[i] else 0)\n",
    "#     print(i, pred, np.round(pred), y_train[i], acc)\n",
    "print('Train Accuracy', np.mean(train_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_acc = []\n",
    "rng = np.random.choice(y_test.shape[0], 190, replace=False)\n",
    "prediction = []\n",
    "for i in rng:\n",
    "    pred = siamese_net.predict([x_test[None,i,0,:,:,None], x_test[None,i,1,:,:,None]])\n",
    "    prediction.append(pred)\n",
    "    acc = 'correct' if np.round(pred)==y_train[i] else \"\"\n",
    "    test_acc.append(1 if np.round(pred)==y_train[i] else 0)\n",
    "#     print(i, pred, np.round(pred), y_test[i], acc)\n",
    "print('Test Accuracy', np.mean(test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_acc = []\n",
    "for k in range(len(Y_val)):\n",
    "    i = np.random.randint(len(Y_val))\n",
    "    j = np.random.randint(len(Y_val))\n",
    "    pred = siamese_net.predict([X_val[None,i,:,:,None], X_val[None,j,:,:,None]])\n",
    "    correct = 1 if Y_val[i]==Y_val[j] and np.round(pred)==0 or Y_val[i]!=Y_val[j] and np.round(pred)==1  else 0\n",
    "    val_acc.append(correct)\n",
    "    print(i, pred, Y_val[i], Y_val[j], correct)\n",
    "print('Validation accuracy:', np.mean(val_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def EEGNet(nb_classes, Chans = 64, Samples = 128, \n",
    "             dropoutRate = 0.25, kernLength = 64, F1 = 4, \n",
    "             D = 2, F2 = 8, dropoutType = 'Dropout'):\n",
    "    \"\"\" Keras Implementation of EEGNet (https://arxiv.org/abs/1611.08024)\n",
    "\n",
    "    Inputs:\n",
    "        \n",
    "      nb_classes      : int, number of classes to classify\n",
    "      Chans, Samples  : number of channels and time points in the EEG data\n",
    "      dropoutRate     : dropout fraction\n",
    "      kernLength      : length of temporal convolution in first layer. We found\n",
    "                        that setting this to be half the sampling rate worked\n",
    "                        well in practice. For the SMR dataset in particular\n",
    "                        since the data was high-passed at 4Hz we used a kernel\n",
    "                        length of 32.     \n",
    "      F1, F2          : number of temporal filters (F1) and number of pointwise\n",
    "                        filters (F2) to learn. Default: F1 = 4, F2 = F1 * D. \n",
    "      D               : number of spatial filters to learn within each temporal\n",
    "                        convolution. Default: D = 2\n",
    "      dropoutType     : Either SpatialDropout2D or Dropout, passed as a string.\n",
    "\n",
    "    \"\"\"\n",
    "    \n",
    "    if dropoutType == 'SpatialDropout2D':\n",
    "        dropoutType = SpatialDropout2D\n",
    "    elif dropoutType == 'Dropout':\n",
    "        dropoutType = Dropout\n",
    "    else:\n",
    "        raise ValueError('dropoutType must be one of SpatialDropout2D '\n",
    "                         'or Dropout, passed as a string.')\n",
    "    \n",
    "    input1   = Input(shape = (1, Chans, Samples))\n",
    "\n",
    "    ##################################################################\n",
    "    block1       = Conv2D(F1, (1, kernLength), padding = 'same',\n",
    "                                   input_shape = (1, Chans, Samples),\n",
    "                                   use_bias = False, data_format='channels_first')(input1)\n",
    "    block1       = BatchNormalization(axis = 1)(block1)\n",
    "    block1       = DepthwiseConv2D((Chans, 1), use_bias = False, \n",
    "                                   depth_multiplier = D,\n",
    "                                   depthwise_constraint = max_norm(1.), data_format='channels_first')(block1)\n",
    "    block1       = BatchNormalization(axis = 1)(block1)\n",
    "    block1       = Activation('elu')(block1)\n",
    "    block1       = AveragePooling2D((1, 4), data_format='channels_first')(block1)\n",
    "    block1       = dropoutType(dropoutRate)(block1)\n",
    "    \n",
    "    block2       = SeparableConv2D(F2, (1, 16),\n",
    "                                   use_bias = False,\n",
    "                                   padding = 'same',\n",
    "                                   data_format='channels_first')(block1)\n",
    "    block2       = BatchNormalization(axis = 1)(block2)\n",
    "    block2       = Activation('elu')(block2)\n",
    "    block2       = AveragePooling2D((1, 8), data_format='channels_first')(block2)\n",
    "    block2       = dropoutType(dropoutRate)(block2)\n",
    "        \n",
    "    flatten      = Flatten(name = 'flatten')(block2)\n",
    "    \n",
    "    dense        = Dense(nb_classes, name = 'dense', \n",
    "                         kernel_constraint = max_norm(0.25))(flatten)\n",
    "    softmax      = Activation('softmax', name = 'softmax')(dense)\n",
    "    \n",
    "    return Model(inputs=input1, outputs=softmax)\n",
    "\n",
    "model  = EEGNet(nb_classes = 2,\n",
    "                Chans = 60,\n",
    "                Samples = 128,\n",
    "                kernLength = 125)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def TrainTestVal_Split(leng, traintest, testval):\n",
    "    leng = 100\n",
    "    x = np.arange(leng)\n",
    "    train_id, test_id, val_id = np.split(\n",
    "        x,[np.round(leng*traintest).astype(int),\n",
    "           np.round(leng*testval).astype(int)])\n",
    "    return train_id, test_id, val_id\n",
    "\n",
    "# train_id, test_id, val_id = TrainTestVal_Split(100, 0.8, 0.9)\n",
    "# print(train_id)\n",
    "# print(test_id)\n",
    "# print(val_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in range(xx_train.shape[0]):\n",
    "    hit = []\n",
    "    for i in range(X.shape[0]):\n",
    "        if np.linalg.norm(X[i,:,:] - xx_train[j,0,:,:])==0:\n",
    "            hit.append(i)\n",
    "            print(hit)\n",
    "            print('success' if np.linalg.norm(Y[i] - yy_train[j])==0 else 'FAIL!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_augment(xx_train, yy_train):\n",
    "    print('Before augmenting', xx_train.shape)\n",
    "    temp_xx = np.concatenate([xx_train, xx_train, xx_train, xx_train, xx_train], axis=0)\n",
    "    print(temp_xx.shape)\n",
    "    temp_xx_0 = temp_xx[:,0,:,:]\n",
    "    temp_xx_0 = temp_xx_0.reshape(temp_xx_0.shape + (1,)).transpose(0,3,1,2)\n",
    "    print(temp_xx_0.shape)\n",
    "\n",
    "    temp_xx_1 = temp_xx[:,1,:,:]\n",
    "    np.random.shuffle(temp_xx_1)\n",
    "    temp_xx_1 = temp_xx_1.reshape(temp_xx_1.shape + (1,)).transpose(0,3,1,2)\n",
    "    print(temp_xx_1.shape)\n",
    "\n",
    "    xx_train_aug = np.concatenate([temp_xx_0, temp_xx_1], axis=1)\n",
    "    print('After augmenting', xx_train_aug.shape)\n",
    "    print(xx_train_aug.shape)\n",
    "\n",
    "    yy_train_aug = np.concatenate([yy_train, yy_train, yy_train, yy_train, yy_train], axis=0)\n",
    "    print('yy_train_aug', yy_train_aug.shape)\n",
    "    return xx_train_aug, yy_train_aug\n",
    "\n",
    "if AUGMENT==True:\n",
    "    xx_train, yy_train = train_augment(xx_train, yy_train)\n",
    "    print('\\nAfter augmenting:\\n', xx_train.shape,'\\n', yy_train.shape )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a,b = [],[]\n",
    "for i in range(len(X)):\n",
    "    if np.array_equal(Y[i],np.array([0,1])):\n",
    "        a.append(i)\n",
    "    if np.array_equal(Y[i],np.array([1,0])):\n",
    "        b.append(i)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "XX_a = np.zeros([len(a),2,X.shape[1],X.shape[2]])\n",
    "XX_b = np.zeros([len(b),2,X.shape[1],X.shape[2]])\n",
    "XX = np.zeros([len(a)+len(b),2,X.shape[1],X.shape[2]])\n",
    "\n",
    "YY_a = np.zeros([len(a),2])\n",
    "YY_b = np.zeros([len(b),2])\n",
    "YY = np.zeros([len(b)+len(b),2])\n",
    "\n",
    "XX_a[:,0,:,:] = X[a,:,:]\n",
    "XX_a[:,1,:,:] = X[np.random.permutation(a),:,:]\n",
    "YY_a = Y[a]\n",
    "XX_b[:,0,:,:] = X[b,:,:]\n",
    "XX_b[:,1,:,:] = X[np.random.permutation(b),:,:]\n",
    "YY_b = Y[b]\n",
    "\n",
    "XX = np.concatenate([XX_a, XX_b], axis=0)\n",
    "YY = np.concatenate([YY_a, YY_b], axis=0)\n",
    "\n",
    "print('XX_a\\t', XX_a.shape)\n",
    "print('XX_b\\t', XX_b.shape)\n",
    "print('XX\\t', XX.shape)\n",
    "\n",
    "train_id, test_id, val_id = train_test_val(XX,YY, 0.2, 0.1, shuffle=SHUFFLE)\n",
    "\n",
    "xx_train = XX[train_id,:,:,:]\n",
    "yy_train = YY[train_id]\n",
    "\n",
    "xx_test = XX[test_id,:,:,:]\n",
    "yy_test = YY[test_id]\n",
    "\n",
    "xx_val = XX[val_id,:,:,:]\n",
    "yy_val = YY[val_id]\n",
    "\n",
    "print('\\nXX_trai\\t', xx_train.shape)\n",
    "print('XX_test\\t', xx_test.shape)\n",
    "print('XX_val\\t', xx_val.shape)\n",
    "print('yy_trai\\t', yy_train.shape)\n",
    "print('yy_test\\t', yy_test.shape)\n",
    "print('yy_val\\t', yy_val.shape)\n",
    "\n",
    "print('\\nIf you use RANDOM SEED, check if you generate the same numbers every time:')\n",
    "print(train_id)\n",
    "print(test_id)\n",
    "print(val_id)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "569px",
    "left": "637.074px",
    "right": "20px",
    "top": "126.989px",
    "width": "603px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}