Siamese-L1-ver-3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make sure you don't hog all the video memory\n",
    "import tensorflow as tf\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)\n",
    "from keras import backend as K\n",
    "K.set_session(sess)\n",
    "###################################\n",
    "\n",
    "from keras.layers import Input, Lambda, merge, Dense, DepthwiseConv2D, Activation, AveragePooling2D\n",
    "from keras.layers import Flatten,Conv2D, MaxPooling2D, Dropout, BatchNormalization, SeparableConv2D\n",
    "from keras.constraints import max_norm\n",
    "from keras.models import Model, Sequential\n",
    "from keras.regularizers import l2\n",
    "from keras.initializers import RandomUniform\n",
    "from keras.optimizers import SGD,Adam\n",
    "from keras.losses import binary_crossentropy\n",
    "import numpy.random as rng\n",
    "import numpy as np\n",
    "import os\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "# import seaborn as sns\n",
    "from sklearn.utils import shuffle\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original data shape: (450, 60, 128)\n",
      "Original labels shape: (450, 1)\n"
     ]
    }
   ],
   "source": [
    "SHUFFLE = False\n",
    "Seed = 4\n",
    "SCRAMBLE_FOR_TEST = False\n",
    "AUGMENT = True\n",
    "\n",
    "file = 'Merged456-197-289_ICA(-eyes)+AUDpreproc.mat, DS2=64Hz, FIR=2-30Hz, centnorm=1, step=2, win=2, TD, 1-93.mat' #0.75(vanilla EEGnet), 0. (SiameseL1)\n",
    "# file = 'Merged456-1-94_ICA(-2,3ICs)+AUDpreproc.mat, DS2=64Hz, FIR=2-30Hz, centnorm=1, step=2, win=2, TD, 1-95.mat' #0.61(vanilla EEGnet)\n",
    "# get the Dataset:\n",
    "import scipy.io as sio\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "\n",
    "path = '/home/amplifier/home/DATASETS/' + file\n",
    "mat_contents = sio.loadmat(path)\n",
    "X = mat_contents['X']\n",
    "Y = mat_contents['Z']\n",
    "\n",
    "if X.shape[1]<X.shape[2]:\n",
    "    X = np.transpose(X,[0,2,1])\n",
    "\n",
    "if Y.shape[1] > Y.shape[0]:\n",
    "    Y = Y.T\n",
    "\n",
    "    \n",
    "# # one hot encode the labels:\n",
    "# onehot_encoder = preprocessing.OneHotEncoder(sparse=False)\n",
    "# Y = onehot_encoder.fit_transform(Y)\n",
    "\n",
    "X = X.transpose(0,2,1)\n",
    "\n",
    "print('Original data shape:', X.shape)\n",
    "print('Original labels shape:', Y.shape)\n",
    "\n",
    "\n",
    "# verify that the model REALLY finds a mapping between the input and the labels. If we get\n",
    "# our accuracy by chance, then we should get the same accuracy on a permuted dataset:\n",
    "if SCRAMBLE_FOR_TEST==True:\n",
    "    Y = np.random.permutation(Y)\n",
    "Y = Y.flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import product\n",
    "from sys import getsizeof\n",
    "\n",
    "    \n",
    "def get_pairs(X,Y):\n",
    "    print('X', X.shape)\n",
    "    print('Y', Y.shape)\n",
    "    \n",
    "    # get the full permutation (very LARGE DATASET!)\n",
    "\n",
    "    temp = []\n",
    "    for x in product(range(Y.shape[0]), repeat=2):\n",
    "        temp.append(np.array(x))\n",
    "    \n",
    "    n_perm = len(temp)\n",
    "    temp = np.array(temp)\n",
    "    print('temp', temp.shape)\n",
    "\n",
    "    XX = np.zeros([n_perm,2,60,128])\n",
    "    XX[:,0,:,:] = X[[temp[i,0] for i in range(n_perm)],:,:]\n",
    "    XX[:,1,:,:] = X[[temp[i,1] for i in range(n_perm)],:,:]\n",
    "\n",
    "    print('Data size in memory (GB):', np.round(getsizeof(XX)/1024/1024/1024, 2))\n",
    "\n",
    "    YY = np.zeros([n_perm, 2])\n",
    "    YY[:,0] = Y[[temp[i,0] for i in range(n_perm)]].flatten()\n",
    "    YY[:,1] = Y[[temp[i,1] for i in range(n_perm)]].flatten()\n",
    "\n",
    "    YYY = []\n",
    "    for i in range(len(YY)):\n",
    "        YYY.append(0 if YY[i,0]==YY[i,1] else 1)\n",
    "    YYY = np.array(YYY).flatten()\n",
    "    \n",
    "    # diag:\n",
    "#     print (XX.shape, YYY.shape)\n",
    "#     for i in range(XX.shape[0]):\n",
    "#         print(temp[i,0], YY[i,0], temp[i,1], YY[i,1], YYY[i])\n",
    "\n",
    "    print('Left Col', np.mean(YY[:,0]))\n",
    "    print('Right Col', np.mean(YY[:,1]))\n",
    "    print('Labels', np.mean(YYY), '\\n')\n",
    "    \n",
    "    print(\"\\nPairs\", XX.shape)\n",
    "    print(\"Labels\", YYY.shape)\n",
    "    \n",
    "    return XX, YYY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(405, 60, 128)\n",
      "(45, 60, 128)\n",
      "(405,)\n",
      "(45,)\n"
     ]
    }
   ],
   "source": [
    "x_train, x_test, y_train, y_test, = train_test_split(X, Y, test_size=0.1, shuffle=True)\n",
    "\n",
    "print(x_train.shape)\n",
    "print(x_test.shape)\n",
    "\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X (45, 60, 128)\n",
      "Y (45,)\n",
      "temp (2025, 2)\n",
      "Data size in memory (GB): 0.23\n",
      "Left Col 0.5777777777777777\n",
      "Right Col 0.5777777777777777\n",
      "Labels 0.48790123456790124 \n",
      "\n",
      "\n",
      "Pairs (2025, 2, 60, 128)\n",
      "Labels (2025,)\n",
      "X (405, 60, 128)\n",
      "Y (405,)\n",
      "temp (164025, 2)\n",
      "Data size in memory (GB): 18.77\n",
      "Left Col 0.49135802469135803\n",
      "Right Col 0.49135802469135803\n",
      "Labels 0.49985063252552964 \n",
      "\n",
      "\n",
      "Pairs (164025, 2, 60, 128)\n",
      "Labels (164025,)\n"
     ]
    }
   ],
   "source": [
    "X_test, Y_test = get_pairs(x_test, y_test)\n",
    "X_train, Y_train = get_pairs(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(X.shape)\n",
    "# val_set = np.random.choice(range(len(Y)), 50, replace=False)\n",
    "# print(val_set.shape)\n",
    "# X_val = X[val_set,:,:]\n",
    "# Y_val = Y[val_set]\n",
    "# print('Validation Set')\n",
    "# print('X_val:', X_val.shape)\n",
    "# print('Y_val:', Y_val.shape)\n",
    "\n",
    "# X = np.delete(X,val_set,0)\n",
    "# Y = np.delete(Y,val_set,0)\n",
    "# print('Train & Test')\n",
    "# print(X.shape)\n",
    "# print(Y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SZ = 195\n",
    "# idx_a = []\n",
    "# idx_b = []\n",
    "# idy_a = []\n",
    "# idy_b = []\n",
    "# YYY = []\n",
    "# for i in range(2):\n",
    "#     addr = np.array(np.where(Y==i)).flatten()\n",
    "#     a = np.random.choice(addr, size=SZ, replace=False).tolist()\n",
    "#     b = np.random.choice(addr, size=SZ, replace=False).tolist()\n",
    "#     idx_a = idx_a + a\n",
    "#     idx_b = idx_b + b\n",
    "#     idy_a = idy_a + a\n",
    "#     idy_b = idy_b + b\n",
    "\n",
    "# for i in range(2):\n",
    "#     addr_a = np.array(np.where(Y==i)).flatten()\n",
    "#     addr_b = np.array(np.where(Y!=i)).flatten()\n",
    "#     a = np.random.choice(addr_a, size=SZ, replace=False).tolist()\n",
    "#     b = np.random.choice(addr_b, size=SZ, replace=False).tolist()\n",
    "#     idx_a = idx_a + a\n",
    "#     idx_b = idx_b + b\n",
    "#     idy_a = idy_a + a\n",
    "#     idy_b = idy_b + b\n",
    "    \n",
    "# XX = [X[idx_a], X[idx_b]]\n",
    "# YY = [Y[idy_a], Y[idy_b]]\n",
    "\n",
    "# for i in range(len(idx_a)):\n",
    "#     YYY.append(0 if YY[0][i]==YY[1][i] else 1)\n",
    "# YYY = np.array(YYY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(len(idy_a)):\n",
    "#     print(i, idy_a[i], idy_b[i], YY[0][i], YY[1][i], YYY[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rng = np.arange(len(idx_a))\n",
    "# train_test = 0.8\n",
    "\n",
    "# # sample ordinal numbers of \n",
    "# train_id = np.random.choice(rng ,np.round(len(rng)*train_test).astype('int64'), replace=False)\n",
    "# test_id = rng[np.isin(rng,train_id, invert=True)]\n",
    "\n",
    "# x_train = [X[np.array(idx_a)[train_id]], X[np.array(idx_b)[train_id]]]\n",
    "# x_test = [X[np.array(idx_a)[test_id]], X[np.array(idx_b)[test_id]]]\n",
    "# y_train = YYY[train_id]\n",
    "# y_test = YYY[test_id]\n",
    "\n",
    "# print(len(idx_a))\n",
    "# print(len(y_train))\n",
    "# print(len(y_test))\n",
    "\n",
    "# print(np.mean(y_train))\n",
    "# print(np.mean(y_test))\n",
    "\n",
    "# x_train = np.array(x_train).transpose(1,0,2,3)\n",
    "# x_test = np.array(x_test).transpose(1,0,2,3)\n",
    "# y_train = np.array(y_train).flatten()\n",
    "# y_test = np.array(y_test).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # get the full permutation (very LARGE DATASET!)\n",
    "# from itertools import product\n",
    "# from sys import getsizeof\n",
    "\n",
    "# temp = []\n",
    "# for x in product(range(Y.shape[0]), repeat=2):\n",
    "#     temp.append(np.array(x))\n",
    "\n",
    "# XX = np.zeros([len(temp), 2, 60, 128])\n",
    "# XX[:,0,:,:] = X[[temp[i][0] for i in range(len(temp))],:,:]\n",
    "# XX[:,1,:,:] = X[[temp[i][1] for i in range(len(temp))],:,:]\n",
    "\n",
    "\n",
    "# print('Data size in memory (GB):', np.round(getsizeof(XX)/1024/1024/1024, 2))\n",
    "\n",
    "# YY = np.zeros([len(temp), 2])\n",
    "# YY[:,0] = Y[[temp[i][0] for i in range(len(temp))]].flatten()\n",
    "# YY[:,1] = Y[[temp[i][1] for i in range(len(temp))]].flatten()\n",
    "\n",
    "# YYY = []\n",
    "# for i in range(len(YY)):\n",
    "#     YYY.append(1 if YY[i,0]==YY[i,1] else 0)\n",
    "# YYY = np.array(YYY).flatten()\n",
    "\n",
    "# x_train, x_test, y_train, y_test = train_test_split(XX,YYY, test_size=0.1, shuffle=True, random_state=10)\n",
    "# print (x_train.shape, y_train.shape, x_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def EEGNet_my(input1, nb_classes, Chans = 64, Samples = 128, \n",
    "             dropoutRate = 0.25, kernLength = 64, F1 = 4, \n",
    "             D = 2, F2 = 8, dropoutType = 'Dropout'):\n",
    "    \n",
    "    \"\"\" Keras Implementation of EEGNet (https://arxiv.org/abs/1611.08024)\n",
    "\n",
    "    Inputs:\n",
    "        \n",
    "      nb_classes      : int, number of classes to classify\n",
    "      Chans, Samples  : number of channels and time points in the EEG data\n",
    "      dropoutRate     : dropout fraction\n",
    "      kernLength      : length of temporal convolution in first layer. We found\n",
    "                        that setting this to be half the sampling rate worked\n",
    "                        well in practice. For the SMR dataset in particular\n",
    "                        since the data was high-passed at 4Hz we used a kernel\n",
    "                        length of 32.     \n",
    "      F1, F2          : number of temporal filters (F1) and number of pointwise\n",
    "                        filters (F2) to learn. Default: F1 = 4, F2 = F1 * D. \n",
    "      D               : number of spatial filters to learn within each temporal\n",
    "                        convolution. Default: D = 2\n",
    "      dropoutType     : Either SpatialDropout2D or Dropout, passed as a string.\n",
    "\n",
    "    \"\"\"\n",
    "    \n",
    "    if dropoutType == 'SpatialDropout2D':\n",
    "        dropoutType = SpatialDropout2D\n",
    "    elif dropoutType == 'Dropout':\n",
    "        dropoutType = Dropout\n",
    "    else:\n",
    "        raise ValueError('dropoutType must be one of SpatialDropout2D '\n",
    "                         'or Dropout, passed as a string.')\n",
    "    \n",
    "    init = RandomUniform(minval=-0.1, maxval=0.1, seed=29)\n",
    "    net = Sequential()\n",
    "    net.add (Conv2D(F1, (1, kernLength), padding = 'same',\n",
    "                                   input_shape = (Chans, Samples,1),\n",
    "                                   use_bias = False, bias_initializer=init, kernel_initializer=init))\n",
    "    net.add (BatchNormalization())\n",
    "    net.add (DepthwiseConv2D((Chans, 1), use_bias = False, \n",
    "                                   depth_multiplier = D,\n",
    "                                   depthwise_constraint = max_norm(1.), bias_initializer=init, kernel_initializer=init))\n",
    "    net.add (BatchNormalization())\n",
    "    net.add (Activation('elu'))\n",
    "    net.add (AveragePooling2D((1, 4)))\n",
    "    net.add (dropoutType(dropoutRate))\n",
    "    net.add (SeparableConv2D(F2, (1, 16), use_bias = False, padding = 'same', bias_initializer=init, kernel_initializer=init))\n",
    "    net.add (BatchNormalization())\n",
    "    net.add (Activation('elu'))\n",
    "    net.add (AveragePooling2D((1, 8)))\n",
    "    net.add (dropoutType(dropoutRate))\n",
    "    net.add (Flatten())\n",
    "    net.add (Dense(nb_classes, kernel_constraint = max_norm(0.25), bias_initializer=init, kernel_initializer=init))\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_7 (InputLayer)            (None, 60, 128, 1)   0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_8 (InputLayer)            (None, 60, 128, 1)   0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sequential_7 (Sequential)       (None, 10)           1338        input_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "sequential_8 (Sequential)       (None, 10)           1338        input_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_4 (Lambda)               (None, 10)           0           sequential_7[1][0]               \n",
      "                                                                 sequential_8[1][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (None, 1)            11          lambda_4[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 2,687\n",
      "Trainable params: 2,607\n",
      "Non-trainable params: 80\n",
      "__________________________________________________________________________________________________\n",
      "1.067009 1.067009\n"
     ]
    }
   ],
   "source": [
    "from keras.engine.topology import Layer\n",
    "from keras.layers import Concatenate\n",
    "\n",
    "# def triplet_loss(y_true, y_pred):\n",
    "#     norm1 = K.sqrt(K.sum(K.square(y_pred[0] - y_pred[1]), axis=-1, keepdims=True))\n",
    "#     norm2 = K.sqrt(K.sum(K.square(y_pred[0] - y_pred[2]), axis=-1, keepdims=True))\n",
    "#     loss = norm1 - norm2 + 0.2\n",
    "#     return loss\n",
    "\n",
    "input_shape = X_train[-1,-1,:,:].shape + (1,)\n",
    "a_input = Input(input_shape)\n",
    "r_input = Input(input_shape)\n",
    "\n",
    "#call the convnet Sequential model on each of the input tensors so params will be shared\n",
    "# encoded NOT as Sequential (stack of layers), but as a Tensor!!!! if you add an argument, a Tensor is returned\n",
    "encoded_a = EEGNet_my(input_shape, 10, Chans=input_shape[0])(a_input)\n",
    "encoded_r = EEGNet_my(input_shape, 10, Chans=input_shape[0])(r_input)\n",
    "\n",
    "L1_distance = Lambda(lambda tensors: (K.abs(tensors[0] - tensors[1])))([encoded_a, encoded_r])\n",
    "L1_distance = Dense(1,activation='sigmoid', use_bias=True)(L1_distance)\n",
    "\n",
    "# siamese_net = Model(inputs=[a_input, r_input], outputs=[encoded_a, encoded_r, L1_distance])\n",
    "siamese_net = Model(inputs=[a_input, r_input], outputs=[L1_distance])\n",
    "optimizer = Adam(0.00003)\n",
    "# optimizer = 'adam'\n",
    "siamese_net.compile(loss=['binary_crossentropy'], optimizer=optimizer, metrics=['accuracy'])\n",
    "siamese_net.summary()\n",
    "\n",
    "a1 = np.linalg.norm(siamese_net.layers[2].layers[13].get_weights()[0])\n",
    "a2 = np.linalg.norm(siamese_net.layers[3].layers[13].get_weights()[0])\n",
    "print(a1, a2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 164025 samples, validate on 2025 samples\n",
      "Epoch 1/100\n",
      "164025/164025 [==============================] - 363s 2ms/step - loss: 0.6948 - acc: 0.5009 - val_loss: 0.6938 - val_acc: 0.4948\n",
      "\n",
      "Epoch 00001: val_acc improved from -inf to 0.49481, saving model to /home/amplifier/home/NEW_DL/weights/Siam3_wts.h5\n",
      "Epoch 2/100\n",
      " 68300/164025 [===========>..................] - ETA: 1:41 - loss: 0.6940 - acc: 0.4985"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164025/164025 [==============================] - 64s 388us/step - loss: 0.6933 - acc: 0.5039 - val_loss: 0.6941 - val_acc: 0.4884\n",
      "\n",
      "Epoch 00005: val_acc did not improve from 0.49728\n",
      "Epoch 6/100\n",
      "164025/164025 [==============================] - 64s 389us/step - loss: 0.6933 - acc: 0.5028 - val_loss: 0.6939 - val_acc: 0.4869\n",
      "\n",
      "Epoch 00006: val_acc did not improve from 0.49728\n",
      "Epoch 7/100\n",
      "129150/164025 [======================>.......] - ETA: 13s - loss: 0.6931 - acc: 0.5044"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164025/164025 [==============================] - 66s 404us/step - loss: 0.6915 - acc: 0.5218 - val_loss: 0.6941 - val_acc: 0.4775\n",
      "\n",
      "Epoch 00011: val_acc did not improve from 0.49728\n",
      "Epoch 12/100\n",
      "164025/164025 [==============================] - 66s 405us/step - loss: 0.6889 - acc: 0.5403 - val_loss: 0.6942 - val_acc: 0.4973\n",
      "\n",
      "Epoch 00012: val_acc did not improve from 0.49728\n",
      "Epoch 13/100\n",
      "164025/164025 [==============================] - 66s 400us/step - loss: 0.6811 - acc: 0.5662 - val_loss: 0.6882 - val_acc: 0.5526\n",
      "\n",
      "Epoch 00013: val_acc improved from 0.49728 to 0.55259, saving model to /home/amplifier/home/NEW_DL/weights/Siam3_wts.h5\n",
      "Epoch 14/100\n",
      " 19550/164025 [==>...........................] - ETA: 56s - loss: 0.6700 - acc: 0.5936"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164025/164025 [==============================] - 67s 406us/step - loss: 0.5992 - acc: 0.6782 - val_loss: 0.6960 - val_acc: 0.5817\n",
      "\n",
      "Epoch 00017: val_acc improved from 0.55951 to 0.58173, saving model to /home/amplifier/home/NEW_DL/weights/Siam3_wts.h5\n",
      "Epoch 18/100\n",
      "164025/164025 [==============================] - 66s 404us/step - loss: 0.5806 - acc: 0.6952 - val_loss: 0.7001 - val_acc: 0.5926\n",
      "\n",
      "Epoch 00018: val_acc improved from 0.58173 to 0.59259, saving model to /home/amplifier/home/NEW_DL/weights/Siam3_wts.h5\n",
      "Epoch 19/100\n",
      "164025/164025 [==============================] - 66s 404us/step - loss: 0.5595 - acc: 0.7136 - val_loss: 0.6996 - val_acc: 0.6104\n",
      "\n",
      "Epoch 00019: val_acc improved from 0.59259 to 0.61037, saving model to /home/amplifier/home/NEW_DL/weights/Siam3_wts.h5\n",
      "Epoch 20/100\n",
      " 98000/164025 [================>.............] - ETA: 26s - loss: 0.5399 - acc: 0.7302"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164025/164025 [==============================] - 104s 636us/step - loss: 0.4737 - acc: 0.7774 - val_loss: 0.7446 - val_acc: 0.5798\n",
      "\n",
      "Epoch 00023: val_acc did not improve from 0.61037\n",
      "Epoch 24/100\n",
      "164025/164025 [==============================] - 106s 646us/step - loss: 0.4517 - acc: 0.7917 - val_loss: 0.7499 - val_acc: 0.5872\n",
      "\n",
      "Epoch 00024: val_acc did not improve from 0.61037\n",
      "Epoch 25/100\n",
      " 88000/164025 [===============>..............] - ETA: 49s - loss: 0.4313 - acc: 0.8060"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164025/164025 [==============================] - 106s 649us/step - loss: 0.3728 - acc: 0.8396 - val_loss: 0.7920 - val_acc: 0.5719\n",
      "\n",
      "Epoch 00028: val_acc did not improve from 0.61037\n",
      "Epoch 29/100\n",
      "164025/164025 [==============================] - 107s 652us/step - loss: 0.3538 - acc: 0.8505 - val_loss: 0.8150 - val_acc: 0.5575\n",
      "\n",
      "Epoch 00029: val_acc did not improve from 0.61037\n",
      "Epoch 30/100\n",
      "  5800/164025 [>.............................] - ETA: 1:42 - loss: 0.3323 - acc: 0.8653- ETA: 1:46 - los"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164025/164025 [==============================] - 92s 561us/step - loss: 0.3059 - acc: 0.8747 - val_loss: 0.8513 - val_acc: 0.5714\n",
      "\n",
      "Epoch 00032: val_acc did not improve from 0.61037\n",
      "Epoch 33/100\n",
      "164025/164025 [==============================] - 66s 405us/step - loss: 0.2929 - acc: 0.8807 - val_loss: 0.8597 - val_acc: 0.5723\n",
      "\n",
      "Epoch 00033: val_acc did not improve from 0.61037\n",
      "Epoch 34/100\n",
      "164025/164025 [==============================] - 67s 406us/step - loss: 0.2777 - acc: 0.8886 - val_loss: 0.8918 - val_acc: 0.5714\n",
      "\n",
      "Epoch 00034: val_acc did not improve from 0.61037\n",
      "Epoch 35/100\n",
      " 15500/164025 [=>............................] - ETA: 1:00 - loss: 0.2685 - acc: 0.8934"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164025/164025 [==============================] - 66s 404us/step - loss: 0.2320 - acc: 0.9106 - val_loss: 0.9774 - val_acc: 0.5931\n",
      "\n",
      "Epoch 00038: val_acc did not improve from 0.61037\n",
      "Epoch 39/100\n",
      "164025/164025 [==============================] - 66s 405us/step - loss: 0.2226 - acc: 0.9144 - val_loss: 0.9735 - val_acc: 0.5960\n",
      "\n",
      "Epoch 00039: val_acc did not improve from 0.61037\n",
      "Epoch 40/100\n",
      "164025/164025 [==============================] - 66s 402us/step - loss: 0.2141 - acc: 0.9194 - val_loss: 1.0084 - val_acc: 0.5965\n",
      "\n",
      "Epoch 00040: val_acc did not improve from 0.61037\n",
      "Epoch 41/100\n",
      "101150/164025 [=================>............] - ETA: 25s - loss: 0.2063 - acc: 0.9220"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164025/164025 [==============================] - 66s 404us/step - loss: 0.1735 - acc: 0.9368 - val_loss: 1.1250 - val_acc: 0.5926\n",
      "\n",
      "Epoch 00045: val_acc did not improve from 0.61037\n",
      "Epoch 46/100\n",
      "164025/164025 [==============================] - 66s 405us/step - loss: 0.1643 - acc: 0.9410 - val_loss: 1.1816 - val_acc: 0.5857\n",
      "\n",
      "Epoch 00046: val_acc did not improve from 0.61037\n",
      "Epoch 47/100\n",
      "164025/164025 [==============================] - 66s 404us/step - loss: 0.1591 - acc: 0.9430 - val_loss: 1.2006 - val_acc: 0.5664\n",
      "\n",
      "Epoch 00047: val_acc did not improve from 0.61037\n",
      "Epoch 48/100\n",
      " 34750/164025 [=====>........................] - ETA: 51s - loss: 0.1585 - acc: 0.9430"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "early_stopping = EarlyStopping(monitor='val_acc', patience=40, mode='min')\n",
    "checkpointer = ModelCheckpoint(filepath='/home/amplifier/home/NEW_DL/weights/Siam3_wts.h5',\n",
    "                               verbose=1,\n",
    "                               monitor='val_acc',\n",
    "                               save_best_only=True)\n",
    "\n",
    "train_history = siamese_net.fit([X_train[:,0,:,:,None], X_train[:,1,:,:,None]], Y_train,\n",
    "                epochs=100,\n",
    "                batch_size=50,\n",
    "                verbose=1,\n",
    "                shuffle=True,\n",
    "                validation_data=([X_test[:,0,:,:,None], X_test[:,1,:,:,None]], Y_test),\n",
    "                callbacks=[checkpointer, early_stopping])\n",
    "\n",
    "# model.save('/home/amplifier/home/NEW_DL/models/Siam3.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "siamese_net.load_weights('/home/amplifier/home/NEW_DL/weights/Siam3_wts.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd4VNXWwOHfSiGht4ReAtJ7CV2aKE0EKVJFAQWxi+Wq1979vHZFFBURAQHBgoKi2OhIQu+EQCCEEhISSkib2d8fJ2iEQAKZyZmZrPd55glz5pwzKyE7a3YXYwxKKaWUp/GzOwCllFIqJ5qglFJKeSRNUEoppTySJiillFIeSROUUkopj6QJSimllEfSBOVlRGSMiKwowPfbLyLXFtT7KVVQtCx5Pk1QHkRE/hCREyIS5KL7PSsiGSJyKuuxW0TeF5HK+bjfTFfEppQ7aVnyDZqgPISIhAGdAQP0d+Gt5xpjSgLlgIFAJSDySguWUp5Oy5Lv0ATlOW4B1gDTgVvPHRSR8iKyUEROishfwFXZLxKRd0TkYNbrkSLSOaebG2MyjDHbgGFAPPBQtnv0E5GNIpIkIqtEpNn514tIb+C/wDAROS0im7KOjxWRHVmfKqNF5I78/iCUyictSz5CE5TnuAWYlfXoJSIVs45PBlKBysC4rEd264AWWJ/qZgNfiUjwxd7EGOMAvsP6hImItAKmAXcA5YGPgIXnN40YY34CXsb6FFnCGNM866VjQD+gFDAWeCvrnkrZRcuSj9AE5QFE5GqgJjDPGBMJ7AVGiog/MBh42hhzxhizFfg8+7XGmJnGmARjTKYx5g0gCKify1vGYRVCgPHAR8aYtcYYhzHmcyANaJ+X2I0xi4wxe43lT+BnsgqsUgVNy5Jv0QTlGW4FfjbGHM96PjvrWCgQABzMdm5M9gtF5KGsZoFkEUkCSgMhubxfVSAx6981gYeymiSSsu5RHaiSl8BFpI+IrBGRxKxr++bh/ZVyFy1LPiTA7gAKOxEpCgwF/EXkSNbhIKAMUBHIxPol35n1Wo1s13YGHgV6ANuMMU4ROQHIJd7PD7gBWJp16CDwkjHmpTyE+6+l77OaLhZgNal8Z4zJEJFvL/X+SrmLliXfozUo+90IOIBGWO3fLYCGwHKsX9avgWdFpJiINCJbpy9QEqvQxQMBIvI0Vvv1BUQkUEQaAl9ijT56M+ulj4GJItJOLMVF5HoRKZnDbY4CYVkFE6AI1h+AeCBTRPoAPa/op6BU/mlZ8jGaoOx3K/CZMeaAMebIuQfwPjAKuAcoARzBGpX0WbZrlwA/AruxmitS+XcTBmSNFAKSgIVAAtDaGBMHYIyJwGo7fx84AUQBYy4S61dZXxNEZL0x5hRwHzAv69qRWe+hlB20LPkY0Q0LlVJKeSKtQSmllPJImqCUUkp5JE1QSimlPJImKKWUUh7J6+ZBhYSEmLCwMLvDUIVUZGTkcWNMqN1xuIKWJWWXvJYjr0tQYWFhRERE2B2GKqREJCb3s7yDliVll7yWI23iU0op5ZE0QSmllPJImqCUUkp5JK/rg8pJRkYGsbGxpKam2h2KRwsODqZatWoEBgbaHYryUFqWLk3LUMHyiQQVGxtLyZIlCQsLQ6RQL/57UcYYEhISiI2NpVatWnaHozyUlqWL0zJU8HyiiS81NZXy5ctrgboEEaF8+fL6yVhdkpali9MyVPB8IkEBWqDyQH9GKi/09+Ti9GdTsHwmQSl1RTLOwo7vIWEv6Mr+SuWbw2n4dcdRImNO5PtePtEH5QlKlCjB6dOn7Q5DXa41U+DX56x/F68A1dtCv7egRAV741LKixhj2J+QwrcbDjEv4iCHk1O5oXkVWtcsm6/7aoJShdu2b6BSMwgfCwfWwuGNEFzG7qiU8niJZ9JZtOUwK/ccJyLmBMdPpyECXeqG8swNjenRMP8f8tzWxCci00TkmIhsvcjro0Rkc9ZjlYg0d1csBckYwyOPPEKTJk1o2rQpc+fOBeDw4cN06dKFFi1a0KRJE5YvX47D4WDMmDF/n/vWW2/ZHH0hk7AXjmyG5sMhfBwM+gjuXgsBReyOrFC78cYbad26NY0bN2bq1KkA/PTTT7Rq1YrmzZvTo0cPAE6fPs3YsWNp2rQpzZo1Y8GCBXaGXSgcP53G1+tjGTd9HW1fWspT325la1wyneuG8OKNTVj+n+58Pq4tvZtUItA//+nFnTWo6VhbH8+4yOv7gK7GmBMi0geYCrTL75s+9/02tsedzO9t/qVRlVI8c0PjPJ379ddfs3HjRjZt2sTx48dp06YNXbp0Yfbs2fTq1YsnnngCh8NBSkoKGzdu5NChQ2zdauXwpKQkl8atcrHtG+trowH2xuGh7CpL06ZNo1y5cpw9e5Y2bdowYMAAxo8fz7Jly6hVqxaJiYkAvPDCC5QuXZotW7YAcOJE/vs81IUyHE5mrz3A1+tj2XwoGWOgculgbutcixtbVKVh5VJue2+3JShjzDIRCbvE66uyPV0DVHNXLAVpxYoVjBgxAn9/fypWrEjXrl1Zt24dbdq0Ydy4cWRkZHDjjTfSokULateuTXR0NPfeey/XX389PXv2tDt835V0AP76GDo/CEWz2sW3fwvV2kJpn/jV8xnvvvsu33xjfXg4ePAgU6dOpUuXLn/PPSpXrhwAS5cuZc6cOX9fV7Zs/vo71IXWRCfwzHfb2HX0FM2rlebBa+vRvUEFGlUuhZ+f+0c0ekof1G3Ajxd7UUQmABMAatSocckb5bWm4y7mIiPBunTpwrJly1i0aBGjR4/mkUce4ZZbbmHTpk0sWbKEyZMnM2/ePKZNm1bAERcCqSdh1lCI3wFJMXDT51nNe1ug1yt2R+ex7ChLf/zxB0uXLmX16tUUK1aMbt260bx5c3bt2nXBucYYHfbtBk6nYUXUcWas3s/SHceoWqYoH41uTc9GFQv85237MHMR6Y6VoB692DnGmKnGmHBjTHhoqGdvxdOlSxfmzp2Lw+EgPj6eZcuW0bZtW2JiYqhQoQLjx4/ntttuY/369Rw/fhyn08ngwYN54YUXWL9+vd3h+x5HJswfBwl7oMkQ2P4dbJgJ27V5zxMlJydTtmxZihUrxs6dO1mzZg1paWn8+eef7Nu3D+DvJr6ePXvy/vvv/32tNvHlT3qmk5lrYujx5p/cMu0vNhxIYtK19Vj6YFd6Na5ky4cBW2tQItIM+AToY4xJsDMWVxk4cCCrV6+mefPmiAivvfYalSpV4vPPP+d///sfgYGBlChRghkzZnDo0CHGjh2L0+kE4JVX9NO8y/38BET9Aje8Ay1vgTPH4MdHoXgIVG8HpavaHaHKpnfv3nz44Yc0a9aM+vXr0759e0JDQ5k6dSqDBg3C6XRSoUIFfvnlF5588knuvvtumjRpgr+/P8888wyDBg2y+1vwOg6n4ftNcbz5y24OJKbQonoZ3h7Wgj5NKxEU4G9rbHKxJimX3Nzqg/rBGNMkh9dqAL8Bt5zXH3VJ4eHh5vxN1nbs2EHDhg3zF2whUah+VvtXwvS+0P5u6P2ydexkHEzpCGdPQO9Xof2dl3VLEYk0xoS7IdoCp2XpyvjCz+hkagY/bjnM8j3HWbU3gcQz6TSsXIr/9KpPt/qhbq8t5bUcua0GJSJfAt2AEBGJBZ4BAgGMMR8CTwPlgQ+yfhiZvlLwlYfYtRj8i8A1T/xzrFQVuHEK/PQ4NB5oX2xK2SRifyL3z9nIoaSzhJYMolv9UHo2qkjPRpUKZODD5XDnKL4Rubx+O3C7u95fKaKWQs2OUKT4v4/X72M9lCpEHE7DlD+ieGvpHqqWKcq8OzrQJqysRw808ZRRfEq5VtJBiN8JLUfbHYlStjLG8Mv2o7z5y252HjlF/+ZVeGlgE0oGe/6eVpqglG+KWmp9rXOtvXEoZaMNB07w7MJtbIpNplZIcd4f2ZLrm1b26FpTdpqglG+KWgqlq0NofbsjUarAZTicvPdbFJN/j6JCySBeG9yMQa2qEuCC5YcKkiYo5Xsy0yH6T2g6GLzkk6JSrrLj8EkeW7CZTbHJDGpZlWcHNKaUFzTn5UQTlPI9B9dC+imoc53dkShVYOKSzvLGz7v5ekMspYsGMnlkK65vVtnusPJFE5QNLrV31P79++nXr9/fC8iqKxC1FPwCoFYXuyNRbqb7sFmDIGauPcCLP2zHABM61+aubnUoXcw7a03ZaYJSvidqKdToAMHuW2VZKU+Qlungme+2MWfdQbrWC+WlgU2oVraY3WG5jO8lqB8fsxYBdaVKTaHPqxd9+dFHH6VmzZrcddddADz77LOICMuWLePEiRNkZGTw4osvMmDA5a37lpqayp133klERAQBAQG8+eabdO/enW3btjF27FjS09NxOp0sWLCAKlWqMHToUGJjY3E4HDz11FMMGzYsX9+218hMg99fhlNHIP00HN0K1z5nd1QuIyK9gXcAf+ATY8yr571eE5gGhAKJwM3GmNh8v7GXl6XTp08zYMCAHK+bMWMGr7/+OiJCs2bN+OKLLzh69CgTJ04kOjoagClTptCxY0cXfNPusf/4GR6ct5H1B5K4p3sdJl1XD38Pm2ibX76XoGwwfPhwHnjggb8L1bx58/jpp5+YNGkSpUqV4vjx47Rv357+/ftf1vDOyZMnA7BlyxZ27txJz5492b17Nx9++CH3338/o0aNIj09HYfDweLFi6lSpQqLFi0CrEU3C42Ns2Dl29aovaCSENYZmgy2OyqXEBF/YDJwHRALrBORhcaY7dlOex2YYYz5XESuAV4BvHICmCvLUnBwMN98880F123fvp2XXnqJlStXEhIS8vfis/fddx9du3blm2++weFweGzT4Ykz6bz3WxRfrNlPEX8/PhjVir5Nvbuv6WJ8L0Fd4tOZu7Rs2ZJjx44RFxdHfHw8ZcuWpXLlykyaNIlly5bh5+fHoUOHOHr0KJUqVcrzfVesWMG9994LQIMGDahZsya7d++mQ4cOvPTSS8TGxjJo0CDq1q1L06ZNefjhh3n00Ufp168fnTt3dte361mcDlj1HlRpBeN/88VRe22BKGNMNICIzAEGANkTVCNgUta/fwe+dck7e3lZMsbw3//+94LrfvvtN4YMGUJISAjwz/5Sv/32GzNmWPur+vv7U7p0afd+s5fJGMM3Gw7x7MJtnE7LZFib6ky6th4VSgXbHZrb+F6CssmQIUOYP38+R44cYfjw4cyaNYv4+HgiIyMJDAwkLCyM1NTUy7rnxRbyHTlyJO3atWPRokX06tWLTz75hGuuuYbIyEgWL17M448/Ts+ePXn66add8a15tp0/QGK0tceT7yUngKrAwWzPY7lw5+lNwGCsZsCBQEkRKZ/TDgGXs7eaXVxVli52nTfuI5V8NoMnv93K95viCK9ZlhcHNqFBJd/vY/WuWVsebPjw4cyZM4f58+czZMgQkpOTqVChAoGBgfz+++/ExMRc9j27dOnCrFmzANi9ezcHDhygfv36REdHU7t2be677z769+/P5s2biYuLo1ixYtx88808/PDDhWNvKWNgxdtQthY0vMHuaNwlp7+k539yeRjoKiIbgK7AISAzp5t5w95qripLF7uuR48ezJs3j4QEK3+fa+Lr0aMHU6ZMAcDhcHDypGu3u79Se46eou87y/lxy2Ee6VWfuXd0KBTJCbQG5TKNGzfm1KlTVK1alcqVKzNq1ChuuOEGwsPDadGiBQ0aNLjse951111MnDiRpk2bEhAQwPTp0wkKCmLu3LnMnDmTwMBAKlWqxNNPP826det45JFH8PPzIzAw8O+C5tNiVkLcerj+TfCzd98aN4oFqmd7Xg2Iy36CMSYOGAQgIiWAwcYYr+2EdFVZuth1jRs35oknnqBr1674+/vTsmVLpk+fzjvvvMOECRP49NNP8ff3Z8qUKXTo0MGd32qu9h0/w8hP1gIw/86OtKhextZ4Cppb94NyB93DJn986mc16yY4tB4mbYXAogXylgW9H5SIBAC7gR5YNaN1wEhjzLZs54QAicYYp4i8BDiMMbm272pZujIF9TM6mJjC0I9Wk5bpZO6E9tStWNLt71lQ8lqOtIlPeaf1M2DPz9BuYoElJzsYYzKBe4AlwA5gnjFmm4g8LyL9s07rBuwSkd1AReAlW4JVLnP0ZCojP1lDSrqDmbe186nkdDm0ic8mW7ZsYfTof48EDgoKYu3atTZF5EX++hgWP2ytVN7xHrujcTtjzGJg8XnHns727/nA/IKOy1P4Wlk6lZrBrdP+IvF0OrPHt6dRlcLR35QTn0lQ3jYyp2nTpmzcuLFA39PbmnNztOo9+PlJqH893PQZBATZHZHP0bJ0ce4uQ+mZTibOjCTq2GmmjWlD80LW53Q+n2jiCw4OJiEhwTf+ALuJMYaEhASCg714zsSh9VZyajQAhn6uyckNtCxdnLvLkMNpeHTBZlZGJfB/g5vRpZ5njrIsSD5Rg6pWrRqxsbHEx8fbHYpHCw4Oplq1anaHceWWvwHBpaH/++Dv/QtheiItS5fmrjJ0Ji2TB+Zu5JftR3mkV30Gt/bicupCPpGgAgMDqVWrlt1hKHc6ut2alNvlP7oIrBtpWSp4h5PPctv0CHYeOclz/Rtza8cwu0PyGD6RoFQhsOJNCCwO7e+0OxKlXCYm4Qw3fbialHQH08a0oVv9CnaH5FE0QSnPl7AXti6A9ndBsXJ2R6OUSyScTuPWaX+R4XCy4M6O1K9UOIeSX4omKOX5Vr4NfoHQ8V67I1HKJVLSMxn3eQRHTqYye3x7TU4X4ROj+JQPS46FjV9Cy5uhZN5XglfKU2U4nNwzewNbYpN4b0QrWtUoa3dIHktrUMqzLX/T+nr1pEufp5QXyHA4ue/LDfy28xgvD2zKdY0q2h2SR9MalPJcybHWkkatRkOZ6rmfr5QHy3Q4eWDuRn7ceoSn+jViZDvP3O7Ek2iCUp7r79rTg/bGoVQ+OZ2Gh7/axKLNh/lv3wbcdrUO5c8LTVDKM2ntSfmQKX/u5duNcTzcsx4Tulxldzhew20JSkSmicgxEdl6kddFRN4VkSgR2SwirdwVi/JCWntSPmLFnuO88fMu+jevwt3d69gdjldxZw1qOtD7Eq/3AepmPSYAhWCHPZUnJw/Dhi+skXtae1Je7FDSWe6bs4E6FUrwyqCmXrUIrydwW4IyxiwDEi9xygBghrGsAcqISGV3xaO8yNop4MyETvfbHYlSVyzD4eSuWetJz3Ty4c2tKR6kg6Yvl519UFWBg9mex2Ydu4CITBCRCBGJ0EUsfVzqSYj4zFqxvJx2JCvv9fHyaDYdTOLVwU2pHVrC7nC8kp0JKqe6bo5r/Btjphpjwo0x4aGhugS9T4ucDmknoeN9dkei1BWLjj/N20v30LtxJfo1q2J3OF7LzgQVC2TvYKgGxNkUi/IEmemw5gOo1QWq6pgZ5Z2cTsNjX28hOMCP5wc0tjscr2Zno+hC4B4RmQO0A5KNMYdtjEfZYfNXkHEGanaCg3/BqcPWfk9KeanZfx3gr32JvDa4GRVKefEGoR7AbQlKRL4EugEhIhILPAMEAhhjPgQWA32BKCAFGOuuWJSHiv4Tvr492wGBCo2hTg/bQlIqPw4nn+XVH3fSqU55bgrXTQfzy20JyhgzIpfXDXC3u95febj0M7DwXihXG4bNhNgIiP0Lmo8AHYqrvJAxhqe+3Uqm08krA5vpkHIX0HGPyh6/vgBJMTBmMVRsbD1a32p3VEpdsUVbDrN0xzGe6NuQGuWL2R2OT9CljlTBO7AG1n4IbSdAWCe7o1Eq35JS0nl24TaaVi3N2E5hdofjM7QGpQrW6WPwzR1Qujr0eMbuaJRyiRcX7SApJYMZ49oR4K+f+11FE5QqOKknYeZgOHUUbv0egnTyovJ+P245zPzIWO7pXodGVUrZHY5P0QSlCkZmGswdBUe3wYg5UL2N3REplW9xSWd57OstNK9WmvuvrWt3OD5HE5RyP0cmLLgd9i2DgR9BvZ52R6RUvjmchklzN5LhcPLO8JYEatOey2mCUu7ldMJ3d8GOhdDrFWg+3O6IlHKJD//cy9p9ibx+U3PCQorbHY5P0pSv3McY+OEB2DwXrnkSOtxld0RKuYS11t5urm9WmcGtclzjWrmAJijlPr8+B+s/h84PQZdH7I5GKZcwxvDMwm0EB/jzzA2NdEKuG2mCUu4RtxFWvA0tR8M1T9kdjVIu8+PWIyzfc5yHetajQklda8+dNEEp13M6YfEjUDwEer6oSxcpn3EmLZMXfthOw8qluLl9TbvD8Xk6SEK53uY51rp6N06BomXsjkYpl3nvtygOJ6fy3oiWOiG3AOhPWLlWajL88jRUawvNdMSeq4hIbxHZJSJRIvJYDq/XEJHfRWSDiGwWkb52xOnLEk6nMX3VPga2rEp4WDm7wykUNEEp1/rj/+DMcej7P/DTXy9XEBF/YDLQB2gEjBCRRued9iQwzxjTEhgOfFCwUfq+z1fHkJrh5O7uV9kdSqGhf0GU65yMg3UfQ8uboUoLu6PxJW2BKGNMtDEmHZgDDDjvHAOcW2enNLo7tUulpGcyY/V+rm1YkToVStodTqGhCUq5zoq3wDh1SLnrVQUOZnsem3Usu2eBm7M2B10M3JvTjURkgohEiEhEfHy8O2L1SfPWHSQpJYM7u9W2O5RCRROUco2TcRA5HVqMgrI6usnFchoGac57PgKYboyphrVT9RcickH5NsZMNcaEG2PCQ0ND3RCq78lwOPl4+T7Ca5aldU3teypImqCUa5yrPXV+yO5IfFEsUD3b82pc2IR3GzAPwBizGggGQgokOh+3eMthDiWdZWJX7XsqaJqgVP5p7cnd1gF1RaSWiBTBGgSx8LxzDgA9AESkIVaC0ja8fHI6DVP+2EudCiW4pkEFu8MpdDRBqfxJjIbv7tbakxsZYzKBe4AlwA6s0XrbROR5EemfddpDwHgR2QR8CYwxxpzfDKgu0y87jrLzyCnu6V4HPz+dcF7QdKKuujKnjsKfr8L6GeAXYK0YobUntzHGLMYa/JD92NPZ/r0d6FTQcfkyYwzv/rqHWiHF6desst3hFEqaoNTli9sAs4dDSgK0HgOdH4ZSWoCVb/l1xzG2xZ3k9Zua66oRNtEEpS7PzkXW5oPFysMdf0LFxnZHpJTLGWN497c91ChXjAEtqtgdTqGlHwtU3kV8BnNGQWgDuP1XTU7KZ/2xO57Nscnc3f0q3SnXRlqDUnmzcxEsehDqXgc3fQ5FitkdkVJuca7vqWqZogxsWc3ucAo1/WigchcbCfNvgyotNTkpn7dsz3E2HEji7u51KBKgfyLtpDUodXGOTGvbjLmjoUQFGDFXk5PyacYY3l66m6plijKktdae7KYJSl3o8Gb4/WWIWQlpJ6FoObh5AZTQpXGUbztXe3p5YFOtPXkAtyYoEekNvAP4A58YY1497/UawOdAmaxzHsua76HskppsDYTIOAONB0LtrlC7OxTTNciUb9Pak+dxW4LKtofNdVhria0TkYVZEwrPObeHzZSs/W0WA2HuiknlwY+PwclYGLcEqre1OxqlCozWnjyPO/8XdA8bb7P9O9g025p4q8lJFSLZR+5p7clzuLOJL6c9bNqdd86zwM8ici9QHLg2pxuJyARgAkCNGjVcHmihdXQbrP8CgkpaE2///D9rpF7X/9gdmVIF6q99iUTGnOC5/o219uRB3JmgLmcPmzdEpAPWHjZNjDHOf11kzFRgKkB4eLgugOkKB9bArKGQeRYcGYCBoFIw6GPwD7Q7Op8kIvcAs4wxJ+yORf3bB3/spXzxIgwNr577yarAuDNB5XUPm95g7WEjIuf2sDnmxrhU1K8w92YoWRlu+Q5KVYGzJyAgyKpNKXephNUXux6YBizRFcftty0umT93x/Nwz3oULeJvdzgqG3fWZXUPG09jjLVv0+xhUO4qGPcTlKkOfv5QPESTk5sZY54E6gKfAmOAPSLysojoTng2mvLHXkoEBTC6Q5jdoajzuC1B6R42HiYlEeaNhu/vh7BOMOZ7a/KtKlBZv99Hsh6ZQFlgvoi8ZmtghdT+42dYvOUwo9rXoHRRbdr2NG6dB6V72HiIw5utWtOZeLjueehwL/hpR3BBE5H7gFuB48AnwCPGmAwR8QP2ADo6pYB9tCyaAH8/butUy+5QVA50JQlfd2wnfHEjBBSF23+xRukpu4QAg4wxMdkPGmOcItLPppgKraMnU1kQGcuQ8GpUKBVsdzgqB/ox2pclRsOMAdaOt7cu1ORkv8VA4rknIlJSRNoBGGN22BZVIfXpin1kOp3c0aW23aGoi9AE5YvSz8C2b6zk5EiD0d9Cee2H9wBTgNPZnp/JOqYKWHJKBrPWxNCvWRVqli9udzjqIrSJz5ckx8LPT8Kun6z5TSUrw+hvoGIjuyNTFsk+CCiraU/LoA1mrN7PmXQHE7vqBzdPpoXDVxzeDLOHQtopaDkKGt0INTtaQ8iVp4jOGihxrtZ0FxBtYzyF0tl0B5+t2k/3+qE0qlIq9wuUbTRB+YI9S+GrWyG4NNz2s27F7rkmAu9iLZJsgF/JWsJLFZx5EQdJPJPOXd3r2B2KyoUmKG+3dQEsGG814438CkpVtjsidRHGmGNYE9aVjeasO0jzaqVpE6ZbyHg6TVDebMt8+Ho81OgAI+fqShAeLmspr9uAxlirpgBgjBlnW1CFzM4jJ9lx+CTP9ddWBm+go/i8kTGw+aus5NQRRn2lyck7fIG1Hl8v4E+s9SlP2RpRIfPNhkME+An9mmlLgzfIUw1KRO4HPsMqTJ8ALbF2v/3ZjbGp7FKTYcXbcCgCjmyFs4kQ1tmqORXRYbJeoo4x5iYRGWCM+VxEZmMtBaYKgMNp+G5DHF3rhVK+RJDd4ag8yGsNapwx5iTQEwgFxgKvXvoS5TKHN8PUbrDyHWuOU8N+cP0bMHKeJifvkpH1NUlEmmBt0hlmXziFy9roBI6cTOXGllXtDkXlUV77oM7t7dQX+MwYs0lEctrvSeWHMdbqDwfWgCMdAoLh9BH4/RUoVg7GLIKaHeyOUl25qSJSFmsU30KgBPCUvSEVHl9vOESJoACua1TR7lD1aPwWAAAgAElEQVRUHuU1QUWKyM9ALeBxESkJOHO5pkDFxh5gy/pVlPB3UCLAQVE/J34B/ohfIOLnj5+f4Af4iUHEqjqKgMEP/Pwx4of4BeDvH4D4+fF3Tvbzwy+gCP6BQQT4+RFABoEmkwCTgTgzrETizLSSC0BI3bwP8z55GI5stna2PboNYlbBqRx2vb/qGmsjweIhLvhJKTtkLQh7MmuzwmWArq9TgM6mO/hp6xH6NKlEcKDODfQWeU1QtwEtgGhjTIqIlMNq5vMYx7Yvo8/6u+0Ow1KpKTQfCU0GQclK/xx3OuDAati9BKKWwrHt/7xWujrUaA9hV1uPoJKQmQpOJ5SrrauPe7msVSPuAebZHUthtHTHUU6nZTKwlTbveZO8JqgOwEZjzBkRuRloBbzjvrAuX/OOvTlV8ztSHIGccviRkiE4nQ6cjkyMIxMn4HCC0xiMASeC0xj8MIhxgsnEOB0YRybGOP9u0zROBzgyMI4MnA4HaSaANAI46/DjVKYfJzP8OHoqgwOJZzl5Np22fjsZm7iasCWPw5LHoXILqNvTaqrbuRhSjoNfoNVUd93zUL0dVGhoTbJVvu4XEXkYmIu1Dh8AxpjEi1+iXGF+ZCyVSwfTvlZ5u0NRlyGvCWoK0FxEmmPtWfMpMAPo6q7ALpd/iRBK1u9GScCuFub4U2nM+esAN67sR2jaPh6rtZce/ptg+esQWAzq9YKG/aFODx0WXjidm++Uvapv0OY+t4pLOsuyPfHc270Ofn7ade5N8pqgMo0xRkQGAO8YYz4VkVvdGZg3Ci0ZxL096jLu6lq8uGgHt/1VjY9veYjragdbAx4CdGhrYWaM0V3xbLAgMhZjYEjr6naHoi5TXhPUKRF5HBgNdBYRf0D3R76I4kEBPNe/MRsPJvH415tp9UAXygdrcirsROSWnI4bY2bkcl1vrCZ1f+ATY8yr573+FtA962kxoIIxpkz+I/Z+Tqfhq8hYOtQuT43yxewOR12mvPa8DwPSsOZDHQGqAv9zW1Q+oEiAH28Pa8HJs5k8/vUWsu2yoAqvNtkenYFngf6XuiDrw+BkoA/QCBghIv/aP8UYM8kY08IY0wJ4D/ja9aF7pzX7EjiQmMLQNtXsDkVdgTwlqKykNAsonbU1dWpun/oU1K9Ukod71ePn7UeZHxlrdzjKZsaYe7M9xmOtyFIkl8vaAlHGmGhjTDowBxhwifNHAF+6JmLv91VELCWDA+jTRJc28kZ5SlAiMhT4C7gJGAqsFZEh7gzMV9x2dW3a1irH899vJy7prN3hKM+SAtTN5ZyqwMFsz2Ozjl1ARGpizVX87WI3E5EJIhIhIhHx8fGXGa53OZmaweIth+nfvIrOffJSeW3iewJoY4y51RhzC9anOp0Bnwf+fsLrQ5rjMIb/zN+sTX2FmIh8LyILsx4/ALuA73K7LIdjF/slGg7MN8Y4LnYzY8xUY0y4MSY8NDQ0b4F7qYUb40jLdDI0XAdHeKu8DpLwy9rL5pwEdCX0PKtRvhhPXN+QJ77Zysy1BxjdvqbdISl7vJ7t35lAjDEmt7bfWCD7X9hqQA7LjQBWgvKQ2er2MsYwc00MDSuXolk1nWPorfKaZH4SkSUiMkZExgCLgMXuC8v3jGxbg851Q3hl8Q5iEs7kfoHyRQeAtcaYP40xK4EEEQnL5Zp1QF0RqSUiRbCS0MLzTxKR+kBZYLVrQ/ZOkTEn2HnkFKPb10SXDfVeeR0k8QgwFWgGNAemGmMedWdgvkZE+L/BzfD3E+74IpJTqRm5X6R8zVf8ew1LR9axizLGZAL3YG3LsQOYZ4zZJiLPi0j2EYAjgDlG25AB+GJNDCWDArixZRW7Q1H5kOcddY0xC4AFbozF51UpU5TJI1sxbvo67py5nmlj2lAkQFtKC5GArJF4ABhj0rNqRZdkjFnMeS0Wxpinz3v+rKuC9Hbxp9JYvOUwo9rVpFgR3TTcm13yr6OInBKRkzk8TonIyYIK0pd0qRfKK4OasiLqOI99rYMmCpn47LWerJVZjtsYj0+aF3GQDIfhZu3r9XqX/HhhjNEF49zgpvDqxCWl8tbS3VQoGcxjfRrYHZIqGBOBWSLyftbzWCDH1SXUlXE4DbPWxNCpTnnqVChhdzgqn9zaviQivUVkl4hEichjFzlnqIhsF5FtWVtgFwr39ajDyHY1+PDPvUz+PcrucFQBMMbsNca0x1oRorExpqMxRv/zXei3nceIS07VkbI+wm0JKi9LtIhIXeBxoJMxpjHwgLvi8TQiwosDmnBjiyr8b8kuPlu5z+6QlJuJyMsiUsYYc9oYc0pEyorIi3bH5Uu+/OsAFUoGcW1D3TXXF7izBpWXJVrGA5OzdhnlvLlWPs/PT3j9pub0alyR577fzgJdDsnX9THGJJ17kvV739fGeHzKkeRU/th1jCGtqxHgr4OPfIE7/xfzskRLPaCeiKwUkTVZqzZfwJeXZwnw9+PdES3pVKc8jy7YzKoo7TP3Yf4i8vey9iJSFNBl7l1kwfpYnAZdOcKHuDNB5WWJlgCstci6Yc3j+ERELtgmwNeXZwkK8GfKza2pHVqcO2ZGsufoKbtDUu4xE/hVRG4TkduAX4DPbY7JJzidhnkRB2lXqxxhIcXtDke5iDsTVF6WaIkFvjPGZBhj9mGtTZbb4pk+qVRwINPGtCE40J8xn60j/lSa3SEpFzPGvAa8CDTE6pf9CdDefBdYuy+RmIQUhrXR2pMvcWeCyssSLd+StdGaiIRgNflFuzEmj1atbDE+vTWchDNp3DN7PZkOZ+4XKW9zBGs1icFAD6zVIVQ+zYs4qNtq+CC3Jag8LtGyBGs9su3A78AjxpgEd8XkDZpVK8PLA5uydl8iby/dY3c4ygVEpJ6IPC0iO4D3sfpmxRjT3Rjzfi6Xq1wkn7W21RjQogpFi+i2Gr7EreuA5LZES9a6YQ9mPVSWQa2q8de+RN7/PYrWYWXpXr+C3SGp/NkJLAduODfvSUQm2RuS71i0+TBpmU6GhdewOxTlYjoW00M9278xDSqVZNLcjbrRofcbjNW097uIfCwiPch5EJG6Aj9sjqN2aHGaVC1ldyjKxTRBeajgQGtkX0amk0fmb9I1+7yYMeYbY8wwoAHwBzAJqCgiU0Skp63Bebnjp9NYE53A9U0r67YaPkgTlAerFVKc/17fkJVRCcxae8DucFQ+GWPOGGNmGWP6YY1q3QjkuASYypsl247gNNC3qQ6O8EWaoDzcuY0OX168g4OJKXaHo1zEGJNojPnIGHON3bF4s8VbDlM7pDgNKum61r5IE5SH+3ujQxEemb8Jp1Ob+pQCSDidxuq9CfTV5j2fpQnKC1QpU5Sn+jViTXQin6wotNPElPqXJduOavOej9ME5SVuCq9GnyaV+L+fdrFuf6Ld4Shlu8VbDlMrpDgNK2vznq/SBOUlRIT/G9KM6mWLcs/s9Rw/rUshqcIr8Uw6q6MT6Nu0kjbv+TBNUF6kVHAgH4xqTVJKBvfP2YBD+6NUIbV0x1EcTqNLG/k4TVBeplGVUrwwoAkroxJ465fddoejlC3W7E2gfPEiNK6ik3N9mSYoLzS0TXWGhVfn/d+jWLr9qN3hKFWgjDGsiU6gfe3y2rzn4zRBeannBjSmadXSTJq3kf3Hz9gdjlIFJvbEWeKSU2lfu5zdoSg30wTlpYID/flgVCv8/YSJMyNJSc+0OySlCsTqaGvDg/a1y9sciXI3TVBerHq5YrwzvCW7jp7i8a+36Hp9qlBYE231P9WpUMLuUJSbaYLycl3rhfLQdfX4bmMc01fttzscpdzKGMPa6ETtfyokNEH5gLu61eG6RhV5adEO1kYX6v0elY+LPXGWQ0lntf+pkNAE5QP8/IQ3hjanRrli3D17A0dPptodklJuca7/qZ32PxUKmqB8RKngQD4a3ZozaZlMmrtRJ/Eqn7QmOoFyxYtQV/ufCgVNUD6kbsWSPNu/Eav2JvDhn3vtDkcpl/qn/6mc9j8VEpqgfMzQ8Or0a1aZN3/ZTWSMLiqrfMc//U9X0Lx39gQc2er6oJRbaYLyMSLCy4OaUqVMMPd9uZHksxl2h6SUS0TGnACgTdgVDJBY/B/4+Bo4o4OIvIkmKB9UKjiQd4e35OjJVP77jc6PUr5hz7FT+PsJV4VeZv9TSiJs/xYcabBxpnuC8wYH1sIvz0Bmut2R5JkmKB/VskZZJl1Xj0WbD/NVZKzd4ah8EJHeIrJLRKJE5LGLnDNURLaLyDYRmV3QMRaEqGOnqVm+GEUCLvPP1qYvwZEOZcMg4jNwOt0Sn0c7HQ9zb4aVb8N3d3vNz0ATlA+b2PUq2tcux7MLtxEdf9rucNQVEBF/YDLQB2gEjBCRRuedUxd4HOhkjGkMPFDggRaAqGOnqXO5tSdjIHI6VGsD1zwFJ/ZB9O9uic9jGQPf3QWpydB6LGyZB0ufsTuqPNEE5cP8/YS3hrWgSIAf98/ZSFqmw+6Q1OVrC0QZY6KNMenAHGDAeeeMByYbY04AGGOOFXCMbpfhcBKTkHL5yxsdWAPHd0PrMdDwBigWAhHT3BKjx/rrY9jzM/R8Efq9BW1uh1XvwpopdkeWK01QPq5y6aK8NrgZWw4l8/z32+0OR12+qsDBbM9js45lVw+oJyIrRWSNiPS+2M1EZIKIRIhIRHx8vBvCdY+YhBQynebyE1TkdAgqBY0HQkAQtLwZdv0IyYfcEqdHOHUUNs2F1R/Ary/Az09C3V7QdjyIQJ/XrGS95L8Qs9ruaC9JE1Qh0LNxJSZ2vYpZaw8wb93B3C9QniSnCT/nj3oJAOoC3YARwCciUianmxljphpjwo0x4aGhoS4N1J2ijllN1JeVoFISYds30PQmKFLcOhY+FowT1s9wQ5Q2O7AG5o+DtxrDNxNgyeOw/A0oXwcGTLaSE4CfP9w4BcrUhK/Hw9kke+O+BLcmqLx07madN0REjIiEuzOewuzhnvW4uk4IT363lc2xnvsLqS4QC1TP9rwaEJfDOd8ZYzKMMfuAXVgJy2fszepDvawRfOtnWCP3Wo/551jZMKhzLfw1FRL3uTRG2xgDS5+Fab1gz1KrpjRxBTy6H55OhLtWQYnzPowElYTBn8DJOFj0oHWP7M4kwILb4csRkGZf/7XbElReOnezzisJ3AesdVcsCgL8/Xh3REtCSwQx8YtIXa/Pe6wD6opILREpAgwHFp53zrdAdwARCcFq8osu0CjdLOrYaaqUDqZ4UEDeLjixH/78P6jbEyo3+/drvV8BDMwcBGeOuzpU98pIhY2z/x33stdhxVvQ6lZ4aIf1/VVqCkXLgt8l/sRXC4fuj8PWBbD2IyspGWMluSkdYNu3sHsJzBwMqSdzvsfBdXDUfV0H7qxB5aVzF+AF4DVA/2K6WbniRfhodGuSz2Yw+tO1nDjjPfMhCitjTCZwD7AE2AHMM8ZsE5HnRaR/1mlLgAQR2Q78DjxijPGpGalRx05zVV6b94yB7+8H8bMGBZwvpC6MmGvVHmbdBOm57Ei98UvY+vWFx0/HWyPjCtLaD+HbO+HtprDkCfjzNfj9RWg+Avq9/U9TZl5d/SDUvBp+ehT+VxterQmzBkPRcjDhdxjyKRyKsJL5+U2BTifMHQVfjbmwBuYiefw4ckVy6txtl/0EEWkJVDfG/CAiD7sxFpWlSdXSfHxLOGOmr2PM9HXMur0dJfL6qVTZwhizGFh83rGns/3bAA9mPXyO02nYG3+aYW2q534ywMZZEP0HXP8GlK6W8zk12sGQz6w/sAvGw/BZ//TRZLf2I/jxP1ayK1YOanezjsfvsprUHJlWk1qHu6F4yBV8d5chM80aeVetLZSrbf3bOKDRAOj//qVrSxfj5w83z7d+Xon7IDEaSlSAjvdBYLBVE/MLtJLQ1+Nh1Ff/XHsoAk4ftR4H10KN9q76Tv8Jz+V3/MclO3dFxA94C3go1xt56cgjT9WxTgjvj2jJ1kPJTJgRQXqmd0zaU4XT4ZOppKQ78tb/dOqINTqtZidoPe7S5zboC9c+B7sWwY7vL3x9/RdWcqrfF0LqwVdjIekAnDxsNXv5BUKdHlbz2ttNrZqWO22eC6ePWM1ygz6CeyOsWtOgT8A/Hx8yA4tC/T7Q4S64/nXo+h8rOZ3TsB90fsgaqp504J/jOxeBXwAUKWGNlsxu27cu6eNzZ4LKrXO3JNAE+ENE9gPtgYU5DZTw1pFHnqxn40q8NrgZq/Ym8IQuh6Q82GWN4FvzgdVkd8O7eatRtL8LKjS2klp6yj/HN38FC++Fq3rATdNh+GxwOmDOKJg1xFp89ub5MPRzuHstVG5uDTZw18ALpxNWvguVmkHt7taxcrWtUYkBRdzzntk1H2593Tzvn2M7F0FYZ2g21BotedZaK5FjO+HrCfDbi/l+W3cmqEt27hpjko0xIcaYMGNMGLAG6G+MiXBjTCqbwa2rcd81dfgqMpZPlvvIiCblc/KcoIyxakK1ukBInbzd3D8A+v4Pkg/CijetY399bDVn1ewEw2Za86fKXwWDpsKRzRC/E4Z9YSUlgND6MPhTEH+r78sdH/Z2/wgJe6DT/Tk3RbpbuVpQoyNsmmN9f/G7rXgaXG+NksxMtZK6IwO+uQOCSmQNRskft3U+GGMyReRc564/MO1c5y4QYYw5fySSssED19YjKv40L/+4g9qhxenRsKLdISn1L1HHTlOmWCDli+dSUzi2w+pD6Xjv5b1BWCdoOhRWvmPNnYr41GrWGzLNav46p35vq98quDRcdc2/71G6KvR8Hn6YZA1vb33r5cWQm5XvQJka0OhG1973cjQfDt/fB4fWw/5l1rH6fa3vvXILq5nvbCIc3ghDZ1h9Wfnk1t7x3Dp3zzvezZ2xqJz5+Qlv3NSCg4mrue/LDcy/syMNK5eyOyyl/rY3aw2+XDcp3PE9IFD/+st/k+ueh12LreTUcrTVt5NTv06TQRe/R6sx1mi/n5+EutdBqSo5n2cM7P7JqnWUqw2lq0NKgpVckw9C8QrW8aJlrfO2LrAGIfT5X/76mvKr8Y2w+BFr8d3DG6FKSys5gVWL+uEBOLbdSvaNchqwffl0+JaiaBF/Pr4lnAGTV3Db9HV8e08nKpQMzv1CpQpAVPxpejbKQ81+5/dQvR2UvIJWgFKVrYmrSQf/WRLocvn5wQ3vwJSO1sTZQVNzPi/yM6umlVch9ayFbsPHXn5MrhRc2mrS2zwX0k5B9yf+ea3pEGvYe3Ap6Puay95SE5QCoFLpYD69tQ03fbia8TMimTuhPcGB/naHpQq5xDPpJJ5Jz73/KXEfHNliLYh6per3ufJrzyl/lbUY65oPoMt/LuwLO7wJfnzMWs2ixzPW6upJB60h6uVqQ6mqcCbeqk2dOgJhV1tDvT1li/vmI2Bb1pywBtlqqkElYcSXUKy8VfNzEU1Q6m9NqpbmneEtuGNmJA/N28R7I1ri5+chBUMVSnuOngLIfZLuzh+srw36uTmiPOh0P6z7xFoHb2C2FcNTT1rziYqVh4EfWUnp/FUuwGo2q9KiwMK9LFddA8VDrQnBFRr++7XaXV3+drpYrPqXno0r8d8+DVm05TDPLNymw8+Vrf7al4gINKta+tIn7vgBKja1RpvZrUQFCB9nNYUlZq045ci0Ngo8EWMNvnD3pF538Q+wRiwO+KBAanWaoNQFxnepzR1da/PFmhje/GW33eGoQmz5nuM0qVKa8iWCLn7SqayVDBp6QO3pnE73W5NYl79hNdV9fgPsWAjXPQc1O9gdXf7U7mqNfCwA2sSncvRY7wYkp2Tw3m9RlAwOYEKXq+wOSRUyp1IzWH/gBBO61L70iWunAMYzmvfOKVnJGtkW8am14Gr6GRj0sTWpVeWZJiiVIxHhpYFNOZWaycuLd3IkOY3/9m1AgL9WulXBWBOdSKbT0LnuJVaP2TLfWmqoxSio1KTggsuLqx+w5kQFl4Fbv7+wz0blShOUuih/P+Gd4S2oWCqYaSv3sTf+NO+NbEmp4EC7Q1OFwPI98RQr4k+rUEfOJxxcB9/eZa1w0O/tgg0uL0pVgXvWWYMiihSzOxqvpB+H1SUF+Pvx9A2NeHlgU1ZGHWfoh6tJStFtOpT7Ld9znNFV4gh6s57VTJZdcizMGWnNXxo2s2DWo7sSZaprcsoHTVAqT0a2q8FnY9sQHX+GsdPXcSYt0+6QlA87mJjCvuNn6FbmGGBg0UP/7NvkdFhbZGSkWPs6FS9va6zKfTRBqTzrXDeU90a2ZNPBJCbOjCQt8yJNL0rl07I91rY6DYNPWIuwJh+EP7IWH13+JhxYZe33VKGBjVEqd9MEpS5Lr8aV+L/BzVi+5zh3z9pASrrWpJTrLd99nKplilI6Lc5aYaHVLbD6A4iYZiWqJkOg2TC7w1RupglKXbabwqvzwoDG/LbzKIM+WMXBxJTcL1IqjzIdTlbuPU7nuiFIUgyUrWltLFi0rLWGXemq0O9Nz1n+R7mNJih1RUZ3COOzsW2JSzpL//dXsGrvcbtDUj5i/YEkTqVmcnXdEGsH1zI1re3W+/7PGrI96GNr4VLl8zRBqSvWtV4o391zNSElghgzbR1Lth2xOyTlAxZExlI00J9uYcHWLq1la1ovNBkE/4mGGu3tDVAVGE1QKl9qhRRn/sSONKpSirtmrefbDYfsDkl5sZT0TH7YHMf1zSpTIiXrd6lMjX9O8NMV9gsTTVAq30oXC2Tm7e1oG1aOSfM2MmP1frtDUl5q8ZYjnEl3MDS8urWwKlhNfKpQ0gSlXKJEUACfjW1DjwYVePq7bfz3my2kZzrtDkt5ma8iDhJWvhhtwspa/U8AZcNsjUnZRxOUcpngQH8+Gh3Ond2uYvbaA4z4eA3HTqbaHZbyEjEJZ1i7L5Gbwqtb27snxUCREi7dAE95F01QyqX8/YRHezdg8shWbI87Sd93V7AqSkf4qdzNj4zFT2BQq6rWgRMxVvOeDicvtDRBKbe4vlllvr27E6WLBjDq07W8+ctuHE7d/FDlzOE0zI+MpXPdUCqXLmodPDcHShVamqCU29SvVJLv772aQS2r8e6vexg8ZRXrD5ywOyzlgSb/HsXh5FSGtaluHTAmaw5UjUtfqHyaJijlVsWKBPDG0Oa8PawFh5LOMuiDVTwwZwNHkrVvSlnmR8by5i+7GdSqKn2aVLIOpiRC+mkdwVfIaYJSBeLGllX5/eFu3N39KhZvPULvd5bpxF7F8j3xPLZgM1fXCeHVQc2swREASfutr9rEV6hpglIFpkRQAI/0asBP93emWtmi3PFFJE99u5XUDF0VvTD6edsR7py5njoVSvDBza0oEpDtz9Hfc6C0ia8w0wSlClzt0BIsuLMjt19diy/WxNDzrWX8sv0oxuggisIgNcPBM99tZcIXkYSFFOOzsW0u3KX53BwobeIr1DRBKVsEBfjzZL9GzL69HUEBfoyfEcGtn61jz9FTdoem3OivfYkM/GAVn6+OYVynWiy4s+M/o/ayS4qx5j8Flyr4IJXH0ASlbNWxTgiL7+/M0/0aseHACXq9vYyHv9rEoaSzdofmMUSkt4jsEpEoEXksh9fHiEi8iGzMetxuR5znSzyTzsHEFA4mprAlNpnxMyIY+tFqTpxJ59Nbw3n6hkYEBVxkbb1zc6BUoRbgzpuLSG/gHcAf+MQY8+p5rz8I3A5kAvHAOGNMjDtjUp4n0N+PcVfXYmDLqnzwRxSfr45h4cY4butci3uvqUOxIm79NfVoIuIPTAauA2KBdSKy0Biz/bxT5xpj7imouHYeOUnE/hPsOHySXUdOUSwogHoVSlArtDh7j51hRVQ8u4+e/tc1Vh9kfcZ1qkXRIrks+poUAxUaufE7UN7AbSU/jwVrAxBujEkRkTuB1wDdJrOQKlu8CE9c34gxnWrxxpJdTPljLws3xvFUv0b0alzxnxFehUtbIMoYEw0gInOAAcD5CcplZqzejzHQvX4FapQv9vfx02mZLN58mFlrY9gUmwxAyeAAGlQqScLpNL6ITiAt00mRAD/ahpVjYMtqhJQoAlgrjHSpF0pIiaDcA3A6Iekg1O/jjm9PeRF3fjTNtWAZY37Pdv4a4GY3xqO8RNUyRXlzWAuGt63BU99uZeLMSPo0qcTLA5tStngRu8MraFWBg9mexwLtcjhvsIh0AXYDk4wxB3M4J09+3HKE1dEJPMM2aocWp2yxIhxITCH+VBoAdSuU4Jl+Dbm2USWqlS369wcHh9MQl3SW0JJBBAfmY1uM00fBkaZNfMqtCSqvBeuc24Afc3pBRCYAEwBq1NBhp4VF21rl+OG+q/l4eTRv/bKb9QdO8PpNzelcN9Tu0ApSTtXG84c7fg98aYxJE5GJwOfANTneLA9l6cvbwomJi+O3Aw7+3B3P2XQH3euHUqNcMdrVLk941WLIp9dBYgu44d2/r/P3E6qXK5bjPS9Lwh7rqyaoQs+dCSovBcs6UeRmIBzomtPrxpipwFSA8PBwHYtciAT6+3FXtzp0qRvKA3M3MvrTv2hXqxx9m1amV+NKVCodbHeI7hYLVM/2vBoQl/0EY0xCtqcfA/93sZvlWpaMgc/6ULNoGcaO+oqxnWpdeJNV78ORzdYjtAF0uPsyvp082L4QAoJ151zl1lF8uRYsABG5FngC6G+MSXNjPMqLNalamh/uvZqHrqtH4pl0nlm4jQ6v/sqD8zZy1Le39FgH1BWRWiJSBBgOLMx+gohUzva0P7Djit9NBBpcD3t+hug/L3z97AlY9j+46hpo0A9+fgr2Lbvit7uAIwO2LrD6n3SIeaHnzgSVl4LVEvgIKzkdc2MsygcEB/pzb4+6/PJgV5Y+2JXxnWvzw6bDdH/9Dyb/HsWJM+l2h+hyxphM4B5gCVbimWeM2SYiz4tI/6zT7hORbSKyCbgPGJOvN203EUpXh5+ftAYsZLfsdUhNhutegIEfQvk68CzW2KUAAAhUSURBVNUYOLAWMi/y8z8ZBx91gS3zc3/vqF/hbCI007FSCsSds/dFpC/wNtYw82nGmJdE5HkgwhizUESWAk2Bw1mXHDDG9L/I7QCrWSIiIsJtMSvvEpNwhpcW7eDn7Ufx9xPa1SpH7yaVGNyqGsWDXN+CLSL/397dB1tR13Ecf3948PoAVyIFeVApRYWgi0ZME/mEg4MoDyXiI2PpVM5QaVGhk42TTU1N48M4MaVmDSgpRmBEk6JAGkyCgEAgEEgwgheRRJ6Kq3i//bFrXRLhjp17d8/u5zXDcPZ39uz8fufc73zO7tn97dKIGFjxDWfgsLW0YhrM/BJ89gGoS8Ni5yb46Seh/1gYPSlp27EeHrwIGnZB2xo4qT+cPxHOuDh5vrERHh6V7GW1Pw6+/ByccPr7d2r6DfDyfJiwDtqV7oSY0mhuHbVoQLUEB5Qdyqqtu/jjqnqeXLWNl1/fR9faGiYOO4vRA3rQpk3lTk8vTUA1NsKDFySzio9fDPUrYP4PYMsS+NoyqO3+33X3bodNC2DrUvjbU7Dz7zDml9B3FCy4F565Ay78DvxlUnL79hufPnT4NOyBn/SGAdfAZXe3xJAtJxxQVlpLNr3BnbNfYuWWXdT1PJ6rBp3C0L5dm3cNzhGUJqAg+Q1qykioqYWG3dDuGLj4+zDoi+//mv27YOoVSZCd9034811w5nAYOwXW/B4eHweDb4Gh33vva5c/Ck/cBDfMgVMOd8KvVTsHlJVaY2Mw88Wt3DdvPZv/8U/aCAb26syFZ3bhvDNOoG+32g904W+pAgrgDxOSi2b7j0lOXKjpeOQNN+yBX18JmxdCbQ+4aQEc2zl5btZXYdnDcPWj770Qd8poeGMj3LzCt3kvOAeUGRARrKnfw5OrtzFn9TbWbksmo+3SsYZh/U7i0v7dGNirM22beRiwdAH1Qb21D+b/EPpdDj3OObj9V8PhtdUwdnJyxiAkvzs98jk4dwIMub1l+mS54YAyO4Ttu/fz3PodzF3zGvPXbWf/24106VjDJf1OYngzwsoBVQH/ehMeuRzql8Nl98CmhbDyMeh8Gnx+9sG/b1khOaDMjmBfwwHmrd3O7JWv8qd1r9NwIAmrJ8YPpnunQ9wCAgdUxezfDVPHwCuLoE17+MzXk72n9oW/8Npofh2Vd5poK73jatoxoq47I+q6szcNq4Xrd9Ct+LNTZO/oWrhuBjz/M+gzArqclXWPLIccUGYkt4IYWdedkXU+vNRqajrA+d/KuheWY75hoZmZ5ZIDyszMcskBZWZmueSAMjOzXHJAmZlZLjmgzMwslxxQZmaWSw4oMzPLpaqb6kjS68Dm93n6BGBHK3Yna2Uab17GempEnJh1JyrBtfQfHmvra1YdVV1AHY6kJUWZJ605yjTeMo01D8r0fnus+eVDfGZmlksOKDMzy6WiBdQDWXeglZVpvGUaax6U6f32WHOqUL9BmZlZcRRtD8rMzArCAWVmZrlUmICSNEzSOkkbJN2adX8qSdLJkuZLWiNptaSb0/bOkp6WtD79/0NZ97VSJLWV9KKk2enyRyQtSsc6TdJRWfexiIpcR1C+Wqr2OipEQElqC0wCLgH6AldL6pttryrqADAhIvoAnwLGp+O7FZgbEb2BuelyUdwMrGmy/GPgnnSsO4EbM+lVgZWgjqB8tVTVdVSIgAIGARsiYmNEvAU8BozKuE8VExH1EbEsfbyH5A+uB8kYJ6erTQZGZ9PDypLUE7gU+EW6LGAIMD1dpTBjzZlC1xGUq5aKUEdFCagewCtNlrekbYUjqRdwNrAI6BoR9ZAUHtAlu55V1L3At4HGdPnDwJsRcSBdLuznm7HS1BGUopaqvo6KElA6RFvhzp+X1AH4LXBLROzOuj8tQdJlwPaIWNq0+RCrFu7zzYHSvM9Fr6Wi1FG7rDtQIVuAk5ss9wRezagvLUJSe5KCmhoRM9Lm1yR1i4h6Sd2A7dn1sGIGAyMlDQeOBmpJvgl2ktQu/fZXuM83JwpfR1CaWipEHRVlD+oFoHd6hspRwFXArIz7VDHpseOHgDURcXeTp2YB16ePrwd+19p9q7SIuC0iekZEL5LPcV5EXAvMB8akqxVirDlU6DqC8tRSUeqoEAGVfhv4CvAUyY+ej0fE6mx7VVGDgXHAEEnL03/DgR8BQyWtB4amy0U1EfiGpA0kx9Ifyrg/hVOCOgLXUlXVkac6MjOzXCrEHpSZmRWPA8rMzHLJAWVmZrnkgDIzs1xyQJmZWS45oOwgki54d+ZjM/vgXEv/PweUmZnlkgOqSkm6TtLi9ELD+9P7vuyVdJekZZLmSjoxXXeApOclrZQ089173Ug6XdIzklakrzkt3XwHSdMlrZU0Nb363qyQXEv55YCqQpL6AFcCgyNiAPAOcC1wHLAsIs4BngXuSF8yBZgYER8H/tqkfSowKSLqgE8D9Wn72cAtJPcE+ijJ1fdmheNayreiTBZbNhcBnwBeSL+QHUMyuWUjMC1d5xFghqTjgU4R8WzaPhn4jaSOQI+ImAkQEfsB0u0tjogt6fJyoBewoOWHZdbqXEs55oCqTgImR8RtBzVK3/2f9Q43j9XhDjU0NHn8Dv47seJyLeWYD/FVp7nAGEldACR1lnQqyef57kzF1wALImIXsFPSuWn7OODZ9B44WySNTrdRI+nYVh2FWfZcSznmNK9CEfGSpNuBOZLaAG8D44F9wMckLQV2kRxbh2Ra/Z+nRbMR+ELaPg64X9Kd6TauaMVhmGXOtZRvns28QCTtjYgOWffDrNq5lvLBh/jMzCyXvAdlZma55D0oMzPLJQeUmZnlkgPKzMxyyQFlZma55IAyM7Nc+jehJFcml9jsmQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# print(np.array(history.losses))\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "loss = train_history.history['loss']\n",
    "val_loss = train_history.history['val_loss']\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.plot(loss)\n",
    "plt.plot(val_loss)\n",
    "plt.legend(['loss', 'val_loss'])\n",
    "plt.title('AdaDelta')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "\n",
    "acc = train_history.history['acc']\n",
    "val_acc = train_history.history['val_acc']\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.plot(acc)\n",
    "plt.plot(val_acc)\n",
    "plt.legend(['acc', 'val_acc'])\n",
    "plt.title('AdaDelta')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy 0.8051282051282052\n"
     ]
    }
   ],
   "source": [
    "train_acc = []\n",
    "rng = np.random.choice(Y_train.shape[0], 195, replace=False)\n",
    "prediction = []\n",
    "for i in rng:\n",
    "    pred = siamese_net.predict([X_train[None,i,0,:,:,None], X_train[None,i,1,:,:,None]])\n",
    "    prediction.append(pred)\n",
    "    acc = 'correct' if np.round(pred)==Y_train[i] else \"\"\n",
    "    train_acc.append(1 if np.round(pred)==Y_train[i] else 0)\n",
    "#     print(i, pred, np.round(pred), y_train[i], acc)\n",
    "print('Train Accuracy', np.mean(train_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy 0.6105263157894737\n"
     ]
    }
   ],
   "source": [
    "test_acc = []\n",
    "rng = np.random.choice(Y_test.shape[0], 190, replace=False)\n",
    "prediction = []\n",
    "for i in rng:\n",
    "    pred = siamese_net.predict([X_test[None,i,0,:,:,None], X_test[None,i,1,:,:,None]])\n",
    "    prediction.append(pred)\n",
    "    acc = 'correct' if np.round(pred)==Y_test[i] else \"\"\n",
    "    test_acc.append(1 if np.round(pred)==Y_test[i] else 0)\n",
    "#     print(i, pred, np.round(pred), y_test[i], acc)\n",
    "print('Test Accuracy', np.mean(test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 [[0.9992372]] [1] [1] 0\n",
      "7 [[0.00024588]] [1] [1] 1\n",
      "21 [[0.46680015]] [0] [0] 1\n",
      "14 [[0.01244232]] [1] [0] 0\n",
      "36 [[0.04232696]] [1] [0] 0\n",
      "29 [[0.99990094]] [1] [0] 1\n",
      "19 [[0.1772979]] [1] [1] 1\n",
      "12 [[0.9069365]] [1] [1] 0\n",
      "37 [[0.7378922]] [0] [1] 1\n",
      "36 [[0.29272014]] [1] [1] 1\n",
      "15 [[0.04256096]] [0] [0] 1\n",
      "28 [[0.53658324]] [0] [0] 0\n",
      "29 [[0.9999231]] [1] [0] 1\n",
      "18 [[0.00088351]] [0] [1] 0\n",
      "30 [[0.9805331]] [0] [0] 0\n",
      "10 [[0.9360994]] [0] [0] 0\n",
      "18 [[0.99991226]] [0] [0] 0\n",
      "44 [[0.00152636]] [1] [0] 0\n",
      "37 [[0.9978452]] [0] [0] 0\n",
      "47 [[0.78717244]] [0] [0] 0\n",
      "2 [[0.9994961]] [0] [1] 1\n",
      "12 [[0.8994222]] [1] [1] 0\n",
      "41 [[0.7775418]] [1] [1] 0\n",
      "36 [[0.99999964]] [1] [1] 0\n",
      "35 [[0.00529279]] [1] [1] 1\n",
      "0 [[0.9932267]] [0] [0] 0\n",
      "8 [[0.9999838]] [0] [1] 1\n",
      "28 [[0.00052391]] [0] [1] 0\n",
      "8 [[0.9998846]] [0] [1] 1\n",
      "18 [[0.9999815]] [0] [1] 1\n",
      "12 [[0.46604905]] [1] [0] 0\n",
      "3 [[0.8300947]] [1] [1] 0\n",
      "25 [[0.00095356]] [1] [1] 1\n",
      "43 [[0.9842917]] [1] [1] 0\n",
      "11 [[8.524662e-05]] [0] [0] 1\n",
      "3 [[0.9992888]] [1] [1] 0\n",
      "44 [[0.9985189]] [1] [1] 0\n",
      "27 [[0.0054677]] [0] [1] 0\n",
      "36 [[0.00521404]] [1] [1] 1\n",
      "42 [[0.9952041]] [1] [0] 1\n",
      "4 [[0.96593416]] [0] [0] 0\n",
      "38 [[0.9997105]] [0] [1] 1\n",
      "2 [[0.9987847]] [0] [0] 0\n",
      "4 [[0.9698879]] [0] [0] 0\n",
      "39 [[5.106873e-05]] [0] [0] 1\n",
      "29 [[0.4841347]] [1] [0] 0\n",
      "13 [[1.5899724e-06]] [1] [0] 0\n",
      "19 [[0.00050936]] [1] [1] 1\n",
      "15 [[0.04388002]] [0] [0] 1\n",
      "1 [[0.4396221]] [1] [1] 1\n",
      "Validation accuracy: 0.44\n"
     ]
    }
   ],
   "source": [
    "val_acc = []\n",
    "for k in range(len(Y_val)):\n",
    "    i = np.random.randint(len(Y_val))\n",
    "    j = np.random.randint(len(Y_val))\n",
    "    pred = siamese_net.predict([X_val[None,i,:,:,None], X_val[None,j,:,:,None]])\n",
    "    correct = 1 if Y_val[i]==Y_val[j] and np.round(pred)==0 or Y_val[i]!=Y_val[j] and np.round(pred)==1  else 0\n",
    "    val_acc.append(correct)\n",
    "    print(i, pred, Y_val[i], Y_val[j], correct)\n",
    "print('Validation accuracy:', np.mean(val_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_33 (InputLayer)        (None, 1, 60, 128)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_26 (Conv2D)           (None, 4, 60, 128)        500       \n",
      "_________________________________________________________________\n",
      "batch_normalization_28 (Batc (None, 4, 60, 128)        16        \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_10 (Depthwi (None, 8, 1, 128)         480       \n",
      "_________________________________________________________________\n",
      "batch_normalization_29 (Batc (None, 8, 1, 128)         32        \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 8, 1, 128)         0         \n",
      "_________________________________________________________________\n",
      "average_pooling2d_14 (Averag (None, 8, 1, 32)          0         \n",
      "_________________________________________________________________\n",
      "dropout_14 (Dropout)         (None, 8, 1, 32)          0         \n",
      "_________________________________________________________________\n",
      "separable_conv2d_7 (Separabl (None, 8, 1, 32)          192       \n",
      "_________________________________________________________________\n",
      "batch_normalization_30 (Batc (None, 8, 1, 32)          32        \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 8, 1, 32)          0         \n",
      "_________________________________________________________________\n",
      "average_pooling2d_15 (Averag (None, 8, 1, 4)           0         \n",
      "_________________________________________________________________\n",
      "dropout_15 (Dropout)         (None, 8, 1, 4)           0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 2)                 66        \n",
      "_________________________________________________________________\n",
      "softmax (Activation)         (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 1,318\n",
      "Trainable params: 1,278\n",
      "Non-trainable params: 40\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def EEGNet(nb_classes, Chans = 64, Samples = 128, \n",
    "             dropoutRate = 0.25, kernLength = 64, F1 = 4, \n",
    "             D = 2, F2 = 8, dropoutType = 'Dropout'):\n",
    "    \"\"\" Keras Implementation of EEGNet (https://arxiv.org/abs/1611.08024)\n",
    "\n",
    "    Inputs:\n",
    "        \n",
    "      nb_classes      : int, number of classes to classify\n",
    "      Chans, Samples  : number of channels and time points in the EEG data\n",
    "      dropoutRate     : dropout fraction\n",
    "      kernLength      : length of temporal convolution in first layer. We found\n",
    "                        that setting this to be half the sampling rate worked\n",
    "                        well in practice. For the SMR dataset in particular\n",
    "                        since the data was high-passed at 4Hz we used a kernel\n",
    "                        length of 32.     \n",
    "      F1, F2          : number of temporal filters (F1) and number of pointwise\n",
    "                        filters (F2) to learn. Default: F1 = 4, F2 = F1 * D. \n",
    "      D               : number of spatial filters to learn within each temporal\n",
    "                        convolution. Default: D = 2\n",
    "      dropoutType     : Either SpatialDropout2D or Dropout, passed as a string.\n",
    "\n",
    "    \"\"\"\n",
    "    \n",
    "    if dropoutType == 'SpatialDropout2D':\n",
    "        dropoutType = SpatialDropout2D\n",
    "    elif dropoutType == 'Dropout':\n",
    "        dropoutType = Dropout\n",
    "    else:\n",
    "        raise ValueError('dropoutType must be one of SpatialDropout2D '\n",
    "                         'or Dropout, passed as a string.')\n",
    "    \n",
    "    input1   = Input(shape = (1, Chans, Samples))\n",
    "\n",
    "    ##################################################################\n",
    "    block1       = Conv2D(F1, (1, kernLength), padding = 'same',\n",
    "                                   input_shape = (1, Chans, Samples),\n",
    "                                   use_bias = False, data_format='channels_first')(input1)\n",
    "    block1       = BatchNormalization(axis = 1)(block1)\n",
    "    block1       = DepthwiseConv2D((Chans, 1), use_bias = False, \n",
    "                                   depth_multiplier = D,\n",
    "                                   depthwise_constraint = max_norm(1.), data_format='channels_first')(block1)\n",
    "    block1       = BatchNormalization(axis = 1)(block1)\n",
    "    block1       = Activation('elu')(block1)\n",
    "    block1       = AveragePooling2D((1, 4), data_format='channels_first')(block1)\n",
    "    block1       = dropoutType(dropoutRate)(block1)\n",
    "    \n",
    "    block2       = SeparableConv2D(F2, (1, 16),\n",
    "                                   use_bias = False,\n",
    "                                   padding = 'same',\n",
    "                                   data_format='channels_first')(block1)\n",
    "    block2       = BatchNormalization(axis = 1)(block2)\n",
    "    block2       = Activation('elu')(block2)\n",
    "    block2       = AveragePooling2D((1, 8), data_format='channels_first')(block2)\n",
    "    block2       = dropoutType(dropoutRate)(block2)\n",
    "        \n",
    "    flatten      = Flatten(name = 'flatten')(block2)\n",
    "    \n",
    "    dense        = Dense(nb_classes, name = 'dense', \n",
    "                         kernel_constraint = max_norm(0.25))(flatten)\n",
    "    softmax      = Activation('softmax', name = 'softmax')(dense)\n",
    "    \n",
    "    return Model(inputs=input1, outputs=softmax)\n",
    "\n",
    "model  = EEGNet(nb_classes = 2,\n",
    "                Chans = 60,\n",
    "                Samples = 128,\n",
    "                kernLength = 125)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24\n",
      " 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49\n",
      " 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74\n",
      " 75 76 77 78 79]\n",
      "[80 81 82 83 84 85 86 87 88 89]\n",
      "[90 91 92 93 94 95 96 97 98 99]\n"
     ]
    }
   ],
   "source": [
    "def TrainTestVal_Split(leng, traintest, testval):\n",
    "    leng = 100\n",
    "    x = np.arange(leng)\n",
    "    train_id, test_id, val_id = np.split(\n",
    "        x,[np.round(leng*traintest).astype(int),\n",
    "           np.round(leng*testval).astype(int)])\n",
    "    return train_id, test_id, val_id\n",
    "\n",
    "# train_id, test_id, val_id = TrainTestVal_Split(100, 0.8, 0.9)\n",
    "# print(train_id)\n",
    "# print(test_id)\n",
    "# print(val_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 834,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[418]\n",
      "success\n",
      "[437]\n",
      "success\n",
      "[75]\n",
      "success\n",
      "[405]\n",
      "success\n",
      "[96]\n",
      "success\n",
      "[216]\n",
      "success\n",
      "[193]\n",
      "success\n",
      "[246]\n",
      "success\n",
      "[144]\n",
      "success\n",
      "[155]\n",
      "success\n",
      "[162]\n",
      "success\n",
      "[191]\n",
      "success\n",
      "[258]\n",
      "success\n",
      "[309]\n",
      "success\n",
      "[126]\n",
      "success\n",
      "[299]\n",
      "success\n",
      "[52]\n",
      "success\n",
      "[340]\n",
      "success\n",
      "[156]\n",
      "success\n",
      "[281]\n",
      "success\n",
      "[121]\n",
      "success\n",
      "[163]\n",
      "success\n",
      "[107]\n",
      "success\n",
      "[104]\n",
      "success\n",
      "[279]\n",
      "success\n",
      "[10]\n",
      "success\n",
      "[264]\n",
      "success\n",
      "[376]\n",
      "success\n",
      "[428]\n",
      "success\n",
      "[101]\n",
      "success\n",
      "[217]\n",
      "success\n",
      "[168]\n",
      "success\n",
      "[39]\n",
      "success\n",
      "[303]\n",
      "success\n",
      "[304]\n",
      "success\n",
      "[61]\n",
      "success\n",
      "[18]\n",
      "success\n",
      "[305]\n",
      "success\n",
      "[212]\n",
      "success\n",
      "[272]\n",
      "success\n",
      "[188]\n",
      "success\n",
      "[67]\n",
      "success\n",
      "[86]\n",
      "success\n",
      "[354]\n",
      "success\n",
      "[411]\n",
      "success\n",
      "[439]\n",
      "success\n",
      "[291]\n",
      "success\n",
      "[82]\n",
      "success\n",
      "[365]\n",
      "success\n",
      "[149]\n",
      "success\n",
      "[238]\n",
      "success\n",
      "[59]\n",
      "success\n",
      "[146]\n",
      "success\n",
      "[209]\n",
      "success\n",
      "[170]\n",
      "success\n",
      "[298]\n",
      "success\n",
      "[140]\n",
      "success\n",
      "[268]\n",
      "success\n",
      "[307]\n",
      "success\n",
      "[100]\n",
      "success\n",
      "[420]\n",
      "success\n",
      "[192]\n",
      "success\n",
      "[23]\n",
      "success\n",
      "[416]\n",
      "success\n",
      "[175]\n",
      "success\n",
      "[80]\n",
      "success\n",
      "[438]\n",
      "success\n",
      "[345]\n",
      "success\n",
      "[399]\n",
      "success\n",
      "[447]\n",
      "success\n",
      "[359]\n",
      "success\n",
      "[445]\n",
      "success\n",
      "[444]\n",
      "success\n",
      "[377]\n",
      "success\n",
      "[390]\n",
      "success\n",
      "[250]\n",
      "success\n",
      "[374]\n",
      "success\n",
      "[85]\n",
      "success\n",
      "[157]\n",
      "success\n",
      "[371]\n",
      "success\n",
      "[247]\n",
      "success\n",
      "[98]\n",
      "success\n",
      "[352]\n",
      "success\n",
      "[159]\n",
      "success\n",
      "[327]\n",
      "success\n",
      "[311]\n",
      "success\n",
      "[321]\n",
      "success\n",
      "[48]\n",
      "success\n",
      "[134]\n",
      "success\n",
      "[45]\n",
      "success\n",
      "[15]\n",
      "success\n",
      "[114]\n",
      "success\n",
      "[38]\n",
      "success\n",
      "[331]\n",
      "success\n",
      "[316]\n",
      "success\n",
      "[368]\n",
      "success\n",
      "[95]\n",
      "success\n",
      "[68]\n",
      "success\n",
      "[357]\n",
      "success\n",
      "[5]\n",
      "success\n",
      "[150]\n",
      "success\n",
      "[277]\n",
      "success\n",
      "[92]\n",
      "success\n",
      "[91]\n",
      "success\n",
      "[130]\n",
      "success\n",
      "[446]\n",
      "success\n",
      "[154]\n",
      "success\n",
      "[60]\n",
      "success\n",
      "[11]\n",
      "success\n",
      "[369]\n",
      "success\n",
      "[412]\n",
      "success\n",
      "[391]\n",
      "success\n",
      "[220]\n",
      "success\n",
      "[40]\n",
      "success\n",
      "[125]\n",
      "success\n",
      "[346]\n",
      "success\n",
      "[266]\n",
      "success\n",
      "[20]\n",
      "success\n",
      "[424]\n",
      "success\n",
      "[417]\n",
      "success\n",
      "[128]\n",
      "success\n",
      "[373]\n",
      "success\n",
      "[152]\n",
      "success\n",
      "[83]\n",
      "success\n",
      "[115]\n",
      "success\n",
      "[50]\n",
      "success\n",
      "[103]\n",
      "success\n",
      "[245]\n",
      "success\n",
      "[72]\n",
      "success\n",
      "[4]\n",
      "success\n",
      "[241]\n",
      "success\n",
      "[322]\n",
      "success\n",
      "[169]\n",
      "success\n",
      "[105]\n",
      "success\n",
      "[441]\n",
      "success\n",
      "[136]\n",
      "success\n",
      "[214]\n",
      "success\n",
      "[223]\n",
      "success\n",
      "[106]\n",
      "success\n",
      "[206]\n",
      "success\n",
      "[6]\n",
      "success\n",
      "[199]\n",
      "success\n",
      "[434]\n",
      "success\n",
      "[26]\n",
      "success\n",
      "[27]\n",
      "success\n",
      "[400]\n",
      "success\n",
      "[392]\n",
      "success\n",
      "[178]\n",
      "success\n",
      "[19]\n",
      "success\n",
      "[347]\n",
      "success\n",
      "[3]\n",
      "success\n",
      "[421]\n",
      "success\n",
      "[242]\n",
      "success\n",
      "[153]\n",
      "success\n",
      "[261]\n",
      "success\n",
      "[138]\n",
      "success\n",
      "[116]\n",
      "success\n",
      "[124]\n",
      "success\n",
      "[33]\n",
      "success\n",
      "[172]\n",
      "success\n",
      "[301]\n",
      "success\n",
      "[436]\n",
      "success\n",
      "[73]\n",
      "success\n",
      "[360]\n",
      "success\n",
      "[265]\n",
      "success\n",
      "[313]\n",
      "success\n",
      "[263]\n",
      "success\n",
      "[275]\n",
      "success\n",
      "[269]\n",
      "success\n",
      "[406]\n",
      "success\n",
      "[442]\n",
      "success\n",
      "[167]\n",
      "success\n",
      "[225]\n",
      "success\n",
      "[119]\n",
      "success\n",
      "[333]\n",
      "success\n",
      "[290]\n",
      "success\n",
      "[90]\n",
      "success\n",
      "[203]\n",
      "success\n",
      "[278]\n",
      "success\n",
      "[410]\n",
      "success\n",
      "[81]\n",
      "success\n",
      "[53]\n",
      "success\n",
      "[71]\n",
      "success\n",
      "[55]\n",
      "success\n",
      "[36]\n",
      "success\n",
      "[147]\n",
      "success\n",
      "[213]\n",
      "success\n",
      "[21]\n",
      "success\n",
      "[342]\n",
      "success\n",
      "[102]\n",
      "success\n",
      "[379]\n",
      "success\n",
      "[398]\n",
      "success\n",
      "[289]\n",
      "success\n",
      "[141]\n",
      "success\n",
      "[179]\n",
      "success\n",
      "[387]\n",
      "success\n",
      "[335]\n",
      "success\n",
      "[44]\n",
      "success\n",
      "[28]\n",
      "success\n",
      "[312]\n",
      "success\n",
      "[284]\n",
      "success\n",
      "[380]\n",
      "success\n",
      "[388]\n",
      "success\n",
      "[394]\n",
      "success\n",
      "[227]\n",
      "success\n",
      "[184]\n",
      "success\n",
      "[88]\n",
      "success\n",
      "[415]\n",
      "success\n",
      "[431]\n",
      "success\n",
      "[160]\n",
      "success\n",
      "[252]\n",
      "success\n",
      "[367]\n",
      "success\n",
      "[164]\n",
      "success\n",
      "[64]\n",
      "success\n",
      "[427]\n",
      "success\n",
      "[132]\n",
      "success\n",
      "[430]\n",
      "success\n",
      "[117]\n",
      "success\n",
      "[228]\n",
      "success\n",
      "[389]\n",
      "success\n",
      "[12]\n",
      "success\n",
      "[393]\n",
      "success\n",
      "[423]\n",
      "success\n",
      "[348]\n",
      "success\n",
      "[24]\n",
      "success\n",
      "[324]\n",
      "success\n",
      "[171]\n",
      "success\n",
      "[257]\n",
      "success\n",
      "[186]\n",
      "success\n",
      "[113]\n",
      "success\n",
      "[123]\n",
      "success\n",
      "[432]\n",
      "success\n",
      "[185]\n",
      "success\n",
      "[13]\n",
      "success\n",
      "[32]\n",
      "success\n",
      "[200]\n",
      "success\n",
      "[51]\n",
      "success\n",
      "[255]\n",
      "success\n",
      "[402]\n",
      "success\n",
      "[180]\n",
      "success\n",
      "[295]\n",
      "success\n",
      "[69]\n",
      "success\n",
      "[161]\n",
      "success\n",
      "[361]\n",
      "success\n",
      "[166]\n",
      "success\n",
      "[280]\n",
      "success\n",
      "[308]\n",
      "success\n",
      "[449]\n",
      "success\n",
      "[326]\n",
      "success\n",
      "[205]\n",
      "success\n",
      "[294]\n",
      "success\n",
      "[395]\n",
      "success\n",
      "[142]\n",
      "success\n",
      "[131]\n",
      "success\n",
      "[234]\n",
      "success\n",
      "[350]\n",
      "success\n",
      "[201]\n",
      "success\n",
      "[259]\n",
      "success\n",
      "[386]\n",
      "success\n",
      "[239]\n",
      "success\n",
      "[253]\n",
      "success\n",
      "[120]\n",
      "success\n",
      "[229]\n",
      "success\n",
      "[414]\n",
      "success\n",
      "[190]\n",
      "success\n",
      "[34]\n",
      "success\n",
      "[237]\n",
      "success\n",
      "[385]\n",
      "success\n",
      "[84]\n",
      "success\n",
      "[111]\n",
      "success\n",
      "[8]\n",
      "success\n",
      "[174]\n",
      "success\n",
      "[118]\n",
      "success\n",
      "[409]\n",
      "success\n",
      "[182]\n",
      "success\n",
      "[57]\n",
      "success\n",
      "[62]\n",
      "success\n",
      "[270]\n",
      "success\n",
      "[37]\n",
      "success\n",
      "[122]\n",
      "success\n",
      "[419]\n",
      "success\n",
      "[425]\n",
      "success\n",
      "[292]\n",
      "success\n",
      "[317]\n",
      "success\n",
      "[77]\n",
      "success\n",
      "[97]\n",
      "success\n",
      "[293]\n",
      "success\n",
      "[338]\n",
      "success\n",
      "[65]\n",
      "success\n",
      "[343]\n",
      "success\n",
      "[332]\n",
      "success\n",
      "[16]\n",
      "success\n",
      "[240]\n",
      "success\n",
      "[110]\n",
      "success\n",
      "[74]\n",
      "success\n",
      "[165]\n",
      "success\n",
      "[204]\n",
      "success\n",
      "[151]\n",
      "success\n",
      "[403]\n",
      "success\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-834-dae8bb9c8bf1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mhit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinalg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mxx_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m             \u001b[0mhit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/numpy/linalg/linalg.py\u001b[0m in \u001b[0;36mnorm\u001b[0;34m(x, ord, axis, keepdims)\u001b[0m\n\u001b[1;32m   2165\u001b[0m                 \u001b[0msqnorm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreal\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreal\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimag\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimag\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2166\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2167\u001b[0;31m                 \u001b[0msqnorm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2168\u001b[0m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msqnorm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2169\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for j in range(xx_train.shape[0]):\n",
    "    hit = []\n",
    "    for i in range(X.shape[0]):\n",
    "        if np.linalg.norm(X[i,:,:] - xx_train[j,0,:,:])==0:\n",
    "            hit.append(i)\n",
    "            print(hit)\n",
    "            print('success' if np.linalg.norm(Y[i] - yy_train[j])==0 else 'FAIL!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before augmenting (324, 2, 60, 128)\n",
      "(1620, 2, 60, 128)\n",
      "(1620, 1, 60, 128)\n",
      "(1620, 1, 60, 128)\n",
      "After augmenting (1620, 2, 60, 128)\n",
      "(1620, 2, 60, 128)\n",
      "yy_train_aug (1620, 2)\n",
      "\n",
      "After augmenting:\n",
      " (1620, 2, 60, 128) \n",
      " (1620, 2)\n"
     ]
    }
   ],
   "source": [
    "def train_augment(xx_train, yy_train):\n",
    "    print('Before augmenting', xx_train.shape)\n",
    "    temp_xx = np.concatenate([xx_train, xx_train, xx_train, xx_train, xx_train], axis=0)\n",
    "    print(temp_xx.shape)\n",
    "    temp_xx_0 = temp_xx[:,0,:,:]\n",
    "    temp_xx_0 = temp_xx_0.reshape(temp_xx_0.shape + (1,)).transpose(0,3,1,2)\n",
    "    print(temp_xx_0.shape)\n",
    "\n",
    "    temp_xx_1 = temp_xx[:,1,:,:]\n",
    "    np.random.shuffle(temp_xx_1)\n",
    "    temp_xx_1 = temp_xx_1.reshape(temp_xx_1.shape + (1,)).transpose(0,3,1,2)\n",
    "    print(temp_xx_1.shape)\n",
    "\n",
    "    xx_train_aug = np.concatenate([temp_xx_0, temp_xx_1], axis=1)\n",
    "    print('After augmenting', xx_train_aug.shape)\n",
    "    print(xx_train_aug.shape)\n",
    "\n",
    "    yy_train_aug = np.concatenate([yy_train, yy_train, yy_train, yy_train, yy_train], axis=0)\n",
    "    print('yy_train_aug', yy_train_aug.shape)\n",
    "    return xx_train_aug, yy_train_aug\n",
    "\n",
    "if AUGMENT==True:\n",
    "    xx_train, yy_train = train_augment(xx_train, yy_train)\n",
    "    print('\\nAfter augmenting:\\n', xx_train.shape,'\\n', yy_train.shape )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "a,b = [],[]\n",
    "for i in range(len(X)):\n",
    "    if np.array_equal(Y[i],np.array([0,1])):\n",
    "        a.append(i)\n",
    "    if np.array_equal(Y[i],np.array([1,0])):\n",
    "        b.append(i)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XX_a\t (225, 2, 60, 128)\n",
      "XX_b\t (225, 2, 60, 128)\n",
      "XX\t (450, 2, 60, 128)\n",
      "\n",
      "XX_trai\t (324, 2, 60, 128)\n",
      "XX_test\t (81, 2, 60, 128)\n",
      "XX_val\t (45, 2, 60, 128)\n",
      "yy_trai\t (324, 2)\n",
      "yy_test\t (81, 2)\n",
      "yy_val\t (45, 2)\n",
      "\n",
      "If you use RANDOM SEED, check if you generate the same numbers every time:\n",
      "[291 144  54 215 210 167 184 442 273  96  62 420 255  76 278  86 270 152\n",
      " 122 325  48 326 264 110 436  31 410 214 361 257 357  73 407 205 115  97\n",
      " 172 303 285   9 242  59 366 119 118 348 267 141  26 345 447 437 224 446\n",
      " 178 189 359 245 107 262 108 330 432 272 364 250 279 346  80 388 439 206\n",
      " 171  33  29 164 258 235 336  13  74  45 406 415 434 317 183 374  92 193\n",
      " 208 219 428  99 225 368  51 256 342  19 394 237  44 360 132 192 315  11\n",
      " 296 414 431 424 169 367   4 412 133 283  68 234 397   5 395 328 101 288\n",
      "  40 165 248 112 449 307 148 308 377 157 309 338 204 320 339 127  64 221\n",
      " 135 378 401  77 159 353 236 138 381  93  25 408 337 413 117 260  58 444\n",
      "  79 266 306 318  47 386 227 251  72 292 380 203 387 176 179 399  55 198\n",
      " 123  70 438 261  36 284 150  60   8 196 441  21  41  82  63  85  12 313\n",
      " 389 271  27 423 400  69 147 168 295 268 411 425 191 393 409 290 358  81\n",
      "  32 201 376 116 402 244 429 143 222 324 182 254 427 247 422 124 104   0\n",
      "  66 289 263 158 103  56 259 146 341 344 299  95 293 207 240 362 109 153\n",
      " 329 298  22 355  75 404 443 331 311  37 213 416 333 276 310 120 239 175\n",
      " 275 321  15 391 161 156  30 121 177 160 130 233  14 220 281 134 419 105\n",
      " 332  53 145 216 327 417 316 113 356 323  49 371 249  98 142  20  89  71\n",
      " 398  28  18   7 243 154 229 194 287 369 162 231 253 382 246 190 166 365]\n",
      "[269  34 373 200 319 418 372 277 128 197 312  38 209  83 195 140 218 370\n",
      " 379 151   3 343 131  39  50  46 232 106 403  17 352   2 445  35 375 302\n",
      " 347 390 170 282  94 217  16 340  67 430 228 392 354 349  87 137 322 334\n",
      " 435 102 212  57 363 125  84 351 350 199 405 186 139 136 286 173  23 280\n",
      "  42 185  90 265 226  52 180 335 384]\n",
      "[300  43 211 188 129 238  88 174 294 230 149 305  24 163 385  91  65 274\n",
      "  78 426  61 433 100 114 297 304 202 223   6 155 181 126 314 440 448 187\n",
      " 301 252   1 383  10 421 396 111 241]\n"
     ]
    }
   ],
   "source": [
    "XX_a = np.zeros([len(a),2,X.shape[1],X.shape[2]])\n",
    "XX_b = np.zeros([len(b),2,X.shape[1],X.shape[2]])\n",
    "XX = np.zeros([len(a)+len(b),2,X.shape[1],X.shape[2]])\n",
    "\n",
    "YY_a = np.zeros([len(a),2])\n",
    "YY_b = np.zeros([len(b),2])\n",
    "YY = np.zeros([len(b)+len(b),2])\n",
    "\n",
    "XX_a[:,0,:,:] = X[a,:,:]\n",
    "XX_a[:,1,:,:] = X[np.random.permutation(a),:,:]\n",
    "YY_a = Y[a]\n",
    "XX_b[:,0,:,:] = X[b,:,:]\n",
    "XX_b[:,1,:,:] = X[np.random.permutation(b),:,:]\n",
    "YY_b = Y[b]\n",
    "\n",
    "XX = np.concatenate([XX_a, XX_b], axis=0)\n",
    "YY = np.concatenate([YY_a, YY_b], axis=0)\n",
    "\n",
    "print('XX_a\\t', XX_a.shape)\n",
    "print('XX_b\\t', XX_b.shape)\n",
    "print('XX\\t', XX.shape)\n",
    "\n",
    "train_id, test_id, val_id = train_test_val(XX,YY, 0.2, 0.1, shuffle=SHUFFLE)\n",
    "\n",
    "xx_train = XX[train_id,:,:,:]\n",
    "yy_train = YY[train_id]\n",
    "\n",
    "xx_test = XX[test_id,:,:,:]\n",
    "yy_test = YY[test_id]\n",
    "\n",
    "xx_val = XX[val_id,:,:,:]\n",
    "yy_val = YY[val_id]\n",
    "\n",
    "print('\\nXX_trai\\t', xx_train.shape)\n",
    "print('XX_test\\t', xx_test.shape)\n",
    "print('XX_val\\t', xx_val.shape)\n",
    "print('yy_trai\\t', yy_train.shape)\n",
    "print('yy_test\\t', yy_test.shape)\n",
    "print('yy_val\\t', yy_val.shape)\n",
    "\n",
    "print('\\nIf you use RANDOM SEED, check if you generate the same numbers every time:')\n",
    "print(train_id)\n",
    "print(test_id)\n",
    "print(val_id)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}