Siamese-L1-ver.2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# make sure you don't hog all the video memory\n",
    "import tensorflow as tf\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)\n",
    "from keras import backend as K\n",
    "K.set_session(sess)\n",
    "###################################\n",
    "\n",
    "from keras.layers import Input, Lambda, merge, Dense, DepthwiseConv2D, Activation, AveragePooling2D\n",
    "from keras.layers import Flatten,Conv2D, MaxPooling2D, Dropout, BatchNormalization, SeparableConv2D\n",
    "from keras.constraints import max_norm\n",
    "from keras.models import Model, Sequential\n",
    "from keras.regularizers import l2\n",
    "from keras.initializers import RandomUniform\n",
    "from keras.optimizers import SGD,Adam\n",
    "from keras.losses import binary_crossentropy\n",
    "import numpy.random as rng\n",
    "import numpy as np\n",
    "import os\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "# import seaborn as sns\n",
    "from sklearn.utils import shuffle\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original data shape: (450, 60, 128)\n",
      "Original labels shape: (450, 1)\n"
     ]
    }
   ],
   "source": [
    "SHUFFLE = False\n",
    "Seed = 4\n",
    "SCRAMBLE_FOR_TEST = False\n",
    "AUGMENT = True\n",
    "\n",
    "file = 'Merged456-197-289_ICA(-eyes)+AUDpreproc.mat, DS2=64Hz, FIR=2-30Hz, centnorm=1, step=2, win=2, TD, 1-93.mat' #0.75(vanilla EEGnet), 0. (SiameseL1)\n",
    "# file = 'Merged456-1-94_ICA(-2,3ICs)+AUDpreproc.mat, DS2=64Hz, FIR=2-30Hz, centnorm=1, step=2, win=2, TD, 1-95.mat' #0.61(vanilla EEGnet)\n",
    "# get the Dataset:\n",
    "import scipy.io as sio\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "\n",
    "path = '/home/amplifier/home/DATASETS/' + file\n",
    "mat_contents = sio.loadmat(path)\n",
    "X = mat_contents['X']\n",
    "Y = mat_contents['Z']\n",
    "\n",
    "if X.shape[1]<X.shape[2]:\n",
    "    X = np.transpose(X,[0,2,1])\n",
    "\n",
    "if Y.shape[1] > Y.shape[0]:\n",
    "    Y = Y.T\n",
    "\n",
    "    \n",
    "# # one hot encode the labels:\n",
    "# onehot_encoder = preprocessing.OneHotEncoder(sparse=False)\n",
    "# Y = onehot_encoder.fit_transform(Y)\n",
    "\n",
    "X = X.transpose(0,2,1)\n",
    "\n",
    "print('Original data shape:', X.shape)\n",
    "print('Original labels shape:', Y.shape)\n",
    "\n",
    "\n",
    "# verify that the model REALLY finds a mapping between the input and the labels. If we get\n",
    "# our accuracy by chance, then we should get the same accuracy on a permuted dataset:\n",
    "if SCRAMBLE_FOR_TEST==True:\n",
    "    Y = np.random.permutation(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment_train(X, Y, train_id, leng):\n",
    "    X = X[train_id,:,:]\n",
    "    Y = Y[train_id]\n",
    "\n",
    "    id1 = np.random.choice(range(len(X)), size=leng,  replace=True)\n",
    "    id2 = np.random.choice(range(len(X)), size=leng,  replace=True)\n",
    "    XX = [X[id1,:,:], X[id2,:,:]]\n",
    "    YY = [Y[id1],Y[id2]]\n",
    "    targets = np.zeros(leng,)\n",
    "    for i in range(leng):\n",
    "        targets[i] = 1 if YY[0][i]==YY[1][i] else 0\n",
    "    return XX, targets\n",
    "\n",
    "def preproc(X,Y):\n",
    "    mask = np.arange(len(X))\n",
    "    mask_perm = np.random.permutation(mask)\n",
    "    X2 = X[mask_perm,:,:]\n",
    "    Y2 = Y[mask_perm]\n",
    "\n",
    "    XX = [X, X2]\n",
    "    YY = np.concatenate([Y,Y2], axis=1)\n",
    "    targets = np.zeros(len(X),)\n",
    "    for i in range(len(X)):\n",
    "        targets[i] = 1 if YY[i][0]==YY[i][1] else 0\n",
    "    return XX, targets\n",
    "\n",
    "def train_test_val(X,Y, test=0.2, val=0.1, shuffle=False):\n",
    "    np.random.seed(3)\n",
    "    if len(X) != len(Y):\n",
    "        print('SIZES ARE DIFFERENT!')\n",
    "    leng = np.arange(len(X))\n",
    "#     if shuffle==True:\n",
    "#         leng = np.random.permutation(leng)\n",
    "    x_train_id, x_val_id, y_train_id, y_val_id = train_test_split(leng, leng, test_size=val, shuffle=shuffle, random_state=10)\n",
    "    x_train_id, x_test_id, y_train_id, y_test_id = train_test_split(x_train_id, y_train_id, test_size=test, shuffle=shuffle, random_state=10)\n",
    "    train_id = x_train_id\n",
    "    test_id = x_test_id\n",
    "    val_id = x_val_id\n",
    "    return train_id, test_id, val_id\n",
    "\n",
    "# train_id, test_id, val_id = train_test_val(X,Y, 0.2, 0.1, shuffle=SHUFFLE)\n",
    "# print(train_id,'\\n', test_id, '\\n', val_id,'\\n')\n",
    "# np.isin(train_id, test_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(len(val_id)):\n",
    "#     print(Y[val_id[i]], '\\t', yy_val[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(train_id,'\\n++++', test_id,'\\n++++', val_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(324, 1) (81, 1) (45, 1)\n"
     ]
    }
   ],
   "source": [
    "train_id, test_id, val_id = train_test_val(X,Y, 0.2, 0.1, shuffle=False)\n",
    "\n",
    "# xx_train, yy_train = preproc(X[train_id], Y[train_id]) #############!!!!!!!!!!!!!!!!\n",
    "# xx_train, yy_train = augment_train(X, Y, train_id, 1000)\n",
    "# xx_test, yy_test = preproc(X[test_id], Y[test_id])\n",
    "# xx_val, yy_val = preproc(X[val_id], Y[val_id])\n",
    "\n",
    "\n",
    "def onehot(Y):\n",
    "    temp = np.zeros([len(Y),2]) \n",
    "    for i in range(len(Y)):\n",
    "        temp[i,:] = np.array([1,0]) if Y[i]==1 else np.array([0,1])\n",
    "    return temp\n",
    "\n",
    "\n",
    "xx_train, yy_train = X[train_id], Y[train_id]\n",
    "xx_test, yy_test = X[test_id], Y[test_id]\n",
    "xx_val, yy_val = X[val_id], Y[val_id]\n",
    "\n",
    "\n",
    "print(yy_train.shape,yy_test.shape,yy_val.shape,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batch(X,Y, size=10):\n",
    "    a = np.random.choice(len(Y), size=size, replace=True)\n",
    "    b = np.random.choice(len(Y), size=size, replace=True)\n",
    "    siam_labels = (Y[a].flatten() != Y[b].flatten())*1\n",
    "    return a, b, X[a], X[b], onehot(Y[a]), onehot(Y[b]), siam_labels\n",
    "#     return a,b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "size = [90000, 2000, 100]\n",
    "\n",
    "a_train, b_train, x_train_a, x_train_b, y_train_a, y_train_b, siam_y_train = get_batch(xx_train, yy_train, size=100000)\n",
    "a_test, b_test, x_test_a, x_test_b, y_test_a, y_test_b, siam_y_test = get_batch(xx_test, yy_test, size=6500)\n",
    "a_val, b_val, x_val_a, x_val_b, y_val_a, y_val_b, siam_y_val = get_batch(xx_val, yy_val, size=2000)\n",
    "\n",
    "# diagnostics:\n",
    "for i in range(len(a_val)):\n",
    "    print(a_val[i],b_val[i],y_val_a[i],y_val_b[i],siam_y_val[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "def EEGNet_my(input1, nb_classes, Chans = 64, Samples = 128, \n",
    "             dropoutRate = 0.25, kernLength = 64, F1 = 4, \n",
    "             D = 2, F2 = 8, dropoutType = 'Dropout'):\n",
    "    \n",
    "    \"\"\" Keras Implementation of EEGNet (https://arxiv.org/abs/1611.08024)\n",
    "\n",
    "    Inputs:\n",
    "        \n",
    "      nb_classes      : int, number of classes to classify\n",
    "      Chans, Samples  : number of channels and time points in the EEG data\n",
    "      dropoutRate     : dropout fraction\n",
    "      kernLength      : length of temporal convolution in first layer. We found\n",
    "                        that setting this to be half the sampling rate worked\n",
    "                        well in practice. For the SMR dataset in particular\n",
    "                        since the data was high-passed at 4Hz we used a kernel\n",
    "                        length of 32.     \n",
    "      F1, F2          : number of temporal filters (F1) and number of pointwise\n",
    "                        filters (F2) to learn. Default: F1 = 4, F2 = F1 * D. \n",
    "      D               : number of spatial filters to learn within each temporal\n",
    "                        convolution. Default: D = 2\n",
    "      dropoutType     : Either SpatialDropout2D or Dropout, passed as a string.\n",
    "\n",
    "    \"\"\"\n",
    "    \n",
    "    if dropoutType == 'SpatialDropout2D':\n",
    "        dropoutType = SpatialDropout2D\n",
    "    elif dropoutType == 'Dropout':\n",
    "        dropoutType = Dropout\n",
    "    else:\n",
    "        raise ValueError('dropoutType must be one of SpatialDropout2D '\n",
    "                         'or Dropout, passed as a string.')\n",
    "    \n",
    "    init = RandomUniform(minval=-0.1, maxval=0.1, seed=29)\n",
    "    net = Sequential()\n",
    "    net.add (Conv2D(F1, (1, kernLength), padding = 'same',\n",
    "                                   input_shape = (Chans, Samples,1),\n",
    "                                   use_bias = False, bias_initializer=init, kernel_initializer=init))\n",
    "    net.add (BatchNormalization())\n",
    "    net.add (DepthwiseConv2D((Chans, 1), use_bias = False, \n",
    "                                   depth_multiplier = D,\n",
    "                                   depthwise_constraint = max_norm(1.), bias_initializer=init, kernel_initializer=init))\n",
    "    net.add (BatchNormalization())\n",
    "    net.add (Activation('elu'))\n",
    "    net.add (AveragePooling2D((1, 4)))\n",
    "    net.add (dropoutType(dropoutRate))\n",
    "    net.add (SeparableConv2D(F2, (1, 16), use_bias = False, padding = 'same', bias_initializer=init, kernel_initializer=init))\n",
    "    net.add (BatchNormalization())\n",
    "    net.add (Activation('elu'))\n",
    "    net.add (AveragePooling2D((1, 8)))\n",
    "    net.add (dropoutType(dropoutRate))\n",
    "    net.add (Flatten())\n",
    "    net.add (Dense(nb_classes, kernel_constraint = max_norm(0.25), bias_initializer=init, kernel_initializer=init))\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def SS(x, activate=True):\n",
    "#     temp = np.abs(x[0] - x[1])\n",
    "#     print(x[0])\n",
    "#     print(x[1])\n",
    "#     print(temp)\n",
    "#     out = np.dot(temp, np.array([1,1,1,1]))\n",
    "#     return 1/(1+np.exp(-out)) if activate==True else out\n",
    "\n",
    "# x = [np.array([1,2,3,5]),np.array([1,2,3,5])]\n",
    "# print(SS(x, activate=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_17 (InputLayer)           (None, 60, 128, 1)   0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_18 (InputLayer)           (None, 60, 128, 1)   0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sequential_17 (Sequential)      (None, 2)            1074        input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "sequential_18 (Sequential)      (None, 2)            1074        input_18[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_9 (Lambda)               (None, 2)            0           sequential_17[1][0]              \n",
      "                                                                 sequential_18[1][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_27 (Dense)                (None, 1)            3           lambda_9[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 2,151\n",
      "Trainable params: 2,071\n",
      "Non-trainable params: 80\n",
      "__________________________________________________________________________________________________\n",
      "0.501751 0.501751\n"
     ]
    }
   ],
   "source": [
    "from keras.engine.topology import Layer\n",
    "from keras.layers import Concatenate\n",
    "\n",
    "# def triplet_loss(y_true, y_pred):\n",
    "#     norm1 = K.sqrt(K.sum(K.square(y_pred[0] - y_pred[1]), axis=-1, keepdims=True))\n",
    "#     norm2 = K.sqrt(K.sum(K.square(y_pred[0] - y_pred[2]), axis=-1, keepdims=True))\n",
    "#     loss = norm1 - norm2 + 0.2\n",
    "#     return loss\n",
    "\n",
    "# input_shape = (64, 128, 1)\n",
    "input_shape = xx_train[-1,:,:].shape + (1,)\n",
    "# input_shape = xx_train[0][0].shape + (1,)\n",
    "a_input = Input(input_shape)\n",
    "r_input = Input(input_shape)\n",
    "\n",
    "#call the convnet Sequential model on each of the input tensors so params will be shared\n",
    "# encoded NOT as Sequential (stack of layers), but as a Tensor!!!! if you add an argument, a Tensor is returned\n",
    "encoded_a = EEGNet_my(input_shape, 2, Chans=input_shape[0])(a_input)\n",
    "encoded_r = EEGNet_my(input_shape, 2, Chans=input_shape[0])(r_input)\n",
    "\n",
    "# class MergeLegs(Layer):\n",
    "#     def __init__(self, **kwargs):\n",
    "#         super(MergeLegs, self).__init__(**kwargs)\n",
    "\n",
    "#     def call(self ,x ,mask=None):\n",
    "#         a = x[0]\n",
    "#         r = x[1]\n",
    "#         norm1 = K.sqrt(K.sum(K.square(a - r), axis=-1, keepdims=True))\n",
    "#         norm2 = K.sqrt(K.sum(K.square(a - n), axis=-1, keepdims=True))\n",
    "#         loss = norm1 - norm2 + 0.5\n",
    "#         self.add_loss(loss, x)\n",
    "#         #you can output whatever you need, just update output_shape adequately\n",
    "#         #But this is probably useful\n",
    "#         return K.concatenate([a,r,n], axis=1)\n",
    "\n",
    "#     def get_output_shape_for(self, input_shape):\n",
    "#         return (input_shape[0][0],1)\n",
    "    \n",
    "\n",
    "L1_distance = Lambda(lambda tensors: (K.abs(tensors[0] - tensors[1])))([encoded_a, encoded_r])\n",
    "L1_distance = Dense(1,activation='sigmoid', use_bias=True)(L1_distance)\n",
    "\n",
    "# siamese_net = Model(inputs=[a_input, r_input], outputs=[encoded_a, encoded_r, L1_distance])\n",
    "siamese_net = Model(inputs=[a_input, r_input], outputs=[L1_distance])\n",
    "optimizer = Adam(0.00006)\n",
    "# optimizer = 'adam'\n",
    "# siamese_net.compile(loss=['binary_crossentropy','binary_crossentropy','binary_crossentropy'], optimizer=optimizer)\n",
    "siamese_net.compile(loss=['binary_crossentropy'], optimizer=optimizer)\n",
    "siamese_net.summary()\n",
    "\n",
    "a1 = np.linalg.norm(siamese_net.layers[2].layers[13].get_weights()[0])\n",
    "a2 = np.linalg.norm(siamese_net.layers[3].layers[13].get_weights()[0])\n",
    "print(a1, a2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 100000 samples, validate on 6500 samples\n",
      "Epoch 1/10\n",
      "100000/100000 [==============================] - 176s 2ms/step - loss: 0.6929 - val_loss: 0.6929\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.69290, saving model to /home/amplifier/home/NEW_DL/weights/Siamese_L1_ver2.h5\n",
      "Epoch 2/10\n",
      "100000/100000 [==============================] - 176s 2ms/step - loss: 0.6929 - val_loss: 0.6930\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.69290\n",
      "Epoch 3/10\n",
      "100000/100000 [==============================] - 195s 2ms/step - loss: 0.6928 - val_loss: 0.6932\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.69290\n",
      "Epoch 4/10\n",
      "100000/100000 [==============================] - 273s 3ms/step - loss: 0.6926 - val_loss: 0.6931\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.69290\n",
      "Epoch 5/10\n",
      "100000/100000 [==============================] - 176s 2ms/step - loss: 0.6923 - val_loss: 0.6932\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.69290\n",
      "Epoch 6/10\n",
      "100000/100000 [==============================] - 176s 2ms/step - loss: 0.6911 - val_loss: 0.6936\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.69290\n",
      "Epoch 7/10\n",
      "100000/100000 [==============================] - 175s 2ms/step - loss: 0.6865 - val_loss: 0.6948\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.69290\n",
      "Epoch 8/10\n",
      "100000/100000 [==============================] - 176s 2ms/step - loss: 0.6718 - val_loss: 0.6969\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.69290\n",
      "Epoch 9/10\n",
      "100000/100000 [==============================] - 175s 2ms/step - loss: 0.6291 - val_loss: 0.7020\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.69290\n",
      "Epoch 10/10\n",
      "100000/100000 [==============================] - 174s 2ms/step - loss: 0.5701 - val_loss: 0.7367\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.69290\n"
     ]
    }
   ],
   "source": [
    "# Training time!\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=10, mode='min')\n",
    "checkpointer = ModelCheckpoint(filepath='/home/amplifier/home/NEW_DL/weights/Siamese_L1_ver2.h5',\n",
    "                               verbose=1,\n",
    "                               monitor='val_loss',\n",
    "                               save_best_only=True)\n",
    "\n",
    "train_history = siamese_net.fit(x = [x_train_a[:,:,:,None], x_train_b[:,:,:,None]],\n",
    "                y=[siam_y_train] ,\n",
    "                epochs=10,\n",
    "                batch_size=10,\n",
    "                verbose=1,\n",
    "                shuffle=True,\n",
    "                validation_data=([x_test_a[:,:,:,None], x_test_b[:,:,:,None]], [siam_y_test]),\n",
    "                callbacks=[checkpointer, early_stopping])\n",
    "\n",
    "# model.save('/home/amplifier/home/NEW_DL/models/EEGnet.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "siamese_net.load_weights('/home/amplifier/home/NEW_DL/weights/Siamese_L1_ver2.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8FfW9//HXJxsBwk7YkUVAZFGsAbcrClbFVlGrRXCFtljXqtdS5ddFa7Xtvb1X21up1lq3igqiVloXtBVBrQuBgmyCIagEEEJYBUK2z++PmcBJSEggOZks7+fjcR45853vzPnMAfLm+505c8zdEREROVIJURcgIiINm4JERERqREEiIiI1oiAREZEaUZCIiEiNKEhERKRGFCQiNWRmE83s3Tp8vc/M7Ot19XoiVVGQiByCmb1tZtvMrFkt7e9uMys0s13hY7WZPWhmXWuwv6drozaRI6UgEamEmfUGTgccGFuLu57h7q2A9sDFQBdg4ZGGiUjUFCQilbsa+AB4ArimtNHMOpjZbDPbaWYfAUfHbmRmvzOzdeH6hWZ2ekU7d/dCd18OXAbkArfH7ON8M1tsZtvN7F9mdlz57c1sDPD/gMvM7CszWxK2TzKzleGIJ9vMvl/TN0LkUBQkIpW7GpgePs41s85h+zQgH+gKfCd8xFoADCMYcTwDPG9mqZW9iLsXAy8TjH4ws68BjwHfBzoAfwRml59ec/fXgV8SjHDS3P34cNVm4HygNTAJeCDcp0hcKEhEKmBm/wH0Ama6+0JgDXC5mSUClwA/c/fd7r4MeDJ2W3d/2t3z3L3I3f8XaAYcU8VLbiAIHoDJwB/d/UN3L3b3J4F9wMnVqd3dX3H3NR6YB7xBGFIi8aAgEanYNcAb7r4lXH4mbEsHkoB1MX0/j93QzG4Pp5Z2mNl2oA3QsYrX6w5sDZ/3Am4Pp7W2h/voCXSrTuFmdp6ZfWBmW8Ntv1GN1xc5YklRFyBS35hZc2AckGhmX4bNzYC2QGegiOAX+yfhuqNitj0duAM4C1ju7iVmtg2wQ7xeAnAB8I+waR1wn7vfV41yy9y+O5z+eoFgWu5ldy80s78e6vVFakojEpGDXQQUA4MIznUMA44F3iH4Bf0icLeZtTCzQcSciAdaEQRNLpBkZj8jOFdxEDNLNrNjgWcJrty6P1z1J+A6MzvJAi3N7Jtm1qqC3WwCeodhBJBCEHq5QJGZnQecc0Tvgkg1KUhEDnYN8Li7f+HuX5Y+gAeBK4CbgDTgS4Iruh6P2XYO8BqwmmDKK5+y02AQXmUFbAdmA3nAie6+AcDdMwnOkzwIbAOygImV1Pp8+DPPzBa5+y7gB8DMcNvLw9cQiRvTF1uJiEhNaEQiIiI1oiAREZEaUZCIiEiNKEhERKRGmsTnSDp27Oi9e/eOugwRkQZl4cKFW9w9vap+TSJIevfuTWZmZtRliIg0KGb2edW9NLUlIiI1pCAREZEaUZCIiEiNNIlzJBUpLCwkJyeH/Pz8qEup11JTU+nRowfJyclRlyIi9VSTDZKcnBxatWpF7969MdONUSvi7uTl5ZGTk0OfPn2iLkdE6qkmO7WVn59Phw4dFCKHYGZ06NBBozYROaQmGySAQqQa9B6JSFWadJCIiDRau7fA61OhcG/cX0pBEqG0tLSoSxCRxqioAGZcBZmPQV5W3F+uyZ5sFxFplNzh1R/CF/+CS/4MXYbG/SU1IqkH3J0pU6YwZMgQhg4dyowZMwDYuHEjI0eOZNiwYQwZMoR33nmH4uJiJk6cuL/vAw88EHH1IlKvfPQnWPQknH47DL20Tl5SIxLg539bzooNO2t1n4O6teauCwZXq++LL77I4sWLWbJkCVu2bGH48OGMHDmSZ555hnPPPZcf//jHFBcXs2fPHhYvXsz69etZtmwZANu3b6/VukWkAct+G16/E475Boz6SZ29rEYk9cC7777LhAkTSExMpHPnzpxxxhksWLCA4cOH8/jjj3P33XezdOlSWrVqRd++fcnOzubmm2/m9ddfp3Xr1lGXLyL1Qd4amHkNdBwA33oEEuru17tGJFDtkUO8uHuF7SNHjmT+/Pm88sorXHXVVUyZMoWrr76aJUuWMGfOHKZNm8bMmTN57LHH6rhiEalX8nfAsxPAEmDCs9CsVZ2+vEYk9cDIkSOZMWMGxcXF5ObmMn/+fEaMGMHnn39Op06dmDx5Mt/97ndZtGgRW7ZsoaSkhEsuuYRf/OIXLFq0KOryRSRKJcXwwmTYugbGPQXt6/4uFBqR1AMXX3wx77//Pscffzxmxn//93/TpUsXnnzySX7zm9+QnJxMWloaTz31FOvXr2fSpEmUlJQA8Ktf/Sri6kUkUv/8OXw6B775v9Dn9EhKsMqmVRqTjIwML//FVitXruTYY4+NqKKGRe+VSD21ZAa8dC1kfBfOv7/Wd29mC909o6p+mtoSEWmIcjJh9s3Q+3Q4778iLUVBIiLS0OzcAM9dAa26wLefhMRov+YhrkFiZmPMbJWZZZnZnRWsf8DMFoeP1Wa2PWwfZmbvm9lyM/vYzC6L2eYJM1sbs92weB6DiEi9UrgXnrscCr6CCc9Byw5RVxS/k+1mlghMA84GcoAFZjbb3VeU9nH322L63wycEC7uAa5290/NrBuw0MzmuHvpp++muPuseNUuIlIvuQfTWRsWw/hnoPOgqCsC4jsiGQFkuXu2uxcAzwEXHqL/BOBZAHdf7e6fhs83AJuB9DjWKiJS/737ACx9Hkb/BAZ+I+pq9otnkHQH1sUs54RtBzGzXkAf4K0K1o0AUoA1Mc33hVNeD5hZs0r2ea2ZZZpZZm5u7pEeg4hI/bDqNfjnPTDkkuA+WvVIPIOkom9Equxa4/HALHcvLrMDs67AX4BJ7l4SNk8FBgLDgfbAHRXt0N0fcfcMd89IT9dgRkQasM0r4YXvQdfjYeyDUM++cC6eQZID9IxZ7gFsqKTveMJprVJm1hp4BfiJu39Q2u7uGz2wD3icYAqt0TvUd5d89tlnDBkypA6rEZE6s2crPDseUloG50VSWkRd0UHiGSQLgP5m1sfMUgjCYnb5TmZ2DNAOeD+mLQV4CXjK3Z8v179r+NOAi4BlcTsCEZEoFRfCzKth50a4bDq0qfDsQOTidtWWuxeZ2U3AHCAReMzdl5vZPUCmu5eGygTgOS/7EftxwEigg5lNDNsmuvtiYLqZpRNMnS0Grqtxsa/dCV8urfFuyugyFM77daWr77jjDnr16sUNN9wAwN13342ZMX/+fLZt20ZhYSH33nsvF154qOsTDpafn8/1119PZmYmSUlJ3H///YwaNYrly5czadIkCgoKKCkp4YUXXqBbt26MGzeOnJwciouL+elPf8pll11W9YuISN14fSp89g5c9DD0HB51NZWK67223P1V4NVybT8rt3x3Bds9DTxdyT5H12KJkRk/fjy33nrr/iCZOXMmr7/+OrfddhutW7dmy5YtnHzyyYwdOxY7jPnQadOmAbB06VI++eQTzjnnHFavXs3DDz/MLbfcwhVXXEFBQQHFxcW8+uqrdOvWjVdeeQWAHTt21P6BisiRyXwMFvwJTr0Zhk2IuppD0k0b4ZAjh3g54YQT2Lx5Mxs2bCA3N5d27drRtWtXbrvtNubPn09CQgLr169n06ZNdOnSpdr7fffdd7n55psBGDhwIL169WL16tWccsop3HfffeTk5PCtb32L/v37M3ToUH74wx9yxx13cP7553P66dHc8E1EyvnsXXh1CvQ7G77+86irqZJukRKhSy+9lFmzZjFjxgzGjx/P9OnTyc3NZeHChSxevJjOnTuTn59/WPus7Cacl19+ObNnz6Z58+ace+65vPXWWwwYMICFCxcydOhQpk6dyj333FMbhyUiNbHtM5hxFbTvC5f+GRISo66oShqRRGj8+PFMnjyZLVu2MG/ePGbOnEmnTp1ITk5m7ty5fP7554e9z5EjRzJ9+nRGjx7N6tWr+eKLLzjmmGPIzs6mb9++/OAHPyA7O5uPP/6YgQMH0r59e6688krS0tJ44oknav8gRaT69u0KvqDKi4Pbn6S2ibqialGQRGjw4MHs2rWL7t2707VrV6644gouuOACMjIyGDZsGAMHDjzsfd5www1cd911DB06lKSkJJ544gmaNWvGjBkzePrpp0lOTqZLly787Gc/Y8GCBUyZMoWEhASSk5N56KGH4nCUIlItJSXw4vchdxVcOQs6HB11RdWm7yORKum9EqkDb90L838DY/4LTq75xai1Qd9HIiLSUCx7IQiRE66Ck74fdTWHTVNbDcjSpUu56qqryrQ1a9aMDz/8MKKKRKTGNvwb/nojHHUKfPP+enf7k+po0kHi7of1GY2oDR06lMWLF9fpazaFqU+RyOzaFHxBVcuOMO4vkJQSdUVHpMlObaWmppKXl6dflIfg7uTl5ZGamhp1KSKNT9E+mHEF7N0W3EMrreHeXLbJjkh69OhBTk4OusX8oaWmptKjR4+oyxBpXNzhb7dCzgIY9xR0PS7qimqkyQZJcnIyffr0iboMEWmK3p8GS56BM6fCoMO7n1591GSntkREIvHpm/DmT+HYsTDyR1FXUysUJCIidSV3Ncz6DnQaDBc/DAmN41dw4zgKEZH6bu+24AuqElNgwjPBF1U1Ek32HImISJ0pLoLnJ8H2L+Cav0Hbo6KuqFYpSERE4u3Nn0L2XBj7e+h1StTV1DpNbYmIxNOiv8AHf4CTroevXR11NXGhIBERiZcvPoC/3wZ9R8E590ZdTdwoSERE4mH7OphxJbTtCd9+HBIb75mExntkIiJRKdgNz00IboMy8VVo3i7qiuJKQSIiUlsK8yH3k+CW8F8ugyueh/QBUVcVd3ENEjMbA/wOSAQedfdfl1v/ADAqXGwBdHL3tuG6a4CfhOvudfcnw/YTgSeA5sCrwC2uOy+KSF1yDy7l3bwCNi2DTcuDR14WeEnQ55z7oP/Z0dZZR+IWJGaWCEwDzgZygAVmNtvdV5T2cffbYvrfDJwQPm8P3AVkAA4sDLfdBjwEXAt8QBAkY4DX4nUcItLE5e+EzSvLBsbmFbBv54E+7XpD5yEw+GLoNCi4CWP7vpGVXNfiOSIZAWS5ezaAmT0HXAisqKT/BILwADgXeNPdt4bbvgmMMbO3gdbu/n7Y/hRwEQoSEampkmLIWwOblx8IjE3LgpFHqWZtoPNgOO4y6DwoCI9Ox0KzVtHVXQ/EM0i6A+tilnOAkyrqaGa9gD7AW4fYtnv4yKmgvaJ9XkswcuGooxrXp0hFpIZ2bwlHGCsOBEbuJ1CUH6y3ROjYH3oMhxMnBvfG6jwY2vRokN9gGG/xDJKK3u3KzmWMB2a5e3EV21Z7n+7+CPAIQEZGhs6hiDRFRfsgd9XB5zK+2nSgT8tOQUgM/17ws/Ng6HgMJOsL3aornkGSA/SMWe4BbKik73jgxnLbnllu27fD9h7l2ivbp4g0dsVFwc0Q926FPVthzxbY8umBwNiyGkr/f5rYDDoNhH5fD85jlIZGWqdoj6ERiGeQLAD6m1kfYD1BWFxevpOZHQO0A96PaZ4D/NLMSi++PgeY6u5bzWyXmZ0MfAhcDfw+jscgInWlMP9AIOwPhrzw+baY5zHt+Tsq3lebo4KQGPiNMDCGQPujG/WHAqMUt3fV3YvM7CaCUEgEHnP35WZ2D5Dp7rPDrhOA52Iv4Q0D4xcEYQRwT+mJd+B6Dlz++xo60S5Sv7gHH8iL/cW/NwyC/SGRVy4wtkLh7sr3mZIGzdtDi3bQokNwlVSL9sHz5u3D5+2D5+16Q/O2dXW0AlhT+AhGRkaGZ2ZmRl2GSP1TVACFe8LH3iAACvcGv9QL90LBnpj1e8LlmPWxbfk7DoRHcUHlr5na9uAQKB8GLTqUbU9qVnfviexnZgvdPaOqfhrnidSWkpJgPr6kKLiU1IuDn/ufV9VeUq5PUfX2WVxQvV/6ZdaHIVBSdHjHaAmQ3BKSm0NKC0gufTSH9n2g+9cqGCnEPE9tq+mlRkh/oofw/qzfUrJ5FUYJFl4cZu6AYzEPcMxjnuPgYJSEywdvh4ft+/sEy0F7SXh5mlf4egkeu00JCWGfhHA5qLNk/36C5wf6l9lnab/99R54fqDfgecBK3MJpFtwJAfag5+OYaVbWbn1ZfpSrg3cEoJtwz520D5K37/S9z/mvQxH2Qf/eZQeQ3Dc7B+NH3ifOehnScxzDl4X+wu9PkhMOfDLPSX8BZ/cMpgaSuscLlewPrl58I19B62PCYqUlsH+dfmrlKMgOYTma9/kmN0LKBcB4a/s2F/lZX6llX14BW1QbrvK20r2/0ygNBpK+7mXXS4O24LlpDLbhdFQdvuY5yVhDDlGiR+8PibqDoRq+V/mB7WFy1ZRn9K9lV/2MvuorE/597vkoFco/2dW7s/Py7YFRYahVT4MLWH/egvbzQwsgYSURBISkkhISiIxMXwkJZMULiclB8+Tk5JJSk4mOTmJ5KQUUpKTSElOJjklhWbJycHz5GQsIRESEiEhKfgsQ+myHaI9MfnAL3z9b18ioL91hzBsyitRl3DE3D38ZRk+h3A5bI99Xq4P5daV7N+X7/+PeWlb6U/K9It5HrPf2G2IbSN2P05JaX0ePi9Xc3FJsFxc4hS746XL7pSU+P72ktifJU5xuM8y60uosG9JWFvxQfso7QtFxSXsLSxmT0ExewuK2VtYzN49xewpKGJPQTH7ikoO688swQppnlxC8xSnRYrTIqWE1OQSWqQEj+YpTotkp3lKCc1TSmiRHPxs39Lo16mEo9OhZWJt/O0ROTwKkkbKzGJmIDQVEYXiEic/Jmj2FAYBk18QtO0pLGZvGDp7Cor39w36B+17C4Ntt+8pDEOraH9oFRYffKFM97bN6dcpjX6d0ugf/uzXKY22LVIieAekqVCQiMRJYoLRslkSLZvF559ZYXEJewqKyd2VT9bmr8ja/BWfhj8/XJtHfuGBEVHHtGb7g6V/5zT6pafRr3Ma6WnNwqk8kSOnIBFpoJITE2jTPIE2zZPp16nsTQNLSpz12/fy6eZdQcBs+oqs3K/467/Xs2vfgQsDWqcm0b9zK/qlBwFzdDiS6damOQkJChipHn2ORKQJcXc279oXBMvmXftHMFmbvyJv94HPfjRPTtw/LRY7VXZU+xYkJeobupsKfY5ERA5iZnRunUrn1qn8R/+OZdZt3V0QM0UWjGQ+yM7jpX+v398nJTGBPh1blg2Yzmn06diSZkk6099UKUhEBID2LVMY0ac9I/q0L9O+K7+QNbm79wfMms1fsWzDDl5dtnH/FXsJBqce3ZHbzu7Pib3aV7B3acwUJCJySK1SkxnWsy3Depa9f1V+YTHZubvJyv2KTzbuZGbmOi556H1GHZPO7eccw5DubSKqWOqazpGISK3YU1DEk//6nIfnrWHH3kLGDO7CbWcP4JguTfvbAxuy6p4jUZCISK3amV/IY++u5dF31rK7oIixx3fj1q8PoE/HllGXJodJQRJDQSJS97btLuCRd7J54r3PKCgu4dKv9eDms/rRo12LqEuTalKQxFCQiEQnd9c+/vB2FtM/+ALHmTDiKG4c1Y/OrfVVtvWdgiSGgkQkehu27+XBuVnMXLCOxATj6lN6cd0ZR9MhTd81Ul8pSGIoSETqjy/y9vC7f37KS//OITU5ke+c1ofJp/elTYvkqEuTchQkMRQkIvVP1uav+O0/VvP3jzfSOjWJa0f2ZeJpfUiL073J5PApSGIoSETqrxUbdnL/m6v5x8pNtG+ZwvVnHM1Vp/QiNVmflI+agiSGgkSk/lu8bjv/+8Yq3vl0C51aNeOm0f24bHhP3XolQtUNkrjefc3MxpjZKjPLMrM7K+kzzsxWmNlyM3smbBtlZotjHvlmdlG47gkzWxuzblg8j0FE6sawnm35y3dPYsa1J9O7Q0t+9vJyRv/PPGYs+IKi4sP7kjCpW3EbkZhZIrAaOBvIARYAE9x9RUyf/sBMYLS7bzOzTu6+udx+2gNZQA9332NmTwB/d/dZ1a1FIxKRhsXdeTdrC//zxmqWrNtO7w4tuO3sAZx/XDcSdXv7OlMfRiQjgCx3z3b3AuA54MJyfSYD09x9G0D5EAldCrzm7nviWKuI1CNmxun90/nrDafyp6szSE1O5JbnFnPe7+bz+rKNNIUp+YYknkHSHVgXs5wTtsUaAAwws/fM7AMzG1PBfsYDz5Zru8/MPjazB8yswovQzexaM8s0s8zc3NwjPQYRiZCZcfagzrz6g9N58PITKCpxrnt6ERc8+C5zP9msQKkn4hkkFY0/y/+pJwH9gTOBCcCjZrb/FqNm1hUYCsyJ2WYqMBAYDrQH7qjoxd39EXfPcPeM9PT0Iz0GEakHEhKM84/rxhu3juR/vn08O/YWMumJBVzy0L/4V9aWqMtr8uIZJDlAz5jlHsCGCvq87O6F7r4WWEUQLKXGAS+5e2Fpg7tv9MA+4HGCKTQRaQKSEhO49MQe/PM/z+S+i4ewYXs+lz/6IZf/6QMWfr416vKarHgGyQKgv5n1MbMUgimq2eX6/BUYBWBmHQmmurJj1k+g3LRWOErBzAy4CFgWl+pFpN5KSUrgipN68faUM/nZ+YNYvWkXlzz0PpMe/4h1W3U6ta7FLUjcvQi4iWBaaiUw092Xm9k9ZjY27DYHyDOzFcBcYIq75wGYWW+CEc28cruebmZLgaVAR+DeeB2DiNRvqcmJfOc/+jD/R6O4Y8xAMj/bxnVPL6RQlwvXKX0gUUQajdeXbeS6pxdx+9kDuPms/lVvIIdUHy7/FRGpU2OGdOWC47vxf299ysqNO6Mup8lQkIhIo/LzsYNp0zyZHz6/RFNcdURBIiKNSvuWKdx70VCWb9jJQ2+vibqcJkFBIiKNzpghXbjg+G78XlNcdUJBIiKNkqa46o6CREQaJU1x1R0FiYg0WmOGdGGsprjiTkEiIo2aprjiT0EiIo1aO01xxZ2CREQaPU1xxZeCRESahGCKK0VTXHGgIBGRJqFdyxTuu3gIyzfs5A9zNcVVmxQkItJknDv4wBTXig2a4qotChIRaVJ+PnYwbVtoiqs2VStIzOwWM2ttgT+b2SIzOyfexYmI1LbSKa4VGzXFVVuqOyL5jrvvBM4B0oFJwK/jVpWISBydO7gLFw7TFFdtqW6QWPjzG8Dj7r4kpk1EpMG5+wJNcdWW6gbJQjN7gyBI5phZK0DvvIg0WJriqj3VDZLvAncCw919D5BMML0lItJgaYqrdlQ3SE4BVrn7djO7EvgJsCN+ZYmI1A1NcdVcdYPkIWCPmR0P/Aj4HHgqblWJiNSRdi1T+GU4xTVtblbU5TRI1Q2SInd34ELgd+7+O6BVVRuZ2RgzW2VmWWZ2ZyV9xpnZCjNbbmbPxLQXm9ni8DE7pr2PmX1oZp+a2QwzS6nmMYiIVOicwV24aFg3Hnwri+UbNNlyuKobJLvMbCpwFfCKmSUSnCepVNhnGnAeMAiYYGaDyvXpD0wFTnP3wcCtMav3uvuw8DE2pv2/gAfcvT+wjeD8jYhIjdy1f4rrY01xHabqBsllwD6Cz5N8CXQHflPFNiOALHfPdvcC4DmCEU2sycA0d98G4O6bD7VDMzNgNDArbHoSuKiaxyAiUqnSKa6VmuI6bNUKkjA8pgNtzOx8IN/dqzpH0h1YF7OcE7bFGgAMMLP3zOwDMxsTsy7VzDLD9tKw6ABsd/eiQ+wTADO7Ntw+Mzc3t+qDFJEmT1NcR6a6t0gZB3wEfBsYB3xoZpdWtVkFbV5uOQnoD5wJTAAeNbO24bqj3D0DuBz4rZkdXc19Bo3uj7h7hrtnpKenV1GqiEjg7rGDadcymOIqKNIUV3VUd2rrxwSfIbnG3a8mmLb6aRXb5AA9Y5Z7ABsq6POyuxe6+1pgFUGw4O4bwp/ZwNvACcAWoK2ZJR1inyIiR6xtixR+efFQVm7cyR/e1hRXdVQ3SBLKnb/Iq8a2C4D+4VVWKcB4YHa5Pn8FRgGYWUeCqa5sM2tnZs1i2k8DVoRXjs0FSkdD1wAvV/MYRESq5exBnTXFdRiqGySvm9kcM5toZhOBV4BXD7VBeB7jJmAOsBKY6e7LzeweMyu9CmsOkGdmKwgCYoq75wHHAplmtiRs/7W7rwi3uQP4TzPLIjhn8ufqHqyISHVpiqv6LPhPfjU6ml1CMDIwYL67vxTPwmpTRkaGZ2ZmRl2GiDQwb67YxOSnMrnlrP7cdvaAqMupc2a2MDxXfUhJVXUo5e4vAC/UqCoRkQbk7EGdufiE7kybm8U5gzszuFubqEuqlw45tWVmu8xsZwWPXWamO5yJSKN31wWDNMVVhUMGibu3cvfWFTxauXvruipSRCQqsVdx6YOKFdN3touIVCF2iktXcR1MQSIiUg2lU1y3z1yiKa5yFCQiItXQtkUKv7p4KJ98uUtTXOUoSEREqunrmuKqkIJEROQwaIrrYAoSEZHDEDvF9aCmuAAFiYjIYfv6oM5864Tu/GFuFsvWa4pLQSIicgTuumAw7Vum8MPnNcWlIBEROQJtWiTzS01xAQoSEZEjpimugIJERKQGNMWlIBERqZE2LZL51bea9hSXgkREpIbOOrYz3/pa053iUpCIiNSCu85vulNcChIRkVpQZorrrU+jLqdOKUhERGpJ6RTXtLfXsHJj0/nuPwWJiEgtuuv8wbRMSeR3/2g6oxIFiYhILWrTIpmJp/bm9eVfsnrTrqjLqRNxDRIzG2Nmq8wsy8zurKTPODNbYWbLzeyZsG2Ymb0ftn1sZpfF9H/CzNaa2eLwMSyexyAicrgmndaHFimJTeZ7S+IWJGaWCEwDzgMGARPMbFC5Pv2BqcBp7j4YuDVctQe4OmwbA/zWzNrGbDrF3YeFj8XxOgYRkSPRrmUKV53ci78t2cDaLbujLifu4jkiGQFkuXu2uxcAzwEXluszGZjm7tsA3H1z+HO1u38aPt8AbAbS41iriEit+u7pfUhOTOChtxv/qCSeQdIdWBeznBO2xRoADDCz98zsAzMbU34nZjYCSAHWxDTfF055PWBmzSp6cTO71swyzSwzNze3ZkciInKYOrVKZcKIo3h2tLVYAAAOOUlEQVRx0XrWb98bdTlxFc8gsQravNxyEtAfOBOYADwaO4VlZl2BvwCT3L30Ez5TgYHAcKA9cEdFL+7uj7h7hrtnpKdrMCMide/akX0xgz/OW1N15wYsnkGSA/SMWe4BbKigz8vuXujua4FVBMGCmbUGXgF+4u4flG7g7hs9sA94nGAKTUSk3unWtjmXfK0Hzy1Yx+ad+VGXEzfxDJIFQH8z62NmKcB4YHa5Pn8FRgGYWUeCqa7ssP9LwFPu/nzsBuEoBTMz4CJgWRyPQUSkRq4/82iKikv40zvZUZcSN3ELEncvAm4C5gArgZnuvtzM7jGzsWG3OUCema0A5hJcjZUHjANGAhMruMx3upktBZYCHYF743UMIiI11atDSy4c1p3pH37B1t0FUZcTF+Ze/rRF45ORkeGZmZlRlyEiTdSnm3Zx9gPzuXl0P24/55ioy6k2M1vo7hlV9dMn20VE4qx/51acN6QLT7z3GTv2FkZdTq1TkIiI1IEbR/Vj174i/vL+Z1GXUusUJCIidWBI9zaMHtiJP7+7lt37iqIup1YpSERE6siNo/qxbU8hz370RdSl1CoFiYhIHTmxVztOPboDf5yfTX5hcdTl1BoFiYhIHbppdD9yd+3j+cx1VXduIBQkIiJ16JS+HTixVzsenpdNYXHj+G53BYmISB0yM24a3Y/12/fy0r/XR11OrVCQiIjUsTMHpDOke2v+MDeL4pKG/6FwBYmISB0zM24a1Y/P8vbw94/L38u24VGQiIhE4JxBXejfKY1pc7MoaeCjEgWJiEgEEhKCcyWrN33FGys2RV1OjShIREQi8s2hXenVoQXT5mbRkG+gqyAREYlIUmICN5x5NEvX72De6ob7leAKEhGRCF18Qg+6tUnl92813FGJgkREJEIpSQlcd+bRLPx8Gx9kb426nCOiIBERidi4jJ6kt2rGtLlZUZdyRBQkIiIRS01OZPLpfXg3awuLvtgWdTmHTUEiIlIPXHFSL9q2SGbaWw1vVKIgERGpB1o2S+K7p/Xhn59sZvmGHVGXc1jiGiRmNsbMVplZlpndWUmfcWa2wsyWm9kzMe3XmNmn4eOamPYTzWxpuM//MzOL5zGIiNSVq0/tTatmSfxh7pqoSzkscQsSM0sEpgHnAYOACWY2qFyf/sBU4DR3HwzcGra3B+4CTgJGAHeZWbtws4eAa4H+4WNMvI5BRKQutWmezNWn9uLVZRvJ2rwr6nKqLZ4jkhFAlrtnu3sB8BxwYbk+k4Fp7r4NwN03h+3nAm+6+9Zw3ZvAGDPrCrR29/c9uOD6KeCiOB6DiEid+s5pfUhNSmxQo5J4Bkl3IPYrwHLCtlgDgAFm9p6ZfWBmY6rYtnv4/FD7BMDMrjWzTDPLzM1tuJ8YFZGmpUNaM6446SheXrKBz/N2R11OtcQzSCo6d1H+Y5tJBNNTZwITgEfNrO0htq3OPoNG90fcPcPdM9LT06tdtIhI1CaP7EtigvHwvIYxKolnkOQAPWOWewDlb7yfA7zs7oXuvhZYRRAslW2bEz4/1D5FRBq0zq1TGZfRg1kLc9i4Y2/U5VQpnkGyAOhvZn3MLAUYD8wu1+evwCgAM+tIMNWVDcwBzjGzduFJ9nOAOe6+EdhlZieHV2tdDbwcx2MQEYnE90cejTv8cV521KVUKW5B4u5FwE0EobASmOnuy83sHjMbG3abA+SZ2QpgLjDF3fPcfSvwC4IwWgDcE7YBXA88CmQBa4DX4nUMIiJR6dm+BRef0J1nP/qC3F37oi7nkKyh3m3ycGRkZHhmZmbUZYiIHJbs3K/4+v3zmDyyL1PPO7bOX9/MFrp7RlX99Ml2EZF6qm96Gucf142n3/+c7XsKoi6nUgoSEZF67MZR/dhdUMzj730WdSmVUpCIiNRjx3RpxTmDOvP4e2vZlV8YdTkVUpCIiNRzN43ux878Iv7ywedRl1IhBYmISD13XI+2nDEgnT+/s5a9BcVRl3MQBYmISANw0+h+5O0u4NmPvoi6lIMoSEREGoDhvdtzUp/2/HH+GvYV1a9RiYJERKSBuHl0fzbt3MeshTlVd65DChIRkQbitH4dGNazLQ+9vYbC4pKoy9lPQSIi0kCYGTeN6kfOtr3MXlx/7lerIBERaUDOOrYTx3ZtzbS3syguqR+3uFKQiIg0IKWjkuzc3by2bGPU5QAKEhGRBmfMkC4cnd6SB9/Koj7ceFdBIiLSwCQmGDeO6scnX+7inys3R12OgkREpCEae3w3erZvzu/nRj8qUZCIiDRASYkJXH9GP5as2867WVsirUVBIiLSQF1yYne6tE7l929lRVqHgkREpIFqlpTI98/oy0drt/LR2q1VbxAnChIRkQZs/PCj6NAyhQfnRjcqUZCIiDRgzVMS+d7pfZm/Opcl67ZHUoOCRESkgbvy5KNo0zw5slFJXIPEzMaY2SozyzKzOytYP9HMcs1scfj4Xtg+KqZtsZnlm9lF4bonzGxtzLph8TwGEZH6rlVqMpNO682bKzbxyZc76/z14xYkZpYITAPOAwYBE8xsUAVdZ7j7sPDxKIC7zy1tA0YDe4A3YraZErPN4ngdg4hIQzHx1N60TElk2tw1df7a8RyRjACy3D3b3QuA54ALj2A/lwKvufueWq1ORKQRadsihatO6c3fP97Amtyv6vS14xkk3YF1Mcs5YVt5l5jZx2Y2y8x6VrB+PPBsubb7wm0eMLNmFb24mV1rZplmlpmbm3tEByAi0pB87/Q+NEtK4KG363ZUEs8gsQrayn+O/29Ab3c/DvgH8GSZHZh1BYYCc2KapwIDgeFAe+COil7c3R9x9wx3z0hPTz+yIxARaUA6pjVjwoijeOnf61m3te4mceIZJDlA7AijB1Dmm1jcPc/d94WLfwJOLLePccBL7l4Ys81GD+wDHieYQhMREeDakX1JNOPheXU3KolnkCwA+ptZHzNLIZiimh3bIRxxlBoLrCy3jwmUm9Yq3cbMDLgIWFbLdYuINFhd2zTnkhN78HxmDpt25tfJa8YtSNy9CLiJYFpqJTDT3Zeb2T1mNjbs9gMzW25mS4AfABNLtzez3gQjmnnldj3dzJYCS4GOwL3xOgYRkYbo+jOOptidR+Zn18nrWdS3H64LGRkZnpmZGXUZIiJ15j9nLubVpRt5747RdEir8JqkKpnZQnfPqKpf0hHtXURE6rUbzuzH9j2F7N5XTIe0+L6WgkREpBHq1ymNxyYOr5PX0r22RESkRhQkIiJSIwoSERGpEQWJiIjUiIJERERqREEiIiI1oiAREZEaUZCIiEiNNIlbpJhZLvD5EW7eEdhSi+U0dHo/DtB7UZbej7Iaw/vRy92r/B6OJhEkNWFmmdW510xToffjAL0XZen9KKspvR+a2hIRkRpRkIiISI0oSKr2SNQF1DN6Pw7Qe1GW3o+ymsz7oXMkIiJSIxqRiIhIjShIRESkRhQkh2BmY8xslZllmdmdUdcTFTPraWZzzWylmS03s1uirqk+MLNEM/u3mf096lqiZmZtzWyWmX0S/j05JeqaomJmt4X/TpaZ2bNmlhp1TfGmIKmEmSUC04DzgEHABDMbFG1VkSkCbnf3Y4GTgRub8HsR6xZgZdRF1BO/A15394HA8TTR98XMugM/ADLcfQiQCIyPtqr4U5BUbgSQ5e7Z7l4APAdcGHFNkXD3je6+KHy+i+CXRPdoq4qWmfUAvgk8GnUtUTOz1sBI4M8A7l7g7tujrSpSSUBzM0sCWgAbIq4n7hQklesOrItZzqGJ//IEMLPewAnAh9FWErnfAj8CSqIupB7oC+QCj4dTfY+aWcuoi4qCu68H/gf4AtgI7HD3N6KtKv4UJJWzCtqa9LXSZpYGvADc6u47o64nKmZ2PrDZ3RdGXUs9kQR8DXjI3U8AdgNN8pyimbUjmLnoA3QDWprZldFWFX8KksrlAD1jlnvQBIaolTGzZIIQme7uL0ZdT8ROA8aa2WcEU56jzezpaEuKVA6Q4+6lo9RZBMHSFH0dWOvuue5eCLwInBpxTXGnIKncAqC/mfUxsxSCE2azI64pEmZmBPPfK939/qjriZq7T3X3Hu7em+DvxVvu3uj/11kZd/8SWGdmx4RNZwErIiwpSl8AJ5tZi/DfzVk0gQsPkqIuoL5y9yIzuwmYQ3DlxWPuvjzisqJyGnAVsNTMFodt/8/dX42wJqlfbgamh//pygYmRVxPJNz9QzObBSwiuNrx3zSBW6XoFikiIlIjmtoSEZEaUZCIiEiNKEhERKRGFCQiIlIjChIREakRBYlIPWdmZ+oOw1KfKUhERKRGFCQitcTMrjSzj8xssZn9Mfy+kq/M7H/NbJGZ/dPM0sO+w8zsAzP72MxeCu/RhJn1M7N/mNmScJujw92nxXzfx/TwU9Mi9YKCRKQWmNmxwGXAae4+DCgGrgBaAovc/WvAPOCucJOngDvc/ThgaUz7dGCaux9PcI+mjWH7CcCtBN+N05fgbgMi9YJukSJSO84CTgQWhIOF5sBmgtvMzwj7PA28aGZtgLbuPi9sfxJ43sxaAd3d/SUAd88HCPf3kbvnhMuLgd7Au/E/LJGqKUhEaocBT7r71DKNZj8t1+9Q9yQ61HTVvpjnxejfrtQjmtoSqR3/BC41s04AZtbezHoR/Bu7NOxzOfCuu+8AtpnZ6WH7VcC88DtecszsonAfzcysRZ0ehcgR0P9qRGqBu68ws58Ab5hZAlAI3EjwJU+DzWwhsIPgPArANcDDYVDE3i33KuCPZnZPuI9v1+FhiBwR3f1XJI7M7Ct3T4u6DpF40tSWiIjUiEYkIiJSIxqRiIhIjShIRESkRhQkIiJSIwoSERGpEQWJiIjUyP8H1YmPckOqhUAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# print(np.array(history.losses))\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "loss = train_history.history['loss']\n",
    "val_loss = train_history.history['val_loss']\n",
    "plt.plot(loss)\n",
    "plt.plot(val_loss)\n",
    "plt.legend(['loss', 'val_loss'])\n",
    "plt.title('AdaDelta')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35199335 0.34024575\n"
     ]
    }
   ],
   "source": [
    "a1 = np.linalg.norm(siamese_net.layers[2].layers[13].get_weights()[0])\n",
    "a2 = np.linalg.norm(siamese_net.layers[3].layers[13].get_weights()[0])\n",
    "print(a1, a2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error 0.34970559469480067\n",
      "Train acc 0.85792\n"
     ]
    }
   ],
   "source": [
    "pred = siamese_net.predict([x_train_a[:,:,:,None], x_train_b[:,:,:,None]])\n",
    "err = siam_y_train - pred.flatten()\n",
    "print('Error', np.sum(np.abs(err))/len(err))\n",
    "\n",
    "acc = []\n",
    "for i in range(len(siam_y_train)):\n",
    "    if siam_y_train[i]==np.round(pred[i]):\n",
    "        acc.append(1)\n",
    "    else:\n",
    "        acc.append(0)\n",
    "print(\"Train acc\", np.mean(acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error 0.4998481442928314\n",
      "Test Acc 0.5027692307692307\n"
     ]
    }
   ],
   "source": [
    "pred = siamese_net.predict([x_test_a[:,:,:,None], x_test_b[:,:,:,None]])\n",
    "err = siam_y_test - pred.flatten()\n",
    "print('Error', np.sum(np.abs(err))/len(err))\n",
    "\n",
    "acc = []\n",
    "for i in range(len(siam_y_test)):\n",
    "    if siam_y_test[i]==np.round(pred[i]):\n",
    "        acc.append(1)\n",
    "    else:\n",
    "        acc.append(0)\n",
    "print(\"Test Acc\", np.mean(acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "152 208 [1. 0.] [0. 1.] 1 [[0.00483108]]\n",
      "249 67 [1. 0.] [1. 0.] 0 [[0.9341727]]\n",
      "131 104 [1. 0.] [1. 0.] 0 [[0.02960188]]\n",
      "256 75 [0. 1.] [0. 1.] 0 [[0.02166533]]\n",
      "277 106 [1. 0.] [0. 1.] 1 [[0.9300978]]\n",
      "147 26 [0. 1.] [0. 1.] 0 [[0.02859868]]\n",
      "202 37 [0. 1.] [1. 0.] 1 [[0.9780969]]\n",
      "169 208 [0. 1.] [0. 1.] 0 [[0.00180813]]\n",
      "138 283 [0. 1.] [1. 0.] 1 [[0.96969765]]\n",
      "119 143 [0. 1.] [0. 1.] 0 [[0.00714612]]\n",
      "148 47 [0. 1.] [0. 1.] 0 [[0.76573807]]\n",
      "93 175 [1. 0.] [0. 1.] 1 [[0.919874]]\n",
      "26 203 [0. 1.] [0. 1.] 0 [[0.04041896]]\n",
      "150 250 [1. 0.] [1. 0.] 0 [[0.01187098]]\n",
      "322 218 [0. 1.] [1. 0.] 1 [[0.8838622]]\n",
      "258 6 [0. 1.] [1. 0.] 1 [[0.0279586]]\n",
      "316 119 [0. 1.] [0. 1.] 0 [[0.9744778]]\n",
      "129 290 [1. 0.] [0. 1.] 1 [[0.00440152]]\n",
      "69 21 [1. 0.] [0. 1.] 1 [[0.00995419]]\n",
      "110 11 [0. 1.] [1. 0.] 1 [[0.03733028]]\n"
     ]
    }
   ],
   "source": [
    "for i in range(20):\n",
    "    a = x_train_a[None,a_train[i],:,:,None]\n",
    "    b = x_train_b[None,b_train[i],:,:,None]\n",
    "    # b = np.random.randn(*a.shape)\n",
    "    pred = siamese_net.predict([a,b])\n",
    "\n",
    "    print(a_train[i],b_train[i],y_train_a[i],y_train_b[i],siam_y_train[i], pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# now test sample by sample\n",
    "cor_test = []\n",
    "L = xx_test.shape\n",
    "for sample_no in range(xx_test.shape[0]):\n",
    "    a = xx_test[None,sample_no,0,:,:,None]\n",
    "    b = np.random.randn(*a.shape)\n",
    "    y = yy_test[sample_no]\n",
    "    if oneleg == False:\n",
    "        pred = siamese_net.predict([a,b])[1]\n",
    "    else:\n",
    "        pred = siamese_net.predict([a,b])\n",
    "    if ((pred[0][0]<pred[0][1]) and (y[0] < y[1])) or ((pred[0][0]>pred[0][1]) and (y[0] > y[1])):\n",
    "        cor_test.append(1)\n",
    "    else:\n",
    "        cor_test.append(0)\n",
    "\n",
    "cor_train = []\n",
    "L = xx_train.shape\n",
    "for sample_no in range(xx_train.shape[0]):\n",
    "    a = xx_train[None,sample_no,0,:,:,None]\n",
    "    b = np.random.randn(*a.shape)\n",
    "    y = yy_train[sample_no]\n",
    "    if oneleg == False:\n",
    "        pred = siamese_net.predict([a,b])[1]\n",
    "    else:\n",
    "        pred = siamese_net.predict([a,b])\n",
    "    if ((pred[0][0]<pred[0][1]) and (y[0] < y[1])) or ((pred[0][0]>pred[0][1]) and (y[0] > y[1])):\n",
    "        cor_train.append(1)\n",
    "    else:\n",
    "        cor_train.append(0)\n",
    "        \n",
    "cor_val = []\n",
    "L = xx_val.shape\n",
    "for sample_no in range(xx_val.shape[0]):\n",
    "    a = xx_val[None,sample_no,0,:,:,None]\n",
    "    b = np.random.randn(*a.shape)\n",
    "    y = yy_val[sample_no]\n",
    "    if oneleg == False:\n",
    "        pred = siamese_net.predict([a,b])[1]\n",
    "    else:\n",
    "        pred = siamese_net.predict([a,b])\n",
    "    if ((pred[0][0]<pred[0][1]) and (y[0] < y[1])) or ((pred[0][0]>pred[0][1]) and (y[0] > y[1])):\n",
    "        cor_val.append(1)\n",
    "    else:\n",
    "        cor_val.append(0)\n",
    "\n",
    "# Report accuracies\n",
    "print('Accuracy on the training data:', np.mean(cor_train))\n",
    "print('Accuracy on the test data:', np.mean(cor_test))\n",
    "print('Accuracy on the validation data:', np.mean(cor_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation trial 0 \t -> 0.4927 0.4892 [ 1.  0.] Correct: True\n",
      "Validation trial 1 \t -> 0.3450 0.6100 [ 0.  1.] Correct: True\n",
      "Validation trial 2 \t -> 0.1880 0.7277 [ 0.  1.] Correct: True\n",
      "Validation trial 3 \t -> 0.4959 0.4871 [ 0.  1.] Correct: False\n",
      "Validation trial 4 \t -> 0.6038 0.4120 [ 0.  1.] Correct: False\n",
      "Validation trial 5 \t -> 0.6666 0.3850 [ 1.  0.] Correct: True\n",
      "Validation trial 6 \t -> 0.2217 0.6993 [ 0.  1.] Correct: True\n",
      "Validation trial 7 \t -> 0.4258 0.5433 [ 0.  1.] Correct: True\n",
      "Validation trial 8 \t -> 0.6632 0.3904 [ 1.  0.] Correct: True\n",
      "Validation trial 9 \t -> 0.7864 0.2785 [ 1.  0.] Correct: True\n",
      "Validation trial 10 \t -> 0.7415 0.3238 [ 0.  1.] Correct: False\n",
      "Validation trial 11 \t -> 0.5459 0.4581 [ 1.  0.] Correct: True\n",
      "Validation trial 12 \t -> 0.7882 0.2765 [ 0.  1.] Correct: False\n",
      "Validation trial 13 \t -> 0.4053 0.5488 [ 0.  1.] Correct: True\n",
      "Validation trial 14 \t -> 0.7553 0.3066 [ 1.  0.] Correct: True\n",
      "Validation trial 15 \t -> 0.7026 0.3425 [ 0.  1.] Correct: False\n",
      "Validation trial 16 \t -> 0.6931 0.3549 [ 0.  1.] Correct: False\n",
      "Validation trial 17 \t -> 0.6808 0.3767 [ 1.  0.] Correct: True\n",
      "Validation trial 18 \t -> 0.5261 0.4750 [ 0.  1.] Correct: False\n",
      "Validation trial 19 \t -> 0.7105 0.3387 [ 1.  0.] Correct: True\n",
      "Validation trial 20 \t -> 0.5706 0.4501 [ 0.  1.] Correct: False\n",
      "Validation trial 21 \t -> 0.7188 0.3417 [ 1.  0.] Correct: True\n",
      "Validation trial 22 \t -> 0.5755 0.4352 [ 0.  1.] Correct: False\n",
      "Validation trial 23 \t -> 0.6920 0.3587 [ 0.  1.] Correct: False\n",
      "Validation trial 24 \t -> 0.5575 0.4588 [ 1.  0.] Correct: True\n",
      "Validation trial 25 \t -> 0.4086 0.5548 [ 1.  0.] Correct: False\n",
      "Validation trial 26 \t -> 0.4905 0.4956 [ 0.  1.] Correct: True\n",
      "Validation trial 27 \t -> 0.3545 0.5884 [ 0.  1.] Correct: True\n",
      "Validation trial 28 \t -> 0.3992 0.5565 [ 0.  1.] Correct: True\n",
      "Validation trial 29 \t -> 0.2394 0.6909 [ 0.  1.] Correct: True\n",
      "Validation trial 30 \t -> 0.6976 0.3504 [ 0.  1.] Correct: False\n",
      "Validation trial 31 \t -> 0.4621 0.5196 [ 0.  1.] Correct: True\n",
      "Validation trial 32 \t -> 0.3602 0.5908 [ 1.  0.] Correct: False\n",
      "Validation trial 33 \t -> 0.4415 0.5338 [ 1.  0.] Correct: False\n",
      "Validation trial 34 \t -> 0.9122 0.1609 [ 1.  0.] Correct: True\n",
      "Validation trial 35 \t -> 0.6590 0.3946 [ 0.  1.] Correct: False\n",
      "Validation trial 36 \t -> 0.4265 0.5436 [ 1.  0.] Correct: False\n",
      "Validation trial 37 \t -> 0.7065 0.3422 [ 1.  0.] Correct: True\n",
      "Validation trial 38 \t -> 0.3919 0.5708 [ 0.  1.] Correct: True\n",
      "Validation trial 39 \t -> 0.4672 0.5246 [ 1.  0.] Correct: False\n",
      "Validation trial 40 \t -> 0.3899 0.5659 [ 0.  1.] Correct: True\n",
      "Validation trial 41 \t -> 0.7328 0.3324 [ 1.  0.] Correct: True\n",
      "Validation trial 42 \t -> 0.6615 0.3784 [ 1.  0.] Correct: True\n",
      "Validation trial 43 \t -> 0.6983 0.3503 [ 0.  1.] Correct: False\n",
      "Validation trial 44 \t -> 0.6723 0.3758 [ 1.  0.] Correct: True\n",
      "Accuracy on the VALIDATION data: 0.6\n"
     ]
    }
   ],
   "source": [
    "# The moment of truth. Check Validation accuracy (completely unseen data):\n",
    "cor_val = []\n",
    "for i in range (len(xx_val)):\n",
    "    a = xx_val[None,i,0,:,:,None]\n",
    "#     a = np.random.randn(*a.shape)\n",
    "\n",
    "    # YOU CAN FEED INTO THE SECOND LEG ANYTHING YOU WANT,\n",
    "    # THE DENSE LAYER IS CONNECTED TO LEG 1, BUT THE WEIGHTS \n",
    "    # IN LEG 1 AND LEG 2 ARE THE SAME, I.E. SHARED.\n",
    "#     b = xx_val[None,i,1,:,:,None]\n",
    "    b = np.random.randn(*a.shape) \n",
    "    pred = siamese_net.predict([a, b])[1]\n",
    "       \n",
    "    if ((pred[0][0]<pred[0][1]) and (yy_val[i][0] < yy_val[i][1]) or\n",
    "       ((pred[0][0]>pred[0][1]) and (yy_val[i][0] > yy_val[i][1]))):\n",
    "        cor_val.append(1)\n",
    "        success = True\n",
    "    else:\n",
    "        cor_val.append(0)\n",
    "        success = False\n",
    "    \n",
    "    print('Validation trial {} \\t -> {:1.4f} {:1.4f} {} Correct: {}'.format (i, pred[0][0],pred[0][1], yy_val[i], success))\n",
    "print('Accuracy on the VALIDATION data:', np.mean(cor_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 805,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_indexes = lambda x, xs: [i for (y, i) in zip(xs, range(len(xs))) if np.array_equal(x, y)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 806,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "setting an array element with a sequence.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-806-7d64cf01bfe9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mind_A\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     pred_diff[i] = siamese_net.predict([xx_train[None,ind_A[i],0,:,:,None],\n\u001b[0;32m----> 9\u001b[0;31m                                         xx_train[None,ind_B[i],0,:,:,None]])\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     pred_same[i] = siamese_net.predict([xx_train[None,ind_A[i],     0,:,:,None],\n",
      "\u001b[0;31mValueError\u001b[0m: setting an array element with a sequence."
     ]
    }
   ],
   "source": [
    "ind_A = get_indexes(np.array([0, 1]),yy_train)\n",
    "ind_B = get_indexes(np.array([1, 0]),yy_train)\n",
    "ind_A_perm = np.random.permutation(ind_A)\n",
    "pred_same = np.zeros(len(ind_A))\n",
    "pred_diff = np.zeros(len(ind_A))\n",
    "\n",
    "for i in range(len(ind_A)):\n",
    "    pred_diff[i] = siamese_net.predict([xx_train[None,ind_A[i],0,:,:,None],\n",
    "                                        xx_train[None,ind_B[i],0,:,:,None]])\n",
    "\n",
    "    pred_same[i] = siamese_net.predict([xx_train[None,ind_A[i],     0,:,:,None],\n",
    "                                        xx_train[None,ind_A_perm[i],0,:,:,None]])\n",
    "\n",
    "print ('Pred_diff_train', np.sum(pred_diff))\n",
    "print ('Pred_same_train', np.sum(pred_same))\n",
    "ind_A = get_indexes(np.array([0, 1]),yy_test)\n",
    "ind_B = get_indexes(np.array([1, 0]),yy_test)\n",
    "ind_A_perm = np.random.permutation(ind_A)\n",
    "pred_same = np.zeros(len(ind_A))\n",
    "pred_diff = np.zeros(len(ind_A))\n",
    "    \n",
    "for i in range(len(ind_A)):\n",
    "    pred_diff[i] = siamese_net.predict([xx_test[None,ind_A[i],0,:,:,None],\n",
    "                                        xx_test[None,ind_B[i],0,:,:,None]])\n",
    "\n",
    "    pred_same[i] = siamese_net.predict([xx_test[None,ind_A[i],     0,:,:,None],\n",
    "                                        xx_test[None,ind_A_perm[i],0,:,:,None]])\n",
    "#     print('yy_train A', ind_A[i], yy_train[ind_A[i]], 'yy_train B', ind_B[i], yy_train[ind_B[i]], pred[i])\n",
    "   \n",
    "print ('Pred_diff_test', np.sum(pred_diff))\n",
    "print ('Pred_same_test', np.sum(pred_same))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 860,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_73 (InputLayer)        (None, 60, 128, 1)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_73 (Conv2D)           (None, 60, 128, 4)        256       \n",
      "_________________________________________________________________\n",
      "batch_normalization_217 (Bat (None, 60, 128, 4)        16        \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_73 (Depthwi (None, 1, 128, 8)         480       \n",
      "_________________________________________________________________\n",
      "batch_normalization_218 (Bat (None, 1, 128, 8)         32        \n",
      "_________________________________________________________________\n",
      "activation_145 (Activation)  (None, 1, 128, 8)         0         \n",
      "_________________________________________________________________\n",
      "average_pooling2d_145 (Avera (None, 1, 32, 8)          0         \n",
      "_________________________________________________________________\n",
      "dropout_145 (Dropout)        (None, 1, 32, 8)          0         \n",
      "_________________________________________________________________\n",
      "separable_conv2d_73 (Separab (None, 1, 32, 8)          192       \n",
      "_________________________________________________________________\n",
      "batch_normalization_219 (Bat (None, 1, 32, 8)          32        \n",
      "_________________________________________________________________\n",
      "activation_146 (Activation)  (None, 1, 32, 8)          0         \n",
      "_________________________________________________________________\n",
      "average_pooling2d_146 (Avera (None, 1, 4, 8)           0         \n",
      "_________________________________________________________________\n",
      "dropout_146 (Dropout)        (None, 1, 4, 8)           0         \n",
      "_________________________________________________________________\n",
      "flatten_73 (Flatten)         (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_145 (Dense)            (None, 10)                330       \n",
      "_________________________________________________________________\n",
      "dense_148 (Dense)            (None, 2)                 22        \n",
      "=================================================================\n",
      "Total params: 1,360\n",
      "Trainable params: 1,320\n",
      "Non-trainable params: 40\n",
      "_________________________________________________________________\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_73 (InputLayer)        (None, 60, 128, 1)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_74 (Conv2D)           (None, 60, 128, 4)        256       \n",
      "_________________________________________________________________\n",
      "batch_normalization_220 (Bat (None, 60, 128, 4)        16        \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_74 (Depthwi (None, 1, 128, 8)         480       \n",
      "_________________________________________________________________\n",
      "batch_normalization_221 (Bat (None, 1, 128, 8)         32        \n",
      "_________________________________________________________________\n",
      "activation_147 (Activation)  (None, 1, 128, 8)         0         \n",
      "_________________________________________________________________\n",
      "average_pooling2d_147 (Avera (None, 1, 32, 8)          0         \n",
      "_________________________________________________________________\n",
      "dropout_147 (Dropout)        (None, 1, 32, 8)          0         \n",
      "_________________________________________________________________\n",
      "separable_conv2d_74 (Separab (None, 1, 32, 8)          192       \n",
      "_________________________________________________________________\n",
      "batch_normalization_222 (Bat (None, 1, 32, 8)          32        \n",
      "_________________________________________________________________\n",
      "activation_148 (Activation)  (None, 1, 32, 8)          0         \n",
      "_________________________________________________________________\n",
      "average_pooling2d_148 (Avera (None, 1, 4, 8)           0         \n",
      "_________________________________________________________________\n",
      "dropout_148 (Dropout)        (None, 1, 4, 8)           0         \n",
      "_________________________________________________________________\n",
      "flatten_74 (Flatten)         (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_146 (Dense)            (None, 10)                330       \n",
      "_________________________________________________________________\n",
      "dense_148 (Dense)            (None, 2)                 22        \n",
      "=================================================================\n",
      "Total params: 1,360\n",
      "Trainable params: 1,320\n",
      "Non-trainable params: 40\n",
      "_________________________________________________________________\n",
      "['sequential_83', 'sequential_84']\n",
      "['dense_145', 'dense_146']\n",
      "0.735211 0.597782\n",
      "\n",
      "In the models made out the layers take from the siamese model:\n",
      "dense_145 dense_146\n",
      "0.735211 0.597782\n"
     ]
    }
   ],
   "source": [
    "testmod_1 = Sequential()\n",
    "testmod_1.add(siamese_net.layers[0])\n",
    "for i in range(14):\n",
    "    testmod_1.add(siamese_net.layers[2].layers[i])\n",
    "testmod_1.add(siamese_net.layers[6])\n",
    "testmod_1.summary()\n",
    "\n",
    "testmod_2 = Sequential()\n",
    "testmod_2.add(siamese_net.layers[0])\n",
    "for i in range(14):\n",
    "    testmod_2.add(siamese_net.layers[3].layers[i])\n",
    "testmod_2.add(siamese_net.layers[6])\n",
    "testmod_2.summary()\n",
    "    \n",
    "pred_1 = testmod_1.predict(xx_train[:,0,:,:,None])\n",
    "pred_2 = testmod_2.predict(xx_train[:,0,:,:,None])\n",
    "\n",
    "\n",
    "# check if the weights in the siamese are really the same in all the three leg of the model:\n",
    "a1 = np.linalg.norm(siamese_net.layers[2].layers[13].get_weights()[0])\n",
    "a2 = np.linalg.norm(siamese_net.layers[3].layers[13].get_weights()[0])\n",
    "\n",
    "print([siamese_net.layers[x].name for x in range(2,4)])\n",
    "print([siamese_net.layers[x].layers[13].name for x in range(2,4)])\n",
    "print(a1,a2)\n",
    "print('\\nIn the models made out the layers take from the siamese model:')\n",
    "a1 = np.linalg.norm(testmod_1.layers[14].get_weights()[0])\n",
    "a2 = np.linalg.norm(testmod_2.layers[14].get_weights()[0])\n",
    "print(testmod_1.layers[14].name, testmod_2.layers[14].name)\n",
    "print(a1,a2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labs10 = []\n",
    "labs01 = []\n",
    "\n",
    "for i in range(len(xx_train)):\n",
    "    pred_1 = testmod_1.predict(xx_train[None,i,0,:,:,None])\n",
    "    pred_2 = testmod_2.predict(xx_train[None,i,0,:,:,None])\n",
    "    print('result:\\t', i, yy_train[i], pred_1, pred_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc_01 = 0\n",
    "for i in range(len(labs01)):\n",
    "    acc_01 += np.linalg.norm(pred_1-testmod_1.predict(xx_train[None,labs01[i],:,:,None]))\n",
    "\n",
    "acc_10 = 0\n",
    "for i in range(len(labs10)):\n",
    "    acc_10 += np.linalg.norm(pred_1-testmod_1.predict(xx_train[None,labs10[i],:,:,None]))\n",
    "\n",
    "print('acc_01:', acc_01)\n",
    "print('acc_10:', acc_10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_33 (InputLayer)        (None, 1, 60, 128)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_26 (Conv2D)           (None, 4, 60, 128)        500       \n",
      "_________________________________________________________________\n",
      "batch_normalization_28 (Batc (None, 4, 60, 128)        16        \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_10 (Depthwi (None, 8, 1, 128)         480       \n",
      "_________________________________________________________________\n",
      "batch_normalization_29 (Batc (None, 8, 1, 128)         32        \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 8, 1, 128)         0         \n",
      "_________________________________________________________________\n",
      "average_pooling2d_14 (Averag (None, 8, 1, 32)          0         \n",
      "_________________________________________________________________\n",
      "dropout_14 (Dropout)         (None, 8, 1, 32)          0         \n",
      "_________________________________________________________________\n",
      "separable_conv2d_7 (Separabl (None, 8, 1, 32)          192       \n",
      "_________________________________________________________________\n",
      "batch_normalization_30 (Batc (None, 8, 1, 32)          32        \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 8, 1, 32)          0         \n",
      "_________________________________________________________________\n",
      "average_pooling2d_15 (Averag (None, 8, 1, 4)           0         \n",
      "_________________________________________________________________\n",
      "dropout_15 (Dropout)         (None, 8, 1, 4)           0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 2)                 66        \n",
      "_________________________________________________________________\n",
      "softmax (Activation)         (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 1,318\n",
      "Trainable params: 1,278\n",
      "Non-trainable params: 40\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def EEGNet(nb_classes, Chans = 64, Samples = 128, \n",
    "             dropoutRate = 0.25, kernLength = 64, F1 = 4, \n",
    "             D = 2, F2 = 8, dropoutType = 'Dropout'):\n",
    "    \"\"\" Keras Implementation of EEGNet (https://arxiv.org/abs/1611.08024)\n",
    "\n",
    "    Inputs:\n",
    "        \n",
    "      nb_classes      : int, number of classes to classify\n",
    "      Chans, Samples  : number of channels and time points in the EEG data\n",
    "      dropoutRate     : dropout fraction\n",
    "      kernLength      : length of temporal convolution in first layer. We found\n",
    "                        that setting this to be half the sampling rate worked\n",
    "                        well in practice. For the SMR dataset in particular\n",
    "                        since the data was high-passed at 4Hz we used a kernel\n",
    "                        length of 32.     \n",
    "      F1, F2          : number of temporal filters (F1) and number of pointwise\n",
    "                        filters (F2) to learn. Default: F1 = 4, F2 = F1 * D. \n",
    "      D               : number of spatial filters to learn within each temporal\n",
    "                        convolution. Default: D = 2\n",
    "      dropoutType     : Either SpatialDropout2D or Dropout, passed as a string.\n",
    "\n",
    "    \"\"\"\n",
    "    \n",
    "    if dropoutType == 'SpatialDropout2D':\n",
    "        dropoutType = SpatialDropout2D\n",
    "    elif dropoutType == 'Dropout':\n",
    "        dropoutType = Dropout\n",
    "    else:\n",
    "        raise ValueError('dropoutType must be one of SpatialDropout2D '\n",
    "                         'or Dropout, passed as a string.')\n",
    "    \n",
    "    input1   = Input(shape = (1, Chans, Samples))\n",
    "\n",
    "    ##################################################################\n",
    "    block1       = Conv2D(F1, (1, kernLength), padding = 'same',\n",
    "                                   input_shape = (1, Chans, Samples),\n",
    "                                   use_bias = False, data_format='channels_first')(input1)\n",
    "    block1       = BatchNormalization(axis = 1)(block1)\n",
    "    block1       = DepthwiseConv2D((Chans, 1), use_bias = False, \n",
    "                                   depth_multiplier = D,\n",
    "                                   depthwise_constraint = max_norm(1.), data_format='channels_first')(block1)\n",
    "    block1       = BatchNormalization(axis = 1)(block1)\n",
    "    block1       = Activation('elu')(block1)\n",
    "    block1       = AveragePooling2D((1, 4), data_format='channels_first')(block1)\n",
    "    block1       = dropoutType(dropoutRate)(block1)\n",
    "    \n",
    "    block2       = SeparableConv2D(F2, (1, 16),\n",
    "                                   use_bias = False,\n",
    "                                   padding = 'same',\n",
    "                                   data_format='channels_first')(block1)\n",
    "    block2       = BatchNormalization(axis = 1)(block2)\n",
    "    block2       = Activation('elu')(block2)\n",
    "    block2       = AveragePooling2D((1, 8), data_format='channels_first')(block2)\n",
    "    block2       = dropoutType(dropoutRate)(block2)\n",
    "        \n",
    "    flatten      = Flatten(name = 'flatten')(block2)\n",
    "    \n",
    "    dense        = Dense(nb_classes, name = 'dense', \n",
    "                         kernel_constraint = max_norm(0.25))(flatten)\n",
    "    softmax      = Activation('softmax', name = 'softmax')(dense)\n",
    "    \n",
    "    return Model(inputs=input1, outputs=softmax)\n",
    "\n",
    "model  = EEGNet(nb_classes = 2,\n",
    "                Chans = 60,\n",
    "                Samples = 128,\n",
    "                kernLength = 125)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24\n",
      " 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49\n",
      " 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74\n",
      " 75 76 77 78 79]\n",
      "[80 81 82 83 84 85 86 87 88 89]\n",
      "[90 91 92 93 94 95 96 97 98 99]\n"
     ]
    }
   ],
   "source": [
    "def TrainTestVal_Split(leng, traintest, testval):\n",
    "    leng = 100\n",
    "    x = np.arange(leng)\n",
    "    train_id, test_id, val_id = np.split(\n",
    "        x,[np.round(leng*traintest).astype(int),\n",
    "           np.round(leng*testval).astype(int)])\n",
    "    return train_id, test_id, val_id\n",
    "\n",
    "# train_id, test_id, val_id = TrainTestVal_Split(100, 0.8, 0.9)\n",
    "# print(train_id)\n",
    "# print(test_id)\n",
    "# print(val_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in range(xx_train.shape[0]):\n",
    "    hit = []\n",
    "    for i in range(X.shape[0]):\n",
    "        if np.linalg.norm(X[i,:,:] - xx_train[j,0,:,:])==0:\n",
    "            hit.append(i)\n",
    "            print(hit)\n",
    "            print('success' if np.linalg.norm(Y[i] - yy_train[j])==0 else 'FAIL!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before augmenting (324, 2, 60, 128)\n",
      "(1620, 2, 60, 128)\n",
      "(1620, 1, 60, 128)\n",
      "(1620, 1, 60, 128)\n",
      "After augmenting (1620, 2, 60, 128)\n",
      "(1620, 2, 60, 128)\n",
      "yy_train_aug (1620, 2)\n",
      "\n",
      "After augmenting:\n",
      " (1620, 2, 60, 128) \n",
      " (1620, 2)\n"
     ]
    }
   ],
   "source": [
    "def train_augment(xx_train, yy_train):\n",
    "    print('Before augmenting', xx_train.shape)\n",
    "    temp_xx = np.concatenate([xx_train, xx_train, xx_train, xx_train, xx_train], axis=0)\n",
    "    print(temp_xx.shape)\n",
    "    temp_xx_0 = temp_xx[:,0,:,:]\n",
    "    temp_xx_0 = temp_xx_0.reshape(temp_xx_0.shape + (1,)).transpose(0,3,1,2)\n",
    "    print(temp_xx_0.shape)\n",
    "\n",
    "    temp_xx_1 = temp_xx[:,1,:,:]\n",
    "    np.random.shuffle(temp_xx_1)\n",
    "    temp_xx_1 = temp_xx_1.reshape(temp_xx_1.shape + (1,)).transpose(0,3,1,2)\n",
    "    print(temp_xx_1.shape)\n",
    "\n",
    "    xx_train_aug = np.concatenate([temp_xx_0, temp_xx_1], axis=1)\n",
    "    print('After augmenting', xx_train_aug.shape)\n",
    "    print(xx_train_aug.shape)\n",
    "\n",
    "    yy_train_aug = np.concatenate([yy_train, yy_train, yy_train, yy_train, yy_train], axis=0)\n",
    "    print('yy_train_aug', yy_train_aug.shape)\n",
    "    return xx_train_aug, yy_train_aug\n",
    "\n",
    "if AUGMENT==True:\n",
    "    xx_train, yy_train = train_augment(xx_train, yy_train)\n",
    "    print('\\nAfter augmenting:\\n', xx_train.shape,'\\n', yy_train.shape )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "a,b = [],[]\n",
    "for i in range(len(X)):\n",
    "    if np.array_equal(Y[i],np.array([0,1])):\n",
    "        a.append(i)\n",
    "    if np.array_equal(Y[i],np.array([1,0])):\n",
    "        b.append(i)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XX_a\t (225, 2, 60, 128)\n",
      "XX_b\t (225, 2, 60, 128)\n",
      "XX\t (450, 2, 60, 128)\n",
      "\n",
      "XX_trai\t (324, 2, 60, 128)\n",
      "XX_test\t (81, 2, 60, 128)\n",
      "XX_val\t (45, 2, 60, 128)\n",
      "yy_trai\t (324, 2)\n",
      "yy_test\t (81, 2)\n",
      "yy_val\t (45, 2)\n",
      "\n",
      "If you use RANDOM SEED, check if you generate the same numbers every time:\n",
      "[291 144  54 215 210 167 184 442 273  96  62 420 255  76 278  86 270 152\n",
      " 122 325  48 326 264 110 436  31 410 214 361 257 357  73 407 205 115  97\n",
      " 172 303 285   9 242  59 366 119 118 348 267 141  26 345 447 437 224 446\n",
      " 178 189 359 245 107 262 108 330 432 272 364 250 279 346  80 388 439 206\n",
      " 171  33  29 164 258 235 336  13  74  45 406 415 434 317 183 374  92 193\n",
      " 208 219 428  99 225 368  51 256 342  19 394 237  44 360 132 192 315  11\n",
      " 296 414 431 424 169 367   4 412 133 283  68 234 397   5 395 328 101 288\n",
      "  40 165 248 112 449 307 148 308 377 157 309 338 204 320 339 127  64 221\n",
      " 135 378 401  77 159 353 236 138 381  93  25 408 337 413 117 260  58 444\n",
      "  79 266 306 318  47 386 227 251  72 292 380 203 387 176 179 399  55 198\n",
      " 123  70 438 261  36 284 150  60   8 196 441  21  41  82  63  85  12 313\n",
      " 389 271  27 423 400  69 147 168 295 268 411 425 191 393 409 290 358  81\n",
      "  32 201 376 116 402 244 429 143 222 324 182 254 427 247 422 124 104   0\n",
      "  66 289 263 158 103  56 259 146 341 344 299  95 293 207 240 362 109 153\n",
      " 329 298  22 355  75 404 443 331 311  37 213 416 333 276 310 120 239 175\n",
      " 275 321  15 391 161 156  30 121 177 160 130 233  14 220 281 134 419 105\n",
      " 332  53 145 216 327 417 316 113 356 323  49 371 249  98 142  20  89  71\n",
      " 398  28  18   7 243 154 229 194 287 369 162 231 253 382 246 190 166 365]\n",
      "[269  34 373 200 319 418 372 277 128 197 312  38 209  83 195 140 218 370\n",
      " 379 151   3 343 131  39  50  46 232 106 403  17 352   2 445  35 375 302\n",
      " 347 390 170 282  94 217  16 340  67 430 228 392 354 349  87 137 322 334\n",
      " 435 102 212  57 363 125  84 351 350 199 405 186 139 136 286 173  23 280\n",
      "  42 185  90 265 226  52 180 335 384]\n",
      "[300  43 211 188 129 238  88 174 294 230 149 305  24 163 385  91  65 274\n",
      "  78 426  61 433 100 114 297 304 202 223   6 155 181 126 314 440 448 187\n",
      " 301 252   1 383  10 421 396 111 241]\n"
     ]
    }
   ],
   "source": [
    "XX_a = np.zeros([len(a),2,X.shape[1],X.shape[2]])\n",
    "XX_b = np.zeros([len(b),2,X.shape[1],X.shape[2]])\n",
    "XX = np.zeros([len(a)+len(b),2,X.shape[1],X.shape[2]])\n",
    "\n",
    "YY_a = np.zeros([len(a),2])\n",
    "YY_b = np.zeros([len(b),2])\n",
    "YY = np.zeros([len(b)+len(b),2])\n",
    "\n",
    "XX_a[:,0,:,:] = X[a,:,:]\n",
    "XX_a[:,1,:,:] = X[np.random.permutation(a),:,:]\n",
    "YY_a = Y[a]\n",
    "XX_b[:,0,:,:] = X[b,:,:]\n",
    "XX_b[:,1,:,:] = X[np.random.permutation(b),:,:]\n",
    "YY_b = Y[b]\n",
    "\n",
    "XX = np.concatenate([XX_a, XX_b], axis=0)\n",
    "YY = np.concatenate([YY_a, YY_b], axis=0)\n",
    "\n",
    "print('XX_a\\t', XX_a.shape)\n",
    "print('XX_b\\t', XX_b.shape)\n",
    "print('XX\\t', XX.shape)\n",
    "\n",
    "train_id, test_id, val_id = train_test_val(XX,YY, 0.2, 0.1, shuffle=SHUFFLE)\n",
    "\n",
    "xx_train = XX[train_id,:,:,:]\n",
    "yy_train = YY[train_id]\n",
    "\n",
    "xx_test = XX[test_id,:,:,:]\n",
    "yy_test = YY[test_id]\n",
    "\n",
    "xx_val = XX[val_id,:,:,:]\n",
    "yy_val = YY[val_id]\n",
    "\n",
    "print('\\nXX_trai\\t', xx_train.shape)\n",
    "print('XX_test\\t', xx_test.shape)\n",
    "print('XX_val\\t', xx_val.shape)\n",
    "print('yy_trai\\t', yy_train.shape)\n",
    "print('yy_test\\t', yy_test.shape)\n",
    "print('yy_val\\t', yy_val.shape)\n",
    "\n",
    "print('\\nIf you use RANDOM SEED, check if you generate the same numbers every time:')\n",
    "print(train_id)\n",
    "print(test_id)\n",
    "print(val_id)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}