3_Spectrograms_CI4R_Datasets_Multi_Class_LSTM_with_Flatten.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mertege/Thesis_Experiments/blob/main/3_Spectrograms_CI4R_Datasets_Multi_Class_LSTM_with_Flatten.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "kN4ixvH39HW6"
      },
      "outputs": [],
      "source": [
        "# LSTM and GRU based architecure is running in Gürbüz dataset with all frequency dataset.\n",
        "# Instead of Attention, flatten is used.\n",
        "# 4 LSTM and GRU blocks are used.\n",
        "# Bidirectional(GRU) ve Bidirectional(LSTM) for each radar are running in parallel and attention applied seperately. Then, these paths are combined with concatenation. \n",
        "# Resolution is 128x170.\n",
        "# It was run 5-times. Results are shared on below:\n",
        "# 5 defa run edildi ve değerler aşağıda paylaşıldı.\n",
        "# Mean test accuracy is 0.937, max test accuracy is 0.949, min test accuracy is 0.930,  std of test accuracy is 0.006.\n",
        "# Time elapsed through all process: 470.058 sec\n",
        "# Time elapsed during test time: 17.070 sec"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "lzJwN6GfkN8Z"
      },
      "outputs": [],
      "source": [
        "from keras.models import Sequential\n",
        "from tensorflow.keras import layers\n",
        "from keras.models import Model\n",
        "from keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization, Normalization, Input, Conv2D, MaxPooling2D, Concatenate, GRU, LSTM, GRU, TimeDistributed, Bidirectional\n",
        "from keras.regularizers import l2, l1\n",
        "import tensorflow as tf\n",
        "import scipy.io\n",
        "import numpy as np\n",
        "import cv2\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "from numpy.random import seed\n",
        "from sklearn.model_selection import KFold, StratifiedKFold\n",
        "import time\n",
        "from sklearn.metrics import precision_recall_fscore_support\n",
        "from keras.callbacks import EarlyStopping\n",
        "from numpy import array\n",
        "from numpy import argmax\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from keras import backend as K \n",
        "import gc\n",
        "import os"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8FERVnWrkrwE",
        "outputId": "8caac0b8-e73a-475d-cf0d-7b7cc73bdf70"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "KArX2XozykI3"
      },
      "outputs": [],
      "source": [
        "from keras.layers import *\n",
        "from keras.models import *\n",
        "from keras import backend as K\n",
        "\n",
        "class attention(Layer):\n",
        "    \n",
        "    def __init__(self, return_sequences=True):\n",
        "        self.return_sequences = return_sequences\n",
        "        super(attention,self).__init__()\n",
        "        \n",
        "    def build(self, input_shape):\n",
        "        \n",
        "        self.W=self.add_weight(name=\"att_weight\", shape=(input_shape[-1],1),\n",
        "                               initializer=\"normal\")\n",
        "        self.b=self.add_weight(name=\"att_bias\", shape=(input_shape[1],1),\n",
        "                               initializer=\"zeros\")\n",
        "        \n",
        "        super(attention,self).build(input_shape)\n",
        "        \n",
        "    def call(self, x):\n",
        "        \n",
        "        e = K.tanh(K.dot(x,self.W)+self.b)\n",
        "        a = K.softmax(e, axis=1)\n",
        "        # a = K.sigmoid(e)\n",
        "        output = x*a\n",
        "        \n",
        "        if self.return_sequences:\n",
        "            return output\n",
        "        \n",
        "        return K.sum(output, axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "o19JHieQ81cS"
      },
      "outputs": [],
      "source": [
        "# 77 GHz\n",
        "bending_77GHz                 = np.load('/content/drive/MyDrive/bending_77Ghz.npy')\n",
        "label_bending_77GHz           = np.zeros((bending_77GHz.shape[0],1))\n",
        "label_bending_77GHz[:]        = 1\n",
        "crawling_77GHz                = np.load('/content/drive/MyDrive/crawling_77Ghz.npy')\n",
        "label_crawling_77GHz          = np.zeros((crawling_77GHz.shape[0],1))\n",
        "label_crawling_77GHz[:]       = 2\n",
        "kneeling_77GHz                = np.load('/content/drive/MyDrive/kneeling_77Ghz.npy')\n",
        "label_kneeling_77GHz          = np.zeros((kneeling_77GHz.shape[0],1))\n",
        "label_kneeling_77GHz[:]       = 3\n",
        "limping_with_RL_Stiff_77GHz   = np.load('/content/drive/MyDrive/limping_with_RL_Stiff_77Ghz.npy')\n",
        "label_limping_with_RL_Stiff_77GHz           = np.zeros((limping_with_RL_Stiff_77GHz.shape[0],1))\n",
        "label_limping_with_RL_Stiff_77GHz[:]        = 4\n",
        "picking_up_an_object_77GHz    = np.load('/content/drive/MyDrive/picking_up_an_object_77Ghz.npy')\n",
        "label_picking_up_an_object_77GHz          = np.zeros((picking_up_an_object_77GHz.shape[0],1))\n",
        "label_picking_up_an_object_77GHz[:]       = 5\n",
        "scissors_gait_77GHz           = np.load('/content/drive/MyDrive/scissors_gait_77Ghz.npy')\n",
        "label_scissors_gait_77GHz          = np.zeros((scissors_gait_77GHz.shape[0],1))\n",
        "label_scissors_gait_77GHz[:]       = 6\n",
        "short_steps_77GHz             = np.load('/content/drive/MyDrive/short_steps_77Ghz.npy')\n",
        "label_short_steps_77GHz          = np.zeros((short_steps_77GHz.shape[0],1))\n",
        "label_short_steps_77GHz[:]       = 7\n",
        "sitting_77GHz                 = np.load('/content/drive/MyDrive/sitting_77Ghz.npy')\n",
        "label_sitting_77GHz          = np.zeros((sitting_77GHz.shape[0],1))\n",
        "label_sitting_77GHz[:]       = 8\n",
        "walking_away_from_Radar_77GHz = np.load('/content/drive/MyDrive/walking_away_from_Radar_77Ghz.npy')\n",
        "label_walking_away_from_Radar_77GHz         = np.zeros((walking_away_from_Radar_77GHz.shape[0],1))\n",
        "label_walking_away_from_Radar_77GHz[:]       = 9\n",
        "Walking_on_both_toes_77GHz    = np.load('/content/drive/MyDrive/Walking_on_both_toes_77Ghz.npy')\n",
        "label_Walking_on_both_toes_77GHz         = np.zeros((Walking_on_both_toes_77GHz.shape[0],1))\n",
        "label_Walking_on_both_toes_77GHz[:]       = 10\n",
        "Walking_towards_radar_77GHz   = np.load('/content/drive/MyDrive/Walking_towards_radar_77Ghz.npy')\n",
        "label_Walking_towards_radar_77GHz          = np.zeros((Walking_towards_radar_77GHz.shape[0],1))\n",
        "label_Walking_towards_radar_77GHz[:]       = 11\n",
        "# 24 GHz\n",
        "bending_24GHz                 = np.load('/content/drive/MyDrive/bending_24GHz.npy')\n",
        "label_bending_24GHz           = np.zeros((bending_24GHz.shape[0],1))\n",
        "label_bending_24GHz[:]        = 1\n",
        "crawling_24GHz                = np.load('/content/drive/MyDrive/crawling_24GHz.npy')\n",
        "label_crawling_24GHz          = np.zeros((crawling_24GHz.shape[0],1))\n",
        "label_crawling_24GHz[:]       = 2\n",
        "kneeling_24GHz                = np.load('/content/drive/MyDrive/kneeling_24GHz.npy')\n",
        "label_kneeling_24GHz          = np.zeros((kneeling_24GHz.shape[0],1))\n",
        "label_kneeling_24GHz[:]       = 3\n",
        "limping_with_RL_Stiff_24GHz   = np.load('/content/drive/MyDrive/limping with right stiff_24GHz.npy')\n",
        "label_limping_with_RL_Stiff_24GHz           = np.zeros((limping_with_RL_Stiff_24GHz.shape[0],1))\n",
        "label_limping_with_RL_Stiff_24GHz[:]        = 4\n",
        "picking_up_an_object_24GHz    = np.load('/content/drive/MyDrive/picking up object_24GHz.npy')\n",
        "label_picking_up_an_object_24GHz          = np.zeros((picking_up_an_object_24GHz.shape[0],1))\n",
        "label_picking_up_an_object_24GHz[:]       = 5\n",
        "scissors_gait_24GHz           = np.load('/content/drive/MyDrive/scissors gait_24GHz.npy')\n",
        "label_scissors_gait_24GHz          = np.zeros((scissors_gait_24GHz.shape[0],1))\n",
        "label_scissors_gait_24GHz[:]       = 6\n",
        "short_steps_24GHz             = np.load('/content/drive/MyDrive/short steps_24GHz.npy')\n",
        "label_short_steps_24GHz          = np.zeros((short_steps_24GHz.shape[0],1))\n",
        "label_short_steps_24GHz[:]       = 7\n",
        "sitting_24GHz                 = np.load('/content/drive/MyDrive/sitting_24GHz.npy')\n",
        "label_sitting_24GHz          = np.zeros((sitting_24GHz.shape[0],1))\n",
        "label_sitting_24GHz[:]       = 8\n",
        "walking_away_from_Radar_24GHz = np.load('/content/drive/MyDrive/walking away from radar_24GHz.npy')\n",
        "label_walking_away_from_Radar_24GHz         = np.zeros((walking_away_from_Radar_24GHz.shape[0],1))\n",
        "label_walking_away_from_Radar_24GHz[:]       = 9\n",
        "Walking_on_both_toes_24GHz    = np.load('/content/drive/MyDrive/walking on toes both_24GHz.npy')\n",
        "label_Walking_on_both_toes_24GHz         = np.zeros((Walking_on_both_toes_24GHz.shape[0],1))\n",
        "label_Walking_on_both_toes_24GHz[:]       = 10\n",
        "Walking_towards_radar_24GHz   = np.load('/content/drive/MyDrive/walking towards radar_24GHz.npy')\n",
        "label_Walking_towards_radar_24GHz          = np.zeros((Walking_towards_radar_24GHz.shape[0],1))\n",
        "label_Walking_towards_radar_24GHz[:]       = 11\n",
        "# Xethru\n",
        "bending_Xethru                 = np.load('/content/drive/MyDrive/bending_Xethru.npy')\n",
        "label_bending_Xethru           = np.zeros((bending_Xethru.shape[0],1))\n",
        "label_bending_Xethru[:]        = 1\n",
        "crawling_Xethru                = np.load('/content/drive/MyDrive/crawling_Xethru.npy')\n",
        "label_crawling_Xethru          = np.zeros((crawling_Xethru.shape[0],1))\n",
        "label_crawling_Xethru[:]       = 2\n",
        "kneeling_Xethru                = np.load('/content/drive/MyDrive/kneeling_Xethru.npy')\n",
        "label_kneeling_Xethru          = np.zeros((kneeling_Xethru.shape[0],1))\n",
        "label_kneeling_Xethru[:]       = 3\n",
        "limping_with_RL_Stiff_Xethru   = np.load('/content/drive/MyDrive/limping with right leg stiff_Xethru.npy')\n",
        "label_limping_with_RL_Stiff_Xethru           = np.zeros((limping_with_RL_Stiff_Xethru.shape[0],1))\n",
        "label_limping_with_RL_Stiff_Xethru[:]        = 4\n",
        "picking_up_an_object_Xethru    = np.load('/content/drive/MyDrive/ picking up an object_Xethru.npy')\n",
        "label_picking_up_an_object_Xethru          = np.zeros((picking_up_an_object_Xethru.shape[0],1))\n",
        "label_picking_up_an_object_Xethru[:]       = 5\n",
        "scissors_gait_Xethru           = np.load('/content/drive/MyDrive/scissor Gait_Xethru.npy')\n",
        "label_scissors_gait_Xethru          = np.zeros((scissors_gait_Xethru.shape[0],1))\n",
        "label_scissors_gait_Xethru[:]       = 6\n",
        "short_steps_Xethru             = np.load('/content/drive/MyDrive/short Step_Xethru.npy')\n",
        "label_short_steps_Xethru          = np.zeros((short_steps_Xethru.shape[0],1))\n",
        "label_short_steps_Xethru[:]       = 7\n",
        "sitting_Xethru                 = np.load('/content/drive/MyDrive/sitting_Xethru.npy')\n",
        "label_sitting_Xethru          = np.zeros((sitting_Xethru.shape[0],1))\n",
        "label_sitting_Xethru[:]       = 8\n",
        "walking_away_from_Radar_Xethru = np.load('/content/drive/MyDrive/Walking away from radar_Xethru.npy')\n",
        "label_walking_away_from_Radar_Xethru         = np.zeros((walking_away_from_Radar_Xethru.shape[0],1))\n",
        "label_walking_away_from_Radar_Xethru[:]       = 9\n",
        "Walking_on_both_toes_Xethru    = np.load('/content/drive/MyDrive/walking on both toes_Xethru.npy')\n",
        "label_Walking_on_both_toes_Xethru         = np.zeros((Walking_on_both_toes_Xethru.shape[0],1))\n",
        "label_Walking_on_both_toes_Xethru[:]       = 10\n",
        "Walking_towards_radar_Xethru   = np.load('/content/drive/MyDrive/Walking towards the radar_Xethru.npy')\n",
        "label_Walking_towards_radar_Xethru          = np.zeros((Walking_towards_radar_Xethru.shape[0],1))\n",
        "label_Walking_towards_radar_Xethru[:]       = 11"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "fchoRm_5UtO2"
      },
      "outputs": [],
      "source": [
        "min_length = min(label_bending_77GHz.shape[0],label_bending_24GHz.shape[0],label_bending_Xethru.shape[0])\n",
        "label_concat = label_bending_77GHz[:min_length]\n",
        "bending_77GHz = bending_77GHz[:min_length,:,:,:]\n",
        "bending_24GHz = bending_24GHz[:min_length,:,:,:]\n",
        "bending_Xethru = bending_Xethru[:min_length,:,:,:]\n",
        "\n",
        "min_length = min(label_crawling_77GHz.shape[0],label_crawling_24GHz.shape[0],label_crawling_Xethru.shape[0])\n",
        "label_concat = np.concatenate((label_concat,label_crawling_77GHz[:min_length]),axis=0)  \n",
        "crawling_77GHz = crawling_77GHz[:min_length,:,:,:]\n",
        "crawling_24GHz = crawling_24GHz[:min_length,:,:,:]\n",
        "crawling_Xethru = crawling_Xethru[:min_length,:,:,:]\n",
        "\n",
        "min_length = min(label_kneeling_77GHz.shape[0],label_kneeling_24GHz.shape[0],label_kneeling_Xethru.shape[0])\n",
        "label_concat = np.concatenate((label_concat,label_kneeling_77GHz[:min_length]),axis=0)  \n",
        "kneeling_77GHz = kneeling_77GHz[:min_length,:,:,:]\n",
        "kneeling_24GHz = kneeling_24GHz[:min_length,:,:,:]\n",
        "kneeling_Xethru = kneeling_Xethru[:min_length,:,:,:]\n",
        "\n",
        "min_length = min(label_limping_with_RL_Stiff_77GHz.shape[0],label_limping_with_RL_Stiff_24GHz.shape[0],label_limping_with_RL_Stiff_Xethru.shape[0])\n",
        "label_concat = np.concatenate((label_concat,label_limping_with_RL_Stiff_77GHz[:min_length]),axis=0)  \n",
        "limping_with_RL_Stiff_77GHz = limping_with_RL_Stiff_77GHz[:min_length,:,:,:]\n",
        "limping_with_RL_Stiff_24GHz = limping_with_RL_Stiff_24GHz[:min_length,:,:,:]\n",
        "limping_with_RL_Stiff_Xethru = limping_with_RL_Stiff_Xethru[:min_length,:,:,:]\n",
        "\n",
        "min_length = min(label_picking_up_an_object_77GHz.shape[0],label_picking_up_an_object_24GHz.shape[0],label_picking_up_an_object_Xethru.shape[0])\n",
        "label_concat = np.concatenate((label_concat,label_picking_up_an_object_77GHz[:min_length]),axis=0)  \n",
        "picking_up_an_object_77GHz = picking_up_an_object_77GHz[:min_length,:,:,:]\n",
        "picking_up_an_object_24GHz = picking_up_an_object_24GHz[:min_length,:,:,:]\n",
        "picking_up_an_object_Xethru = picking_up_an_object_Xethru[:min_length,:,:,:]\n",
        "\n",
        "min_length = min(label_scissors_gait_77GHz.shape[0],label_scissors_gait_24GHz.shape[0],label_scissors_gait_Xethru.shape[0])\n",
        "label_concat = np.concatenate((label_concat,label_scissors_gait_77GHz[:min_length]),axis=0)  \n",
        "scissors_gait_77GHz = scissors_gait_77GHz[:min_length,:,:,:]\n",
        "scissors_gait_24GHz = scissors_gait_24GHz[:min_length,:,:,:]\n",
        "scissors_gait_Xethru = scissors_gait_Xethru[:min_length,:,:,:]\n",
        "\n",
        "min_length = min(label_short_steps_77GHz.shape[0],label_short_steps_24GHz.shape[0],label_short_steps_Xethru.shape[0])\n",
        "label_concat = np.concatenate((label_concat,label_short_steps_77GHz[:min_length]),axis=0)  \n",
        "short_steps_77GHz = short_steps_77GHz[:min_length,:,:,:]\n",
        "short_steps_24GHz = short_steps_24GHz[:min_length,:,:,:]\n",
        "short_steps_Xethru = short_steps_Xethru[:min_length,:,:,:]\n",
        "\n",
        "min_length = min(label_sitting_77GHz.shape[0],label_sitting_24GHz.shape[0],label_sitting_Xethru.shape[0])\n",
        "label_concat = np.concatenate((label_concat,label_sitting_77GHz[:min_length]),axis=0)  \n",
        "sitting_77GHz = sitting_77GHz[:min_length,:,:,:]\n",
        "sitting_24GHz = sitting_24GHz[:min_length,:,:,:]\n",
        "sitting_Xethru = sitting_Xethru[:min_length,:,:,:]\n",
        "\n",
        "min_length = min(label_walking_away_from_Radar_77GHz.shape[0],label_walking_away_from_Radar_24GHz.shape[0],label_walking_away_from_Radar_Xethru.shape[0])\n",
        "label_concat = np.concatenate((label_concat,label_walking_away_from_Radar_77GHz[:min_length]),axis=0)  \n",
        "walking_away_from_Radar_77GHz = walking_away_from_Radar_77GHz[:min_length,:,:,:]\n",
        "walking_away_from_Radar_24GHz = walking_away_from_Radar_24GHz[:min_length,:,:,:]\n",
        "walking_away_from_Radar_Xethru = walking_away_from_Radar_Xethru[:min_length,:,:,:]\n",
        "\n",
        "min_length = min(label_Walking_on_both_toes_77GHz.shape[0],label_Walking_on_both_toes_24GHz.shape[0],label_Walking_on_both_toes_Xethru.shape[0])\n",
        "label_concat = np.concatenate((label_concat,label_Walking_on_both_toes_77GHz[:min_length]),axis=0)  \n",
        "Walking_on_both_toes_77GHz = Walking_on_both_toes_77GHz[:min_length,:,:,:]\n",
        "Walking_on_both_toes_24GHz = Walking_on_both_toes_24GHz[:min_length,:,:,:]\n",
        "Walking_on_both_toes_Xethru = Walking_on_both_toes_Xethru[:min_length,:,:,:]\n",
        "\n",
        "min_length = min(label_Walking_towards_radar_77GHz.shape[0],label_Walking_towards_radar_24GHz.shape[0],label_Walking_towards_radar_Xethru.shape[0])\n",
        "label_concat = np.concatenate((label_concat,label_Walking_towards_radar_77GHz[:min_length]),axis=0)  \n",
        "Walking_towards_radar_77GHz = Walking_towards_radar_77GHz[:min_length,:,:,:]\n",
        "Walking_towards_radar_24GHz = Walking_towards_radar_24GHz[:min_length,:,:,:]\n",
        "Walking_towards_radar_Xethru = Walking_towards_radar_Xethru[:min_length,:,:,:]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "_upOlnklMNhD"
      },
      "outputs": [],
      "source": [
        "# 77 GHz\n",
        "spectrogram_77GHz_concat = np.concatenate((bending_77GHz,crawling_77GHz),axis=0)\n",
        "del bending_77GHz\n",
        "del crawling_77GHz\n",
        "spectrogram_77GHz_concat = np.concatenate((spectrogram_77GHz_concat,kneeling_77GHz               ),axis=0)\n",
        "del kneeling_77GHz\n",
        "spectrogram_77GHz_concat = np.concatenate((spectrogram_77GHz_concat,limping_with_RL_Stiff_77GHz  ),axis=0)\n",
        "del limping_with_RL_Stiff_77GHz\n",
        "spectrogram_77GHz_concat = np.concatenate((spectrogram_77GHz_concat,picking_up_an_object_77GHz   ),axis=0)\n",
        "del picking_up_an_object_77GHz\n",
        "spectrogram_77GHz_concat = np.concatenate((spectrogram_77GHz_concat,scissors_gait_77GHz          ),axis=0)\n",
        "del scissors_gait_77GHz\n",
        "spectrogram_77GHz_concat = np.concatenate((spectrogram_77GHz_concat,short_steps_77GHz            ),axis=0)\n",
        "del short_steps_77GHz\n",
        "spectrogram_77GHz_concat = np.concatenate((spectrogram_77GHz_concat,sitting_77GHz                ),axis=0)\n",
        "del sitting_77GHz\n",
        "spectrogram_77GHz_concat = np.concatenate((spectrogram_77GHz_concat,walking_away_from_Radar_77GHz),axis=0)\n",
        "del walking_away_from_Radar_77GHz\n",
        "spectrogram_77GHz_concat = np.concatenate((spectrogram_77GHz_concat,Walking_on_both_toes_77GHz   ),axis=0)\n",
        "del Walking_on_both_toes_77GHz\n",
        "spectrogram_77GHz_concat = np.concatenate((spectrogram_77GHz_concat,Walking_towards_radar_77GHz  ),axis=0)\n",
        "del Walking_towards_radar_77GHz\n",
        "\n",
        "# 24 GHz\n",
        "spectrogram_24GHz_concat = np.concatenate((bending_24GHz,crawling_24GHz),axis=0)\n",
        "del bending_24GHz\n",
        "del crawling_24GHz\n",
        "spectrogram_24GHz_concat = np.concatenate((spectrogram_24GHz_concat,kneeling_24GHz               ),axis=0)\n",
        "del kneeling_24GHz\n",
        "spectrogram_24GHz_concat = np.concatenate((spectrogram_24GHz_concat,limping_with_RL_Stiff_24GHz  ),axis=0)\n",
        "del limping_with_RL_Stiff_24GHz\n",
        "spectrogram_24GHz_concat = np.concatenate((spectrogram_24GHz_concat,picking_up_an_object_24GHz   ),axis=0)\n",
        "del picking_up_an_object_24GHz\n",
        "spectrogram_24GHz_concat = np.concatenate((spectrogram_24GHz_concat,scissors_gait_24GHz          ),axis=0)\n",
        "del scissors_gait_24GHz\n",
        "spectrogram_24GHz_concat = np.concatenate((spectrogram_24GHz_concat,short_steps_24GHz            ),axis=0)\n",
        "del short_steps_24GHz\n",
        "spectrogram_24GHz_concat = np.concatenate((spectrogram_24GHz_concat,sitting_24GHz                ),axis=0)\n",
        "del sitting_24GHz\n",
        "spectrogram_24GHz_concat = np.concatenate((spectrogram_24GHz_concat,walking_away_from_Radar_24GHz),axis=0)\n",
        "del walking_away_from_Radar_24GHz\n",
        "spectrogram_24GHz_concat = np.concatenate((spectrogram_24GHz_concat,Walking_on_both_toes_24GHz   ),axis=0)\n",
        "del Walking_on_both_toes_24GHz\n",
        "spectrogram_24GHz_concat = np.concatenate((spectrogram_24GHz_concat,Walking_towards_radar_24GHz  ),axis=0)\n",
        "del Walking_towards_radar_24GHz\n",
        "\n",
        "\n",
        "# Xethru\n",
        "spectrogram_Xethru_concat = np.concatenate((bending_Xethru,crawling_Xethru),axis=0)\n",
        "del bending_Xethru\n",
        "del crawling_Xethru\n",
        "spectrogram_Xethru_concat = np.concatenate((spectrogram_Xethru_concat,kneeling_Xethru               ),axis=0)\n",
        "del kneeling_Xethru\n",
        "spectrogram_Xethru_concat = np.concatenate((spectrogram_Xethru_concat,limping_with_RL_Stiff_Xethru  ),axis=0)\n",
        "del limping_with_RL_Stiff_Xethru\n",
        "spectrogram_Xethru_concat = np.concatenate((spectrogram_Xethru_concat,picking_up_an_object_Xethru   ),axis=0)\n",
        "del picking_up_an_object_Xethru\n",
        "spectrogram_Xethru_concat = np.concatenate((spectrogram_Xethru_concat,scissors_gait_Xethru          ),axis=0)\n",
        "del scissors_gait_Xethru\n",
        "spectrogram_Xethru_concat = np.concatenate((spectrogram_Xethru_concat,short_steps_Xethru            ),axis=0)\n",
        "del short_steps_Xethru\n",
        "spectrogram_Xethru_concat = np.concatenate((spectrogram_Xethru_concat,sitting_Xethru                ),axis=0)\n",
        "del sitting_Xethru\n",
        "spectrogram_Xethru_concat = np.concatenate((spectrogram_Xethru_concat,walking_away_from_Radar_Xethru),axis=0)\n",
        "del walking_away_from_Radar_Xethru\n",
        "spectrogram_Xethru_concat = np.concatenate((spectrogram_Xethru_concat,Walking_on_both_toes_Xethru   ),axis=0)\n",
        "del Walking_on_both_toes_Xethru\n",
        "spectrogram_Xethru_concat = np.concatenate((spectrogram_Xethru_concat,Walking_towards_radar_Xethru  ),axis=0)\n",
        "del Walking_towards_radar_Xethru\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yq7lpIJPLnXL",
        "outputId": "95af81ec-5591-4c35-d180-7d1f4cc1305d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/preprocessing/_label.py:115: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n"
          ]
        }
      ],
      "source": [
        "# One-Hot Encode Label\n",
        "# integer encode\n",
        "label_encoder = LabelEncoder()\n",
        "integer_encoded = label_encoder.fit_transform(label_concat)\n",
        "# binary encode\n",
        "onehot_encoder = OneHotEncoder(sparse=False)\n",
        "integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n",
        "onehot_encoded_labels = onehot_encoder.fit_transform(integer_encoded)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "wFp_PErnMLN2"
      },
      "outputs": [],
      "source": [
        "# Concat range-doppler data\n",
        "range_doppler_concat = spectrogram_77GHz_concat\n",
        "range_doppler_concat_label = label_concat\n",
        "# Shuffle concat range doppler\n",
        "shuffle_indx = random.sample(range(0, range_doppler_concat.shape[0]), range_doppler_concat.shape[0]) # split validation data\n",
        "range_doppler_concat_shuffle = range_doppler_concat[shuffle_indx,:,:,:]\n",
        "range_doppler_concat_label_shuffle = range_doppler_concat_label[shuffle_indx,:]\n",
        "# Concat range-doppler data\n",
        "spectrogram_concat = spectrogram_24GHz_concat\n",
        "spectrogram_concat_label = onehot_encoded_labels\n",
        "# Shuffle concat range doppler\n",
        "spectrogram_concat_shuffle = spectrogram_concat[shuffle_indx,:,:,:]\n",
        "spectrogram_concat_label_shuffle = spectrogram_concat_label[shuffle_indx,:]\n",
        "# Third Spectrogram\n",
        "third_spectrogram_concat = spectrogram_Xethru_concat\n",
        "third_spectrogram_concat_shuffle = third_spectrogram_concat[shuffle_indx,:,:,:]\n",
        "del range_doppler_concat\n",
        "del spectrogram_77GHz_concat\n",
        "del spectrogram_24GHz_concat\n",
        "del spectrogram_concat\n",
        "del spectrogram_Xethru_concat\n",
        "del third_spectrogram_concat"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "bPov5sO1jGHh"
      },
      "outputs": [],
      "source": [
        "range_doppler_concat_shuffle_new = np.zeros((630,128,170,1))\n",
        "for ii in range(range_doppler_concat_shuffle.shape[0]):\n",
        "  aa = range_doppler_concat_shuffle[ii,:,:,0]\n",
        "  range_doppler_concat_shuffle_new[ii,:,:,0] = cv2.resize(aa, (range_doppler_concat_shuffle_new.shape[2], range_doppler_concat_shuffle_new.shape[1]),interpolation = cv2.INTER_CUBIC)\n",
        "range_doppler_concat_shuffle = range_doppler_concat_shuffle_new\n",
        "del aa\n",
        "del range_doppler_concat_shuffle_new\n",
        "\n",
        "spectrogram_concat_shuffle_new = np.zeros((630,128,170,1))\n",
        "for ii in range(spectrogram_concat_shuffle.shape[0]):\n",
        "  aa = spectrogram_concat_shuffle[ii,:,:,0]\n",
        "  spectrogram_concat_shuffle_new[ii,:,:,0] = cv2.resize(aa, (spectrogram_concat_shuffle_new.shape[2], spectrogram_concat_shuffle_new.shape[1]),interpolation = cv2.INTER_CUBIC)\n",
        "spectrogram_concat_shuffle = spectrogram_concat_shuffle_new\n",
        "del aa\n",
        "del spectrogram_concat_shuffle_new\n",
        "\n",
        "third_spectrogram_concat_shuffle_new = np.zeros((630,128,170,1))\n",
        "for ii in range(third_spectrogram_concat_shuffle.shape[0]):\n",
        "  aa = third_spectrogram_concat_shuffle[ii,:,:,0]\n",
        "  third_spectrogram_concat_shuffle_new[ii,:,:,0] = cv2.resize(aa, (third_spectrogram_concat_shuffle_new.shape[2], third_spectrogram_concat_shuffle_new.shape[1]),interpolation = cv2.INTER_CUBIC)\n",
        "third_spectrogram_concat_shuffle = third_spectrogram_concat_shuffle_new\n",
        "del aa\n",
        "del third_spectrogram_concat_shuffle_new"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "_1aBNkjoLCvI"
      },
      "outputs": [],
      "source": [
        "# ---------------- Augmente and shuffle (train and test) data data ----------------\n",
        "def mixup_augmentation(images,range_doppler_training_data,third_spectrogram_training_data, labels, repeat_of_mixup, alpha, beta):\n",
        "    batch_size = images.shape[0]\n",
        "    concat_images = np.zeros((batch_size*(repeat_of_mixup+1),images.shape[1],images.shape[2],images.shape[3]))\n",
        "    concat_images_range_doppler = np.zeros((batch_size*(repeat_of_mixup+1),images.shape[1],images.shape[2],images.shape[3]))\n",
        "    concat_images_third_spectrogram = np.zeros((batch_size*(repeat_of_mixup+1),images.shape[1],images.shape[2],images.shape[3]))\n",
        "    concat_label = np.zeros((batch_size*(repeat_of_mixup+1),labels.shape[1]))\n",
        "\n",
        "    if repeat_of_mixup == 0:\n",
        "      concat_images = images\n",
        "      concat_images_range_doppler = range_doppler_training_data\n",
        "      concat_images_third_spectrogram = third_spectrogram_training_data\n",
        "      concat_label = labels      \n",
        "    else:\n",
        "      for ii in range(repeat_of_mixup):\n",
        "        # shuffle train dataset\n",
        "        shuffle_indx_1 = random.sample(range(0, images.shape[0]), images.shape[0]) # split validation data\n",
        "        images_shuffled_1 = images[shuffle_indx_1,:,:,:]\n",
        "        range_doppler_training_data_shuffled_1 = range_doppler_training_data[shuffle_indx_1,:,:,:]\n",
        "        third_spectrogram_training_data_shuffled_1 = third_spectrogram_training_data[shuffle_indx_1,:,:,:]\n",
        "        labels_shuffled_1 = labels[shuffle_indx_1,:]\n",
        "\n",
        "        shuffle_indx_2 = random.sample(range(0, images.shape[0]), images.shape[0]) # split validation data\n",
        "        images_shuffled_2 = images[shuffle_indx_2,:,:,:]\n",
        "        range_doppler_training_data_shuffled_2 = range_doppler_training_data[shuffle_indx_2,:,:,:]\n",
        "        third_spectrogram_training_data_shuffled_2 = third_spectrogram_training_data[shuffle_indx_2,:,:,:]\n",
        "        labels_shuffled_2 = labels[shuffle_indx_2,:]\n",
        "\n",
        "        # shuffle_indx_3 = random.sample(range(0, images.shape[0]), images.shape[0]) # split validation data\n",
        "        # images_shuffled_3 = images[shuffle_indx_3,:,:,:]\n",
        "        # range_doppler_training_data_shuffled_3 = range_doppler_training_data[shuffle_indx_3,:,:,:]\n",
        "        # third_spectrogram_training_data_shuffled_3 = third_spectrogram_training_data[shuffle_indx_3,:,:,:]\n",
        "        # labels_shuffled_3 = labels[shuffle_indx_3,:]\n",
        "\n",
        "        # shuffle_indx_4 = random.sample(range(0, images.shape[0]), images.shape[0]) # split validation data\n",
        "        # images_shuffled_4 = images[shuffle_indx_4,:,:,:]\n",
        "        # range_doppler_training_data_shuffled_4 = range_doppler_training_data[shuffle_indx_4,:,:,:]\n",
        "        # third_spectrogram_training_data_shuffled_4 = third_spectrogram_training_data[shuffle_indx_4,:,:,:]\n",
        "        # labels_shuffled_4 = labels[shuffle_indx_4,:]\n",
        "\n",
        "        # Sample lambda and reshape it to do the mixup\n",
        "        # ll = np.random.beta(alpha, beta, (batch_size,1,1,1))\n",
        "        gaussian_mean = 0.2\n",
        "        gaussian_std = 0.02\n",
        "        ll = np.random.normal(gaussian_mean, gaussian_std, (batch_size,1,1,1))\n",
        "        x_l = np.reshape(ll, (batch_size,1,1,1))\n",
        "        y_l = np.reshape(ll, (batch_size,1))\n",
        "      \n",
        "        # ll_2 = np.random.normal(gaussian_mean, gaussian_std, (batch_size,1,1,1))\n",
        "        # x_l_2 = np.reshape(ll_2, (batch_size,1,1,1))\n",
        "        # y_l_2 = np.reshape(ll_2, (batch_size,1))\n",
        "\n",
        "        # ll_3 = np.random.normal(gaussian_mean, gaussian_std, (batch_size,1,1,1))\n",
        "        # x_l_3 = np.reshape(ll_3, (batch_size,1,1,1))\n",
        "        # y_l_3 = np.reshape(ll_3, (batch_size,1))\n",
        "\n",
        "        \n",
        "        # Perform mixup on both images and labels by combining a pair of images/labels\n",
        "        # images_mixup = images_shuffled_1 * x_l + images_shuffled_2 * x_l_2 + images_shuffled_3 * x_l_3 + images_shuffled_4 * (1 - x_l - x_l_2 - x_l_3)\n",
        "        # images_mixup_range_doppler = range_doppler_training_data_shuffled_1 * x_l + range_doppler_training_data_shuffled_2 * x_l_2 +\\\n",
        "        # range_doppler_training_data_shuffled_3 * x_l_3 + range_doppler_training_data_shuffled_4 * (1 - x_l - x_l_2 - x_l_3)\n",
        "        # images_mixup_third_spectrogram = third_spectrogram_training_data_shuffled_1 * x_l + third_spectrogram_training_data_shuffled_2 * x_l_2 +\\\n",
        "        # third_spectrogram_training_data_shuffled_3 * x_l_3 + third_spectrogram_training_data_shuffled_4 * (1 - x_l - x_l_2 - x_l_3)\n",
        "        # labels_mixup = labels_shuffled_1 * y_l + labels_shuffled_2 * y_l_2  + labels_shuffled_3 * y_l_3 + labels_shuffled_4 * (1 - y_l - y_l_2 - y_l_3)\n",
        "\n",
        "        images_mixup = images_shuffled_1 * x_l + images_shuffled_2 * (1 - x_l)\n",
        "        images_mixup_range_doppler = range_doppler_training_data_shuffled_1 * x_l + range_doppler_training_data_shuffled_2 * (1 - x_l)\n",
        "        images_mixup_third_spectrogram = third_spectrogram_training_data_shuffled_1 * x_l + third_spectrogram_training_data_shuffled_2 * (1 - x_l)\n",
        "        labels_mixup = labels_shuffled_1 * y_l + labels_shuffled_2 * (1 - y_l)\n",
        "\n",
        "        concat_images[ii*batch_size:(ii+1)*batch_size,:,:,:] = images_mixup\n",
        "        concat_images_range_doppler[ii*batch_size:(ii+1)*batch_size,:,:,:] = images_mixup_range_doppler\n",
        "        concat_images_third_spectrogram[ii*batch_size:(ii+1)*batch_size,:,:,:] = images_mixup_third_spectrogram\n",
        "        concat_label[ii*batch_size:(ii+1)*batch_size,:] = labels_mixup\n",
        "\n",
        "      concat_images[repeat_of_mixup*batch_size:,:,:,:] = images\n",
        "      concat_images_range_doppler[repeat_of_mixup*batch_size:,:,:,:] = range_doppler_training_data\n",
        "      concat_images_third_spectrogram[repeat_of_mixup*batch_size:,:,:,:] = third_spectrogram_training_data\n",
        "      concat_label[repeat_of_mixup*batch_size:,:] = labels\n",
        "    return (concat_images,concat_images_range_doppler,concat_images_third_spectrogram, concat_label)\n",
        "\n",
        "def split_and_augmentation_of_training(spectrogram_concat_shuffle_train,range_doppler_concat_shuffle_train,third_spectrogram,range_doppler_concat_label_shuffle_train,\\\n",
        "                                       repeat_of_mixup, augmentation_enable):\n",
        "  # ---------------- Parameters ----------------\n",
        "  # to split train and validation with same distribution\n",
        "  alpha = 0.1\n",
        "  beta = 5\n",
        "  dummy_label = np.zeros((spectrogram_concat_shuffle_train.shape[0],1))\n",
        "  for randomlist_for_train_indx, randomlist_for_validation_indx in kfold.split(spectrogram_concat_shuffle_train,dummy_label):   \n",
        "    randomlist_for_validation_indx\n",
        "  # size_of_validation = int(0.2*spectrogram_concat_shuffle_train.shape[0])\n",
        "  # # Split validation\n",
        "  # randomlist_for_validation_indx = random.sample(range(0, range_doppler_concat_shuffle_train.shape[0]), size_of_validation) # split validation data\n",
        "  # randomlist_for_train_indx = np.delete(range(0, range_doppler_concat_shuffle_train.shape[0]), randomlist_for_validation_indx) # split training data\n",
        "  # get validation data\n",
        "  third_spectrogram_validation_data = third_spectrogram[randomlist_for_validation_indx,:,:,:]\n",
        "  spectrogram_validation_data = spectrogram_concat_shuffle_train[randomlist_for_validation_indx,:,:,:]\n",
        "  spectrogram_validation_labels = range_doppler_concat_label_shuffle_train[randomlist_for_validation_indx,:]\n",
        "  range_doppler_validation_data = range_doppler_concat_shuffle_train[randomlist_for_validation_indx,:,:,:]\n",
        "  # get training data\n",
        "  third_spectrogram_training_data = third_spectrogram[randomlist_for_train_indx,:,:,:]\n",
        "  spectrogram_training_data = spectrogram_concat_shuffle_train[randomlist_for_train_indx,:,:,:]\n",
        "  spectrogram_training_labels = spectrogram_concat_label_shuffle_train[randomlist_for_train_indx,:]\n",
        "  range_doppler_training_data = range_doppler_concat_shuffle_train[randomlist_for_train_indx,:,:,:]\n",
        "\n",
        "  (spectrogram_augmented_image,range_doppler_augmented_image,third_spectrogram_augmented_image,spectrograms_label)=\\\n",
        "   mixup_augmentation(spectrogram_training_data,range_doppler_training_data,third_spectrogram_training_data, spectrogram_training_labels, repeat_of_mixup, alpha, beta)\n",
        "\n",
        "  return (spectrogram_augmented_image,range_doppler_augmented_image,third_spectrogram_augmented_image,spectrograms_label,\\\n",
        "     spectrogram_validation_data,range_doppler_validation_data,third_spectrogram_validation_data, spectrogram_validation_labels)\n",
        "\n",
        "\n",
        "\n",
        "def normalize_inputs(range_doppler_concat_shuffle, normalize_inputs_enable):\n",
        "  # ---------------- Normalize Inputs ----------------\n",
        "  if normalize_inputs_enable == True:\n",
        "    layer = Normalization(axis=None)\n",
        "    layer.adapt(range_doppler_concat_shuffle)\n",
        "    range_doppler_concat_shuffle = layer(range_doppler_concat_shuffle)\n",
        "  else:\n",
        "    range_doppler_concat_shuffle = range_doppler_concat_shuffle\n",
        "  return(range_doppler_concat_shuffle)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "9l7kn8DBfgBh"
      },
      "outputs": [],
      "source": [
        "normalize_inputs_enable = 1\n",
        "range_doppler_concat_shuffle = normalize_inputs(range_doppler_concat_shuffle, normalize_inputs_enable)\n",
        "spectrogram_concat_shuffle = normalize_inputs(spectrogram_concat_shuffle, normalize_inputs_enable)\n",
        "third_spectrogram_concat_shuffle = normalize_inputs(third_spectrogram_concat_shuffle, normalize_inputs_enable)\n",
        "range_doppler_concat_shuffle = np.float32(range_doppler_concat_shuffle)\n",
        "spectrogram_concat_shuffle = np.float32(spectrogram_concat_shuffle)\n",
        "third_spectrogram_concat_shuffle = np.float32(third_spectrogram_concat_shuffle)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "FJ8DgKewawvT"
      },
      "outputs": [],
      "source": [
        "n_features = range_doppler_concat_shuffle.shape[1]\n",
        "n_steps = range_doppler_concat_shuffle.shape[2]\n",
        "range_doppler_concat_shuffle = np.transpose(range_doppler_concat_shuffle, axes = (0,2,1,3)) \n",
        "spectrogram_concat_shuffle = np.transpose(spectrogram_concat_shuffle, axes = (0,2,1,3)) \n",
        "third_spectrogram_concat_shuffle = np.transpose(third_spectrogram_concat_shuffle, axes = (0,2,1,3)) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "cellView": "code",
        "id": "rNpkBBIadpPo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "51b20cba-e220-4682-b894-9faa7c885d29"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_7\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " input_6 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 64)           1609792     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 64)           1609792     ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 64)           1609792     ['input_6[0][0]']                \n",
            "                                                                                                  \n",
            " model_3 (Functional)           (None, 64)           1251392     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_4 (Functional)           (None, 64)           1251392     ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " model_5 (Functional)           (None, 64)           1251392     ['input_6[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 384)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]',                \n",
            "                                                                  'model_2[0][0]',                \n",
            "                                                                  'model_3[0][0]',                \n",
            "                                                                  'model_4[0][0]',                \n",
            "                                                                  'model_5[0][0]']                \n",
            "                                                                                                  \n",
            " model_6 (Functional)           (None, 11)           25611       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 8,609,163\n",
            "Trainable params: 8,609,035\n",
            "Non-trainable params: 128\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "4/4 [==============================] - 19s 131ms/step - loss: 0.3137 - accuracy: 0.9048\n",
            "[0.9047619104385376]\n",
            "Model: \"model_7\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " input_6 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 64)           1609792     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 64)           1609792     ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 64)           1609792     ['input_6[0][0]']                \n",
            "                                                                                                  \n",
            " model_3 (Functional)           (None, 64)           1251392     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_4 (Functional)           (None, 64)           1251392     ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " model_5 (Functional)           (None, 64)           1251392     ['input_6[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 384)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]',                \n",
            "                                                                  'model_2[0][0]',                \n",
            "                                                                  'model_3[0][0]',                \n",
            "                                                                  'model_4[0][0]',                \n",
            "                                                                  'model_5[0][0]']                \n",
            "                                                                                                  \n",
            " model_6 (Functional)           (None, 11)           25611       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 8,609,163\n",
            "Trainable params: 8,609,035\n",
            "Non-trainable params: 128\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "4/4 [==============================] - 18s 131ms/step - loss: 0.3148 - accuracy: 0.8889\n",
            "[0.9047619104385376, 0.8888888955116272]\n",
            "Model: \"model_7\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " input_6 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 64)           1609792     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 64)           1609792     ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 64)           1609792     ['input_6[0][0]']                \n",
            "                                                                                                  \n",
            " model_3 (Functional)           (None, 64)           1251392     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_4 (Functional)           (None, 64)           1251392     ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " model_5 (Functional)           (None, 64)           1251392     ['input_6[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 384)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]',                \n",
            "                                                                  'model_2[0][0]',                \n",
            "                                                                  'model_3[0][0]',                \n",
            "                                                                  'model_4[0][0]',                \n",
            "                                                                  'model_5[0][0]']                \n",
            "                                                                                                  \n",
            " model_6 (Functional)           (None, 11)           25611       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 8,609,163\n",
            "Trainable params: 8,609,035\n",
            "Non-trainable params: 128\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "4/4 [==============================] - 18s 138ms/step - loss: 0.0646 - accuracy: 0.9841\n",
            "[0.9047619104385376, 0.8888888955116272, 0.9841269850730896]\n",
            "Model: \"model_7\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " input_6 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 64)           1609792     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 64)           1609792     ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 64)           1609792     ['input_6[0][0]']                \n",
            "                                                                                                  \n",
            " model_3 (Functional)           (None, 64)           1251392     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_4 (Functional)           (None, 64)           1251392     ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " model_5 (Functional)           (None, 64)           1251392     ['input_6[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 384)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]',                \n",
            "                                                                  'model_2[0][0]',                \n",
            "                                                                  'model_3[0][0]',                \n",
            "                                                                  'model_4[0][0]',                \n",
            "                                                                  'model_5[0][0]']                \n",
            "                                                                                                  \n",
            " model_6 (Functional)           (None, 11)           25611       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 8,609,163\n",
            "Trainable params: 8,609,035\n",
            "Non-trainable params: 128\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "4/4 [==============================] - 18s 144ms/step - loss: 0.2700 - accuracy: 0.9206\n",
            "[0.9047619104385376, 0.8888888955116272, 0.9841269850730896, 0.920634925365448]\n",
            "Model: \"model_7\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " input_6 (InputLayer)           [(None, 170, 128)]   0           []                               \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 64)           1609792     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 64)           1609792     ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 64)           1609792     ['input_6[0][0]']                \n",
            "                                                                                                  \n",
            " model_3 (Functional)           (None, 64)           1251392     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_4 (Functional)           (None, 64)           1251392     ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " model_5 (Functional)           (None, 64)           1251392     ['input_6[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 384)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]',                \n",
            "                                                                  'model_2[0][0]',                \n",
            "                                                                  'model_3[0][0]',                \n",
            "                                                                  'model_4[0][0]',                \n",
            "                                                                  'model_5[0][0]']                \n",
            "                                                                                                  \n",
            " model_6 (Functional)           (None, 11)           25611       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 8,609,163\n",
            "Trainable params: 8,609,035\n",
            "Non-trainable params: 128\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "4/4 [==============================] - 18s 141ms/step - loss: 0.1283 - accuracy: 0.9524\n",
            "[0.9047619104385376, 0.8888888955116272, 0.9841269850730896, 0.920634925365448, 0.9523809552192688]\n",
            "[0.9301587343215942]\n",
            "4/4 [==============================] - 18s 135ms/step - loss: 0.0294 - accuracy: 1.0000\n",
            "[1.0]\n",
            "4/4 [==============================] - 18s 138ms/step - loss: 0.3929 - accuracy: 0.8889\n",
            "[1.0, 0.8888888955116272]\n",
            "4/4 [==============================] - 18s 132ms/step - loss: 0.1835 - accuracy: 0.9206\n",
            "[1.0, 0.8888888955116272, 0.920634925365448]\n",
            "4/4 [==============================] - 18s 141ms/step - loss: 0.2938 - accuracy: 0.8968\n",
            "[1.0, 0.8888888955116272, 0.920634925365448, 0.89682537317276]\n",
            "4/4 [==============================] - 17s 142ms/step - loss: 0.0540 - accuracy: 0.9762\n",
            "[1.0, 0.8888888955116272, 0.920634925365448, 0.89682537317276, 0.976190447807312]\n",
            "[0.9301587343215942, 0.9365079283714295]\n",
            "4/4 [==============================] - 18s 136ms/step - loss: 0.1775 - accuracy: 0.9365\n",
            "[0.9365079402923584]\n",
            "4/4 [==============================] - 18s 133ms/step - loss: 0.1131 - accuracy: 0.9603\n",
            "[0.9365079402923584, 0.9603174328804016]\n",
            "4/4 [==============================] - 17s 136ms/step - loss: 0.1569 - accuracy: 0.9603\n",
            "[0.9365079402923584, 0.9603174328804016, 0.9603174328804016]\n",
            "4/4 [==============================] - 18s 133ms/step - loss: 0.1834 - accuracy: 0.9603\n",
            "[0.9365079402923584, 0.9603174328804016, 0.9603174328804016, 0.9603174328804016]\n",
            "4/4 [==============================] - 18s 138ms/step - loss: 0.2206 - accuracy: 0.9286\n",
            "[0.9365079402923584, 0.9603174328804016, 0.9603174328804016, 0.9603174328804016, 0.9285714030265808]\n",
            "[0.9301587343215942, 0.9365079283714295, 0.9492063283920288]\n",
            "4/4 [==============================] - 18s 141ms/step - loss: 0.1905 - accuracy: 0.9444\n",
            "[0.9444444179534912]\n",
            "4/4 [==============================] - 17s 134ms/step - loss: 0.2274 - accuracy: 0.9206\n",
            "[0.9444444179534912, 0.920634925365448]\n",
            "4/4 [==============================] - 17s 136ms/step - loss: 0.2158 - accuracy: 0.9127\n",
            "[0.9444444179534912, 0.920634925365448, 0.9126983880996704]\n",
            "4/4 [==============================] - 17s 136ms/step - loss: 0.0818 - accuracy: 0.9683\n",
            "[0.9444444179534912, 0.920634925365448, 0.9126983880996704, 0.9682539701461792]\n",
            "4/4 [==============================] - 18s 141ms/step - loss: 0.1897 - accuracy: 0.9365\n",
            "[0.9444444179534912, 0.920634925365448, 0.9126983880996704, 0.9682539701461792, 0.9365079402923584]\n",
            "[0.9301587343215942, 0.9365079283714295, 0.9492063283920288, 0.9365079283714295]\n",
            "4/4 [==============================] - 17s 144ms/step - loss: 0.2450 - accuracy: 0.9206\n",
            "[0.920634925365448]\n",
            "4/4 [==============================] - 18s 138ms/step - loss: 0.2490 - accuracy: 0.9048\n",
            "[0.920634925365448, 0.9047619104385376]\n",
            "4/4 [==============================] - 18s 131ms/step - loss: 0.3322 - accuracy: 0.9206\n",
            "[0.920634925365448, 0.9047619104385376, 0.920634925365448]\n",
            "4/4 [==============================] - 18s 139ms/step - loss: 0.1290 - accuracy: 0.9444\n",
            "[0.920634925365448, 0.9047619104385376, 0.920634925365448, 0.9444444179534912]\n",
            "4/4 [==============================] - 18s 140ms/step - loss: 0.0683 - accuracy: 0.9762\n",
            "[0.920634925365448, 0.9047619104385376, 0.920634925365448, 0.9444444179534912, 0.976190447807312]\n",
            "[0.9301587343215942, 0.9365079283714295, 0.9492063283920288, 0.9365079283714295, 0.9333333253860474]\n",
            "Mean test accuracy is 0.937, max test accuracy is 0.949, min test accuracy is 0.930,  std of test accuracy is 0.006.\n",
            "Time elapsed through all process: 470.06, sec\n"
          ]
        }
      ],
      "source": [
        "\n",
        "# ---------- Parameters ----------------\n",
        "augmentation_enable = True\n",
        "normalize_inputs_enable = True\n",
        "num_folds = 5\n",
        "kfold = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state = None) # random_state = 1 ile split run'dan run'a sabit.\n",
        "test_accuracy_per_run = []\n",
        "f1_score_per_run = []\n",
        "epoch_number = 100\n",
        "batch_size = 32\n",
        "dense_size = 128\n",
        "dropout_prob_dense = 0.3\n",
        "repeat_of_mixup = 0\n",
        "number_of_repeat = 5\n",
        "lstm_dropout_rate = 0.25\n",
        "unit_number_of_lstm = 128\n",
        "recurrent_dropout_rate = 0.0\n",
        "recurrent_regularizer_value = l1(0.0000)\n",
        "dense_unit_of_range_doppler_function = 64\n",
        "dropout_prob_dense = 0.1\n",
        "for repeat_run_number in range(number_of_repeat):\n",
        "  test_accuracy_per_fold = []\n",
        "  f1_score_per_fold = []\n",
        "  for train, test in kfold.split(range_doppler_concat_shuffle,range_doppler_concat_label_shuffle):   \n",
        "    gc.collect()\n",
        "    K.clear_session()\n",
        "\n",
        "    randomlist_for_test_indx = test\n",
        "    randomlist_for_train_indx = train\n",
        "    # test data\n",
        "    spectrogram_concat_shuffle_test = spectrogram_concat_shuffle[randomlist_for_test_indx,:,:,:]\n",
        "    spectrogram_concat_label_shuffle_test = spectrogram_concat_label_shuffle[randomlist_for_test_indx,:]\n",
        "    range_doppler_concat_shuffle_test = range_doppler_concat_shuffle[randomlist_for_test_indx,:,:,:]\n",
        "    third_spectrogram_concat_shuffle_test = third_spectrogram_concat_shuffle[randomlist_for_test_indx,:,:,:]\n",
        "    #train data\n",
        "    spectrogram_concat_shuffle_train = spectrogram_concat_shuffle[randomlist_for_train_indx,:,:,:]\n",
        "    spectrogram_concat_label_shuffle_train = spectrogram_concat_label_shuffle[randomlist_for_train_indx,:]\n",
        "    range_doppler_concat_shuffle_train = range_doppler_concat_shuffle[randomlist_for_train_indx,:,:,:]\n",
        "    third_spectrogram_concat_shuffle_train = third_spectrogram_concat_shuffle[randomlist_for_train_indx,:,:,:]\n",
        "    # ---------------- Split labels to equal them during augmentation for Validation ----------------\n",
        "\n",
        "            \n",
        "    (spectrogram_augmented_image,range_doppler_augmented_image,third_spectrogram_augmented_image,spectrogram_concat_label_shuffle_concat,\\\n",
        "     validation_spectrogram,validation_range_doppler,validation_third_spectrogram, spectrogram_validation_labels)  =\\\n",
        "      split_and_augmentation_of_training(spectrogram_concat_shuffle_train,range_doppler_concat_shuffle_train,third_spectrogram_concat_shuffle_train,\\\n",
        "                                         spectrogram_concat_label_shuffle_train,\\\n",
        "                                         repeat_of_mixup, augmentation_enable)\n",
        "\n",
        "\n",
        "    def lstm_encoder_network_1(input_shape):\n",
        "        input = Input(shape=input_shape)\n",
        "        x = Bidirectional(LSTM(unit_number_of_lstm, return_sequences=True, dropout = lstm_dropout_rate,\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(input)                            \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)\n",
        "        x = Bidirectional(LSTM(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)      \n",
        "        x = Bidirectional(LSTM(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)\n",
        "        x = Bidirectional(LSTM(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)        \n",
        "        x = Flatten()(x)\n",
        "        x = Dense(dense_unit_of_range_doppler_function)(x)\n",
        "        x = Activation('LeakyReLU')(x)\n",
        "        x = Dropout(dropout_prob_dense)(x)\n",
        "\n",
        "        return Model(input, x)\n",
        "\n",
        "    def lstm_encoder_network_2(input_shape):\n",
        "        input = Input(shape=input_shape)\n",
        "        x = Bidirectional(LSTM(unit_number_of_lstm, return_sequences=True, dropout = lstm_dropout_rate,\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(input)                            \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)\n",
        "        x = Bidirectional(LSTM(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)      \n",
        "        x = Bidirectional(LSTM(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)\n",
        "        x = Bidirectional(LSTM(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)        \n",
        "        x = Flatten()(x)\n",
        "        x = Dense(dense_unit_of_range_doppler_function)(x)\n",
        "        x = Activation('LeakyReLU')(x)\n",
        "        x = Dropout(dropout_prob_dense)(x)\n",
        "\n",
        "        return Model(input, x)\n",
        "\n",
        "    def lstm_encoder_network_3(input_shape):\n",
        "        input = Input(shape=input_shape)\n",
        "        x = Bidirectional(LSTM(unit_number_of_lstm, return_sequences=True, dropout = lstm_dropout_rate,\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(input)                            \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)\n",
        "        x = Bidirectional(LSTM(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)      \n",
        "        x = Bidirectional(LSTM(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)\n",
        "        x = Bidirectional(LSTM(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)        \n",
        "        x = Flatten()(x)\n",
        "        x = Dense(dense_unit_of_range_doppler_function)(x)\n",
        "        x = Activation('LeakyReLU')(x)\n",
        "        x = Dropout(dropout_prob_dense)(x)\n",
        "\n",
        "        return Model(input, x)\n",
        "\n",
        "    def lstm_encoder_network_1_gru(input_shape):\n",
        "        input = Input(shape=input_shape)\n",
        "        x = Bidirectional(GRU(unit_number_of_lstm, return_sequences=True, dropout = lstm_dropout_rate,\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(input)                    \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)\n",
        "        x = Bidirectional(GRU(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)      \n",
        "        x = Bidirectional(GRU(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)\n",
        "        x = Bidirectional(GRU(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)               \n",
        "        x = Flatten()(x)\n",
        "        x = Dense(dense_unit_of_range_doppler_function)(x)\n",
        "        x = Activation('LeakyReLU')(x)\n",
        "        x = Dropout(dropout_prob_dense)(x)\n",
        "\n",
        "        return Model(input, x)\n",
        "\n",
        "    def lstm_encoder_network_2_gru(input_shape):\n",
        "        input = Input(shape=input_shape)\n",
        "        x = Bidirectional(GRU(unit_number_of_lstm, return_sequences=True, dropout = lstm_dropout_rate,\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(input)                    \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)\n",
        "        x = Bidirectional(GRU(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)      \n",
        "        x = Bidirectional(GRU(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)\n",
        "        x = Bidirectional(GRU(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)               \n",
        "        x = Flatten()(x)\n",
        "        x = Dense(dense_unit_of_range_doppler_function)(x)\n",
        "        x = Activation('LeakyReLU')(x)\n",
        "        x = Dropout(dropout_prob_dense)(x)\n",
        "\n",
        "        return Model(input, x)\n",
        "\n",
        "    def lstm_encoder_network_3_gru(input_shape):\n",
        "        input = Input(shape=input_shape)\n",
        "        x = Bidirectional(GRU(unit_number_of_lstm, return_sequences=True, dropout = lstm_dropout_rate,\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(input)                    \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)\n",
        "        x = Bidirectional(GRU(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)      \n",
        "        x = Bidirectional(GRU(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)         \n",
        "        x = Bidirectional(GRU(int(unit_number_of_lstm), return_sequences=True, dropout = int(lstm_dropout_rate),\n",
        "                              recurrent_dropout= recurrent_dropout_rate, kernel_regularizer=recurrent_regularizer_value))(x) \n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)                            \n",
        "        x = Flatten()(x)\n",
        "        x = Dense(dense_unit_of_range_doppler_function)(x)\n",
        "        x = Activation('LeakyReLU')(x)\n",
        "        x = Dropout(dropout_prob_dense)(x)\n",
        "\n",
        "        return Model(input, x)\n",
        "\n",
        "\n",
        "    def decoder_for_concat(input_shape):\n",
        "      input = Input(shape=input_shape)\n",
        "      # x = BatchNormalization()(input)\n",
        "      # x = Dense(512)(x)\n",
        "      # x = BatchNormalization()(x)\n",
        "      # x = Activation('LeakyReLU')(x)\n",
        "      # x = Dropout(0.25)(x)\n",
        "      x = Dense(64)(input)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('LeakyReLU')(x)\n",
        "      x = Dropout(0.15)(x)\n",
        "      x = Dense(11, activation=\"softmax\")(x)\n",
        "      return Model(input, x)\n",
        "\n",
        "    input_shape = range_doppler_concat_shuffle.shape[1:3]\n",
        "    base_network_lstm_1 = lstm_encoder_network_1(input_shape)\n",
        "    lstm_input_1  = Input(shape=input_shape)\n",
        "    processed_lstm_1  = base_network_lstm_1(lstm_input_1)\n",
        "\n",
        "    input_shape = spectrogram_concat_shuffle.shape[1:3]\n",
        "    base_network_lstm_2 = lstm_encoder_network_2(input_shape)\n",
        "    lstm_input_2  = Input(shape=input_shape)\n",
        "    processed_lstm_2  = base_network_lstm_2(lstm_input_2)\n",
        "\n",
        "    input_shape = third_spectrogram_concat_shuffle.shape[1:3]\n",
        "    base_network_lstm_3 = lstm_encoder_network_3(input_shape)\n",
        "    lstm_input_3  = Input(shape=input_shape)\n",
        "    processed_lstm_3  = base_network_lstm_3(lstm_input_3)\n",
        "\n",
        "\n",
        "    base_network_lstm_1_gru = lstm_encoder_network_1_gru(input_shape)\n",
        "    processed_lstm_1_gru  = base_network_lstm_1_gru(lstm_input_1)\n",
        "\n",
        "    base_network_lstm_2_gru = lstm_encoder_network_2_gru(input_shape)\n",
        "    processed_lstm_2_gru  = base_network_lstm_2_gru(lstm_input_2)\n",
        "\n",
        "    base_network_lstm_3_gru = lstm_encoder_network_3_gru(input_shape)\n",
        "    processed_lstm_3_gru  = base_network_lstm_3_gru(lstm_input_3)    \n",
        "\n",
        "    concat_layer = Concatenate()([processed_lstm_1, processed_lstm_2, processed_lstm_3, processed_lstm_1_gru, processed_lstm_2_gru, processed_lstm_3_gru])\n",
        "\n",
        "    base_decoder_network = decoder_for_concat((concat_layer.shape[1]))\n",
        "    out = base_decoder_network(concat_layer)\n",
        "\n",
        "    model = Model(inputs=[lstm_input_1,lstm_input_2,lstm_input_3], outputs=[out])\n",
        "    if repeat_run_number == 0:\n",
        "      print(model.summary())\n",
        "    # ---------------- Compile and Fit ----------------\n",
        "    t = time.time()\n",
        "    model.compile(loss='categorical_crossentropy',\n",
        "                  optimizer='adam',\n",
        "                  metrics=['accuracy'])\n",
        "    \n",
        "    earlyStopping = EarlyStopping(monitor='val_loss', patience=35, verbose=0,restore_best_weights=True, mode='min')\n",
        "    history = model.fit((range_doppler_augmented_image,spectrogram_augmented_image,third_spectrogram_augmented_image),(spectrogram_concat_label_shuffle_concat),\n",
        "                    epochs=epoch_number,\n",
        "                    batch_size=batch_size,\n",
        "                    shuffle = True,\n",
        "                    verbose = 0,\n",
        "                    callbacks=[earlyStopping],\n",
        "                    validation_data = ((validation_range_doppler,validation_spectrogram,validation_third_spectrogram) , (spectrogram_validation_labels)))\n",
        "    elapsed = time.time() - t\n",
        "    tf.keras.models.load_model\n",
        "    test_loss, test_accuracy  = model.evaluate([range_doppler_concat_shuffle_test,spectrogram_concat_shuffle_test,third_spectrogram_concat_shuffle_test],\\\n",
        "                                               [spectrogram_concat_label_shuffle_test],\n",
        "                  batch_size=batch_size)\n",
        "    test_accuracy_per_fold.append(test_accuracy)\n",
        "    print(test_accuracy_per_fold)\n",
        "\n",
        "  test_accuracy_per_run.append(sum(test_accuracy_per_fold)/num_folds)\n",
        "  print(test_accuracy_per_run)\n",
        "\n",
        "print(f'Mean test accuracy is {\"{:.3f}\".format(sum(test_accuracy_per_run)/number_of_repeat)},\\\n",
        " max test accuracy is {\"{:.3f}\".format(max(test_accuracy_per_run))},\\\n",
        " min test accuracy is {\"{:.3f}\".format(min(test_accuracy_per_run))}, \\\n",
        " std of test accuracy is {\"{:.3f}\".format(np.std(test_accuracy_per_run, axis=0))}.')\n",
        "\n",
        "print(f'Time elapsed through all process: {\"{:.2f}\".format(elapsed)}, sec')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "t_test = time.time()\n",
        "y_test_predicted = model.predict((range_doppler_concat_shuffle_test[0:1,:,:,:],spectrogram_concat_shuffle_test[0:1,:,:,:],third_spectrogram_concat_shuffle_test[0:1,:,:,:]))\n",
        "test_elapsed = time.time() - t_test \n",
        "print(f'Time elapsed during test time: {\"{:.3f}\".format(test_elapsed)}, sec')"
      ],
      "metadata": {
        "id": "M7mr28f9ndza",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bd5d5716-da01-43bf-e9de-18aa76925dd2"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Time elapsed during test time: 17.070, sec\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "0N7v4PDP4Dv2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "defbeea5-2eeb-4e34-9383-f21f7d84e529"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mean test accuracy is 0.937, max test accuracy is 0.949, min test accuracy is 0.930,  std of test accuracy is 0.006.\n",
            "Time elapsed through all process: 470.058 sec\n",
            "Time elapsed during test time: 17.070 sec\n"
          ]
        }
      ],
      "source": [
        "print(f'Mean test accuracy is {\"{:.3f}\".format(sum(test_accuracy_per_run)/number_of_repeat)},\\\n",
        " max test accuracy is {\"{:.3f}\".format(max(test_accuracy_per_run))},\\\n",
        " min test accuracy is {\"{:.3f}\".format(min(test_accuracy_per_run))}, \\\n",
        " std of test accuracy is {\"{:.3f}\".format(np.std(test_accuracy_per_run, axis=0))}.')\n",
        "print(f'Time elapsed through all process: {\"{:.3f}\".format(elapsed)} sec')\n",
        "print(f'Time elapsed during test time: {\"{:.3f}\".format(test_elapsed)} sec')"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "machine_shape": "hm",
      "name": "3_Spectrograms_CI4R_Datasets_Multi_Class_LSTM_with_Flatten.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}