Range_Doppler_and_Spectrogram_LSTM_with_no_MixUp.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Range_Doppler_and_Spectrogram_LSTM_with_no_MixUp.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mertege/Thesis_Experiments/blob/main/Range_Doppler_and_Spectrogram_LSTM_with_no_MixUp.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kN4ixvH39HW6"
      },
      "source": [
        "# LSTM-based architecure is running in Linda dataset.\n",
        "# Mix-up is not used.\n",
        "# 2 LSTM block is used.\n",
        "# Range-doppler and spectrogram data are used in together.\n",
        "# It was run 5-times. Results are shared on below:\n",
        "# Min test accuracy is 0.929, Max test accuracy is 0.976, Mean test accuracy is 0.948, std of test accuracy is 0.019.\n",
        "# Time elapsed during training time: 93.257 sec\n",
        "# Time elapsed during test time: 0.211 sec"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lzJwN6GfkN8Z"
      },
      "source": [
        "from keras.models import Sequential\n",
        "from tensorflow.keras import layers\n",
        "from keras.models import Model\n",
        "from keras.layers import Activation, Dropout, Flatten, Dense, AveragePooling1D, BatchNormalization, Normalization, Input, Conv2D, MaxPooling2D, Concatenate, GRU, LSTM, GRU, TimeDistributed, Bidirectional\n",
        "from keras.regularizers import l2, l1\n",
        "import tensorflow as tf\n",
        "import scipy.io\n",
        "import numpy as np\n",
        "import cv2\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "from numpy.random import seed\n",
        "from sklearn.model_selection import KFold, StratifiedKFold\n",
        "import time\n",
        "from sklearn.metrics import precision_recall_fscore_support\n",
        "from keras.callbacks import EarlyStopping\n",
        "from numpy import array\n",
        "from numpy import argmax\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from keras import backend as K \n",
        "import gc\n",
        "import os"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8FERVnWrkrwE",
        "outputId": "b0dd9c05-0d49-467f-ef17-81c77d0390e7"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o19JHieQ81cS"
      },
      "source": [
        "# Get Range-Doppler data from\n",
        "range_doppler_fast_resized = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_fast_resized.mat')\n",
        "range_doppler_fast_resized = range_doppler_fast_resized['range_doppler_fast_resized']\n",
        "range_doppler_fast_resized = np.transpose(range_doppler_fast_resized, (2, 0, 1))\n",
        "# range_doppler_fast_resized = np.delete(range_doppler_fast_resized,(49), axis=0) # 50th row is deleted since there is no 50th row in spectrogram fast data.\n",
        "range_doppler_fast_label = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_fast_label.mat')\n",
        "range_doppler_fast_label = range_doppler_fast_label['range_doppler_fast_label']  \n",
        "\n",
        "range_doppler_slow_resized = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_slow_resized.mat')\n",
        "range_doppler_slow_resized = range_doppler_slow_resized['range_doppler_slow_resized']\n",
        "range_doppler_slow_resized = np.transpose(range_doppler_slow_resized, (2, 0, 1))\n",
        "range_doppler_slow_label = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_slow_label.mat')\n",
        "range_doppler_slow_label = range_doppler_slow_label['range_doppler_slow_label']  \n",
        "\n",
        "range_doppler_slow_pocket_resized = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_slow_pocket_resized.mat')\n",
        "range_doppler_slow_pocket_resized = range_doppler_slow_pocket_resized['range_doppler_slow_pocket_resized']\n",
        "range_doppler_slow_pocket_resized = np.transpose(range_doppler_slow_pocket_resized, (2, 0, 1))\n",
        "range_doppler_pocket_label = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_pocket_label.mat')\n",
        "range_doppler_pocket_label = range_doppler_pocket_label['range_doppler_pocket_label']  \n",
        "# Get Range-Doppler data from\n",
        "spectrogram_fast_resized = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_fast_full_size.mat')\n",
        "spectrogram_fast_resized = spectrogram_fast_resized['spectrogram_fast']\n",
        "spectrogram_fast_resized = np.transpose(spectrogram_fast_resized, (2, 0, 1))\n",
        "spectrogram_fast_label = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_fast_label.mat')\n",
        "spectrogram_fast_label = spectrogram_fast_label['spectrogram_fast_label']  \n",
        "\n",
        "spectrogram_slow_resized = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_slow_full_size.mat')\n",
        "spectrogram_slow_resized = spectrogram_slow_resized['spectrogram_slow']\n",
        "spectrogram_slow_resized = np.transpose(spectrogram_slow_resized, (2, 0, 1))\n",
        "spectrogram_slow_label = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_slow_label.mat')\n",
        "spectrogram_slow_label = spectrogram_slow_label['spectrogram_slow_label']  \n",
        "\n",
        "spectrogram_slow_pocket_resized = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_slow_pocket_full_size.mat')\n",
        "spectrogram_slow_pocket_resized = spectrogram_slow_pocket_resized['spectrogram_slow_pocket']\n",
        "spectrogram_slow_pocket_resized = np.transpose(spectrogram_slow_pocket_resized, (2, 0, 1))\n",
        "spectrogram_slow_pocket_label = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_slow_pocket_label.mat')\n",
        "spectrogram_slow_pocket_label = spectrogram_slow_pocket_label['spectrogram_slow_pocket_label']  "
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wFp_PErnMLN2"
      },
      "source": [
        "# Concat range-doppler data\n",
        "range_doppler_concat = np.concatenate((range_doppler_fast_resized,range_doppler_slow_resized),axis=0)\n",
        "range_doppler_concat = np.concatenate((range_doppler_concat,range_doppler_slow_pocket_resized),axis=0)\n",
        "range_doppler_concat = range_doppler_concat[:,:,:,np.newaxis] \n",
        "range_doppler_concat_label = np.zeros((range_doppler_concat.shape[0],1))\n",
        "range_doppler_concat_label[:range_doppler_fast_resized.shape[0],:] = 1\n",
        "# Shuffle concat range doppler\n",
        "shuffle_indx = random.sample(range(0, range_doppler_concat.shape[0]), range_doppler_concat.shape[0]) # split validation data\n",
        "range_doppler_concat_shuffle = range_doppler_concat[shuffle_indx,:,:,:]\n",
        "range_doppler_concat_label_shuffle = range_doppler_concat_label[shuffle_indx,:]\n",
        "# Concat range-doppler data\n",
        "spectrogram_concat = np.concatenate((spectrogram_fast_resized,spectrogram_slow_resized),axis=0)\n",
        "spectrogram_concat = np.concatenate((spectrogram_concat,spectrogram_slow_pocket_resized),axis=0)\n",
        "spectrogram_concat = spectrogram_concat[:,:,:,np.newaxis] \n",
        "spectrogram_concat_label = np.zeros((spectrogram_concat.shape[0],1))\n",
        "spectrogram_concat_label[:spectrogram_fast_resized.shape[0],:] = 1\n",
        "# Shuffle concat range doppler\n",
        "# shuffle_indx = random.sample(range(0, spectrogram_concat.shape[0]), spectrogram_concat.shape[0]) # split validation data\n",
        "spectrogram_concat_shuffle = spectrogram_concat[shuffle_indx,:,:,:]\n",
        "spectrogram_concat_label_shuffle = spectrogram_concat_label[shuffle_indx,:]"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_1aBNkjoLCvI"
      },
      "source": [
        "# ---------------- Augmente and shuffle (train and test) data data ----------------\n",
        "data_augmentation = tf.keras.Sequential([\n",
        "  layers.RandomFlip(\"horizontal_and_vertical\"),\n",
        "  layers.RandomRotation(0.1),\n",
        "])\n",
        "\n",
        "\n",
        "def mixup_augmentation(images,range_doppler_training_data, labels, repeat_of_mixup, alpha=0.2):\n",
        "    batch_size = images.shape[0]\n",
        "    concat_images = np.zeros((batch_size*(repeat_of_mixup+1),images.shape[1],images.shape[2],images.shape[3]))\n",
        "    concat_images_range_doppler = np.zeros((batch_size*(repeat_of_mixup+1),range_doppler_training_data.shape[1],\\\n",
        "                                            range_doppler_training_data.shape[2],range_doppler_training_data.shape[3]))\n",
        "    concat_label = np.zeros((batch_size*(repeat_of_mixup+1),labels.shape[1]))\n",
        "    if repeat_of_mixup != 0:\n",
        "      for ii in range(repeat_of_mixup):\n",
        "        # shuffle train dataset\n",
        "        shuffle_indx_1 = random.sample(range(0, images.shape[0]), images.shape[0]) # split validation data\n",
        "        images_shuffled_1 = images[shuffle_indx_1,:,:,:]\n",
        "        range_doppler_training_data_shuffled_1 = range_doppler_training_data[shuffle_indx_1,:,:,:]\n",
        "        labels_shuffled_1 = labels[shuffle_indx_1,:]\n",
        "\n",
        "        shuffle_indx_2 = random.sample(range(0, images.shape[0]), images.shape[0]) # split validation data\n",
        "        images_shuffled_2 = images[shuffle_indx_2,:,:,:]\n",
        "        range_doppler_training_data_shuffled_2 = range_doppler_training_data[shuffle_indx_2,:,:,:]\n",
        "        labels_shuffled_2 = labels[shuffle_indx_2,:]\n",
        "\n",
        "        # Sample lambda and reshape it to do the mixup\n",
        "        gaussian_mean = 0.2\n",
        "        gaussian_std = 0.02\n",
        "        ll = np.random.normal(gaussian_mean, gaussian_std, (batch_size,1,1,1))\n",
        "        x_l = np.reshape(ll, (batch_size,1,1,1))\n",
        "        y_l = np.reshape(ll, (batch_size,1))\n",
        "        \n",
        "        # Perform mixup on both images and labels by combining a pair of images/labels\n",
        "        images_mixup = images_shuffled_1 * x_l + images_shuffled_2 * (1 - x_l)\n",
        "        images_mixup_range_doppler = range_doppler_training_data_shuffled_1 * x_l + range_doppler_training_data_shuffled_2 * (1 - x_l)\n",
        "        labels_mixup = labels_shuffled_1 * y_l + labels_shuffled_2 * (1 - y_l)\n",
        "        concat_images[ii*batch_size:(ii+1)*batch_size,:,:,:] = images_mixup\n",
        "        concat_images_range_doppler[ii*batch_size:(ii+1)*batch_size,:,:,:] = images_mixup_range_doppler\n",
        "        concat_label[ii*batch_size:(ii+1)*batch_size,:] = labels_mixup\n",
        "\n",
        "    concat_images[repeat_of_mixup*batch_size:,:,:,:] = images\n",
        "    concat_images_range_doppler[repeat_of_mixup*batch_size:,:,:,:] = range_doppler_training_data\n",
        "    concat_label[repeat_of_mixup*batch_size:,:] = labels\n",
        "    return (concat_images,concat_images_range_doppler, concat_label)\n",
        "def split_and_augmentation_of_training(spectrogram_concat_shuffle_train,range_doppler_concat_shuffle_train,range_doppler_concat_label_shuffle_train,\\\n",
        "                                       repeat_of_mixup, augmentation_enable):\n",
        "  # ---------------- Parameters ----------------\n",
        "  repeat_of_augmentation_for_fast = 1\n",
        "  repeat_of_augmentation_for_slow = np.floor(repeat_of_augmentation_for_fast/2)\n",
        "  repeat_of_augmentation_for_slow = int(repeat_of_augmentation_for_slow)\n",
        "  # size_of_validation = 30\n",
        "  alpha = 0.2\n",
        "  dummy_label = np.zeros((spectrogram_concat_shuffle_train.shape[0],1))\n",
        "  for randomlist_for_train_indx, randomlist_for_validation_indx in kfold.split(spectrogram_concat_shuffle_train,dummy_label):   \n",
        "    randomlist_for_validation_indx\n",
        "  # Split validation\n",
        "  # randomlist_for_validation_indx = random.sample(range(0, range_doppler_concat_shuffle_train.shape[0]), size_of_validation) # split validation data\n",
        "  # randomlist_for_train_indx = np.delete(range(0, range_doppler_concat_shuffle_train.shape[0]), randomlist_for_validation_indx) # split training data\n",
        "  # get validation data\n",
        "  spectrogram_validation_data = spectrogram_concat_shuffle_train[randomlist_for_validation_indx,:,:,:]\n",
        "  spectrogram_validation_labels = range_doppler_concat_label_shuffle_train[randomlist_for_validation_indx,:]\n",
        "  range_doppler_validation_data = range_doppler_concat_shuffle_train[randomlist_for_validation_indx,:,:,:]\n",
        "  # get training data\n",
        "  spectrogram_training_data = spectrogram_concat_shuffle_train[randomlist_for_train_indx,:,:,:]\n",
        "  spectrogram_training_labels = spectrogram_concat_label_shuffle_train[randomlist_for_train_indx,:]\n",
        "  range_doppler_training_data = range_doppler_concat_shuffle_train[randomlist_for_train_indx,:,:,:]\n",
        "\n",
        "  # Rotate Augmentation\n",
        "  # get slow and fast indexes of training data\n",
        "  slow_indexes = np.where(spectrogram_training_labels == 0)[0]\n",
        "  fast_indexes = np.delete(range(0, spectrogram_training_labels.shape[0]), slow_indexes)  \n",
        "\n",
        "  slow_spectrograms_train = spectrogram_training_data[slow_indexes,:,:,:]\n",
        "  size_of_samples_slow = slow_spectrograms_train.shape[0]\n",
        "\n",
        "  fast_spectrograms_train = spectrogram_training_data[fast_indexes,:,:,:]  \n",
        "  size_of_samples_fast = fast_spectrograms_train.shape[0]\n",
        "\n",
        "  slow_range_train = range_doppler_training_data[slow_indexes,:,:,:]\n",
        "  fast_range_train = range_doppler_training_data[fast_indexes,:,:,:]  \n",
        "\n",
        "  if augmentation_enable == True: \n",
        "    # ---------------- Augmente Train Data for Fast ----------------\n",
        "    augmented_image_fast = np.zeros((size_of_samples_fast*repeat_of_augmentation_for_fast,fast_spectrograms_train.shape[1],fast_spectrograms_train.shape[2],1))\n",
        "    spectrograms_fast_label = np.ones((size_of_samples_fast*(repeat_of_augmentation_for_fast+1),1))\n",
        "    # augmented_image_fast = np.flip(fast_spectrograms_train,axis=2)\n",
        "    for jj in range(repeat_of_augmentation_for_fast):\n",
        "      for ii in range(size_of_samples_fast):\n",
        "        augmented_image_fast[size_of_samples_fast*jj+ii,:,:,:] = data_augmentation(fast_spectrograms_train[ii,:,:,:])\n",
        "    augmented_image_fast = np.concatenate((augmented_image_fast,fast_spectrograms_train),axis=0)   \n",
        "\n",
        "    augmented_image_fast_range = np.zeros((size_of_samples_fast*repeat_of_augmentation_for_fast,fast_range_train.shape[1],fast_range_train.shape[2],1))\n",
        "    # augmented_image_fast_range  = np.flip(fast_range_train,axis=2)\n",
        "    for jj in range(repeat_of_augmentation_for_fast):\n",
        "      for ii in range(size_of_samples_fast):\n",
        "        augmented_image_fast_range[size_of_samples_fast*jj+ii,:,:,:]  =data_augmentation(fast_range_train[ii,:,:,:])\n",
        "    augmented_image_fast_range = np.concatenate((augmented_image_fast_range,fast_range_train),axis=0)   \n",
        "\n",
        "    # ---------------- Augmente Train Data for Slow ----------------\n",
        "    augmented_image_slow = slow_spectrograms_train\n",
        "    augmented_image_slow_range = slow_range_train\n",
        "\n",
        "  else:\n",
        "    augmented_image_fast = fast_spectrograms_train\n",
        "    augmented_image_slow = slow_spectrograms_train\n",
        "    augmented_image_fast_range = fast_range_train\n",
        "    augmented_image_slow_range = slow_range_train\n",
        "    spectrograms_fast_label = np.ones((size_of_samples_fast,1))\n",
        "    \n",
        "  spectrograms_slow_label = np.zeros((size_of_samples_slow,1))\n",
        "\n",
        "  spectrogram_training_data = np.concatenate((augmented_image_fast,augmented_image_slow),axis=0)\n",
        "  range_doppler_training_data = np.concatenate((augmented_image_fast_range,augmented_image_slow_range),axis=0)\n",
        "  spectrogram_training_labels = np.concatenate((spectrograms_fast_label,spectrograms_slow_label),axis=0)\n",
        "\n",
        "  (spectrogram_augmented_image,range_doppler_augmented_image,spectrograms_label)=\\\n",
        "   mixup_augmentation(spectrogram_training_data,range_doppler_training_data, spectrogram_training_labels, repeat_of_mixup, alpha=0.2)\n",
        "\n",
        "  return (spectrogram_augmented_image,range_doppler_augmented_image,spectrograms_label,\\\n",
        "     spectrogram_validation_data,range_doppler_validation_data, spectrogram_validation_labels)\n",
        "def normalize_inputs(range_doppler_concat_shuffle, normalize_inputs_enable):\n",
        "  # ---------------- Normalize Inputs ----------------\n",
        "  if normalize_inputs_enable == True:\n",
        "    layer = Normalization(axis=None)\n",
        "    layer.adapt(range_doppler_concat_shuffle)\n",
        "    range_doppler_concat_shuffle = layer(range_doppler_concat_shuffle)\n",
        "  else:\n",
        "    range_doppler_concat_shuffle = range_doppler_concat_shuffle\n",
        "  return(range_doppler_concat_shuffle)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "range_doppler_concat_shuffle_new = np.zeros((range_doppler_concat_shuffle.shape[0],52,512,1))\n",
        "for ii in range(range_doppler_concat_shuffle.shape[0]):\n",
        "  aa = range_doppler_concat_shuffle[ii,:,:,0]\n",
        "  range_doppler_concat_shuffle_new[ii,:,:,0] = cv2.resize(aa, (range_doppler_concat_shuffle_new.shape[2], range_doppler_concat_shuffle_new.shape[1]),interpolation = cv2.INTER_CUBIC)\n",
        "range_doppler_concat_shuffle = range_doppler_concat_shuffle_new\n",
        "del aa\n",
        "del range_doppler_concat_shuffle_new\n",
        "\n",
        "spectrogram_concat_shuffle_new = np.zeros((spectrogram_concat_shuffle.shape[0],52,423,1))\n",
        "for ii in range(spectrogram_concat_shuffle.shape[0]):\n",
        "  aa = spectrogram_concat_shuffle[ii,:,:,0]\n",
        "  spectrogram_concat_shuffle_new[ii,:,:,0] = cv2.resize(aa, (spectrogram_concat_shuffle_new.shape[2], spectrogram_concat_shuffle_new.shape[1]),interpolation = cv2.INTER_CUBIC)\n",
        "spectrogram_concat_shuffle = spectrogram_concat_shuffle_new\n",
        "del aa\n",
        "del spectrogram_concat_shuffle_new"
      ],
      "metadata": {
        "id": "IXw6G3EYuQbJ"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8CUvxggth7ZQ"
      },
      "source": [
        "normalize_inputs_enable = 1\n",
        "range_doppler_concat_shuffle = normalize_inputs(range_doppler_concat_shuffle, normalize_inputs_enable)\n",
        "spectrogram_concat_shuffle = normalize_inputs(spectrogram_concat_shuffle, normalize_inputs_enable)\n",
        "range_doppler_concat_shuffle = np.float32(range_doppler_concat_shuffle)\n",
        "spectrogram_concat_shuffle = np.float32(spectrogram_concat_shuffle)\n"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "n_features = range_doppler_concat_shuffle.shape[1]\n",
        "n_steps = range_doppler_concat_shuffle.shape[2]\n",
        "range_doppler_concat_shuffle = np.transpose(range_doppler_concat_shuffle, axes = (0,2,1,3)) \n",
        "spectrogram_concat_shuffle = np.transpose(spectrogram_concat_shuffle, axes = (0,2,1,3)) "
      ],
      "metadata": {
        "id": "38jkSYXM4aey"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rNpkBBIadpPo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e0fa5d5d-1197-4480-92a6-cf5912325361"
      },
      "source": [
        "\n",
        "# ---------- Parameters ----------------\n",
        "augmentation_enable = True\n",
        "normalize_inputs_enable = True\n",
        "num_folds = 5\n",
        "kfold = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state = None) # random_state = 1 ile split run'dan run'a sabit.\n",
        "test_accuracy_per_run = []\n",
        "f1_score_per_run = []\n",
        "epoch_number = 100\n",
        "batch_size = 32\n",
        "dense_size = 64\n",
        "dropout_prob_dense = 0.1\n",
        "repeat_of_mixup = 0\n",
        "number_of_repeat = 5\n",
        "unit_number_of_lstm = 16\n",
        "dense_unit_of_range_doppler_function = 256\n",
        "dense_unit_of_spectrogram_function = 8\n",
        "decoder_dense_unit = 256\n",
        "\n",
        "for repeat_run_number in range(number_of_repeat):\n",
        "  test_accuracy_per_fold = []\n",
        "  f1_score_per_fold = []\n",
        "  if repeat_run_number > 0:\n",
        "    del range_doppler_concat_shuffle_test\n",
        "    del spectrogram_concat_shuffle_test\n",
        "    del range_doppler_augmented_image\n",
        "    del range_doppler_concat_shuffle_train\n",
        "    del spectrogram_concat_shuffle_train\n",
        "    del spectrogram_augmented_image\n",
        "   \n",
        "  for randomlist_for_train_indx, randomlist_for_test_indx in kfold.split(range_doppler_concat_shuffle,range_doppler_concat_label_shuffle):   \n",
        "    gc.collect()\n",
        "    K.clear_session()\n",
        "    \n",
        "    # test data\n",
        "    range_doppler_concat_shuffle_test = range_doppler_concat_shuffle[randomlist_for_test_indx,:,:,:]\n",
        "    spectrogram_concat_shuffle_test = spectrogram_concat_shuffle[randomlist_for_test_indx,:,:,:]\n",
        "    range_doppler_concat_label_shuffle_test = range_doppler_concat_label_shuffle[randomlist_for_test_indx,:]\n",
        "    #train data\n",
        "    range_doppler_concat_shuffle_train = range_doppler_concat_shuffle[randomlist_for_train_indx,:,:,:]\n",
        "    spectrogram_concat_shuffle_train = spectrogram_concat_shuffle[randomlist_for_train_indx,:,:,:]\n",
        "    spectrogram_concat_label_shuffle_train = spectrogram_concat_label_shuffle[randomlist_for_train_indx,:]\n",
        "      # ---------------- MixUp Augmentation ----------------\n",
        "    (spectrogram_augmented_image,range_doppler_augmented_image,spectrogram_concat_label_shuffle_concat,\\\n",
        "     validation_spectrogram,validation_range_doppler, spectrogram_validation_labels)  =\\\n",
        "      split_and_augmentation_of_training(spectrogram_concat_shuffle_train,range_doppler_concat_shuffle_train,\\\n",
        "                                         spectrogram_concat_label_shuffle_train,\\\n",
        "                                         repeat_of_mixup, augmentation_enable)\n",
        "    \n",
        "    \n",
        "    # ---------------- Neural Network Architecture ----------------\n",
        "\n",
        "\n",
        "\n",
        "    def lstm_encoder_network_1(input_shape):\n",
        "        input = Input(shape=input_shape)\n",
        "        x = Bidirectional(LSTM(unit_number_of_lstm, return_sequences=True, dropout = 0.5))(input)\n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)\n",
        "        x = Bidirectional(LSTM(unit_number_of_lstm, return_sequences=True, dropout = 0.5))(x)\n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)                  \n",
        "        x = Flatten()(x)\n",
        "        x = BatchNormalization()(x)\n",
        "        x = Dropout(0.5)(x)\n",
        "        x = Dense(dense_unit_of_range_doppler_function)(x)\n",
        "        x = BatchNormalization()(x)\n",
        "        x = Activation('LeakyReLU')(x)\n",
        "        x = Dropout(dropout_prob_dense)(x)\n",
        "        return Model(input, x)\n",
        "\n",
        "    def lstm_encoder_network_2(input_shape):\n",
        "        input = Input(shape=input_shape)\n",
        "        x = Bidirectional(LSTM(unit_number_of_lstm, return_sequences=True, dropout = 0.5))(input)\n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)\n",
        "        x = Bidirectional(LSTM(unit_number_of_lstm, return_sequences=True, dropout = 0.5))(x)\n",
        "        x = AveragePooling1D(pool_size=2, strides=None)(x)                         \n",
        "        x = Flatten()(x)\n",
        "        x = BatchNormalization()(x)\n",
        "        x = Dropout(0.5)(x)\n",
        "        x = Dense(dense_unit_of_spectrogram_function)(x)\n",
        "        x = BatchNormalization()(x)\n",
        "        x = Activation('LeakyReLU')(x)\n",
        "        x = Dropout(dropout_prob_dense)(x)\n",
        "        return Model(input, x)\n",
        "\n",
        "    def decoder_for_concat(input_shape):\n",
        "      input = Input(shape=input_shape)\n",
        "      x = BatchNormalization()(input)\n",
        "      x = Dense(decoder_dense_unit)(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('LeakyReLU')(x)\n",
        "      x = Dropout(0.3)(x)\n",
        "      x = Dense(dense_size)(x)\n",
        "      # x = BatchNormalization()(x)\n",
        "      x = Activation('LeakyReLU')(x)\n",
        "      x = Dropout(dropout_prob_dense)(x)\n",
        "      x = Dense(1, activation=\"sigmoid\")(x)\n",
        "      return Model(input, x)\n",
        "\n",
        "    input_shape = range_doppler_concat_shuffle.shape[1:3]\n",
        "    base_network_lstm = lstm_encoder_network_1(input_shape)\n",
        "    range_doppler_input  = Input(shape=input_shape)\n",
        "    processed_range_doppler  = base_network_lstm(range_doppler_input)\n",
        "\n",
        "    input_shape = spectrogram_concat_shuffle_train.shape[1:3]\n",
        "    base_network_lstm_2 = lstm_encoder_network_2(input_shape)\n",
        "    spectrogram_input  = Input(shape=input_shape)\n",
        "    processed_spectrogram  = base_network_lstm_2(spectrogram_input)\n",
        "\n",
        "    concat_layer = Concatenate()([processed_range_doppler, processed_spectrogram])\n",
        "\n",
        "    base_decoder_network = decoder_for_concat((concat_layer.shape[1]))\n",
        "    out = base_decoder_network(concat_layer)\n",
        "\n",
        "    model = Model(inputs=[range_doppler_input, spectrogram_input], outputs=[out]) \n",
        "    if repeat_run_number == 0:\n",
        "      print(model.summary())\n",
        "    # ---------------- Compile and Fit ----------------\n",
        "    t = time.time()\n",
        "    model.compile(loss='binary_crossentropy',\n",
        "                  optimizer='adam',\n",
        "                  metrics=['accuracy'])\n",
        "    \n",
        "    earlyStopping = EarlyStopping(monitor='val_loss', patience=25, verbose=0,restore_best_weights=True, mode='min')\n",
        "    history = model.fit((range_doppler_augmented_image, spectrogram_augmented_image),(spectrogram_concat_label_shuffle_concat),\n",
        "                    epochs=epoch_number,\n",
        "                    batch_size=batch_size,\n",
        "                    shuffle = True,\n",
        "                    verbose=0,\n",
        "                    callbacks=[earlyStopping],\n",
        "                    validation_data = ((validation_range_doppler, validation_spectrogram) , (spectrogram_validation_labels)))\n",
        "    elapsed = time.time() - t\n",
        "    test_loss, test_accuracy  = model.evaluate([range_doppler_concat_shuffle_test, spectrogram_concat_shuffle_test],\\\n",
        "                                               [range_doppler_concat_label_shuffle_test],\n",
        "                  batch_size=batch_size)\n",
        "    \n",
        "    gc.collect()\n",
        "    # ---------------- Get Test Results ----------------\n",
        "    y_test_predicted = model.predict((range_doppler_concat_shuffle_test, spectrogram_concat_shuffle_test), batch_size=batch_size)\n",
        "    # ----- Binarize y_test_predicted values -----\n",
        "    y_test_predicted_binary = np.zeros(y_test_predicted.size)\n",
        "    for ii in range(y_test_predicted.size):\n",
        "      if y_test_predicted[ii] < 0.5:\n",
        "        y_test_predicted_binary[ii] = 0\n",
        "      else:\n",
        "        y_test_predicted_binary[ii] = 1\n",
        "    \n",
        "    test_precision, test_recall, test_f1_score, support = precision_recall_fscore_support(range_doppler_concat_label_shuffle_test, y_test_predicted_binary, average='macro')\n",
        "\n",
        "    test_accuracy_per_fold.append(test_accuracy)\n",
        "    f1_score_per_fold.append(test_f1_score)\n",
        "  test_accuracy_per_run.append(sum(test_accuracy_per_fold)/num_folds)\n",
        "  f1_score_per_run.append(sum(f1_score_per_fold)/num_folds)\n",
        "  print(test_accuracy_per_run)\n",
        "  print(f1_score_per_run)\n",
        "print(f'Mean test accuracy is {\"{:.3f}\".format(sum(test_accuracy_per_run)/number_of_repeat)}, mean test f1 score is {\"{:.3f}\".format(sum(f1_score_per_run)/number_of_repeat)}, \\\n",
        "max test accuracy is {\"{:.3f}\".format(max(test_accuracy_per_run))}, max test f1 score is {\"{:.3f}\".format(max(f1_score_per_run))}, \\\n",
        "min test accuracy is {\"{:.3f}\".format(min(test_accuracy_per_run))}, min test f1 score is {\"{:.3f}\".format(min(f1_score_per_run))}, \\\n",
        "std of test accuracy is {\"{:.3f}\".format(np.std(test_accuracy_per_run, axis=0))}, std of test f1 score is {\"{:.3f}\".format(np.std(f1_score_per_run, axis=0))}')\n",
        "\n",
        "print(f'Time elapsed through all process: {\"{:.3f}\".format(elapsed)}, sec')"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 512, 52)]    0           []                               \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 423, 52)]    0           []                               \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          1081344     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 8)            55464       ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 264)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            86433       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 1,223,241\n",
            "Trainable params: 1,206,761\n",
            "Non-trainable params: 16,480\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 4s 47ms/step - loss: 0.1228 - accuracy: 0.9706\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 512, 52)]    0           []                               \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 423, 52)]    0           []                               \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          1081344     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 8)            55464       ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 264)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            86433       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 1,223,241\n",
            "Trainable params: 1,206,761\n",
            "Non-trainable params: 16,480\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 3s 36ms/step - loss: 0.0012 - accuracy: 1.0000\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 512, 52)]    0           []                               \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 423, 52)]    0           []                               \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          1081344     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 8)            55464       ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 264)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            86433       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 1,223,241\n",
            "Trainable params: 1,206,761\n",
            "Non-trainable params: 16,480\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 3s 41ms/step - loss: 0.5862 - accuracy: 0.9412\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 512, 52)]    0           []                               \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 423, 52)]    0           []                               \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          1081344     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 8)            55464       ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 264)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            86433       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 1,223,241\n",
            "Trainable params: 1,206,761\n",
            "Non-trainable params: 16,480\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 3s 39ms/step - loss: 0.1520 - accuracy: 0.9412\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 512, 52)]    0           []                               \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 423, 52)]    0           []                               \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          1081344     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 8)            55464       ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 264)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            86433       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 1,223,241\n",
            "Trainable params: 1,206,761\n",
            "Non-trainable params: 16,480\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 3s 40ms/step - loss: 0.7712 - accuracy: 0.8235\n",
            "WARNING:tensorflow:5 out of the last 9 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7fa79e6485f0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
            "[0.9352941155433655]\n",
            "[0.9238844971453666]\n",
            "2/2 [==============================] - 3s 37ms/step - loss: 0.3225 - accuracy: 0.9118\n",
            "WARNING:tensorflow:6 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7fa79fa1e0e0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
            "2/2 [==============================] - 3s 38ms/step - loss: 0.0798 - accuracy: 1.0000\n",
            "2/2 [==============================] - 3s 38ms/step - loss: 0.1347 - accuracy: 0.9412\n",
            "2/2 [==============================] - 3s 40ms/step - loss: 0.1098 - accuracy: 0.9706\n",
            "2/2 [==============================] - 3s 39ms/step - loss: 0.0022 - accuracy: 1.0000\n",
            "[0.9352941155433655, 0.9647058725357056]\n",
            "[0.9238844971453666, 0.959967024810914]\n",
            "2/2 [==============================] - 3s 39ms/step - loss: 0.3086 - accuracy: 0.9412\n",
            "2/2 [==============================] - 3s 41ms/step - loss: 0.1485 - accuracy: 0.9118\n",
            "2/2 [==============================] - 3s 42ms/step - loss: 0.2887 - accuracy: 0.9118\n",
            "2/2 [==============================] - 3s 39ms/step - loss: 0.0683 - accuracy: 0.9706\n",
            "2/2 [==============================] - 3s 38ms/step - loss: 0.2883 - accuracy: 0.9118\n",
            "[0.9352941155433655, 0.9647058725357056, 0.9294117450714111]\n",
            "[0.9238844971453666, 0.959967024810914, 0.9191743021599634]\n",
            "2/2 [==============================] - 3s 39ms/step - loss: 0.0405 - accuracy: 0.9706\n",
            "2/2 [==============================] - 3s 38ms/step - loss: 0.0195 - accuracy: 1.0000\n",
            "2/2 [==============================] - 3s 39ms/step - loss: 0.0988 - accuracy: 0.9706\n",
            "2/2 [==============================] - 3s 39ms/step - loss: 0.5839 - accuracy: 0.9412\n",
            "2/2 [==============================] - 3s 39ms/step - loss: 0.3189 - accuracy: 0.7941\n",
            "[0.9352941155433655, 0.9647058725357056, 0.9294117450714111, 0.9352941036224365]\n",
            "[0.9238844971453666, 0.959967024810914, 0.9191743021599634, 0.9174717599034679]\n",
            "2/2 [==============================] - 3s 40ms/step - loss: 0.1077 - accuracy: 0.9412\n",
            "2/2 [==============================] - 3s 37ms/step - loss: 0.1215 - accuracy: 0.9706\n",
            "2/2 [==============================] - 3s 40ms/step - loss: 0.1086 - accuracy: 1.0000\n",
            "2/2 [==============================] - 3s 40ms/step - loss: 0.0063 - accuracy: 1.0000\n",
            "2/2 [==============================] - 3s 38ms/step - loss: 0.4750 - accuracy: 0.9706\n",
            "[0.9352941155433655, 0.9647058725357056, 0.9294117450714111, 0.9352941036224365, 0.9764705777168274]\n",
            "[0.9238844971453666, 0.959967024810914, 0.9191743021599634, 0.9174717599034679, 0.9731016521765827]\n",
            "Mean test accuracy is 0.948, mean test f1 score is 0.939, max test accuracy is 0.976, max test f1 score is 0.973, min test accuracy is 0.929, min test f1 score is 0.917, std of test accuracy is 0.019, std of test f1 score is 0.023\n",
            "Time elapsed through all process: 93.257, sec\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "t_test = time.time()\n",
        "y_test_predicted = model.predict((range_doppler_concat_shuffle_test[0:1,:,:,:],spectrogram_concat_shuffle_test[0:1,:,:,:]))\n",
        "test_elapsed = time.time() - t_test \n",
        "print(f'Time elapsed during test time: {\"{:.2f}\".format(test_elapsed)}, sec')"
      ],
      "metadata": {
        "id": "okw85Hdg7ClG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "52b25df5-fe9c-4d6c-a4ca-6299ea9d89c4"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Time elapsed during test time: 0.21, sec\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0N7v4PDP4Dv2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0fe4161a-d2c4-4ae3-bef6-78899abe45e7"
      },
      "source": [
        "print(f'Min test accuracy is {\"{:.3f}\".format(min(test_accuracy_per_run))}, \\\n",
        "Max test accuracy is {\"{:.3f}\".format(max(test_accuracy_per_run))}, \\\n",
        "Mean test accuracy is {\"{:.3f}\".format(sum(test_accuracy_per_run)/number_of_repeat)}, \\\n",
        "std of test accuracy is {\"{:.3f}\".format(np.std(test_accuracy_per_run, axis=0))}.')\n",
        "print(f'Time elapsed during training time: {\"{:.3f}\".format(elapsed)} sec')\n",
        "print(f'Time elapsed during test time: {\"{:.3f}\".format(test_elapsed)} sec')"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Min test accuracy is 0.929, Max test accuracy is 0.976, Mean test accuracy is 0.948, std of test accuracy is 0.019.\n",
            "Time elapsed during training time: 93.257 sec\n",
            "Time elapsed during test time: 0.211 sec\n"
          ]
        }
      ]
    }
  ]
}