Range_Doppler_CI4R_Datasets_Multi_Class_CNN_10Ghz.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mertege/Thesis_Experiments/blob/main/Range_Doppler_CI4R_Datasets_Multi_Class_CNN_10Ghz.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kN4ixvH39HW6"
      },
      "outputs": [],
      "source": [
        "# Image-based architecure is running with only 10GHz in Gürbüz dataset.\n",
        "# 5-blocks are used. Reslotion 128x128\n",
        "# Only Range-Doppler plots are used.\n",
        "# It was run 3-times. Results are shared on below:\n",
        "# Min test accuracy is 0.887, Max test accuracy is 0.902, Mean test accuracy is 0.897, std of test accuracy is 0.007.\n",
        "# Time elapsed during training time: 40.898 sec\n",
        "# Time elapsed during test time: 0.179 sec"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "lzJwN6GfkN8Z"
      },
      "outputs": [],
      "source": [
        "from keras.models import Sequential\n",
        "from tensorflow.keras import layers\n",
        "from keras.models import Model\n",
        "from keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization, Normalization, Input, Conv2D, MaxPooling2D, Concatenate\n",
        "from tensorflow.keras import regularizers\n",
        "import tensorflow as tf\n",
        "import scipy.io\n",
        "import numpy as np\n",
        "import cv2\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "from numpy.random import seed\n",
        "from sklearn.model_selection import KFold, StratifiedKFold\n",
        "import time\n",
        "from sklearn.metrics import precision_recall_fscore_support\n",
        "from keras.callbacks import EarlyStopping\n",
        "from numpy import array\n",
        "from numpy import argmax\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from keras import backend as K \n",
        "import gc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8FERVnWrkrwE",
        "outputId": "053b654c-4cf3-4db1-fc17-88a0093a42d0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Range-Doppler plots\n",
        "bending_Xethru = scipy.io.loadmat('/content/drive/MyDrive/rangedopplers10ghz_bending.mat')\n",
        "bending_Xethru = bending_Xethru['rds']\n",
        "bending_Xethru = np.transpose(bending_Xethru, (2, 0, 1))\n",
        "label_bending_Xethru           = np.zeros((bending_Xethru.shape[0],1))\n",
        "label_bending_Xethru[:]        = 1\n",
        "\n",
        "crawling_Xethru = scipy.io.loadmat('/content/drive/MyDrive/rangedopplers10ghz_bending.mat')\n",
        "crawling_Xethru = crawling_Xethru['rds']\n",
        "crawling_Xethru = np.transpose(crawling_Xethru, (2, 0, 1))\n",
        "label_crawling_Xethru           = np.zeros((crawling_Xethru.shape[0],1))\n",
        "label_crawling_Xethru[:]        = 2\n",
        "\n",
        "kneeling_Xethru = scipy.io.loadmat('/content/drive/MyDrive/rangedopplers10ghz_kneeling.mat')\n",
        "kneeling_Xethru = kneeling_Xethru['rds']\n",
        "kneeling_Xethru = np.transpose(kneeling_Xethru, (2, 0, 1))\n",
        "label_kneeling_Xethru           = np.zeros((kneeling_Xethru.shape[0],1))\n",
        "label_kneeling_Xethru[:]        = 3\n",
        "\n",
        "limping_Xethru = scipy.io.loadmat('/content/drive/MyDrive/rangedopplers10ghz_limping.mat')\n",
        "limping_Xethru = limping_Xethru['rds']\n",
        "limping_Xethru = np.transpose(limping_Xethru, (2, 0, 1))\n",
        "label_limping_Xethru           = np.zeros((limping_Xethru.shape[0],1))\n",
        "label_limping_Xethru[:]        = 4\n",
        "\n",
        "pickingobject_Xethru = scipy.io.loadmat('/content/drive/MyDrive/rangedopplers10ghz_pickingobject.mat')\n",
        "pickingobject_Xethru = pickingobject_Xethru['rds']\n",
        "pickingobject_Xethru = np.transpose(pickingobject_Xethru, (2, 0, 1))\n",
        "label_pickingobject_Xethru           = np.zeros((pickingobject_Xethru.shape[0],1))\n",
        "label_pickingobject_Xethru[:]        = 5\n",
        "\n",
        "scissorsgait_Xethru = scipy.io.loadmat('/content/drive/MyDrive/rangedopplers10ghz_scissorsgait.mat')\n",
        "scissorsgait_Xethru = scissorsgait_Xethru['rds']\n",
        "scissorsgait_Xethru = np.transpose(scissorsgait_Xethru, (2, 0, 1))\n",
        "label_scissorsgait_Xethru           = np.zeros((scissorsgait_Xethru.shape[0],1))\n",
        "label_scissorsgait_Xethru[:]        = 6\n",
        "\n",
        "shortstep_Xethru = scipy.io.loadmat('/content/drive/MyDrive/rangedopplers10ghz_shortstep.mat')\n",
        "shortstep_Xethru = shortstep_Xethru['rds']\n",
        "shortstep_Xethru = np.transpose(shortstep_Xethru, (2, 0, 1))\n",
        "label_shortstep_Xethru           = np.zeros((shortstep_Xethru.shape[0],1))\n",
        "label_shortstep_Xethru[:]        = 7\n",
        "\n",
        "sitting_Xethru = scipy.io.loadmat('/content/drive/MyDrive/rangedopplers10ghz_sitting.mat')\n",
        "sitting_Xethru = sitting_Xethru['rds']\n",
        "sitting_Xethru = np.transpose(sitting_Xethru, (2, 0, 1))\n",
        "label_sitting_Xethru           = np.zeros((sitting_Xethru.shape[0],1))\n",
        "label_sitting_Xethru[:]        = 8\n",
        "\n",
        "walkingaway_Xethru = scipy.io.loadmat('/content/drive/MyDrive/rangedopplers10ghz_walkingaway.mat')\n",
        "walkingaway_Xethru = walkingaway_Xethru['rds']\n",
        "walkingaway_Xethru = np.transpose(walkingaway_Xethru, (2, 0, 1))\n",
        "label_walkingaway_Xethru           = np.zeros((walkingaway_Xethru.shape[0],1))\n",
        "label_walkingaway_Xethru[:]        = 9\n",
        "\n",
        "walkingontoes_Xethru = scipy.io.loadmat('/content/drive/MyDrive/rangedopplers10ghz_walkingontoes.mat')\n",
        "walkingontoes_Xethru = walkingontoes_Xethru['rds']\n",
        "walkingontoes_Xethru = np.transpose(walkingontoes_Xethru, (2, 0, 1))\n",
        "label_walkingontoes_Xethru           = np.zeros((walkingontoes_Xethru.shape[0],1))\n",
        "label_walkingontoes_Xethru[:]        = 10\n",
        "\n",
        "walkingtowards_Xethru = scipy.io.loadmat('/content/drive/MyDrive/rangedopplers10ghz_walkingtowards.mat')\n",
        "walkingtowards_Xethru = walkingtowards_Xethru['rds']\n",
        "walkingtowards_Xethru = np.transpose(walkingtowards_Xethru, (2, 0, 1))\n",
        "label_walkingtowards_Xethru           = np.zeros((walkingtowards_Xethru.shape[0],1))\n",
        "label_walkingtowards_Xethru[:]        = 11\n",
        "\n"
      ],
      "metadata": {
        "id": "TBipIq9UZpJZ"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Spectrogram plots\n",
        "bending_Xethru_spectrogam = scipy.io.loadmat('/content/drive/MyDrive/spectrograms10ghz_bending.mat')\n",
        "bending_Xethru_spectrogam = bending_Xethru_spectrogam['rds']\n",
        "bending_Xethru_spectrogam = np.transpose(bending_Xethru_spectrogam, (2, 0, 1))\n",
        "label_bending_Xethru_spectrogam           = np.zeros((bending_Xethru_spectrogam.shape[0],1))\n",
        "label_bending_Xethru_spectrogam[:]        = 1\n",
        "\n",
        "crawling_Xethru_spectrogam = scipy.io.loadmat('/content/drive/MyDrive/spectrograms10ghz_bending.mat')\n",
        "crawling_Xethru_spectrogam = crawling_Xethru['rds']\n",
        "crawling_Xethru_spectrogam = np.transpose(crawling_Xethru_spectrogam, (2, 0, 1))\n",
        "label_crawling_Xethru_spectrogam           = np.zeros((crawling_Xethru_spectrogam.shape[0],1))\n",
        "label_crawling_Xethru_spectrogam[:]        = 2\n",
        "\n",
        "kneeling_Xethru_spectrogam = scipy.io.loadmat('/content/drive/MyDrive/spectrograms10ghz_kneeling.mat')\n",
        "kneeling_Xethru_spectrogam = kneeling_Xethru_spectrogam['rds']\n",
        "kneeling_Xethru_spectrogam = np.transpose(kneeling_Xethru_spectrogam, (2, 0, 1))\n",
        "label_kneeling_Xethru_spectrogam           = np.zeros((kneeling_Xethru_spectrogam.shape[0],1))\n",
        "label_kneeling_Xethru_spectrogam[:]        = 3\n",
        "\n",
        "limping_Xethru_spectrogam = scipy.io.loadmat('/content/drive/MyDrive/spectrograms10ghz_limping.mat')\n",
        "limping_Xethru_spectrogam = limping_Xethru_spectrogam['rds']\n",
        "limping_Xethru_spectrogam = np.transpose(limping_Xethru_spectrogam, (2, 0, 1))\n",
        "label_limping_Xethru_spectrogam           = np.zeros((limping_Xethru_spectrogam.shape[0],1))\n",
        "label_limping_Xethru_spectrogam[:]        = 4\n",
        "\n",
        "pickingobject_Xethru_spectrogam = scipy.io.loadmat('/content/drive/MyDrive/spectrograms10ghz_pickingobject.mat')\n",
        "pickingobject_Xethru_spectrogam = pickingobject_Xethru_spectrogam['rds']\n",
        "pickingobject_Xethru_spectrogam = np.transpose(pickingobject_Xethru_spectrogam, (2, 0, 1))\n",
        "label_pickingobject_Xethru_spectrogam           = np.zeros((pickingobject_Xethru_spectrogam.shape[0],1))\n",
        "label_pickingobject_Xethru_spectrogam[:]        = 5\n",
        "\n",
        "scissorsgait_Xethru_spectrogam = scipy.io.loadmat('/content/drive/MyDrive/spectrograms10ghz_scissorsgait.mat')\n",
        "scissorsgait_Xethru_spectrogam = scissorsgait_Xethru_spectrogam['rds']\n",
        "scissorsgait_Xethru_spectrogam = np.transpose(scissorsgait_Xethru_spectrogam, (2, 0, 1))\n",
        "label_scissorsgait_Xethru_spectrogam           = np.zeros((scissorsgait_Xethru_spectrogam.shape[0],1))\n",
        "label_scissorsgait_Xethru_spectrogam[:]        = 6\n",
        "\n",
        "shortstep_Xethru_spectrogam = scipy.io.loadmat('/content/drive/MyDrive/spectrograms10ghz_shortstep.mat')\n",
        "shortstep_Xethru_spectrogam = shortstep_Xethru_spectrogam['rds']\n",
        "shortstep_Xethru_spectrogam = np.transpose(shortstep_Xethru_spectrogam, (2, 0, 1))\n",
        "label_shortstep_Xethru_spectrogam           = np.zeros((shortstep_Xethru_spectrogam.shape[0],1))\n",
        "label_shortstep_Xethru_spectrogam[:]        = 7\n",
        "\n",
        "sitting_Xethru_spectrogam = scipy.io.loadmat('/content/drive/MyDrive/spectrograms10ghz_sitting.mat')\n",
        "sitting_Xethru_spectrogam = sitting_Xethru_spectrogam['rds']\n",
        "sitting_Xethru_spectrogam = np.transpose(sitting_Xethru_spectrogam, (2, 0, 1))\n",
        "label_sitting_Xethru_spectrogam           = np.zeros((sitting_Xethru_spectrogam.shape[0],1))\n",
        "label_sitting_Xethru_spectrogam[:]        = 8\n",
        "\n",
        "walkingaway_Xethru_spectrogam = scipy.io.loadmat('/content/drive/MyDrive/spectrograms10ghz_walkingaway.mat')\n",
        "walkingaway_Xethru_spectrogam = walkingaway_Xethru_spectrogam['rds']\n",
        "walkingaway_Xethru_spectrogam = np.transpose(walkingaway_Xethru_spectrogam, (2, 0, 1))\n",
        "label_walkingaway_Xethru_spectrogam           = np.zeros((walkingaway_Xethru_spectrogam.shape[0],1))\n",
        "label_walkingaway_Xethru_spectrogam[:]        = 9\n",
        "\n",
        "walkingontoes_Xethru_spectrogam = scipy.io.loadmat('/content/drive/MyDrive/spectrograms10ghz_walkingontoes.mat')\n",
        "walkingontoes_Xethru_spectrogam = walkingontoes_Xethru_spectrogam['rds']\n",
        "walkingontoes_Xethru_spectrogam = np.transpose(walkingontoes_Xethru_spectrogam, (2, 0, 1))\n",
        "label_walkingontoes_Xethru_spectrogam           = np.zeros((walkingontoes_Xethru_spectrogam.shape[0],1))\n",
        "label_walkingontoes_Xethru_spectrogam[:]        = 10\n",
        "\n",
        "walkingtowards_Xethru_spectrogam = scipy.io.loadmat('/content/drive/MyDrive/spectrograms10ghz_walkingtowards.mat')\n",
        "walkingtowards_Xethru_spectrogam = walkingtowards_Xethru_spectrogam['rds']\n",
        "walkingtowards_Xethru_spectrogam = np.transpose(walkingtowards_Xethru_spectrogam, (2, 0, 1))\n",
        "label_walkingtowards_Xethru_spectrogam           = np.zeros((walkingtowards_Xethru_spectrogam.shape[0],1))\n",
        "label_walkingtowards_Xethru_spectrogam[:]        = 11\n",
        "\n"
      ],
      "metadata": {
        "id": "fxNbEDgxn7vo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "Njy107I7J1SQ"
      },
      "outputs": [],
      "source": [
        "spectrogram_concat = np.concatenate((bending_Xethru,crawling_Xethru),axis=0)\n",
        "spectrogram_concat = np.concatenate((spectrogram_concat,kneeling_Xethru               ),axis=0)\n",
        "spectrogram_concat = np.concatenate((spectrogram_concat,limping_Xethru  ),axis=0)\n",
        "spectrogram_concat = np.concatenate((spectrogram_concat,pickingobject_Xethru   ),axis=0)\n",
        "spectrogram_concat = np.concatenate((spectrogram_concat,scissorsgait_Xethru          ),axis=0)\n",
        "spectrogram_concat = np.concatenate((spectrogram_concat,shortstep_Xethru            ),axis=0)\n",
        "spectrogram_concat = np.concatenate((spectrogram_concat,sitting_Xethru                ),axis=0)\n",
        "spectrogram_concat = np.concatenate((spectrogram_concat,walkingaway_Xethru),axis=0)\n",
        "spectrogram_concat = np.concatenate((spectrogram_concat,walkingontoes_Xethru   ),axis=0)\n",
        "spectrogram_concat = np.concatenate((spectrogram_concat,walkingtowards_Xethru  ),axis=0)\n",
        "\n",
        "label_concat = np.concatenate((label_bending_Xethru,label_crawling_Xethru),axis=0)\n",
        "label_concat = np.concatenate((label_concat,label_kneeling_Xethru               ),axis=0)\n",
        "label_concat = np.concatenate((label_concat,label_limping_Xethru  ),axis=0)\n",
        "label_concat = np.concatenate((label_concat,label_pickingobject_Xethru   ),axis=0)\n",
        "label_concat = np.concatenate((label_concat,label_scissorsgait_Xethru          ),axis=0)\n",
        "label_concat = np.concatenate((label_concat,label_shortstep_Xethru           ),axis=0)\n",
        "label_concat = np.concatenate((label_concat,label_sitting_Xethru                ),axis=0)\n",
        "label_concat = np.concatenate((label_concat,label_walkingaway_Xethru),axis=0)\n",
        "label_concat = np.concatenate((label_concat,label_walkingontoes_Xethru  ),axis=0)\n",
        "label_concat = np.concatenate((label_concat,label_walkingtowards_Xethru ),axis=0)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "spectrogram_concat_spectrogam = np.concatenate((bending_Xethru_spectrogam,crawling_Xethru_spectrogam),axis=0)\n",
        "spectrogram_concat_spectrogam = np.concatenate((spectrogram_concat_spectrogam,kneeling_Xethru_spectrogam               ),axis=0)\n",
        "spectrogram_concat_spectrogam = np.concatenate((spectrogram_concat_spectrogam,limping_Xethru_spectrogam  ),axis=0)\n",
        "spectrogram_concat_spectrogam = np.concatenate((spectrogram_concat_spectrogam,pickingobject_Xethru_spectrogam   ),axis=0)\n",
        "spectrogram_concat_spectrogam = np.concatenate((spectrogram_concat_spectrogam,scissorsgait_Xethru_spectrogam          ),axis=0)\n",
        "spectrogram_concat_spectrogam = np.concatenate((spectrogram_concat_spectrogam,shortstep_Xethru_spectrogam            ),axis=0)\n",
        "spectrogram_concat_spectrogam = np.concatenate((spectrogram_concat_spectrogam,sitting_Xethru_spectrogam                ),axis=0)\n",
        "spectrogram_concat_spectrogam = np.concatenate((spectrogram_concat_spectrogam,walkingaway_Xethru_spectrogam),axis=0)\n",
        "spectrogram_concat_spectrogam = np.concatenate((spectrogram_concat_spectrogam,walkingontoes_Xethru_spectrogam   ),axis=0)\n",
        "spectrogram_concat_spectrogam = np.concatenate((spectrogram_concat_spectrogam,walkingtowards_Xethru_spectrogam  ),axis=0)\n",
        "\n",
        "label_concat_spectrogam = np.concatenate((label_bending_Xethru_spectrogam,label_crawling_Xethru_spectrogam),axis=0)\n",
        "label_concat_spectrogam = np.concatenate((label_concat_spectrogam,label_kneeling_Xethru_spectrogam               ),axis=0)\n",
        "label_concat_spectrogam = np.concatenate((label_concat_spectrogam,label_limping_Xethru_spectrogam  ),axis=0)\n",
        "label_concat_spectrogam = np.concatenate((label_concat_spectrogam,label_pickingobject_Xethru_spectrogam   ),axis=0)\n",
        "label_concat_spectrogam = np.concatenate((label_concat_spectrogam,label_scissorsgait_Xethru_spectrogam          ),axis=0)\n",
        "label_concat_spectrogam = np.concatenate((label_concat_spectrogam,label_shortstep_Xethru_spectrogam           ),axis=0)\n",
        "label_concat_spectrogam = np.concatenate((label_concat_spectrogam,label_sitting_Xethru_spectrogam                ),axis=0)\n",
        "label_concat_spectrogam = np.concatenate((label_concat_spectrogam,label_walkingaway_Xethru_spectrogam),axis=0)\n",
        "label_concat_spectrogam = np.concatenate((label_concat_spectrogam,label_walkingontoes_Xethru_spectrogam  ),axis=0)\n",
        "label_concat_spectrogam = np.concatenate((label_concat_spectrogam,label_walkingtowards_Xethru_spectrogam ),axis=0)"
      ],
      "metadata": {
        "id": "vrk3nL7CpBJO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZQ15CVi7xF9_"
      },
      "outputs": [],
      "source": [
        "del bending_Xethru\n",
        "del crawling_Xethru\n",
        "del kneeling_Xethru\n",
        "del limping_with_RL_Stiff_Xethru\n",
        "del picking_up_an_object_Xethru\n",
        "del scissors_gait_Xethru\n",
        "del short_steps_Xethru\n",
        "del sitting_Xethru\n",
        "del walking_away_from_Radar_Xethru\n",
        "del Walking_on_both_toes_Xethru\n",
        "del Walking_towards_radar_Xethru\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lbxCXzor28pI",
        "outputId": "e9f32a39-80a2-4370-e34b-88cc53dd38f7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/preprocessing/_label.py:115: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n"
          ]
        }
      ],
      "source": [
        "# One-Hot Encode Label\n",
        "# integer encode\n",
        "label_encoder = LabelEncoder()\n",
        "integer_encoded = label_encoder.fit_transform(label_concat)\n",
        "# binary encode\n",
        "onehot_encoder = OneHotEncoder(sparse=False)\n",
        "integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n",
        "onehot_encoded_labels = onehot_encoder.fit_transform(integer_encoded)\n",
        "spectrogram_concat_label = onehot_encoded_labels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "7tmFfvNmN6kj"
      },
      "outputs": [],
      "source": [
        "# Shuffle concat range doppler\n",
        "shuffle_indx = random.sample(range(0, spectrogram_concat.shape[0]), spectrogram_concat.shape[0]) # split validation data\n",
        "spectrogram_concat = spectrogram_concat[:,:,:, np.newaxis]\n",
        "spectrogram_concat_shuffle = spectrogram_concat[shuffle_indx,:,:,:]\n",
        "spectrogram_concat_label_shuffle = spectrogram_concat_label[shuffle_indx,:]\n",
        "del spectrogram_concat\n",
        "del spectrogram_concat_label"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "ERSyA8QgNx1L"
      },
      "outputs": [],
      "source": [
        "range_doppler_concat_shuffle = spectrogram_concat_shuffle\n",
        "del spectrogram_concat_shuffle\n",
        "range_doppler_concat_label = label_concat\n",
        "del label_concat\n",
        "range_doppler_concat_label_shuffle = range_doppler_concat_label[shuffle_indx,:]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "_1aBNkjoLCvI"
      },
      "outputs": [],
      "source": [
        "# ---------------- Augmente and shuffle (train and test) data data ----------------\n",
        "def mixup_augmentation(range_doppler_training_data, labels, repeat_of_mixup, alpha=0.2):\n",
        "    batch_size = range_doppler_training_data.shape[0]\n",
        "    concat_images = np.zeros((batch_size*(repeat_of_mixup+1),range_doppler_training_data.shape[1],range_doppler_training_data.shape[2],range_doppler_training_data.shape[3]))\n",
        "    concat_images_range_doppler = np.zeros((batch_size*(repeat_of_mixup+1),range_doppler_training_data.shape[1],range_doppler_training_data\\\n",
        "                                            .shape[2],range_doppler_training_data.shape[3]))\n",
        "    concat_label = np.zeros((batch_size*(repeat_of_mixup+1),labels.shape[1]))\n",
        "    \n",
        "    for ii in range(repeat_of_mixup):\n",
        "      # shuffle train dataset\n",
        "      shuffle_indx_1 = random.sample(range(0, range_doppler_training_data.shape[0]), range_doppler_training_data.shape[0]) # split validation data\n",
        "      images_shuffled_1 = range_doppler_training_data[shuffle_indx_1,:,:,:]\n",
        "      range_doppler_training_data_shuffled_1 = range_doppler_training_data[shuffle_indx_1,:,:,:]\n",
        "      labels_shuffled_1 = labels[shuffle_indx_1,:]\n",
        "\n",
        "      shuffle_indx_2 = random.sample(range(0, range_doppler_training_data.shape[0]), range_doppler_training_data.shape[0]) # split validation data\n",
        "      images_shuffled_2 = range_doppler_training_data[shuffle_indx_2,:,:,:]\n",
        "      range_doppler_training_data_shuffled_2 = range_doppler_training_data[shuffle_indx_2,:,:,:]\n",
        "      labels_shuffled_2 = labels[shuffle_indx_2,:]\n",
        "\n",
        "      # Sample lambda and reshape it to do the mixup\n",
        "      ll = np.random.beta(alpha, alpha, (batch_size,1,1,1))\n",
        "      x_l = np.reshape(ll, (batch_size,1,1,1))\n",
        "      y_l = np.reshape(ll, (batch_size,1))\n",
        "      \n",
        "      # Perform mixup on both images and labels by combining a pair of images/labels\n",
        "      images_mixup_range_doppler = range_doppler_training_data_shuffled_1 * x_l + range_doppler_training_data_shuffled_2 * (1 - x_l)\n",
        "      labels_mixup = labels_shuffled_1 * y_l + labels_shuffled_2 * (1 - y_l)\n",
        "      concat_images_range_doppler[ii*batch_size:(ii+1)*batch_size,:,:,:] = images_mixup_range_doppler\n",
        "      concat_label[ii*batch_size:(ii+1)*batch_size,:] = labels_mixup\n",
        "\n",
        "    concat_images_range_doppler[repeat_of_mixup*batch_size:,:,:,:] = range_doppler_training_data\n",
        "    concat_label[repeat_of_mixup*batch_size:,:] = labels\n",
        "    return (concat_images_range_doppler, concat_label)\n",
        "\n",
        "def split_and_augmentation_of_training(range_doppler_concat_shuffle_train,range_doppler_concat_label_shuffle_train,\\\n",
        "                                       repeat_of_mixup, augmentation_enable):\n",
        "  # ---------------- Parameters ----------------\n",
        "  size_of_validation = 50\n",
        "  # Split validation\n",
        "  randomlist_for_validation_indx = random.sample(range(0, range_doppler_concat_shuffle_train.shape[0]), size_of_validation) # split validation data\n",
        "  randomlist_for_train_indx = np.delete(range(0, range_doppler_concat_shuffle_train.shape[0]), randomlist_for_validation_indx) # split training data\n",
        "  # get validation data\n",
        "  spectrogram_validation_labels = range_doppler_concat_label_shuffle_train[randomlist_for_validation_indx,:]\n",
        "  range_doppler_validation_data = range_doppler_concat_shuffle_train[randomlist_for_validation_indx,:,:,:]\n",
        "  # get training data\n",
        "  spectrogram_training_labels = spectrogram_concat_label_shuffle_train[randomlist_for_train_indx,:]\n",
        "  range_doppler_training_data = range_doppler_concat_shuffle_train[randomlist_for_train_indx,:,:,:]\n",
        "\n",
        "  (range_doppler_augmented_image,spectrograms_label)=\\\n",
        "   mixup_augmentation(range_doppler_training_data, spectrogram_training_labels, repeat_of_mixup, alpha=0.2)\n",
        "\n",
        "  return (range_doppler_augmented_image,spectrograms_label,\\\n",
        "     range_doppler_validation_data, spectrogram_validation_labels)\n",
        "\n",
        "\n",
        "\n",
        "def normalize_inputs(range_doppler_concat_shuffle, normalize_inputs_enable):\n",
        "  # ---------------- Normalize Inputs ----------------\n",
        "  if normalize_inputs_enable == True:\n",
        "    layer = Normalization(axis=None)\n",
        "    layer.adapt(range_doppler_concat_shuffle)\n",
        "    range_doppler_concat_shuffle = layer(range_doppler_concat_shuffle)\n",
        "  else:\n",
        "    range_doppler_concat_shuffle = range_doppler_concat_shuffle\n",
        "  return(range_doppler_concat_shuffle)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "range_doppler_concat_shuffle_new = np.zeros((range_doppler_concat_shuffle.shape[0],128,128,1))\n",
        "for ii in range(range_doppler_concat_shuffle.shape[0]):\n",
        "  aa = range_doppler_concat_shuffle[ii,:,:,0]\n",
        "  range_doppler_concat_shuffle_new[ii,:,:,0] = cv2.resize(aa, (range_doppler_concat_shuffle_new.shape[2], range_doppler_concat_shuffle_new.shape[1]),interpolation = cv2.INTER_CUBIC)\n",
        "range_doppler_concat_shuffle = range_doppler_concat_shuffle_new\n",
        "del aa\n",
        "del range_doppler_concat_shuffle_new"
      ],
      "metadata": {
        "id": "qTb4H2jLG31J"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "8CUvxggth7ZQ"
      },
      "outputs": [],
      "source": [
        "normalize_inputs_enable = 1\n",
        "range_doppler_concat_shuffle = normalize_inputs(range_doppler_concat_shuffle, normalize_inputs_enable)\n",
        "range_doppler_concat_shuffle = np.float32(range_doppler_concat_shuffle)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rNpkBBIadpPo",
        "outputId": "400520f9-8923-4c09-baba-444dc27aecee"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 48ms/step - loss: 0.4120 - accuracy: 0.8992\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 5ms/step - loss: 0.6413 - accuracy: 0.8992\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 5ms/step - loss: 0.4227 - accuracy: 0.8992\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 5ms/step - loss: 0.2955 - accuracy: 0.9070\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 4ms/step - loss: 0.2465 - accuracy: 0.9070\n",
            "[0.9023255944252014]\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 4ms/step - loss: 0.4126 - accuracy: 0.9070\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 5ms/step - loss: 0.4069 - accuracy: 0.8837\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 4ms/step - loss: 0.3754 - accuracy: 0.9070\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 5ms/step - loss: 0.4833 - accuracy: 0.9070\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 5ms/step - loss: 0.3659 - accuracy: 0.9070\n",
            "[0.9023255944252014, 0.9023255944252014]\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 5ms/step - loss: 0.3908 - accuracy: 0.8992\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 5ms/step - loss: 0.6293 - accuracy: 0.8062\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 5ms/step - loss: 0.2733 - accuracy: 0.9070\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 4ms/step - loss: 0.5654 - accuracy: 0.9070\n",
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 128, 128, 1)]     0         \n",
            "                                                                 \n",
            " model (Functional)          (None, 256)               6384      \n",
            "                                                                 \n",
            " model_1 (Functional)        (None, 11)                51339     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 57,723\n",
            "Trainable params: 57,347\n",
            "Non-trainable params: 376\n",
            "_________________________________________________________________\n",
            "None\n",
            "5/5 [==============================] - 0s 4ms/step - loss: 0.4029 - accuracy: 0.9147\n",
            "[0.9023255944252014, 0.9023255944252014, 0.8868217229843139]\n",
            "Mean test accuracy is 0.90,max test accuracy is 0.90,min test accuracy is 0.89, std of test accuracy is 0.01.\n",
            "Time elapsed through all process: 40.90, sec\n"
          ]
        }
      ],
      "source": [
        "\n",
        "# ---------- Parameters ----------------\n",
        "augmentation_enable = True\n",
        "normalize_inputs_enable = True\n",
        "num_folds = 5\n",
        "kfold = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state = None) # random_state = 1 ile split run'dan run'a sabit.\n",
        "test_accuracy_per_run = []\n",
        "f1_score_per_run = []\n",
        "epoch_number = 100\n",
        "batch_size = 32\n",
        "dense_size = 128\n",
        "dropout_prob_cnn = 0.1\n",
        "dropout_prob_dense = 0.3\n",
        "repeat_of_mixup = 2\n",
        "number_of_repeat = 3\n",
        "for repeat_run_number in range(number_of_repeat):\n",
        "  test_accuracy_per_fold = []\n",
        "  f1_score_per_fold = []\n",
        "  if repeat_run_number > 0:\n",
        "    del range_doppler_concat_shuffle_test\n",
        "    del range_doppler_augmented_image\n",
        "    del range_doppler_concat_shuffle_train\n",
        "   \n",
        "  for randomlist_for_train_indx, randomlist_for_test_indx in kfold.split(range_doppler_concat_shuffle,range_doppler_concat_label_shuffle):   \n",
        "    gc.collect()\n",
        "    K.clear_session()\n",
        "    \n",
        "    # test data\n",
        "    range_doppler_concat_shuffle_test = range_doppler_concat_shuffle[randomlist_for_test_indx,:,:,:]\n",
        "    range_doppler_concat_label_shuffle_test = spectrogram_concat_label_shuffle[randomlist_for_test_indx,:]\n",
        "    #train data\n",
        "    range_doppler_concat_shuffle_train = range_doppler_concat_shuffle[randomlist_for_train_indx,:,:,:]\n",
        "    spectrogram_concat_label_shuffle_train = spectrogram_concat_label_shuffle[randomlist_for_train_indx,:]\n",
        "      # ---------------- MixUp Augmentation ----------------\n",
        "    (range_doppler_augmented_image,spectrogram_concat_label_shuffle_concat,\\\n",
        "     validation_range_doppler, spectrogram_validation_labels)  =\\\n",
        "      split_and_augmentation_of_training(range_doppler_concat_shuffle_train,\\\n",
        "                                         spectrogram_concat_label_shuffle_train,\\\n",
        "                                         repeat_of_mixup, augmentation_enable)\n",
        "    \n",
        "    # ---------------- Neural Network Architecture ----------------\n",
        "\n",
        "    def encoder_for_range_doppler(input_shape):\n",
        "      input = Input(shape=input_shape)\n",
        "      x = Conv2D(4, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(input)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)\n",
        "      x = Conv2D(8, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)\n",
        "      x = Conv2D(16, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(0.2)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)   \n",
        "      x = Conv2D(16, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(0.2)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)     \n",
        "      x = Conv2D(16, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(0.2)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)           \n",
        "      x = Flatten()(x)\n",
        "      return Model(input, x)\n",
        "\n",
        "\n",
        "    def decoder_for_concat(input_shape):\n",
        "      input = Input(shape=input_shape)\n",
        "      x = Dense(128,kernel_regularizer='l2')(input)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(0.5)(x)\n",
        "      x = Dense(dense_size,kernel_regularizer='l2')(x)\n",
        "      # x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_dense)(x)\n",
        "      x = Dense(11, activation=\"softmax\")(x)\n",
        "      return Model(input, x)\n",
        "    input_shape = range_doppler_concat_shuffle.shape[1:]\n",
        "    base_network_range_doppler = encoder_for_range_doppler(input_shape)\n",
        "    range_doppler_input  = Input(shape=input_shape)\n",
        "    processed_range_doppler  = base_network_range_doppler(range_doppler_input)\n",
        "\n",
        "\n",
        "    base_decoder_network = decoder_for_concat((processed_range_doppler.shape[1]))\n",
        "    out = base_decoder_network(processed_range_doppler)\n",
        "\n",
        "    model = Model(inputs=[range_doppler_input], outputs=[out]) \n",
        "    print(model.summary())\n",
        "    # ---------------- Compile and Fit ----------------\n",
        "    model.compile(loss='categorical_crossentropy',\n",
        "                  optimizer='rmsprop',\n",
        "                  metrics=['accuracy'])\n",
        "    \n",
        "    earlyStopping = EarlyStopping(monitor='val_loss', patience=35, verbose=0,restore_best_weights=True, mode='min')\n",
        "    t = time.time()\n",
        "    history = model.fit((range_doppler_augmented_image),(spectrogram_concat_label_shuffle_concat),\n",
        "                    epochs=epoch_number,\n",
        "                    batch_size=batch_size,\n",
        "                    shuffle = True,\n",
        "                    verbose=0,\n",
        "                    callbacks=[earlyStopping],\n",
        "                    validation_data = ((validation_range_doppler) , (spectrogram_validation_labels)))\n",
        "    elapsed = time.time() - t\n",
        "    test_loss, test_accuracy  = model.evaluate([range_doppler_concat_shuffle_test],\\\n",
        "                                               [range_doppler_concat_label_shuffle_test],\n",
        "                  batch_size=batch_size)\n",
        "\n",
        "    test_accuracy_per_fold.append(test_accuracy)\n",
        "    # f1_score_per_fold.append(test_f1_score)\n",
        "\n",
        "  test_accuracy_per_run.append(sum(test_accuracy_per_fold)/num_folds)\n",
        "  # f1_score_per_run.append(sum(f1_score_per_fold)/num_folds)\n",
        "  print(test_accuracy_per_run)\n",
        "  # print(f1_score_per_run)\n",
        "print(f'Mean test accuracy is {\"{:.2f}\".format(sum(test_accuracy_per_run)/number_of_repeat)},\\\n",
        "max test accuracy is {\"{:.2f}\".format(max(test_accuracy_per_run))},\\\n",
        "min test accuracy is {\"{:.2f}\".format(min(test_accuracy_per_run))}, \\\n",
        "std of test accuracy is {\"{:.2f}\".format(np.std(test_accuracy_per_run, axis=0))}.')\n",
        "print(f'Time elapsed through all process: {\"{:.2f}\".format(elapsed)}, sec')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IXYxmx1CTbTR",
        "outputId": "dea9ab76-d62e-41d2-f1ce-a01d698857cd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Time elapsed during test time: 0.18, sec\n"
          ]
        }
      ],
      "source": [
        "t_test = time.time()\n",
        "y_test_predicted = model.predict((range_doppler_concat_shuffle_test[0:1,:,:,:]))\n",
        "test_elapsed = time.time() - t_test \n",
        "print(f'Time elapsed during test time: {\"{:.2f}\".format(test_elapsed)}, sec')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0N7v4PDP4Dv2",
        "outputId": "349a6b36-e685-49af-edeb-f00c3d29be51"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Min test accuracy is 0.887, Max test accuracy is 0.902, Mean test accuracy is 0.897, std of test accuracy is 0.007.\n",
            "Time elapsed during training time: 40.898 sec\n",
            "Time elapsed during test time: 0.179 sec\n"
          ]
        }
      ],
      "source": [
        "print(f'Min test accuracy is {\"{:.3f}\".format(min(test_accuracy_per_run))}, \\\n",
        "Max test accuracy is {\"{:.3f}\".format(max(test_accuracy_per_run))}, \\\n",
        "Mean test accuracy is {\"{:.3f}\".format(sum(test_accuracy_per_run)/number_of_repeat)}, \\\n",
        "std of test accuracy is {\"{:.3f}\".format(np.std(test_accuracy_per_run, axis=0))}.')\n",
        "print(f'Time elapsed during training time: {\"{:.3f}\".format(elapsed)} sec')\n",
        "print(f'Time elapsed during test time: {\"{:.3f}\".format(test_elapsed)} sec')"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "machine_shape": "hm",
      "name": "Range_Doppler_CI4R_Datasets_Multi_Class_CNN_10Ghz.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}