Range_Doppler_and_Spectrogram_CNN_with_no_MixUp.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mertege/Thesis_Experiments/blob/main/Range_Doppler_and_Spectrogram_CNN_with_no_MixUp.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kN4ixvH39HW6"
      },
      "outputs": [],
      "source": [
        "# CNN-based architecure is running in Linda dataset.\n",
        "# There is no mix-up augmentation.\n",
        "# 5-blocks are used.\n",
        "# Range-doppler and spectrogram data are used in together.\n",
        "# It was run 5-times. Results are shared on below:\n",
        "# \"mixup\" yöntemi kullanılarak data 5 kere augmente edildi. Toplam 6 kat data train için kullanıldı.\n",
        "# 5 defa run edildi ve değerler aşağıda paylaşıldı.\n",
        "# Mean test accuracy is 0.839, mean test f1 score is 0.805, max test accuracy is 0.900, max test f1 score is 0.891, min test accuracy is 0.782, min test f1 score is 0.751, std of test accuracy is 0.040, std of test f1 score is 0.056\n",
        "# Time elapsed through all process: 8.264, sec\n",
        "# Time elapsed during test time: 0.091, sec"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lzJwN6GfkN8Z"
      },
      "outputs": [],
      "source": [
        "from keras.models import Sequential\n",
        "from tensorflow.keras import layers\n",
        "from keras.models import Model\n",
        "from keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization, Normalization, Input, Conv2D, MaxPooling2D, Concatenate\n",
        "from keras.regularizers import l2\n",
        "import tensorflow as tf\n",
        "import scipy.io\n",
        "import numpy as np\n",
        "import cv2\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "from numpy.random import seed\n",
        "from sklearn.model_selection import KFold, StratifiedKFold\n",
        "import time\n",
        "from sklearn.metrics import precision_recall_fscore_support\n",
        "from keras.callbacks import EarlyStopping\n",
        "from keras import backend as K \n",
        "import gc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8FERVnWrkrwE",
        "outputId": "5b53a85d-f18d-4aac-94fb-f560bee3f662"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o19JHieQ81cS"
      },
      "outputs": [],
      "source": [
        "# Get Range-Doppler data from\n",
        "range_doppler_fast_resized = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_fast_resized.mat')\n",
        "range_doppler_fast_resized = range_doppler_fast_resized['range_doppler_fast_resized']\n",
        "range_doppler_fast_resized = np.transpose(range_doppler_fast_resized, (2, 0, 1))\n",
        "# range_doppler_fast_resized = np.delete(range_doppler_fast_resized,(49), axis=0) # 50th row is deleted since there is no 50th row in spectrogram fast data.\n",
        "range_doppler_fast_label = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_fast_label.mat')\n",
        "range_doppler_fast_label = range_doppler_fast_label['range_doppler_fast_label']  \n",
        "\n",
        "range_doppler_slow_resized = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_slow_resized.mat')\n",
        "range_doppler_slow_resized = range_doppler_slow_resized['range_doppler_slow_resized']\n",
        "range_doppler_slow_resized = np.transpose(range_doppler_slow_resized, (2, 0, 1))\n",
        "range_doppler_slow_label = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_slow_label.mat')\n",
        "range_doppler_slow_label = range_doppler_slow_label['range_doppler_slow_label']  \n",
        "\n",
        "range_doppler_slow_pocket_resized = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_slow_pocket_resized.mat')\n",
        "range_doppler_slow_pocket_resized = range_doppler_slow_pocket_resized['range_doppler_slow_pocket_resized']\n",
        "range_doppler_slow_pocket_resized = np.transpose(range_doppler_slow_pocket_resized, (2, 0, 1))\n",
        "range_doppler_pocket_label = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_pocket_label.mat')\n",
        "range_doppler_pocket_label = range_doppler_pocket_label['range_doppler_pocket_label']  \n",
        "# Get Range-Doppler data from\n",
        "spectrogram_fast_resized = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_fast_resized.mat')\n",
        "spectrogram_fast_resized = spectrogram_fast_resized['spectrogram_fast_resized']\n",
        "spectrogram_fast_resized = np.transpose(spectrogram_fast_resized, (2, 0, 1))\n",
        "spectrogram_fast_label = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_fast_label.mat')\n",
        "spectrogram_fast_label = spectrogram_fast_label['spectrogram_fast_label']  \n",
        "\n",
        "spectrogram_slow_resized = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_slow_resized.mat')\n",
        "spectrogram_slow_resized = spectrogram_slow_resized['spectrogram_slow_resized']\n",
        "spectrogram_slow_resized = np.transpose(spectrogram_slow_resized, (2, 0, 1))\n",
        "spectrogram_slow_label = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_slow_label.mat')\n",
        "spectrogram_slow_label = spectrogram_slow_label['spectrogram_slow_label']  \n",
        "\n",
        "spectrogram_slow_pocket_resized = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_slow_pocket_resized.mat')\n",
        "spectrogram_slow_pocket_resized = spectrogram_slow_pocket_resized['spectrogram_slow_pocket_resized']\n",
        "spectrogram_slow_pocket_resized = np.transpose(spectrogram_slow_pocket_resized, (2, 0, 1))\n",
        "spectrogram_slow_pocket_label = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_slow_pocket_label.mat')\n",
        "spectrogram_slow_pocket_label = spectrogram_slow_pocket_label['spectrogram_slow_pocket_label']  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wFp_PErnMLN2"
      },
      "outputs": [],
      "source": [
        "# Concat range-doppler data\n",
        "range_doppler_concat = np.concatenate((range_doppler_fast_resized,range_doppler_slow_resized),axis=0)\n",
        "range_doppler_concat = np.concatenate((range_doppler_concat,range_doppler_slow_pocket_resized),axis=0)\n",
        "range_doppler_concat = range_doppler_concat[:,:,:,np.newaxis] \n",
        "range_doppler_concat_label = np.zeros((range_doppler_concat.shape[0],1))\n",
        "range_doppler_concat_label[:range_doppler_fast_resized.shape[0],:] = 1\n",
        "# Shuffle concat range doppler\n",
        "shuffle_indx = random.sample(range(0, range_doppler_concat.shape[0]), range_doppler_concat.shape[0]) # split validation data\n",
        "range_doppler_concat_shuffle = range_doppler_concat[shuffle_indx,:,:,:]\n",
        "range_doppler_concat_label_shuffle = range_doppler_concat_label[shuffle_indx,:]\n",
        "# Concat range-doppler data\n",
        "spectrogram_concat = np.concatenate((spectrogram_fast_resized,spectrogram_slow_resized),axis=0)\n",
        "spectrogram_concat = np.concatenate((spectrogram_concat,spectrogram_slow_pocket_resized),axis=0)\n",
        "spectrogram_concat = spectrogram_concat[:,:,:,np.newaxis] \n",
        "spectrogram_concat_label = np.zeros((spectrogram_concat.shape[0],1))\n",
        "spectrogram_concat_label[:spectrogram_fast_resized.shape[0],:] = 1\n",
        "# Shuffle concat range doppler\n",
        "# shuffle_indx = random.sample(range(0, spectrogram_concat.shape[0]), spectrogram_concat.shape[0]) # split validation data\n",
        "spectrogram_concat_shuffle = spectrogram_concat[shuffle_indx,:,:,:]\n",
        "spectrogram_concat_label_shuffle = spectrogram_concat_label[shuffle_indx,:]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U3u3vlz1oARv"
      },
      "outputs": [],
      "source": [
        "range_doppler_concat_shuffle_new = np.zeros((170,51,512,1))\n",
        "for ii in range(range_doppler_concat_shuffle.shape[0]):\n",
        "  aa = range_doppler_concat_shuffle[ii,:,:,0]\n",
        "  range_doppler_concat_shuffle_new[ii,:,:,0] = cv2.resize(aa, (512, 51),interpolation = cv2.INTER_CUBIC)\n",
        "  # range_doppler_concat_shuffle = range_doppler_concat_shuffle[:, np.newaxis] \n",
        "range_doppler_concat_shuffle = range_doppler_concat_shuffle_new\n",
        "del aa\n",
        "del range_doppler_concat_shuffle_new\n",
        "\n",
        "spectrogram_concat_shuffle_new = np.zeros((170,51,422,1))\n",
        "for ii in range(spectrogram_concat_shuffle.shape[0]):\n",
        "  aa = spectrogram_concat_shuffle[ii,:,:,0]\n",
        "  spectrogram_concat_shuffle_new[ii,:,:,0] = cv2.resize(aa, (422, 51),interpolation = cv2.INTER_CUBIC)\n",
        "  # range_doppler_concat_shuffle = range_doppler_concat_shuffle[:, np.newaxis] \n",
        "spectrogram_concat_shuffle = spectrogram_concat_shuffle_new\n",
        "del aa\n",
        "del spectrogram_concat_shuffle_new"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N1YbC9MeVko_"
      },
      "outputs": [],
      "source": [
        "del range_doppler_fast_resized\n",
        "del range_doppler_fast_label\n",
        "del range_doppler_slow_resized\n",
        "del range_doppler_slow_label\n",
        "del range_doppler_slow_pocket_resized\n",
        "del range_doppler_pocket_label\n",
        "del spectrogram_fast_resized\n",
        "del spectrogram_fast_label\n",
        "del spectrogram_slow_resized\n",
        "del spectrogram_slow_label\n",
        "del spectrogram_slow_pocket_resized\n",
        "del spectrogram_slow_pocket_label\n",
        "del range_doppler_concat\n",
        "del range_doppler_concat_label\n",
        "del spectrogram_concat\n",
        "del spectrogram_concat_label"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_1aBNkjoLCvI"
      },
      "outputs": [],
      "source": [
        "# ---------------- Augmente and shuffle (train and test) data data ----------------\n",
        "data_augmentation = tf.keras.Sequential([\n",
        "  layers.RandomFlip(\"horizontal_and_vertical\"),\n",
        "  layers.RandomRotation(0.1),\n",
        "])\n",
        "\n",
        "\n",
        "def mixup_augmentation(images,range_doppler_training_data, labels, repeat_of_mixup, alpha=0.2):\n",
        "    batch_size = images.shape[0]\n",
        "    concat_images = np.zeros((batch_size*(repeat_of_mixup+1),images.shape[1],images.shape[2],images.shape[3]))\n",
        "    concat_images_range_doppler = np.zeros((batch_size*(repeat_of_mixup+1),range_doppler_training_data.shape[1],\\\n",
        "                                            range_doppler_training_data.shape[2],range_doppler_training_data.shape[3]))\n",
        "    concat_label = np.zeros((batch_size*(repeat_of_mixup+1),labels.shape[1]))\n",
        "    for ii in range(repeat_of_mixup):\n",
        "      # shuffle train dataset\n",
        "      shuffle_indx_1 = random.sample(range(0, images.shape[0]), images.shape[0]) # split validation data\n",
        "      images_shuffled_1 = images[shuffle_indx_1,:,:,:]\n",
        "      range_doppler_training_data_shuffled_1 = range_doppler_training_data[shuffle_indx_1,:,:,:]\n",
        "      labels_shuffled_1 = labels[shuffle_indx_1,:]\n",
        "\n",
        "      shuffle_indx_2 = random.sample(range(0, images.shape[0]), images.shape[0]) # split validation data\n",
        "      images_shuffled_2 = images[shuffle_indx_2,:,:,:]\n",
        "      range_doppler_training_data_shuffled_2 = range_doppler_training_data[shuffle_indx_2,:,:,:]\n",
        "      labels_shuffled_2 = labels[shuffle_indx_2,:]\n",
        "\n",
        "      # Sample lambda and reshape it to do the mixup\n",
        "      gaussian_mean = 0.2\n",
        "      gaussian_std = 0.02\n",
        "      ll = np.random.normal(gaussian_mean, gaussian_std, (batch_size,1,1,1))\n",
        "      x_l = np.reshape(ll, (batch_size,1,1,1))\n",
        "      y_l = np.reshape(ll, (batch_size,1))\n",
        "      \n",
        "      # Perform mixup on both images and labels by combining a pair of images/labels\n",
        "      images_mixup = images_shuffled_1 * x_l + images_shuffled_2 * (1 - x_l)\n",
        "      images_mixup_range_doppler = range_doppler_training_data_shuffled_1 * x_l + range_doppler_training_data_shuffled_2 * (1 - x_l)\n",
        "      labels_mixup = labels_shuffled_1 * y_l + labels_shuffled_2 * (1 - y_l)\n",
        "      concat_images[ii*batch_size:(ii+1)*batch_size,:,:,:] = images_mixup\n",
        "      concat_images_range_doppler[ii*batch_size:(ii+1)*batch_size,:,:,:] = images_mixup_range_doppler\n",
        "      concat_label[ii*batch_size:(ii+1)*batch_size,:] = labels_mixup\n",
        "\n",
        "    concat_images[repeat_of_mixup*batch_size:,:,:,:] = images\n",
        "    concat_images_range_doppler[repeat_of_mixup*batch_size:,:,:,:] = range_doppler_training_data\n",
        "    concat_label[repeat_of_mixup*batch_size:,:] = labels\n",
        "    return (concat_images,concat_images_range_doppler, concat_label)\n",
        "def split_and_augmentation_of_training(spectrogram_concat_shuffle_train,range_doppler_concat_shuffle_train,range_doppler_concat_label_shuffle_train,\\\n",
        "                                       repeat_of_mixup, augmentation_enable):\n",
        "  # ---------------- Parameters ----------------\n",
        "  repeat_of_augmentation_for_fast = 1\n",
        "  repeat_of_augmentation_for_slow = np.floor(repeat_of_augmentation_for_fast/2)\n",
        "  repeat_of_augmentation_for_slow = int(repeat_of_augmentation_for_slow)\n",
        "  # size_of_validation = 30\n",
        "  alpha = 0.2\n",
        "  dummy_label = np.zeros((spectrogram_concat_shuffle_train.shape[0],1))\n",
        "  for randomlist_for_train_indx, randomlist_for_validation_indx in kfold.split(spectrogram_concat_shuffle_train,dummy_label):   \n",
        "    randomlist_for_validation_indx\n",
        "  # Split validation\n",
        "  # randomlist_for_validation_indx = random.sample(range(0, range_doppler_concat_shuffle_train.shape[0]), size_of_validation) # split validation data\n",
        "  # randomlist_for_train_indx = np.delete(range(0, range_doppler_concat_shuffle_train.shape[0]), randomlist_for_validation_indx) # split training data\n",
        "  # get validation data\n",
        "  spectrogram_validation_data = spectrogram_concat_shuffle_train[randomlist_for_validation_indx,:,:,:]\n",
        "  spectrogram_validation_labels = range_doppler_concat_label_shuffle_train[randomlist_for_validation_indx,:]\n",
        "  range_doppler_validation_data = range_doppler_concat_shuffle_train[randomlist_for_validation_indx,:,:,:]\n",
        "  # get training data\n",
        "  spectrogram_training_data = spectrogram_concat_shuffle_train[randomlist_for_train_indx,:,:,:]\n",
        "  spectrogram_training_labels = spectrogram_concat_label_shuffle_train[randomlist_for_train_indx,:]\n",
        "  range_doppler_training_data = range_doppler_concat_shuffle_train[randomlist_for_train_indx,:,:,:]\n",
        "\n",
        "  # Rotate Augmentation\n",
        "  # get slow and fast indexes of training data\n",
        "  slow_indexes = np.where(spectrogram_training_labels == 0)[0]\n",
        "  fast_indexes = np.delete(range(0, spectrogram_training_labels.shape[0]), slow_indexes)  \n",
        "\n",
        "  slow_spectrograms_train = spectrogram_training_data[slow_indexes,:,:,:]\n",
        "  size_of_samples_slow = slow_spectrograms_train.shape[0]\n",
        "\n",
        "  fast_spectrograms_train = spectrogram_training_data[fast_indexes,:,:,:]  \n",
        "  size_of_samples_fast = fast_spectrograms_train.shape[0]\n",
        "\n",
        "  slow_range_train = range_doppler_training_data[slow_indexes,:,:,:]\n",
        "  fast_range_train = range_doppler_training_data[fast_indexes,:,:,:]  \n",
        "\n",
        "  if augmentation_enable == True: \n",
        "    # ---------------- Augmente Train Data for Fast ----------------\n",
        "    augmented_image_fast = np.zeros((size_of_samples_fast*repeat_of_augmentation_for_fast,fast_spectrograms_train.shape[1],fast_spectrograms_train.shape[2],1))\n",
        "    spectrograms_fast_label = np.ones((size_of_samples_fast*(repeat_of_augmentation_for_fast+1),1))\n",
        "    # augmented_image_fast = np.flip(fast_spectrograms_train,axis=2)\n",
        "    for jj in range(repeat_of_augmentation_for_fast):\n",
        "      for ii in range(size_of_samples_fast):\n",
        "        augmented_image_fast[size_of_samples_fast*jj+ii,:,:,:] = data_augmentation(fast_spectrograms_train[ii,:,:,:])\n",
        "    augmented_image_fast = np.concatenate((augmented_image_fast,fast_spectrograms_train),axis=0)   \n",
        "\n",
        "    augmented_image_fast_range = np.zeros((size_of_samples_fast*repeat_of_augmentation_for_fast,fast_range_train.shape[1],fast_range_train.shape[2],1))\n",
        "    # augmented_image_fast_range  = np.flip(fast_range_train,axis=2)\n",
        "    for jj in range(repeat_of_augmentation_for_fast):\n",
        "      for ii in range(size_of_samples_fast):\n",
        "        augmented_image_fast_range[size_of_samples_fast*jj+ii,:,:,:]  =data_augmentation(fast_range_train[ii,:,:,:])\n",
        "    augmented_image_fast_range = np.concatenate((augmented_image_fast_range,fast_range_train),axis=0)   \n",
        "\n",
        "    # ---------------- Augmente Train Data for Slow ----------------\n",
        "    augmented_image_slow = slow_spectrograms_train\n",
        "    augmented_image_slow_range = slow_range_train\n",
        "\n",
        "  else:\n",
        "    augmented_image_fast = fast_spectrograms_train\n",
        "    augmented_image_slow = slow_spectrograms_train\n",
        "    augmented_image_fast_range = fast_range_train\n",
        "    augmented_image_slow_range = slow_range_train\n",
        "    spectrograms_fast_label = np.ones((size_of_samples_fast,1))\n",
        "    \n",
        "  spectrograms_slow_label = np.zeros((size_of_samples_slow,1))\n",
        "\n",
        "  spectrogram_training_data = np.concatenate((augmented_image_fast,augmented_image_slow),axis=0)\n",
        "  range_doppler_training_data = np.concatenate((augmented_image_fast_range,augmented_image_slow_range),axis=0)\n",
        "  spectrogram_training_labels = np.concatenate((spectrograms_fast_label,spectrograms_slow_label),axis=0)\n",
        "\n",
        "  (spectrogram_augmented_image,range_doppler_augmented_image,spectrograms_label)=\\\n",
        "   mixup_augmentation(spectrogram_training_data,range_doppler_training_data, spectrogram_training_labels, repeat_of_mixup, alpha=0.2)\n",
        "\n",
        "  return (spectrogram_augmented_image,range_doppler_augmented_image,spectrograms_label,\\\n",
        "     spectrogram_validation_data,range_doppler_validation_data, spectrogram_validation_labels)\n",
        "def normalize_inputs(range_doppler_concat_shuffle, normalize_inputs_enable):\n",
        "  # ---------------- Normalize Inputs ----------------\n",
        "  if normalize_inputs_enable == True:\n",
        "    layer = Normalization(axis=None)\n",
        "    layer.adapt(range_doppler_concat_shuffle)\n",
        "    range_doppler_concat_shuffle = layer(range_doppler_concat_shuffle)\n",
        "  else:\n",
        "    range_doppler_concat_shuffle = range_doppler_concat_shuffle\n",
        "  return(range_doppler_concat_shuffle)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z0i1pvvIE4A6"
      },
      "outputs": [],
      "source": [
        "normalize_inputs_enable = 1\n",
        "range_doppler_concat_shuffle = normalize_inputs(range_doppler_concat_shuffle, normalize_inputs_enable)\n",
        "spectrogram_concat_shuffle = normalize_inputs(spectrogram_concat_shuffle, normalize_inputs_enable)\n",
        "range_doppler_concat_shuffle = np.float32(range_doppler_concat_shuffle)\n",
        "spectrogram_concat_shuffle = np.float32(spectrogram_concat_shuffle)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rNpkBBIadpPo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "86c38fdb-4e39-474c-81f0-fb81d894b68c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 60ms/step - loss: 0.4418 - accuracy: 0.8235\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 8ms/step - loss: 0.0313 - accuracy: 1.0000\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 9ms/step - loss: 0.3480 - accuracy: 0.8529\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 7ms/step - loss: 0.6679 - accuracy: 0.7353\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 7ms/step - loss: 0.6684 - accuracy: 0.6765\n",
            "WARNING:tensorflow:5 out of the last 9 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f3c01ac77a0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
            "[0.8176470518112182]\n",
            "[0.7720237847067116]\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 9ms/step - loss: 0.6314 - accuracy: 0.9118\n",
            "WARNING:tensorflow:6 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f3c016958c0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 8ms/step - loss: 0.5207 - accuracy: 0.9706\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 9ms/step - loss: 0.5201 - accuracy: 0.8824\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 9ms/step - loss: 0.1682 - accuracy: 0.9706\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 8ms/step - loss: 0.6206 - accuracy: 0.7647\n",
            "[0.8176470518112182, 0.899999988079071]\n",
            "[0.7720237847067116, 0.8906189763888357]\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 9ms/step - loss: 0.2069 - accuracy: 0.8824\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 9ms/step - loss: 0.6785 - accuracy: 0.6765\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 9ms/step - loss: 0.4861 - accuracy: 0.8824\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 8ms/step - loss: 0.5701 - accuracy: 0.8529\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 8ms/step - loss: 0.5626 - accuracy: 0.8529\n",
            "[0.8176470518112182, 0.899999988079071, 0.82941175699234]\n",
            "[0.7720237847067116, 0.8906189763888357, 0.7602264845681314]\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 8ms/step - loss: 0.1860 - accuracy: 0.9412\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 8ms/step - loss: 0.5426 - accuracy: 0.8235\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 8ms/step - loss: 0.6270 - accuracy: 0.7941\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 8ms/step - loss: 0.4535 - accuracy: 0.9118\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 9ms/step - loss: 0.3216 - accuracy: 0.8529\n",
            "[0.8176470518112182, 0.899999988079071, 0.82941175699234, 0.8647058725357055]\n",
            "[0.7720237847067116, 0.8906189763888357, 0.7602264845681314, 0.8519012730534739]\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 8ms/step - loss: 0.6632 - accuracy: 0.7059\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 9ms/step - loss: 0.5733 - accuracy: 0.8235\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 8ms/step - loss: 1.4953 - accuracy: 0.7059\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 8ms/step - loss: 0.4640 - accuracy: 0.8235\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 51, 512, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 51, 422, 1)  0           []                               \n",
            "                                ]                                                                 \n",
            "                                                                                                  \n",
            " model (Functional)             (None, 256)          8496        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " model_1 (Functional)           (None, 208)          8496        ['input_4[0][0]']                \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 464)          0           ['model[0][0]',                  \n",
            "                                                                  'model_1[0][0]']                \n",
            "                                                                                                  \n",
            " model_2 (Functional)           (None, 1)            63681       ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 80,673\n",
            "Trainable params: 80,385\n",
            "Non-trainable params: 288\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "2/2 [==============================] - 0s 10ms/step - loss: 0.5393 - accuracy: 0.8529\n",
            "[0.8176470518112182, 0.899999988079071, 0.82941175699234, 0.8647058725357055, 0.7823529481887818]\n",
            "[0.7720237847067116, 0.8906189763888357, 0.7602264845681314, 0.8519012730534739, 0.7510527784663124]\n",
            "Mean test accuracy is 0.84, mean test f1 score is 0.81, max test accuracy is 0.90, max test f1 score is 0.89, min test accuracy is 0.78, min test f1 score is 0.75, std of test accuracy is 0.04, std of test f1 score is 0.06\n",
            "Time elapsed through all process: 8.26, sec\n"
          ]
        }
      ],
      "source": [
        "\n",
        "# ---------- Parameters ----------------\n",
        "augmentation_enable = True\n",
        "normalize_inputs_enable = True\n",
        "num_folds = 5\n",
        "kfold = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state = None) # random_state = 1 ile split run'dan run'a sabit.\n",
        "test_accuracy_per_run = []\n",
        "f1_score_per_run = []\n",
        "epoch_number = 100\n",
        "batch_size = 32\n",
        "dense_size = 32\n",
        "dropout_prob_cnn = 0.1\n",
        "dropout_prob_dense = 0.3\n",
        "repeat_of_mixup = 0\n",
        "number_of_repeat = 5\n",
        "for repeat_run_number in range(number_of_repeat):\n",
        "  test_accuracy_per_fold = []\n",
        "  f1_score_per_fold = []\n",
        "  if repeat_run_number > 0:\n",
        "    del range_doppler_concat_shuffle_test\n",
        "    del spectrogram_concat_shuffle_test\n",
        "    del range_doppler_augmented_image\n",
        "    del range_doppler_concat_shuffle_train\n",
        "    del spectrogram_concat_shuffle_train\n",
        "    del spectrogram_augmented_image\n",
        "   \n",
        "  for randomlist_for_train_indx, randomlist_for_test_indx in kfold.split(range_doppler_concat_shuffle,range_doppler_concat_label_shuffle):   \n",
        "    gc.collect()\n",
        "    K.clear_session()\n",
        "    \n",
        "    # test data\n",
        "    range_doppler_concat_shuffle_test = range_doppler_concat_shuffle[randomlist_for_test_indx,:,:,:]\n",
        "    spectrogram_concat_shuffle_test = spectrogram_concat_shuffle[randomlist_for_test_indx,:,:,:]\n",
        "    range_doppler_concat_label_shuffle_test = range_doppler_concat_label_shuffle[randomlist_for_test_indx,:]\n",
        "    #train data\n",
        "    range_doppler_concat_shuffle_train = range_doppler_concat_shuffle[randomlist_for_train_indx,:,:,:]\n",
        "    spectrogram_concat_shuffle_train = spectrogram_concat_shuffle[randomlist_for_train_indx,:,:,:]\n",
        "    spectrogram_concat_label_shuffle_train = spectrogram_concat_label_shuffle[randomlist_for_train_indx,:]\n",
        "      # ---------------- MixUp Augmentation ----------------\n",
        "    (spectrogram_augmented_image,range_doppler_augmented_image,spectrogram_concat_label_shuffle_concat,\\\n",
        "     validation_spectrogram,validation_range_doppler, spectrogram_validation_labels)  =\\\n",
        "      split_and_augmentation_of_training(spectrogram_concat_shuffle_train,range_doppler_concat_shuffle_train,\\\n",
        "                                         spectrogram_concat_label_shuffle_train,\\\n",
        "                                         repeat_of_mixup, augmentation_enable)\n",
        "\n",
        "\n",
        "    def encoder_for_range_doppler(input_shape):\n",
        "      input = Input(shape=input_shape)\n",
        "      x = Conv2D(8, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(input)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)\n",
        "      x = Conv2D(16, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)\n",
        "      x = Conv2D(16, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)      \n",
        "      x = Conv2D(16, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)  \n",
        "      x = Conv2D(16, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)              \n",
        "      x = Flatten()(x)\n",
        "      return Model(input, x)\n",
        "\n",
        "    def encoder_for_spectrogram(input_shape):\n",
        "      input = Input(shape=input_shape)\n",
        "      x = Conv2D(8, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(input)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)\n",
        "      x = Conv2D(16, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)\n",
        "      x = Conv2D(16, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)      \n",
        "      x = Conv2D(16, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)      \n",
        "      x = Conv2D(16, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)             \n",
        "      x = Flatten()(x)\n",
        "      return Model(input, x)\n",
        "\n",
        "    def decoder_for_concat(input_shape):\n",
        "      input = Input(shape=input_shape)\n",
        "      x = Dense(128, activation=\"relu\")(input)\n",
        "      x = Dropout(dropout_prob_dense)(x) \n",
        "      x = Dense(dense_size, activation=\"relu\")(x)\n",
        "      x = Dropout(dropout_prob_dense)(x)\n",
        "      x = Dense(1, activation=\"sigmoid\")(x)\n",
        "      return Model(input, x)\n",
        "    input_shape = range_doppler_concat_shuffle.shape[1:]\n",
        "    base_network_range_doppler = encoder_for_range_doppler(input_shape)\n",
        "    range_doppler_input  = Input(shape=input_shape)\n",
        "    processed_range_doppler  = base_network_range_doppler(range_doppler_input)\n",
        "\n",
        "    input_shape = spectrogram_concat_shuffle.shape[1:]\n",
        "    base_network_spectrogram = encoder_for_spectrogram(input_shape) \n",
        "    spectrogram_input  = Input(shape=input_shape)\n",
        "    processed_spectrogram  = base_network_spectrogram(spectrogram_input)\n",
        "\n",
        "    concat_layer = Concatenate()([processed_range_doppler, processed_spectrogram])\n",
        "\n",
        "    base_decoder_network = decoder_for_concat((concat_layer.shape[1]))\n",
        "    out = base_decoder_network(concat_layer)\n",
        "\n",
        "    model = Model(inputs=[range_doppler_input, spectrogram_input], outputs=[out])\n",
        "\n",
        "    print(model.summary())\n",
        "    # print(model.summary())\n",
        "    # ---------------- Compile and Fit ----------------\n",
        "    t = time.time()\n",
        "    model.compile(loss='binary_crossentropy',\n",
        "                  optimizer='rmsprop',\n",
        "                  metrics=['accuracy'])\n",
        "    \n",
        "    earlyStopping = EarlyStopping(monitor='val_loss', patience=35, verbose=0,restore_best_weights=True, mode='min')\n",
        "    # earlyStopping = EarlyStopping(monitor='val_accuracy', patience=15, verbose=0,restore_best_weights=True, mode='max')\n",
        "    history = model.fit((range_doppler_augmented_image,spectrogram_augmented_image),(spectrogram_concat_label_shuffle_concat),\n",
        "                    epochs=epoch_number,\n",
        "                    batch_size=batch_size,\n",
        "                    shuffle = True,\n",
        "                    verbose=0,\n",
        "                    callbacks=[earlyStopping],\n",
        "                    validation_data = ((validation_range_doppler,validation_spectrogram) , (spectrogram_validation_labels)))\n",
        "    elapsed = time.time() - t\n",
        "    # tf.keras.models.load_model\n",
        "    test_loss, test_accuracy  = model.evaluate([range_doppler_concat_shuffle_test,spectrogram_concat_shuffle_test],\\\n",
        "                                               [range_doppler_concat_label_shuffle_test],\n",
        "                  batch_size=batch_size)\n",
        "    # ---------------- Get Test Results ----------------\n",
        "    y_test_predicted = model.predict((range_doppler_concat_shuffle_test,spectrogram_concat_shuffle_test), batch_size=batch_size)\n",
        "    # ----- Binarize y_test_predicted values -----\n",
        "    y_test_predicted_binary = np.zeros(y_test_predicted.size)\n",
        "    for ii in range(y_test_predicted.size):\n",
        "      if y_test_predicted[ii] < 0.5:\n",
        "        y_test_predicted_binary[ii] = 0\n",
        "      else:\n",
        "        y_test_predicted_binary[ii] = 1\n",
        "    \n",
        "    test_precision, test_recall, test_f1_score, support = precision_recall_fscore_support(range_doppler_concat_label_shuffle_test, y_test_predicted_binary, average='macro')\n",
        "\n",
        "    test_accuracy_per_fold.append(test_accuracy)\n",
        "    f1_score_per_fold.append(test_f1_score)\n",
        "  test_accuracy_per_run.append(sum(test_accuracy_per_fold)/num_folds)\n",
        "  f1_score_per_run.append(sum(f1_score_per_fold)/num_folds)\n",
        "  print(test_accuracy_per_run)\n",
        "  print(f1_score_per_run)\n",
        "print(f'Mean test accuracy is {\"{:.2f}\".format(sum(test_accuracy_per_run)/number_of_repeat)}, mean test f1 score is {\"{:.2f}\".format(sum(f1_score_per_run)/number_of_repeat)}, \\\n",
        "max test accuracy is {\"{:.2f}\".format(max(test_accuracy_per_run))}, max test f1 score is {\"{:.2f}\".format(max(f1_score_per_run))}, \\\n",
        "min test accuracy is {\"{:.2f}\".format(min(test_accuracy_per_run))}, min test f1 score is {\"{:.2f}\".format(min(f1_score_per_run))}, \\\n",
        "std of test accuracy is {\"{:.2f}\".format(np.std(test_accuracy_per_run, axis=0))}, std of test f1 score is {\"{:.2f}\".format(np.std(f1_score_per_run, axis=0))}')\n",
        "print(f'Time elapsed through all process: {\"{:.2f}\".format(elapsed)}, sec')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "t_test = time.time()\n",
        "y_test_predicted = model.predict((range_doppler_concat_shuffle_test[0:1,:,:,:],spectrogram_concat_shuffle_test[0:1,:,:,:]))\n",
        "test_elapsed = time.time() - t_test \n",
        "print(f'Time elapsed during test time: {\"{:.2f}\".format(test_elapsed)}, sec')"
      ],
      "metadata": {
        "id": "Rm9iV2Qj58PC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "226b9edf-c5c4-4660-a9b3-5f7b5f28a103"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Time elapsed during test time: 0.09, sec\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0N7v4PDP4Dv2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "96ab3a3b-b52a-4816-f6ef-f3b4cb62d83f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mean test accuracy is 0.839, mean test f1 score is 0.805, max test accuracy is 0.900, max test f1 score is 0.891, min test accuracy is 0.782, min test f1 score is 0.751, std of test accuracy is 0.040, std of test f1 score is 0.056\n",
            "Time elapsed through all process: 8.264, sec\n",
            "Time elapsed during test time: 0.091, sec\n"
          ]
        }
      ],
      "source": [
        "print(f'Mean test accuracy is {\"{:.3f}\".format(sum(test_accuracy_per_run)/number_of_repeat)}, mean test f1 score is {\"{:.3f}\".format(sum(f1_score_per_run)/number_of_repeat)}, \\\n",
        "max test accuracy is {\"{:.3f}\".format(max(test_accuracy_per_run))}, max test f1 score is {\"{:.3f}\".format(max(f1_score_per_run))}, \\\n",
        "min test accuracy is {\"{:.3f}\".format(min(test_accuracy_per_run))}, min test f1 score is {\"{:.3f}\".format(min(f1_score_per_run))}, \\\n",
        "std of test accuracy is {\"{:.3f}\".format(np.std(test_accuracy_per_run, axis=0))}, std of test f1 score is {\"{:.3f}\".format(np.std(f1_score_per_run, axis=0))}')\n",
        "print(f'Time elapsed through all process: {\"{:.3f}\".format(elapsed)}, sec')\n",
        "print(f'Time elapsed during test time: {\"{:.3f}\".format(test_elapsed)}, sec')"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "Range_Doppler_and_Spectrogram_CNN_with_no_MixUp.ipynb",
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}