CGR_project_4kmers_Before Features selection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CGR_project_4kmers_WITHOUT.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "4RwUrCdZrFNp"
      },
      "source": [
        "# ***C/NC DNA Detecting Code***\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "\n",
        "> This code to detect the coding and noncoding sequances (4-kmers) using chaos game representation, Chaos game representation (CGR) is an iterative mapping technique that assigns each nucleotide in a DNA sequence to a respective position on the plane in a one-to-one manner. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "WC6O1E3MmPCK",
        "colab": {}
      },
      "source": [
        "%matplotlib inline\n",
        "import numpy as np\n",
        "import os\n",
        "import imageio\n",
        "import matplotlib.pylab as plt\n",
        "import math\n",
        "import pandas as pd\n",
        "from pathlib import Path\n",
        "import matplotlib.cm as cm\n",
        "import seaborn as sn\n",
        "from sklearn import model_selection\n",
        "from sklearn.model_selection import train_test_split\n",
        "from IPython.core.interactiveshell import InteractiveShell\n",
        "InteractiveShell.ast_node_interactivity = \"all\"\n"
      ],
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Tppjl72LktYk",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 57
        },
        "outputId": "8944344d-76ea-4496-a2b8-d4db73768bd6"
      },
      "source": [
        "#Retrieve_the_data\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lo7KJ-z9hUbl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "start = time.time()"
      ],
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "49rzQUhlB7Nu",
        "colab": {}
      },
      "source": [
        "inp_out_needed=['data_coding_df1','data_noncoding_df1','data_coding_df1','data_noncoding_df1']\n",
        "paths={'data_coding_df1':'/content/drive/My Drive/project/new_CGR_data/coding_seq','data_noncoding_df1':'/content/drive/My Drive/project/new_CGR_data/non_codingseq'}"
      ],
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "mnvVW912B7jE",
        "colab": {}
      },
      "source": [
        "# df_image is a subfunction to create data frame from read each CGR image where each row is one image"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "s5_ts-YkB7yJ",
        "colab": {}
      },
      "source": [
        "def df_image(Images):\n",
        "  Images1=sorted(Images) # sort them ascendingly \n",
        "  ImagF=np.zeros((len(Images),497664))  # All Images\n",
        "  for i in range(len(Images1)):\n",
        "      ImagF_1=imageio.imread(Images1[i])\n",
        "      ImagF[i,:]=ImagF_1.reshape((1,497664))\n",
        "  return pd.DataFrame(ImagF)"
      ],
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "y0OGITH1B8DX",
        "colab": {}
      },
      "source": [
        "# clas_assign is a subfunction to assign class 1 (one) to each coding cgr images data frame and class 0 (zero) to  noncoding cgr images data frame"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "iKXOjexjB8Sa",
        "colab": {}
      },
      "source": [
        "def clas_assign(Images,c,b):\n",
        "  c= df_image(Images)\n",
        "  if b.split('_')[1] =='coding':\n",
        "    c['y']=np.ones((len(c)))\n",
        "  elif b.split('_')[1] =='noncoding':\n",
        "    c['y']=np.zeros((len(c)))\n",
        "  return c"
      ],
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0-TyPYc8B8fy",
        "colab": {}
      },
      "source": [
        "#A- For loop to make data frame from taking different types of images (coding & noncoding) from different folder (tain & test)  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "wfolnJN_B8uZ",
        "colab": {}
      },
      "source": [
        "for c in inp_out_needed:\n",
        "  b=c\n",
        "  home_path = os.getcwd()\n",
        "  path=paths[c]\n",
        "  os.chdir(path)\n",
        "  Images=os.listdir()\n",
        "  exec('{} = clas_assign(Images,c,b)'.format(c))"
      ],
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "CsBeSmTlG01L",
        "colab": {}
      },
      "source": [
        "# B- Merge codind & noncoding traing data (after assign class to each one ) in one train data frame and\n",
        "# codind & noncoding testing data (after assign class to each one ) in one test data frame "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "P5rK3ebmG1Bv",
        "colab": {}
      },
      "source": [
        "# merge coding sequence of train data to noncoding sequence of train data to obtain training sequence data frame\n",
        "data_coding_noncoding1=pd.concat([data_coding_df1,data_noncoding_df1])"
      ],
      "execution_count": 70,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GK4pIkLe2F0k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# C - split train dataframe (from step B ) to X_train (data) & y_train (labeles) variables to can start machine learning classification"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gUQnB8wVBlLm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X=data_coding_noncoding1.iloc[:,:-1]\n",
        "y=data_coding_noncoding1.iloc[:,-1]\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42)"
      ],
      "execution_count": 71,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "GrBIKC2cz-do",
        "colab": {}
      },
      "source": [
        "df_resultcol=['NaiveBayes','logistic regression','KNN','Perceptron','SVM','Combination']\n",
        "df_resultrow=['Accuracy','Specificity','Precision','Sensitivity']"
      ],
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "8sYsgkq24xXq",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "outputId": "70482eae-edd4-40c3-841e-3cc8456ccf2b"
      },
      "source": [
        "df_result=pd.DataFrame(columns=df_resultcol,index=df_resultrow)\n",
        "df_result"
      ],
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "      <th>KNN</th>\n",
              "      <th>Perceptron</th>\n",
              "      <th>SVM</th>\n",
              "      <th>Combination</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Accuracy</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Precision</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            NaiveBayes logistic regression  KNN Perceptron  SVM Combination\n",
              "Accuracy           NaN                 NaN  NaN        NaN  NaN         NaN\n",
              "Specificity        NaN                 NaN  NaN        NaN  NaN         NaN\n",
              "Precision          NaN                 NaN  NaN        NaN  NaN         NaN\n",
              "Sensitivity        NaN                 NaN  NaN        NaN  NaN         NaN"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "V2uFGFtf-F98",
        "colab": {}
      },
      "source": [
        "# F- Statistical Data Analysis after Scalling:"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RraTMda3PGXl"
      },
      "source": [
        " Apply different type of classification :\n",
        "\n",
        "1- Naive Bayes Classifier (supervised linear classifier)\n",
        "\n",
        "2- Logistic regression (supervised linear classifier)\n",
        "\n",
        "3- k-nearst neighbor classifier ( supervised nonlinear classifier)\n",
        "\n",
        "4- perceptron (supervised linear classifier)\n",
        "\n",
        "5- SVM (Support Vector Machine ) (supervised classifier)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "CjLik5cxLXBo",
        "colab": {}
      },
      "source": [
        "#1- Naive Bayes Classifier"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0U5wv0HcwVYI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# E- feature scaling\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "sc= StandardScaler()\n",
        "X_train=sc.fit_transform(X_train)\n",
        "X_test=sc.transform(X_test)"
      ],
      "execution_count": 74,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4r69jARAl2lK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "41d25de7-321c-42af-a555-b126cdde97da"
      },
      "source": [
        "#Import Gaussian Naive Bayes model\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "\n",
        "#Create a Gaussian Classifier\n",
        "model_n = GaussianNB()\n",
        "\n",
        "# Train the model using the training sets\n",
        "model_n.fit(X_train, y_train)\n",
        "\n",
        "#Predict Output\n",
        "y_pred_nai = model_n.predict(X_test) "
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GaussianNB(priors=None, var_smoothing=1e-09)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "uuBkSeuIl26R",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 57
        },
        "outputId": "4c68af7a-5f2a-493b-de59-433b049df9da"
      },
      "source": [
        "# confusion matrix\n",
        "os.chdir(home_path)\n",
        "from sklearn.metrics import confusion_matrix\n",
        "cm_naiv=confusion_matrix(y_test,y_pred_nai)\n",
        "cm_naiv"
      ],
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[28,  5],\n",
              "       [ 3, 42]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Aq0gJdTcJVUT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "fff8159b-0501-4c0c-bdd0-2c67d945e773"
      },
      "source": [
        "#Sensitivity_(True positive rate TPR) \n",
        "TP_nav= cm_naiv[0][0]\n",
        "FN_nav= cm_naiv[1][0]\n",
        "TPR_nav= TP_nav/(TP_nav+FN_nav) * 100\n",
        "print(\"Sensitivity \"+str(TPR_nav)+'%')"
      ],
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sensitivity 90.32258064516128%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D4m-RHeUJVgq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "7d9b8352-d33c-4d20-d137-e5a8ab15d5f7"
      },
      "source": [
        "#Specificity(Total Negative Result TNR)\n",
        "TN_nav= cm_naiv[1][1]\n",
        "FP_nav= cm_naiv[0][1] \n",
        "TNR_nav = TN_nav/(TN_nav+FP_nav) * 100\n",
        "print(\"Specificity \"+str(TNR_nav)+'%')"
      ],
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Specificity 89.36170212765957%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5i7btoTGfW62",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "c288e7ef-455b-4d8e-9828-69f7d358e345"
      },
      "source": [
        "percision_nav=(TP_nav/(TP_nav+FP_nav))*100\n",
        "percision_nav"
      ],
      "execution_count": 79,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "84.84848484848484"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "UUB413aiqXMY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        },
        "outputId": "69f9d892-cc25-4aac-cdd7-dcf769106b1f"
      },
      "source": [
        "os.chdir(home_path)\n",
        "cm_picture_nav=sn.heatmap(pd.DataFrame(cm_naiv),cmap=\"Blues\", annot=True)\n",
        "figure1 = cm_picture_nav.get_figure()   \n",
        "plt.title(\"Confusion Matrix Naive Bayes\")\n",
        "figure1.savefig('/content/drive/My Drive/project/CGR_data/Confusion_NaiveBayes_4kmers.jpg', dpi=400)"
      ],
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Confusion Matrix Naive Bayes')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 80
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVoAAAEICAYAAAAeFzyKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAaHElEQVR4nO3de5xVdb3/8dd7BvACIqCCiPfUPFoHLDWveTsadDmomaZWZBRmcdKf5jV/lpWdPJ00j3bqjFfygmJKmpnKz1uJl0RFBOl4JxEUhfCKysDn98da0HaY2WvPsNfsNYv308d6sPdae777M9uZ93z2d122IgIzM8tPU6MLMDMrOwetmVnOHLRmZjlz0JqZ5cxBa2aWMwetmVnOHLTdTNI6kn4v6XVJ16/GOEdLuqOetTWCpD9KGtOg535L0taNeG5bszhoOyDpKEnT0l/G+Wkg7FWHoQ8DhgAbRMQXujpIRFwdEQfVoZ4PkLSvpJA0uc364en6e2oc5weSrsp6XESMiogJXajzq2k9p7RZP1fSvrWMERH9IuK5zj53Rl37Slqe/ty8JeklSWfX8zms53HQtkPSicAvgJ+QhOLmwH8Do+sw/BbAUxHRWoex8vIqsLukDSrWjQGeqtcTKLG6P3+LgFMkrVePmupoXhri/YC9gLGSDm50UdY4Dto2JK0P/BD4dkTcGBFvR8TSiPh9RJycPmYtSb+QNC9dfiFprXTbvmlXdZKkBWk3fEy67WzgLOCItNsZ27bzk7Rl2qn1Su9/VdJzkt6U9LykoyvW31fxdXtIejidknhY0h4V2+6R9CNJU9Nx7pC0YZWX4X3gd8AX069vBo4Arm7zWl0g6UVJb0h6RNLe6fqRwBkV3+fjFXWcI2kq8A6wdbru6+n2X0m6oWL8cyXdKUkd1DkbeAA4sb2NknaV9ICkxen/h4sk9anYHpK2kfQJSS+n3+eKbYdImpHebpJ0mqRnJS2UNEnSoCqv30oR8TxwP7BDDa/bxpLeqfwDJ+ljkl6V1Du9/zVJsyX9XdLtkrZI10vS+enP3BuSnpD0kVpqtPw5aFe1O7A2MLnKY74H7AaMAIYDuwJnVmzfGFgfGAaMBX4paWBEfJ+kS74u7XgurVaIpL7AfwGjImI9YA9gejuPGwT8IX3sBsB5wB/adKRHAccAg4E+wHerPTfwG+Ar6e1PATOBeW0e8zDJazAIuAa4XtLaEXFbm+9zeMXXfBkYB6wHzGkz3knAR9M/InuTvHZjovp54v8XOKGD4FsG/B9gQ5L/rwcA32r7oIh4CHgb2L9i9VHp9wTwb8DBwD7AJsDfgV9WqWklSdsCewIPVqzu6HV7GbgHOLzisV8Gro2IpZJGk/wBOxTYCPgzMDF93EHAJ4HtSH72DgcW1lKj5c9Bu6oNgNcy3tofDfwwIhZExKvA2SS/ECssTbcvjYhbgbeAD3exnuXARyStExHzI2JWO4/5DPB0RFwZEa0RMRH4K/C5isdcHhFPRcQSYBLJL3qHIuJ+YJCkD5ME7m/aecxVEbEwfc6fA2uR/X1eERGz0q9Z2ma8d0hex/OAq4B/i4i5GXVOB6YAp7az7ZGIeDB9rheA/yEJy/ZMBI4ESKciPs0/QuybwPciYm5EvAf8ADhsxbuOdmySdtFvkEy3PASsfPeR8bpNAL6U1tGc1nRlRR3/HhGz05/PnwAj0q52Kckfr+0BpY+Z30F91s0ctKtaCGxY5ZcIkq6mshubk65bOUaboH4H6NfZQiLibZK37N8E5kv6g6Tta6hnRU3DKu6/3IV6rgTGA/vRTocv6bvp29jXJS0m6aSqTUkAvFhtY9pdPgeI5A9CLc4CjpM0pE1920m6JZ0WeIMkmDqq7xrg0HQK6FDg0YhY8ZpuAUxOw3MxyZTFMpL5+/bMi4gBEdEfGAAsIQnQFXVVe91uAnaQtBVwIPB6RPyloo4LKupYRPI6DYuIu4CLSDrtBZJaJPXPeuGsezhoV/UA8B7JW8WOzCP5oV9hc1Z9W12rt4F1K+5vXLkxIm6PiAOBoSRd6sU11LOippe6WNMKV5K81b417TZXSt/an0LyFnVgRAwAXif5xQfo6O1+1cvFSfo2SYc3Lx0/U0T8FbiRZEqn0q9IXrNt09A7o6K+tmM8SfLHaRQfnDaA5I/DqDQ8VyxrR0Tm6xsRr6djfS79/qq+bhHxLskfmC+RdPdXVgz3InBsmzrWSd99EBH/FREfJ5kP3g44Oas+6x4O2jbSX4yzSOZVD5a0rqTekkZJ+o/0YROBMyVtlO5UOovkrW5XTAc+KWlzJTviTl+xQdIQSaPTudr3SKYglrczxq3AdkoOSesl6QiSX7ZbulgTsHJHzj6sGmCQvE1tJTlCoZeks4DKDuoVYEt14sgCSdsBP+YfIXOKpKpTHBXOJpmDHtCmxjeAt9J3AsdljHENcDzJXGflMc6/Bs6p2PG0UTpfmklSP5KdiiumfLJeN0imab4K/CsfDNpfA6dL2jEde31JX0hv75Lu1OtN8sf7Xdr/WbEGcNC2I503O5FkB9erJJ3EeJI98ZCEwTRgBvAE8Gi6rivPNQW4Lh3rET4Yjk1pHfNI3ibuQzthERELgc+S7ExaSNIxfTYiXutKTW3Gvi8i2uvWbwduI5mDnEPyi105LbAiqBZKejTredKpmquAcyPi8Yh4mqQDvTJ9O59V5/MkodS3YvV3SbrTN0neCVyXMcxEktf4rjav3QXAzcAdkt4k2bH1iSrjbKL0OFqS12YQybw+ZL9uRMRUkpCsnL4gIiYD5wLXplMhM0k6cEjC+mKSHXVzSH4Ofpbx/Vo3kS/8bVY8ku4CromISxpdi60+B61ZwUjaheRIis0i4s1G12Orz1MHZgUiaQLw/4ATHLLl4Y7WzCxn7mjNzHJW7aD8uvjkeVPdMtsqfnfsbo0uwQpoUN/mjq5rUbN1dhpfc+Yseeyi1X6+WrijNTPLWe4drZlZt1rtq2/Wn4PWzMqlqTn7Md3MQWtm5dLh5Ysbx0FrZuXiqQMzs5y5ozUzy1kBO9riVWRmtjqk2peahlOzpMck3ZLe30rSQ5KekXSdKj6HriMOWjMrl6bm2pfaHE/yqRornAucHxHbkFyWcmxmSZ3+JszMikxNtS9ZQ0mbknwm3yXpfZF8iOdv04dMoPqnsQCeozWzsqnvzrBfkFxIf730/gbA4orPBJzLBz+br13uaM2sXDrR0UoaJ2laxTJu5TDSZ4EFEfHI6pbkjtbMyqUTRx1ERAvQ0sHmPYF/lfRpYG2Sjwu6ABggqVfa1W5KDR+C6o7WzMqlubn2pYqIOD0iNo2ILUk+YPOuiDgauBs4LH3YGJKPiK/KQWtm5VLnw7vacSpwoqRnSOZsL836Ak8dmFm55HDCQkTcA9yT3n4O2LUzX++gNbNy8Sm4ZmY5K+ApuA5aMysXd7RmZjnzhb/NzHLmqQMzs5x56sDMLGfuaM3McuagNTPLmXeGmZnlzHO0ZmY589SBmVnO3NGameVLDlozs3w5aM3McqYmB62ZWa7c0ZqZ5cxBa2aWMwetmVneipez/nBGMysXSTUvGeOsLekvkh6XNEvS2en6KyQ9L2l6uozIqskdrZmVSlNT3frH94D9I+ItSb2B+yT9Md12ckT8ttaBHLRmVir1mqONiADeSu/2TpfoylieOjCzclHti6RxkqZVLOM+MJTULGk6sACYEhEPpZvOkTRD0vmS1soqyR2tmZVKZzraiGgBWqpsXwaMkDQAmCzpI8DpwMtAn/RrTwV+WO153NGaWanUa2dYpYhYDNwNjIyI+ZF4D7gc2DXr6x20ZlYqalLNS9VxpI3SThZJ6wAHAn+VNDRdJ+BgYGZWTZ46MLNSqeMJC0OBCZKaSZrSSRFxi6S7JG1EMtM7Hfhm1kAOWjMrlToedTAD2Kmd9ft3diwHrZmVik/BNTPLmYPWzCxvxctZB62ZlUsdT8GtGwetmZWKpw7MzPJWvJx10OZlcL8+nDFqOwat25sI+P0TL/Pbx+azzUZ9OelfPkSfZrFsOZx/17PMfvmt7AGtlA75zL+wbt++NDc10dzci8uvvr7RJfV47mjXIMsi+O97n+epBW+zTu9mLvnScB6es5jj9t6CKx74Gw+9sJjdthrIN/fekuOvzzyxxErsl/9zBQMGDmx0GaXRI4NW0vbAaGBYuuol4OaImJ1nYT3dwreXsvDtpQAsWbqMOQvfYaN+fQigb5/kZe/bp5nX3n6/gVWalU+PC1pJpwJHAtcCf0lXbwpMlHRtRPw05/pKYeP+a7Ht4H48+fJbXHjP8/znoTvyrX22RIJvTXyi0eVZA0ni+G9/HSEO/vzhHPz5wxtdUo/XEz9ufCywY0QsrVwp6TxgFtBu0KbXdBwHsM1hJzN099F1KLVnWqd3Ez/63PZceM9zvPP+MkYP35iL7n2ee59eyH7bbcCpB23DiTfManSZ1iC/vuwqBg8ewqJFCzn+uK+zxZZbs9PHd250WT1aETvarAPOlgObtLN+aLqtXRHREhE7R8TOa3LINjeJH31ue6bMfpU/PbMIgJE7DObepxcCcPdTC/mnjfs1skRrsMGDhwAwaNAG7LPfATw5a0aDK+r58rhM4urK6mhPAO6U9DTwYrpuc2AbYHyehZXBqQdtw5xFS5j06LyV6xa+9T4jNu3P9Llv8LHN1mfu4ncbWKE10pIl77B8edC3b1+WLHmHhx68n69947hGl9XjFbChrR60EXGbpO1ILmxbuTPs4fTK49aBj26yHiN3GMyzr77NpV8aDsDFU//Gf0x5hu/stzXNTeL91uX8bMozDa7UGmXRwoWcdtJ3AFi2rJWDRn6G3ffcu8FV9XxFnDrIPOogIpYDD3ZDLaXyxLw3+eR5U9vd9o2rH+/maqyIhm26GVdeN7nRZZROUw/cGWZm1qMUsKF10JpZubijNTPLmTtaM7OcFXFnWPEu3Ghmthqk2pfq42htSX+R9LikWZLOTtdvJekhSc9Iuk5Sn6yaHLRmVipNTU01LxneA/aPiOHACGCkpN2Ac4HzI2Ib4O8kZ9BWr2k1vyczs0KpV0cbiRXXMO2dLgHsD/w2XT8BODirJgetmZVKZ07BlTRO0rSKZVybsZolTQcWAFOAZ4HFEdGaPmQu/ziZq0PeGWZmpdKZfWER0QK0VNm+DBghaQAwGdi+KzU5aM2sVPI46iAiFku6G9gdGCCpV9rVbkpyWYKqPHVgZqVSx6MONko7WSStAxwIzAbuBg5LHzYGuCmrJne0ZlYqdTwzbCgwQVIzSVM6KSJukfQkcK2kHwOPAZdmDeSgNbNSqdfUQUTMAHZqZ/1zJFc0rJmD1sxKpYAnhjlozaxcingKroPWzEqlgDnroDWzcvFlEs3McuapAzOznDlozcxyVsCcddCaWbm4ozUzy1kBc9ZBa2bl4qMOzMxy1lTAltZBa2alUsCcddCaWbl4Z5iZWc4KOEXroDWzcvHOMDOznAkHrZlZrgrY0DpozaxcvDPMzCxnBcxZfwqumZVLk1TzUo2kzSTdLelJSbMkHZ+u/4GklyRNT5dPZ9XkjtbMSqWORx20AidFxKOS1gMekTQl3XZ+RPxnrQM5aM2sVOo1dRAR84H56e03Jc0GhnVlLE8dmFmpdGbqQNI4SdMqlnHtjSlpS5KPHn8oXTVe0gxJl0kamFlT3b47M7MCUCeWiGiJiJ0rlpZVxpP6ATcAJ0TEG8CvgA8BI0g63p9n1eSpAzMrlXoe3iWpN0nIXh0RNwJExCsV2y8Gbskax0FrZqVSr31hShL7UmB2RJxXsX5oOn8LcAgwM2ssB62ZlUodjzrYE/gy8ISk6em6M4AjJY0AAngBODZrIAetmZVKvaYOIuI+aPfCCbd2diwHrZmViq91YGaWM1/rwMwsZ8WLWQetmZVMcwHnDhy0ZlYqnjowM8tZAXPWQWtm5ZJ1+cNGcNCaWakUMGfzD9o7vrNn3k9hPdDAXcY3ugQroCWPXbTaY3iO1swsZ80OWjOzfBXw6C4HrZmVi4PWzCxnnqM1M8uZO1ozs5wVsKF10JpZufQqYNI6aM2sVAqYsw5aMysXn4JrZpazAuYsTY0uwMysnppU+1KNpM0k3S3pSUmzJB2frh8kaYqkp9N/B2bWVJ9vzcysGJqbVPOSoRU4KSJ2AHYDvi1pB+A04M6I2Ba4M71flYPWzEqlXh1tRMyPiEfT228Cs4FhwGhgQvqwCcDBmTWtzjdkZlY06sx/0jhJ0yqWce2OKW0J7AQ8BAyJiPnpppeBIVk1eWeYmZVKZ84Mi4gWoKXaYyT1A24AToiINypP8Y2IkBRZz+OgNbNSqecpuJJ6k4Ts1RFxY7r6FUlDI2K+pKHAgsya6leSmVnjSap5yRhHwKXA7Ig4r2LTzcCY9PYY4KasmtzRmlmpNNevfdwT+DLwhKTp6bozgJ8CkySNBeYAh2cN5KA1s1Kp15lhEXEf0NFgB3RmLAetmZWKL5NoZpazIp6C66A1s1Jp6vDdfuM4aM2sVNzRmpnlrFcBJ2kdtGZWKu5ozcxy5gt/m5nlrIA566A1s3Ip4nUFHLRmViqeOjAzy5mD1swsZ8WLWQetmZVMARtaB62ZlUvWdWYbwUFrZqXiow7MzHLmnWFmZjnz1IGZWc48dWBmlrMidrRFDH8zsy5TJ5bMsaTLJC2QNLNi3Q8kvSRperp8OmscB62ZlUqzVPNSgyuAke2sPz8iRqTLrVmDeOrAzEqlnjMHEfEnSVuu7jjuaM2sVNSZ/6RxkqZVLONqfJrxkmakUwsDsx7soDWzUpFqXyKiJSJ2rlhaaniKXwEfAkYA84GfZ32Bpw7MrFTy/hTciHhlxW1JFwO3ZNdkZlYinelouza+hlbcPQSY2dFjV3BHa2alUs9TcCVNBPYFNpQ0F/g+sK+kEUAALwDHZo3joDWzUqnnp41HxJHtrL60s+M4aM2sVFTAS387aM2sVAp4Bq6Dtju89957HPOVo1n6/vu0LlvGgQd9im+N/06jy7IGaWoSU68+hXkLXufzx/+ay88Zw8d22JylrcuYNnMO48+ZSGvr8kaX2WMVsaP1UQfdoE+fPlxy2QSun3wzk274HVPv+zMzHp/e6LKsQcYftR//+/zKI4S49o8PM/yQH7HzF37COmv35phD9mhgdT1fk2pfuq2m7nuqNZck1u3bF4DW1lZaW1uL+f7Gcjds8ABG7rUjl0++f+W62+97cuXtaTPnMGxw5olGVkWTVPPSbTV12zOt4ZYtW8bhh45mv733YLfd9+Cf/3l4o0uyBvjZyZ/nexf8juXLY5VtvXo1ceRndmXK/U+285VWq3pevateuhy0ko6psm3l+cOXXlzLGW3l19zczKQbb+KOu+5l5hMzePrppxpdknWzUXt/hAWL3uSx2S+2u/2C049g6qPPMPWxZ7u5snIpYke7OjvDzgYub29Der5wC8C7raz6p3sN1r9/f3bZ9RPcf9+f2Xbb7RpdjnWj3UdszWf3+Sgj99qRtfr0pn/ftbnsx1/ha2f+hjPGjWKjgf044seXNLrMHq+Ik3JVg1bSjI42AUPqX045LVq0iF69etG/f3/effddHnzgfo4Z+41Gl2Xd7KwLb+asC28GYO+Pb8sJXzmAr535G756yO4cuMc/MerYC4lwX7LaCpi0WR3tEOBTwN/brBdw/6oPt/a89uoCzjzjNJYvX8by5cFBnxrJPvvu1+iyrCAuPOOL/G3+Iu6ZcBIAN901nX9vua3BVfVcRfwUXFX7CyrpUuDyiLivnW3XRMRRWU/gqQNrz8Bdxje6BCugJY9dtNop+fBzr9ecObtsvX63pHLVjjYixlbZlhmyZmbdrngNrc8MM7NyKeKZYQ5aMyuVAk7ROmjNrFwKmLMOWjMrFxWwpXXQmlmpFDBnHbRmVi4FzFkHrZmVTAGT1lfvMrNSUSf+yxxLukzSAkkzK9YNkjRF0tPpv5nXtXTQmlmp1Pnjxq8ARrZZdxpwZ0RsC9yZ3q/KQWtmpVLPoI2IPwGL2qweDUxIb08ADs4ax0FrZqXSmamDymtnp8u4Gp5iSETMT2+/TA1XMvTOMDMrlc4c3lV57eyuiIiQlHkRG3e0ZlYq3fBRNq9IGgqQ/rsg6wsctGZWLvkn7c3AmPT2GOCmrC/w1IGZlUo9L/wtaSKwL7ChpLnA94GfApMkjQXmAIdnjeOgNbNSqef5ChFxZAebDujMOA5aMyuXAp4Z5qA1s1Lxhb/NzHLmq3eZmeWsgDnroDWzcvGFv83MclbAnHXQmlm5FDBnHbRmVjIFTFoHrZmVig/vMjPLmedozcxy1uSgNTPLW/GS1kFrZqXiqQMzs5wVMGcdtGZWLu5ozcxy5lNwzcxyVryYddCaWckUsKF10JpZudTzzDBJLwBvAsuA1ojYuSvjOGjNrFzq39HuFxGvrc4ADlozK5UCzhzQ1OgCzMzqqUmqealBAHdIekTSuK7W5I7WzEqlMzvD0vCsDNCWiGipuL9XRLwkaTAwRdJfI+JPna3JQWtma6w0VFuqbH8p/XeBpMnArkCng9ZTB2ZWKlLtS/Vx1FfSeituAwcBM7tSkztaMyuVOh7eNQSYnJ5p1gu4JiJu68pADlozK5V6nbAQEc8Bw+sxloPWzErFZ4aZmeXMnxlmZpYzd7RmZjkrYM46aM2sZAqYtA5aMyuVGk+t7VaKiEbXsMaQNK7N6X1m/rlYA/jMsO7V5YtSWKn556LkHLRmZjlz0JqZ5cxB2708D2ft8c9FyXlnmJlZztzRmpnlzEFrZpYzB203kTRS0v9KekbSaY2uxxpP0mWSFkjq0sWkredw0HYDSc3AL4FRwA7AkZJ2aGxVVgBXACMbXYTlz0HbPXYFnomI5yLifeBaYHSDa7IGSz/kb1Gj67D8OWi7xzDgxYr7c9N1ZrYGcNCameXMQds9XgI2q7i/abrOzNYADtru8TCwraStJPUBvgjc3OCazKybOGi7QUS0AuOB24HZwKSImNXYqqzRJE0EHgA+LGmupLGNrsny4VNwzcxy5o7WzCxnDlozs5w5aM3McuagNTPLmYPWzCxnDlozs5w5aM3Mcvb/AZCVHTBHZt+CAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "PsIoZRO6snrY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "6db60fb3-8b38-4edd-c69f-e6dafbaaad99"
      },
      "source": [
        "#Calculate Accuracy and Print it\n",
        "accu_navie=(sum([1 for i, j in zip(y_pred_nai, y_test) if i == j])/len(y_test))*100\n",
        "print(\"Accuracy = \" ,accu_navie, \"%\")"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy =  89.74358974358975 %\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "mL_Kf-HmDx-9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "8f18e56a-412a-4a4a-ebb3-e1d34f63acaa"
      },
      "source": [
        "# calculating the cross validation for a set of predictions:\n",
        "results_naiv = model_selection.cross_val_score(model_n, X_train, y_train,cv=None)*100\n",
        "results_naiv"
      ],
      "execution_count": 82,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([85.71428571, 93.65079365, 83.87096774, 85.48387097, 91.93548387])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WcFjJVfvz3AD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "acc_cross_validation=pd.DataFrame()\n",
        "acc_cross_validation['NaiveBayes']=results_naiv"
      ],
      "execution_count": 83,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "vgOMOspg1xV4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "outputId": "156f85bc-9d1b-452d-db48-4363b4f95f3c"
      },
      "source": [
        "df_result['NaiveBayes']=pd.Series({'Accuracy':accu_navie,'Cross_Validation':results_naiv,'Confusion_Matrix':cm_naiv,'Sensitivity':TPR_nav,'Specificity':TNR_nav,'Precision':percision_nav})\n",
        "df_result"
      ],
      "execution_count": 84,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "      <th>KNN</th>\n",
              "      <th>Perceptron</th>\n",
              "      <th>SVM</th>\n",
              "      <th>Combination</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Accuracy</th>\n",
              "      <td>89.7436</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>89.3617</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Precision</th>\n",
              "      <td>84.8485</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>90.3226</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            NaiveBayes logistic regression  KNN Perceptron  SVM Combination\n",
              "Accuracy       89.7436                 NaN  NaN        NaN  NaN         NaN\n",
              "Specificity    89.3617                 NaN  NaN        NaN  NaN         NaN\n",
              "Precision      84.8485                 NaN  NaN        NaN  NaN         NaN\n",
              "Sensitivity    90.3226                 NaN  NaN        NaN  NaN         NaN"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "fe1RlAJn8HYx",
        "colab": {}
      },
      "source": [
        "#2- logistic regression"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "WfVnmtZYFlDX",
        "colab": {}
      },
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn import linear_model"
      ],
      "execution_count": 85,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "QlCUHpETFoKq",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "outputId": "c52a95ff-ebae-4e61-d11f-d81ca534611e"
      },
      "source": [
        "clf =linear_model.LogisticRegression().fit(X_train, y_train)"
      ],
      "execution_count": 86,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "z1CU1Bl5Fo1K",
        "colab": {}
      },
      "source": [
        "y_pred_log1=clf.predict(X_test)"
      ],
      "execution_count": 87,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "cbLHoghiUlH0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 57
        },
        "outputId": "847f4c51-8133-4145-b5b5-2bcde012f231"
      },
      "source": [
        "# confusion matrix\n",
        "os.chdir(home_path)\n",
        "from sklearn.metrics import confusion_matrix\n",
        "cm_logistic=confusion_matrix(y_test,y_pred_log1)\n",
        "cm_logistic"
      ],
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[32,  1],\n",
              "       [ 2, 43]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 88
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wOdEl8TbJrLb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "6cea9f1d-35e6-4de2-d529-8eb24ee733ab"
      },
      "source": [
        "#Sensitivity_(True positive rate TPR) \n",
        "TP_log= cm_logistic[0][0]\n",
        "FN_log= cm_logistic[1][0]\n",
        "TPR_log= TP_log/(TP_log+FN_log) * 100\n",
        "print(\"Sensitivity \"+str(TPR_log)+'%')"
      ],
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sensitivity 94.11764705882352%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3uJ8dMB1JrYG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "27ce0eaa-f045-4c9c-fa5f-d38dab44f83f"
      },
      "source": [
        "#Specificity(Total Negative Result TNR)\n",
        "TN_log= cm_logistic[1][1]\n",
        "FP_log= cm_logistic[0][1] \n",
        "TNR_log = TN_log/(TN_log+FP_log) * 100\n",
        "print(\"Specificity \"+str(TNR_log)+'%')"
      ],
      "execution_count": 90,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Specificity 97.72727272727273%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wJYPN4WdgC5f",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "030e3251-ca59-4d8a-b2b6-21be97ed7183"
      },
      "source": [
        "percision_log=(TP_log/(TP_log+FP_log))*100\n",
        "percision_log"
      ],
      "execution_count": 91,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "96.96969696969697"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 91
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "U_7iZgxdU839",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        },
        "outputId": "eca1c559-2959-4c34-fa40-9ff0fb662ebc"
      },
      "source": [
        "os.chdir(home_path)\n",
        "cm_picture_logistic=sn.heatmap(pd.DataFrame(cm_logistic),cmap=\"Blues\", annot=True)\n",
        "figure2 = cm_picture_logistic.get_figure()   \n",
        "plt.title(\"Confusion Matrix Logistic Regression\")\n",
        "figure2.savefig('/content/drive/My Drive/project/CGR_data/Confusion_Logistic_4kmers.jpg', dpi=400)"
      ],
      "execution_count": 92,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Confusion Matrix Logistic Regression')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 92
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVoAAAEICAYAAAAeFzyKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAafElEQVR4nO3debwcZZn28d91cgJhEvZAJiTsq4GRoICAIBENBGRTRGUNyrxRxigMyKYMRGQQFERmUDTKJiCLCpJBBCKyiGggQFhiwiJbErJAIBB2Qu73j3oSOodzejnpOl2pXF8+9Ul3VffTdxXd13n6qeoqRQRmZpaftlYXYGZWdg5aM7OcOWjNzHLmoDUzy5mD1swsZw5aM7OcOWhrkLSSpP+T9Iqk3yxFO4dIurWZtbWCpD9KGtnqOgAkrSfpNUm9uvHcn0n6rzzqKipJu0h6rNV1LI9KE7SSDpY0MX3wZqZA2LkJTX8eGACsGREHdreRiLgyInZvQj1LkDRMUki6vsP8rdP8O+psZ4ykK2o9LiL2jIjLulHnEZLubvR5NWp5LiL6RcR7jb52RHwtIr7X6GtKekbSm+l9NkvSpZL6NdpOK0TEXyJi81bXsTwqRdBKOhb4MXAmWSiuB/wU2K8Jza8PPB4RC5rQVl5eAHaUtGbFvJHA4816AWVK8X5pgn0ioh8wFNgGOLnZLyCpvdltWgtFxDI9AasCrwEHVnnMimRB/HyafgysmJYNA6YDxwFzgJnAl9Oy7wLvAO+m1zgSGANcUdH2BkAA7en+EcBTwHzgaeCQivl3VzxvJ+A+4JX0704Vy+4Avgf8NbVzK9C/i3VbVP/PgK+neb2AGcCpwB0Vjz0fmAa8CtwP7JLmj+iwng9V1PHfqY43gU3SvH9Pyy8EflfR/tnAbYA6qXOJ9e+wrNq22BC4K22HPwE/WbT969n2wIeAt4D30rrNS4+9FDij4nX2AyalbfNPYEQXtT4DfLri/g+AP1Tc3wG4B5gHPAQMa3BdjgSeA+5K878CTAFeBm4B1k/zBZxH9p59FXgE2Cot2wv4R3qdGcC3Kt8rFfV8KP3/nAdMBvatWHZpqu8PqZ0JwMat/rwvq1PLC1jqFchCYsGiD1sXjzkd+DuwNrBW+iB8Ly0blp5/OtA7vUnfAFZPy8ewZLB2vL/4ww70TW/6zdOygcCW6fYRpKAB1kgfnMPS8w5K99dMy+9IH/bNgJXS/bO6WLdhZEG7EzAhzdsrfSj/nSWD9lBgzfSaxwGzgD6drVdFHc8BW6bn9GbJoP0Xsl7zEcAuwIvA4C7qXLz+HebX2hZ/A84BVgB2Ttv3A0Fb77aveN1LSUELbE8W8sPJvuUNArboYj2eIQUtMJgs4M5P9wcBc9P2b0vtzQXWamBdfpXWZSWy8H+SLBDbgVOAe9Lj9yD7Y7kaWeh+CBiYls3k/T+iqwMfqXyvpNu9U9vfTvXsRhaom1dsn7lp27QDVwJXt/rzvqxOZfgquCbwYlT/an8IcHpEzImIF8h6qodVLH83LX83Im4i6/l0dyxrIbCVpJUiYmZETO7kMZ8BnoiIyyNiQURcBUwF9ql4zCUR8XhEvAlcS/Y1tUsRcQ+whqTNgcPJPrAdH3NFRMxNr3kuWU+/1npeGhGT03Pe7dDeG2Tb8UfAFcA3ImJ6jfY66nJbSFoP2A44NSLeiYi7gXFV2qpn23fmSODiiBgfEQsjYkZETK3y+N9Lmk/27WAOcFqafyhwU0TclNoZD0wE9mpgXcZExOvp//vXgO9HxJT0/j4TGCppfbL37MrAFmTfIKZExMzUxrvAEEmrRMTLEfFAJ6+zA9CP7A/4OxHxZ+BGsj90i1wfEfem176SGu9B61oZgnYu0L/GmNY6wLMV959N8xa30SGo3yB7EzYkIl4Hvkj2AZkp6Q+StqijnkU1Daq4P6sb9VwOjAY+CVzfcaGkb0mako6gmEc27NK/RpvTqi2MiAlkX9dF9gehUdW2xTrASynQq9bTwLbvzLpk3yDqtX9ErEzWQ9yC97fh+sCBkuYtmsh6rgMbWJfKeesD51e09RLZdh6UgvECsq/3cySNlbRKet4BZL3qZyXdKWnHTl5nHWBaRCysmNeM96B1ogxB+zfgbWD/Ko95nuxNu8h6aV53vE72lXmRf61cGBG3RMRwsg/XVOAXddSzqKYZ3axpkcuB/yDrVVV+oJG0C3AC8AWyYZHVyL4ua1HpXbRZ9fRukr5O1jN+PrXfqGrbYiZZL71ye6/bVUNVtn2tU9RNAzZupOj0eneSfcU+p6KdyyNitYqpb0Sc1cC6VNY6Dfhqh/ZWSt9eiIj/iYiPAkPIhpmOT/Pvi4j9yIbKfk/nfwCfB9btsIOzGe9B68QyH7QR8QrZTp+fSNpf0r9I6i1pT0k/SA+7CjhF0lqS+qfH1zyUqQuTgE+kYzhXpWKPs6QBkvaT1Jcs/F8j+zrb0U3AZumQtHZJXyT7sNzYzZoAiIingV2B73SyeGWysegXgHZJpwKrVCyfDWzQyJEFkjYDziD7ynwYcIKkal8vJalP5USVbRERz5J99R4jaYXUM9uni4arbfvZwGBJK3RR10XAlyV9SlKbpEEN9IZ/DAyXtDXZe2ofSXtI6pXWcZikwY2sS4WfASdL2jKt46qSDky3t5P0MUm9yf74vwUsTG0fImnVNNTzKp2/ByeQ9VJPSJ+XYameq+tcb2vAMh+0AGm88ViynQUvkPUERpP9NYcsDCYCD5PtvHggzevOa40Hrklt3c+S4diW6nie7GversBRnbQxF9ibbIfUXLKe4N4R8WJ3aurQ9t0R0Vlv/RbgZrKdV8+SfTArv6Yu+jHGXEmdjektIQ3VXAGcHREPRcQTZDtWLpe0YhdP24ns6IXK6RWqb4tDgB3TsjPItv3bnbRdbdv/mWyv+ixJH9jGEXEv8GWyvfivAHfywV52p9KY/6/Ixl6nke3A+jbvvw+P5/3PWb3rsqjt68mO5Lha0qvAo8CeafEqZD32l8n+f84FfpiWHQY8k57ztfS6Hdt+hyxY9yTbiflT4PAaY9PWTYrwib9t2SHpGmBqRJxW88EFV6Z1sepK0aO18kpfkTdOX+lHkPUYf1/reUVUpnWxxvjXJ1Z0/wpcR3YY33TgqIh4sLUldVuZ1sUa4KEDM7OceejAzCxnuQ8dbHr8ze4y2wc88v0RrS7BCqhP++LjurttpW1G1505bz54wVK/Xj3cozUzy5l3hplZuRTwbJ4OWjMrl7aGL7iROwetmZWLemTYtSEOWjMrFw8dmJnlzD1aM7OcuUdrZpYz92jNzHLmow7MzHLmoQMzs5x56MDMLGcF7NEWryIzs6WhtvqneprLrv/2oKQb0/0NJU2Q9KSka6pci24xB62ZlUuvXvVP9TkamFJx/2zgvIjYhOyabUfWasBBa2blItU/1WxKg4HPAL9M9wXsBvw2PeQyYP9a7ThozaxcGhg6kDRK0sSKaVSH1n5MdmXmRZdsXxOYFxEL0v3pwKBaJXlnmJmVSwNHHUTEWGBs581ob2BORNwvadjSlOSgNbNyad5RBx8H9pW0F9AHWAU4H1hNUnvq1Q4GZtRqyEMHZlYuTRqjjYiTI2JwRGwAfAn4c0QcAtwOfD49bCRwQ62SHLRmVi5tveqfuudE4FhJT5KN2V5U6wkeOjCzcsnhBwsRcQdwR7r9FLB9I8930JpZufgnuGZmOSvgT3AdtGZWLg5aM7Oc+Xy0ZmY58xitmVnOPHRgZpYz92jNzPIlB62ZWb4ctGZmOVObg9bMLFfu0ZqZ5cxBa2aWMwetmVneipezDlozKxf3aM3MctbW5l+GmZnlqog92uJFv5nZ0lADU7VmpD6S7pX0kKTJkr6b5l8q6WlJk9I0tFZJ7tGaWak0sUf7NrBbRLwmqTdwt6Q/pmXHR8Rv623IQWtmpdKsoI2IAF5Ld3unKbrTlocOzKxU1Kb6J2mUpIkV06gl2pJ6SZoEzAHGR8SEtOi/JT0s6TxJK9aqyT1aMyuVRnq0ETEWGFtl+XvAUEmrAddL2go4GZgFrJCeeyJwerXXcY/WzEpFUt1TvSJiHnA7MCIiZkbmbeAS6rj0uIPWzEqlWUEraa3Uk0XSSsBwYKqkgWmegP2BR2vV5KEDMyuVJh51MBC4TFIvsk7ptRFxo6Q/S1qL7ACxScDXajXkoDWzcmlSzkbEw8A2nczfrdG2HLRmVir+Ca6ZWc6K+BNcB62ZlUvxctZBm5cV2tv49VHbs0J7G+1t4uZHZvM/tz7JuQd9mK0Gr8qChQt5+LlX+K/fTWbBwm792MSWcaeecjJ33XkHa6yxJtfdcGOryymNIvZoizeYURLvLFjI4T+/j33Pu4d9z7uHT2zen6Hrrcq4B59njx/+hc+c+1f69O7FFz42uNWlWovst//nuPDnv2x1GaWTx3G0S6tmj1bSFsB+wKA0awYwLiKm5FlYGbzxznsAtPcS7W0iAu6c+uLi5Q9Nm8eAVfu0qjxrsY9uux0zZkxvdRmls8z1aCWdCFxNNupxb5oEXCXppPzLW7a1Ccb95078/bTd+OsTc3lo2iuLl7W3if0/sg5/eezFKi2YWaMaOddBT6nVoz0S2DIi3q2cKelHwGTgrM6elE7MMApgreHfYNWt92pCqcuehQH7nncPK/dp56cjt2HTAf14YnZ2MqAxnxvCfU+/zMSnX25xlWblssz1aIGFwDqdzB+YlnUqIsZGxLYRse3yGrKV5r+1gAn/fIlPbNEfgNHDN2aNvitw5v9NbXFlZuWzLI7RHgPcJukJYFqatx6wCTA6z8KWdWv07c277wXz31rAiu1t7LTpmvzijqc5cPvB7LJZfw7/+X2EDzYwa7oCdmirB21E3CxpM7Kz01TuDLsvnT7MurDWKivygy9+mLY20Sb440OzuH3KC0w5a3een/cWv/nGDgDc+shsLvjTP1tcrbXCid86lon33cu8eS8zfLdPcNTXv8HnDjiw1WUt84o4dKDIuVu16fE3u99mH/DI90e0ugQroD7tS/9zg81PvKXuzHns7D16JJX9gwUzK5UCdmgdtGZWLm09eNhWvRy0ZlYq7tGameWsiDvDHLRmVioFzFmfVMbMyqWtra3uqRpJfSTdK+khSZMlfTfN31DSBElPSrpG0go1a2rSupmZFYJU/1TD28BuEbE1MBQYIWkH4GzgvIjYBHiZ7FQFVTlozaxUmvUT3HRJ8dfS3d5pCmA34Ldp/mVkV8KtykFrZqXSSI9W0ihJEyumUUu2pV6SJgFzgPHAP4F5EbEgPWQ67/9qtkveGWZmpdLIUQcRMRYYW2X5e8BQSasB1wNbdKcmB62ZlUoeRx1ExDxJtwM7AqtJak+92sFk53+pykMHZlYqbW2qe6pG0lqpJ4uklYDhwBTgduDz6WEjgRtq1eQerZmVShN/sDAQuExSL7JO6bURcaOkfwBXSzoDeBC4qFZDDlozK5Vm5WxEPAxs08n8p8hOHVs3B62ZlYp/gmtmlrMC5qyD1szKxadJNDPLmYcOzMxy5qA1M8tZAXPWQWtm5eIerZlZzgqYsw5aMysXH3VgZpaztgJ2aR20ZlYqBcxZB62ZlYt3hpmZ5ayAQ7QOWjMrF+8MMzPLmXDQmpnlqoAdWl/KxszKpVmXG5e0rqTbJf1D0mRJR6f5YyTNkDQpTXvVqsk9WjMrlSYedLAAOC4iHpC0MnC/pPFp2XkRcU69DTlozaxUmvWDhYiYCcxMt+dLmgIM6lZNTanIzKwgmnUV3EqSNiC7ftiENGu0pIclXSxp9Zo1dWdFzMyKSmpk0ihJEyumUR9sT/2A3wHHRMSrwIXAxsBQsh7vubVq8tCBmZVKI0MHETEWGNvVckm9yUL2yoi4Lj1ndsXyXwA31qyp7orMzJYBamCq2k52WMJFwJSI+FHF/IEVD/ss8GitmtyjNbNSaeK5Dj4OHAY8ImlSmvdt4CBJQ4EAngG+WqshB62ZlUqzfrAQEXfTecf3pkbbctCaWan4XAdmZjnzaRLNzHJWwA6tg9bMysU9WjOznBUvZh20ZlYyvQo4duCgNbNS8dCBmVnOCpizDlozK5dmnSaxmRy0ZlYqBczZ/IP2gTN2z/slbBm0+najW12CFdCbD16w1G14jNbMLGe9HLRmZvkq4NFdDlozKxcHrZlZzjxGa2aWM/dozcxyVsAOra8ZZmbl0i7VPVUjaV1Jt0v6h6TJko5O89eQNF7SE+lfX27czJYvjVxuvIYFwHERMQTYAfi6pCHAScBtEbEpcFu6X5WD1sxKpU2qe6omImZGxAPp9nxgCjAI2A+4LD3sMmD/mjUt1RqZmRVMIz1aSaMkTayYRnXepjYAtgEmAAMiYmZaNAsYUKsm7wwzs1Jp5KiDiBgLjK32GEn9gN8Bx0TEq5WHj0VESIpar+OgNbNSaeaJvyX1JgvZKyPiujR7tqSBETFT0kBgTq12PHRgZqXSpvqnapR1XS8CpkTEjyoWjQNGptsjgRtq1eQerZmVipp31bCPA4cBj0ialOZ9GzgLuFbSkcCzwBdqNeSgNbNSadbIQUTcTdfXevxUI205aM2sVPwTXDOznPmkMmZmOetVwF38DlozKxVfnNHMLGceozUzy1kBO7QOWjMrl7bmHUfbNA5aMysV92jNzHLWXsBBWgetmZWKe7RmZjnz4V1mZjkrYM46aM2sXAr4wzAHrZmVi4cOzMxy5qA1M8tZ8WLWQWtmJVPADm0hx43NzLpNUt1THW1dLGmOpEcr5o2RNEPSpDTtVasdB62ZlUpbA1MdLgVGdDL/vIgYmqabajXioQMzK5Vm7gyLiLskbbC07bhHa2al0sjQgaRRkiZWTKPqfJnRkh5OQwur13qwg9bMSqWRoYOIGBsR21ZMY+t4iQuBjYGhwEzg3FpP8NCBmZVK3hdnjIjZFa/1C+DGWs9xj9bMSkUNTN1qXxpYcfezwKNdPXYR92jNrFR6NbFHK+kqYBjQX9J04DRgmKShQADPAF+t1Y6D1sxKpZkjBxFxUCezL2q0HQetmZWKCvgjXAetmZVKEX+C66A1s1LxVXDNzHLmHq2ZWc58Plozs5wV8GrjDlozKxcfdWBmlrMCjhw4aHvCrFkzOe07J/HS3LlI8NkDvsBBhx7e6rKsRdraxF+vPIHn57zCAUf/jAtPO5iPDFkPIZ58bg7/79TLef3Nd1pd5jLLPdrlVHuvXvzncSewxZAtef311znsSwfwsR13YqONN2l1adYCow/+JI89PZuV+/YB4IRzrmP+628BcPZxn+OoL+3KOZeMb2WJy7QijtH6pDI9oP9aa7PFkC0B6Nu3LxtsuDFz5syu8Swro0Frr8aInbfkkuvvWTxvUcgC9FmxNxHRitJKo02qe+qxmnrslQyA52fM4LGpU9jq37ZudSnWAj88/gC+c/7vWbhwyTD9+ZhDeeZPZ7L5BgP46dV3tqi6csj77F3d0e2glfTlKssWn7X8kl/Wcx7d5cMbb7zOCcd+k+NOOIl+/fq1uhzrYXvushVzXprPg1OmfWDZV8dcwUa7f4epT8/i87t/tAXVlUcRe7RLM0b7XeCSzhaks5SPBZj/9kJ/DwIWvPsuJxx7NCM+sw+7fXr3VpdjLbDj0I3Ye9d/Y8TOW7LiCr1ZpW8fLj7jcL5yyq8AWLgw+M0t93PsyOFcPu7vLa522VXAIdrqQSvp4a4WAQOaX045RQSnn3YKG264EYcefkSry7EWOfV/x3Hq/44DYJePbsoxh3+Kr5zyKzZatz9PTXsRgL13/TCPP+Px+6VSwKSt1aMdAOwBvNxhvoB7Pvhw68xDDz7ATTeOY5NNN+PgAz8LwH988xh23mXXFldmrSaJX55+GCv3XQkJHnl8Bt8885pWl7VMK+JPcFVtD6eki4BLIuLuTpb9OiIOrvUCHjqwzqy9wzdbXYIV0JsPXrDUKXnfU6/UnTnbbbRqj6Ry1Z1hEXFkZyGbltUMWTOzHtfEww7S5cTnSHq0Yt4aksZLeiL968uNm9nyRQ38V4dLgREd5p0E3BYRmwK3pftVOWjNrFSk+qdaIuIu4KUOs/cDLku3LwP2r9WOg9bMSqWRkYPKY/7TNKqOlxgQETPT7VnUcQSWz3VgZqWiBo46qDzmvzsiIiTV3PnmHq2ZlUozhw66MFvSwOy1NBCYU+sJDlozK5UeONfBOGBkuj0SuKHWExy0ZlYuzT286yrgb8DmkqZLOhI4Cxgu6Qng0+l+VR6jNbNSaeaJvyPioC4WfaqRdhy0ZlYqBfwFroPWzMrFQWtmljNfM8zMLGfu0ZqZ5ayAOeugNbOSKWDSOmjNrFSKeOJvB62ZlUrxYtZBa2ZlU8CkddCaWan48C4zs5wVcIjWQWtm5VLAnHXQmlm5NHLi757ioDWzUilgzjpozaxcCpizDlozK5kCJq2D1sxKxYd3mZnlrJljtJKeAeYD7wELImLb7rTjoDWzUmlrfof2kxHx4tI04KA1s5Ip3tCBr4JrZqUiNTJplKSJFdOoDs0FcKuk+ztZVjf3aM2sVBrpz0bEWGBslYfsHBEzJK0NjJc0NSLuarQm92jNrFQa6dHWEhEz0r9zgOuB7btTk4PWzEpFUt1TjXb6Slp50W1gd+DR7tTkoQMzK5Um7gobAFyfArkd+HVE3Nydhhy0ZlYqzTqONiKeArZuRlsOWjMrFf8yzMwsb8XLWQetmZVLAXPWQWtm5eLLjZuZ5ayAOevjaM3M8uYerZmVShF7tA5aMysVH95lZpYz92jNzHLmoDUzy5mHDszMcuYerZlZzgqYsw5aMyuZAiatg9bMSqWIP8FVRLS6huWGpFHpGkVmi/l9UX7+CW7P6vZVNK3U/L4oOQetmVnOHLRmZjlz0PYsj8NZZ/y+KDnvDDMzy5l7tGZmOXPQmpnlzEHbQySNkPSYpCclndTqeqz1JF0saY6kR1tdi+XLQdsDJPUCfgLsCQwBDpI0pLVVWQFcCoxodRGWPwdtz9geeDIinoqId4Crgf1aXJO1WETcBbzU6josfw7anjEImFZxf3qaZ2bLAQetmVnOHLQ9YwawbsX9wWmemS0HHLQ94z5gU0kbSloB+BIwrsU1mVkPcdD2gIhYAIwGbgGmANdGxOTWVmWtJukq4G/A5pKmSzqy1TVZPvwTXDOznLlHa2aWMwetmVnOHLRmZjlz0JqZ5cxBa2aWMwetmVnOHLRmZjn7/yd6xaPpzEs4AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "UVSc0_eSVzgf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "0e77ce92-a0d6-440d-fd21-9eb1916140e5"
      },
      "source": [
        "#Calculate Accuracy and Print it\n",
        "accu_log=(sum([1 for i, j in zip(y_pred_log1, y_test) if i == j])/len(y_test))*100\n",
        "print(\"Accuracy = \" ,accu_log, \"%\")"
      ],
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy =  96.15384615384616 %\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "6mK8nioeLBh1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 561
        },
        "outputId": "26a098ce-7ca4-4c22-c71e-de8a28369e79"
      },
      "source": [
        "# calculating the cross validation for a set of predictions:\n",
        "results_log = model_selection.cross_val_score(clf, X_train, y_train,cv=None )*100\n",
        "results_log"
      ],
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 96.82539683, 100.        , 100.        ,  96.77419355,\n",
              "        96.77419355])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 94
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kTmaxq8s1xmf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "d7fa7390-6a45-4b26-9686-38d29bd6b9ba"
      },
      "source": [
        "acc_cross_validation['logistic regression']=results_log\n",
        "acc_cross_validation"
      ],
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>85.714286</td>\n",
              "      <td>96.825397</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>93.650794</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>83.870968</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>85.483871</td>\n",
              "      <td>96.774194</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>91.935484</td>\n",
              "      <td>96.774194</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   NaiveBayes  logistic regression\n",
              "0   85.714286            96.825397\n",
              "1   93.650794           100.000000\n",
              "2   83.870968           100.000000\n",
              "3   85.483871            96.774194\n",
              "4   91.935484            96.774194"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "OymQu9hT2T1C",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "outputId": "5418a382-94d6-48e5-c368-10506ac3a970"
      },
      "source": [
        "df_result['logistic regression']=pd.Series({'Accuracy': accu_log,'Cross_Validation':results_log,'Confusion_Matrix':cm_logistic,'Sensitivity':TPR_log,'Specificity':TNR_log,'Precision':percision_log})\n",
        "df_result"
      ],
      "execution_count": 96,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "      <th>KNN</th>\n",
              "      <th>Perceptron</th>\n",
              "      <th>SVM</th>\n",
              "      <th>Combination</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Accuracy</th>\n",
              "      <td>89.7436</td>\n",
              "      <td>96.1538</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>89.3617</td>\n",
              "      <td>97.7273</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Precision</th>\n",
              "      <td>84.8485</td>\n",
              "      <td>96.9697</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>90.3226</td>\n",
              "      <td>94.1176</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            NaiveBayes logistic regression  KNN Perceptron  SVM Combination\n",
              "Accuracy       89.7436             96.1538  NaN        NaN  NaN         NaN\n",
              "Specificity    89.3617             97.7273  NaN        NaN  NaN         NaN\n",
              "Precision      84.8485             96.9697  NaN        NaN  NaN         NaN\n",
              "Sensitivity    90.3226             94.1176  NaN        NaN  NaN         NaN"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xSFeTpuuLZYS",
        "colab": {}
      },
      "source": [
        "# 3- KNN:"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "kHwQE46PLfQo",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 78
        },
        "outputId": "37e28564-9247-40ac-c6a6-45f064bcbfe5"
      },
      "source": [
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.model_selection import cross_val_score\n",
        "neigh = KNeighborsClassifier(n_neighbors=25, metric= 'euclidean')\n",
        "neigh.fit(X_train, y_train)\n",
        "\n",
        "preds1 = neigh.predict(X_test)\n",
        "preds1.shape\n"
      ],
      "execution_count": 97,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='euclidean',\n",
              "                     metric_params=None, n_jobs=None, n_neighbors=25, p=2,\n",
              "                     weights='uniform')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 97
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(78,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "vlcsrAjIMIZV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 57
        },
        "outputId": "dca8b524-9326-451d-9a43-8eb1c20dcc8f"
      },
      "source": [
        "# confusion matrix\n",
        "os.chdir(home_path)\n",
        "from sklearn.metrics import confusion_matrix\n",
        "cm_knn=confusion_matrix(y_test,preds1)\n",
        "cm_knn"
      ],
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[33,  0],\n",
              "       [16, 29]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jYOAB1rlJ6vr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "bbb8d5cc-a91c-40b4-c6e4-9dcbc828681b"
      },
      "source": [
        "#Sensitivity_(True positive rate TPR) \n",
        "TP_knn= cm_knn[0][0]\n",
        "FN_knn= cm_knn[1][0]\n",
        "TPR_knn= TP_knn/(TP_knn+FN_knn) * 100\n",
        "print(\"Sensitivity \"+str(TPR_knn)+'%')"
      ],
      "execution_count": 99,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sensitivity 67.3469387755102%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ozvuOhJ_J69z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "a47af145-2759-43c2-9d84-39a0fd2e71c5"
      },
      "source": [
        "#Specificity(Total Negative Result TNR)\n",
        "TN_knn= cm_knn[1][1]\n",
        "FP_knn= cm_knn[0][1] \n",
        "TNR_knn = TN_knn/(TN_knn+FP_knn) * 100\n",
        "print(\"Specificity \"+str(TNR_knn)+'%')"
      ],
      "execution_count": 100,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Specificity 100.0%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qFKEdXFjgbwU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "f09b62e2-3e24-4d4e-a6c5-19055cdf0f7d"
      },
      "source": [
        "percision_knn=(TP_knn/(TP_knn+FP_knn))*100\n",
        "percision_knn"
      ],
      "execution_count": 101,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "100.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 101
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "k78ClrVxOx-w",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        },
        "outputId": "f1197b76-0208-4a64-8bed-46a2643605b5"
      },
      "source": [
        "os.chdir(home_path)\n",
        "cm_picture2=sn.heatmap(pd.DataFrame(cm_knn),cmap=\"Blues\", annot=True)\n",
        "figure3 = cm_picture2.get_figure()   \n",
        "plt.title(\"Confusion Matrix KNN\")\n",
        "figure3.savefig('/content/drive/My Drive/project/CGR_data/Confusion_KNN_4kmers.jpg', dpi=400)"
      ],
      "execution_count": 102,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Confusion Matrix KNN')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 102
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVoAAAEICAYAAAAeFzyKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAW10lEQVR4nO3df7xVdZ3v8df7nAOCYIqgR1AUE7XIEiavU1rjrzJs6iE2lmaZ06U5TSP36uSUztT1Vz+uNmrZ1XHCREkd1MYczUwi0kxLQ40UxNCroCA/FH8EivLjfOaPveCxPZ6z1z6H/WWvs3g/fawHe6+19nd/zgHf53O+68dWRGBmZum0NLsAM7Oyc9CamSXmoDUzS8xBa2aWmIPWzCwxB62ZWWIO2hKSNFjSTyW9IunHWzDOZyT9opG1NYOkn0s6pdl12LbLQdtEkk6S9KCkNZKWZYHwgQYMfTzQDgyPiE/2dZCIuD4ijm5APW8i6XBJIemWLusPzNbfXec450q6Lm+/iDgmIqb3oc6/lXRv1fO3SbpP0s2SBkq6Jqv34Kp9xkqKqud3S3pd0uiqdR+StKi39Vj/5aBtEklfBr4HfJtKKO4J/BtwbAOG3wtYGBEbGjBWKs8D75c0vGrdKcDCRr2BKhryb1zSMGA2sBg4ISLWZZteBL6Z8/JXgf/TiDqsf3LQNoGkHYHzgVMj4icR8WpErI+In0bEV7J9tpP0PUnPZcv3JG2XbTtc0hJJZ0hamXXDn8+2nQecDZyQdcqTu3Z+ksZknVhb9vxvJT0labWkpyV9pmp9dUd3iKQ52ZTEHEmHVG27W9I3so5vtaRfSBpR49uwDvgv4MTs9a3ACcD1Xb5Xl0p6VtKfJT0k6YPZ+onAv1R9nX+squNbku4DXgPenq37Qrb9Ckk3V41/oaTZklTj72sX4C5gHvDZLj/ApgPvkXRYja/1+8CnJe1TYx8rMQdtc7wfGATcUmOfrwHvA8YDBwIHA1+v2r4bsCOwOzAZuFzSsIg4h0qXfGNEDI2Iq2oVImkIlSA4JiJ2AA4B5naz387Az7J9hwOXAD/r0pGeBHwe2BUYCPxTrfcGfgR8Lnv8ESpB9lyXfeZQ+R7sDPwH8GNJgyLizi5f54FVrzkZ6AB2oNKBVjsDeHf2Q+SDVL53p0TP16LvDNwN/A74nxHR2WX7a1kd36rxdS4FrgTOq7GPlZiDtjmGAy/k/Gr/GeD8iFgZEc9T+Z/05Krt67Pt6yPiDmANsH8f6+kEDpA0OCKWRcT8bvb5a+CJiLg2IjZExAzgceDjVftcHRELI2ItcBOVgOxRRPwW2FnS/lQC90fd7HNdRKzK3vNiYDvyv85rImJ+9pr1XcZ7jcr38RLgOuB/RcSSGmONBvbLxuwpjH8A7CnpmBrj/F/g45LelVO7lZCDtjlWASM2/ereg1G8uRtbnK3bPEaXoH4NGNrbQiLiVSq/sv89sEzSzyS9o456NtW0e9Xz5X2o51pgCnAE3XT4kv5J0oJsuuJlKl18rSkJgGdrbYyIB4CnAFH5gVDLH6l05j+XNKGH8d4AvpEtPb3n88BlVKaMbBvjoG2O3wFvAJNq7PMclYNam+zJW3+trterwPZVz3er3hgRMyPiw8BIKl3qlXXUs6mmpX2saZNrgX8A7si6zc2yX+2/CnwKGBYROwGvUAlIgJ46zJq3pJN0KpXO+Lls/Joi4lLgAmCWpAN62O1qYCfgEzWG+lcqP1Dem/eeVi4O2iaIiFeoHLC6XNIkSdtLGiDpGEnfyXabAXxd0i7ZQaWzqfyq2xdzgb+StGd2IO6fN22Q1C7p2Gyu9g0qUxBd5yEB7gD2U+WUtDZJJwDjgNv7WBMAEfE0cBiVOemudgA2UDlDoU3S2cDbqravAMb05swCSftROUvgs1SmEL4qqeYUR1bnd4BLgV9mUx1dt28AzgHOrDHGy8DF1BHuVi4O2ibJ5hu/TOUA1/NUft2dQuVIPFTC4EHgEeBR4GHyTyPq6b1mATdmYz3Em8OxJavjOSqnKh0GfKmbMVYBH6NyMGkVlbD4WES80Jeauox9b0R0163PBO6kcsrXYuB13jwtsOlijFWSHs57n2yq5jrgwoj4Y0Q8QeXMhWs3ndGRU+c3gB8Cs3s4g2AGsCxnmEuBjXnvZeUi3/jbzCwtd7RmZok5aM3MEnPQmpkl5qA1M0us1gnzDTF4whQfbbO3eGnOZc0uwQpoUBs93nOiXr3JnLV/uGyL368e7mjNzBJL3tGamW1VjbkzZkM5aM2sXFpam13BWzhozaxcer61cNM4aM2sXDx1YGaWmDtaM7PE3NGamSXmjtbMLDGfdWBmlpinDszMEvPUgZlZYu5ozcwSc9CamSXW6oNhZmZpeY7WzCyxAk4dFK8iM7MtIdW/1BxGgyT9XtIfJc2XdF62fm9JD0h6UtKNkgbmleSgNbNyUUv9S21vAEdGxIHAeGCipPcBFwLfjYixwEvA5LyBHLRmVi4N6mijYk32dEC2BHAk8J/Z+unApLySHLRmVi4trXUvkjokPVi1dFQPJalV0lxgJTAL+P/AyxGxIdtlCbB7Xkk+GGZm5dKLg2ERMRWYWmP7RmC8pJ2AW4B39KUkB62ZlUuC07si4mVJdwHvB3aS1JZ1tXsAS/Ne76kDMyuXBh0Mk7RL1skiaTDwYWABcBdwfLbbKcCteSW5ozWzcmncebQjgemSWqk0pTdFxO2SHgNukPRN4A/AVXkDOWjNrFwadD/aiHgEmNDN+qeAg3szloPWzMrFl+CamSVWwEtwHbRmVi7uaM3M0pKD1swsLQetmVlianHQmpkl5Y7WzCwxB62ZWWIOWjOz1IqXsw5aMysXd7RmZom1tPjKMDOzpNzRmpmlVrycddCaWbm4ozUzS8xBa2aWmC/BNTNLzB2tmVliDlozs8QctGZmiTlozcxSK17OOmjNrFx8Ca6ZWWJFnDooXvSbmW0J9WKpNYw0WtJdkh6TNF/Sadn6cyUtlTQ3Wz6aV5I72kS2G9jGL686nYED22hrbeWWX/6Bb/77HVxxzkn8xbg9EeLJZ1byd2dfy6tr1zW7XGuS+35zDxde8C06N3Zy3N98ksl/19Hskvq9Bna0G4AzIuJhSTsAD0malW37bkRcVO9ADtpE3li3gYkd3+fVtetoa2vhV9O+zC/ue4yvXvQTVr/6OgAXnvEJvnTiYVx09ayc0ayMNm7cyLe/dT4/uPJq2tvbOemE4zn8iCPZZ+zYZpfWrzUqaCNiGbAse7xa0gJg976MlTt1IOkdks6U9P1sOVPSO/vyZtuaTZ3qgLZW2tpaiYjNIQswaLsBRESzyrMmm/foI4wevRd7jB7NgIEDmfjRv+buu2Y3u6x+T1Jvlg5JD1Yt3f5KIWkMMAF4IFs1RdIjkqZJGpZXU82glXQmcAOV2YzfZ4uAGZLOqvsr30a1tIj7bziLZ2ZfwK/uf5w58xYD8INzP8uiX36b/ce08283/LrJVVqzrFyxgt1G7rb5+a7t7axYsaKJFZWDWlT3EhFTI+KgqmXqW8aThgI3A6dHxJ+BK4B9gPFUOt6L82rK62gnA/8jIi6IiOuy5QLg4Gxb919o1U+JDS/Mz6uhtDo7g/edeAFjP/J1DjpgL8btMxKAL557HW8/+ms8/vRyjj/6vU2u0qxcetPR1jHWACohe31E/AQgIlZExMaI6ASupJKHNeUFbScwqpv1I7Nt3ar+KdE24l15NZTeK2vW8usHF3L0IeM2r+vsDH488yEmHTW+iZVZM+3a3s7yZcs3P1+5YgXt7e1NrKgcGhW0quxwFbAgIi6pWj+yarfjgHl5NeUF7enAbEk/lzQ1W+4EZgOn5Q2+LRsxbCg7Dh0MVOZij/rLd7Bw8QrePnrE5n0+dth7WLjIvypuq951wLt55plFLFnyLOvXrePOO37GYUcc2eyy+j2p/iXHocDJwJFdTuX6jqRHJT0CHAH8Y95ANc86iIg7Je1HpTXedLRtKTAnIjbmlrkN223E27jy/JNpbWmhpUXcPOthfv6b+cyedjo7DBmMBI8uXMr//vaNzS7VmqStrY1//trZfKnjC3R2bmTScX/D2LH7Nrusfq+BZx3cS/dn297R27GU+qj34AlTfFjd3uKlOZc1uwQroEFtW36ngv3PnFl35vzpwo9slcvIfB6tmZVKAa/AddCaWbm0+KNszMzSckdrZpZYEe/e5aA1s1IpYM46aM2sXHzjbzOzxNzRmpkl5jlaM7PECpizDlozKxd3tGZmiRUwZx20ZlYuvjLMzCwxTx2YmSVWwJx10JpZubijNTNLrIA566A1s3LxwTAzs8Q8dWBmlpiD1swssQLmrIPWzMrFHa2ZWWIFzFkHrZmVSxHPOijercjNzLZAi1T3Uouk0ZLukvSYpPmSTsvW7yxplqQnsj+H5dbUoK/NzKwQpPqXHBuAMyJiHPA+4FRJ44CzgNkRsS8wO3tek4PWzEpFUt1LLRGxLCIezh6vBhYAuwPHAtOz3aYDk/JqctCaWam0qP5FUoekB6uWju7GlDQGmAA8ALRHxLJs03KgPa8mHwwzs1LpzcGwiJgKTK21j6ShwM3A6RHx5+pOOCJCUuTWVHdFZmb9gHrxX+5Y0gAqIXt9RPwkW71C0shs+0hgZd44DlozK5XeTB3UokrrehWwICIuqdp0G3BK9vgU4Na8mjx1YGal0sArww4FTgYelTQ3W/cvwAXATZImA4uBT+UN5KA1s1JpVM5GxL3Q4/zCUb0Zy0FrZqWSdyFCMzhozaxUingJroPWzEqlgA2tg9bMysVTB2ZmiRUvZh20ZlYyvvG3mVliBTwW5qA1s3LxWQdmZol56sDMLLECNrQOWjMrF3e0ZmaJFS9mHbRmVjKtBZw7cNCaWal46sDMLLEC5qyD1szKxfc6MDNLrIA5mz5ov3TelNRvYf3QsE/+sNklWAGtveULWzyG52jNzBJrddCamaVVwLO7HLRmVi4OWjOzxDxHa2aWmDtaM7PECtjQ0tLsAszMGqlNqnvJI2mapJWS5lWtO1fSUklzs+WjeeM4aM2sVKT6lzpcA0zsZv13I2J8ttyRN4inDsysVBp5CW5E3CNpzJaO447WzEqlNx2tpA5JD1YtHXW+zRRJj2RTC8PydnbQmlmptKj+JSKmRsRBVcvUOt7iCmAfYDywDLg47wWeOjCzUkl94++IWLHpsaQrgdvzXuOgNbNSSX0eraSREbEse3ocMK/W/uCgNbOSUQM/NUzSDOBwYISkJcA5wOGSxgMBLAK+mDeOg9bMSqWRHW1EfLqb1Vf1dhwHrZmVii/BNTNLzDeVMTNLrLWAJ606aM2sVPzhjGZmiXmO1swssQI2tA5aMyuXlgaeR9soDlozKxV3tGZmibUVcJLWQWtmpeKO1swsMZ/eZWaWWAFz1kFrZuVSwAvDHLRmVi6eOjAzS8xBa2aWWPFi1kFrZiVTwIbWQWtm5eL70ZqZJeazDszMEvPBMDOzxDx1YGaWmKcOzMwSc0drZpZY8WK2mF22mVmftUp1L3kkTZO0UtK8qnU7S5ol6Ynsz2F54zhozaxUpPqXOlwDTOyy7ixgdkTsC8zOntfkoDWzUlEv/ssTEfcAL3ZZfSwwPXs8HZiUN46D1sxKpTcdraQOSQ9WLR11vEV7RCzLHi8H2vNe4INhZlYqvfkU3IiYCkzt63tFREiK/JrMzEqkwXO03VkhaWTlvTQSWJn3AgetmZVKi1T30ke3Aadkj08Bbs17gacOzKxUGvlp45JmAIcDIyQtAc4BLgBukjQZWAx8Km8cB62ZlUo9ZxPUKyI+3cOmo3ozjoPWzEqlgFfgOmhTOWH8boxrH8qaNzbyr3c/vXn9B/YexqFjdiICHlu5htsfe76JVdrWtMfwIfzwtMPYdafBRMC0WY9z+e3zefeYnfl/f38oQwYNYPHKNXz+u3exeu36ZpfbbzWyo20UB20ic555hXuffomTJozavG7s8O05YLehXPTrRWzsDIYObG1ihba1bejs5KxrHmDuU6sYOmgAv714ErPnLuWKf/ggZ01/gHvnL+dzR+3HP056D+fPeKjZ5fZbjZyjbRSfdZDIUy+u5bV1nW9ad8iYnZj9xCo2dlZOu1uzbmMzSrMmWf7SWuY+tQqANa+v5/ElLzNq+BDGjtqRe+cvB+BXc5cy6f1jmlhl/7cVzjrofU1b7Z2MXYYO5O3Dt+e0D+7FqYfsyeidBjW7JGuSPXcZyvi9hzNn4UoWPPsSHz94LwA+ceje7DFiSJOr69/Ui2Vr6XPQSvp8jW2bL2t7ZOZNfX2L0mmR2H5AK5f+ZjE/fWwln3vvqPwXWekMGdTGjDM/xFem3c/qtev54mX30HHMO7nvokkMHTSAdRs68wexHhWxo92SOdrzgKu721B9WduXb3s89/K0bcUrr6/n0WWrAXjm5dcJYMjAVl71FMI2o61VzPjqh7jxnie59f5FACxc+gofP+9OAMaOehvHHDS6iRX2fwWcoq0dtJIe6WkTddxIwd7s0WVrGDtie55c9Rq7DBlAa4scstuYfz/1r/jTkpf5/m2bb2/KLjsO4vlXXkeCs46fwJUzH29ihSVQwKTN62jbgY8AL3VZL+C3SSoqic/+xSjGjtieIQNbOfvD+zDzTy/w+2de5sQJI/nK4XuzsTOY8Ydl+QNZaRzyznY+c8S+PLroRe6/5DgAzrluDmNH7cgXjxkHwK33L+JHsxc2s8x+rz9+Cu7twNCImNt1g6S7k1RUEtc9/Fy3669/2OG6rfrtghUMPu6Hb1k/8+ElXH77/CZUVE7Fi9mcoI2IyTW2ndT4cszMtlABk9YXLJhZqfjKMDOzxAo4ReugNbNyKWDOOmjNrFxUwJbWQWtmpVLAnHXQmlm5FDBnHbRmVjIFTFoHrZmVik/vMjNLzHO0ZmaJOWjNzBLz1IGZWWLuaM3MEitgzjpozaxkGpi0khYBq4GNwIaIOKgv4zhozaxUEtz4+4iIeGFLBnDQmlmpFHHqwB83bmbl0ovPG6/+xO5s6egyWgC/kPRQN9vq5o7WzEqlN6d3VX9idw8+EBFLJe0KzJL0eETc09ua3NGaWalI9S95ImJp9udK4Bbg4L7U5KA1s1LpxcxB7XGkIZJ22PQYOBqYV/tV3fPUgZmVSgNv/N0O3JKN1wb8R0Tc2ZeBHLRmViqNytmIeAo4sBFjOWjNrFSKeHqXg9bMyqWASeugNbNS8d27zMwS8927zMwSa3HQmpmlVrykddCaWal46sDMLLEC5qyD1szKxR2tmVliDbwEt2EctGZWKsWLWQetmZVMARtaB62ZlYuvDDMzS614OeugNbNyKWDOOmjNrFwSfNz4FnPQmlmpFDBn/ZlhZmapuaM1s1IpYkfroDWzUvHpXWZmibmjNTNLzEFrZpaYpw7MzBIrYkfr07vMrFTUiyV3LGmipD9JelLSWX2tyUFrZuXSoKSV1ApcDhwDjAM+LWlcX0ry1IGZlUoDL8E9GHgyIp4CkHQDcCzwWG8HUkQ0qijLIakjIqY2uw4rFv+7aB5JHUBH1aqpm/4uJB0PTIyIL2TPTwb+MiKm9PZ9PHWwdXXk72LbIP+7aJKImBoRB1UtSX7gOWjNzLq3FBhd9XyPbF2vOWjNzLo3B9hX0t6SBgInArf1ZSAfDNu6PA9n3fG/iwKKiA2SpgAzgVZgWkTM78tYPhhmZpaYpw7MzBJz0JqZJeag3UoadSmflYekaZJWSprX7FosLQftVtDIS/msVK4BJja7CEvPQbt1bL6ULyLWAZsu5bNtWETcA7zY7DosPQft1rE78GzV8yXZOjPbBjhozcwSc9BuHQ27lM/M+h8H7dbRsEv5zKz/cdBuBRGxAdh0Kd8C4Ka+Xspn5SFpBvA7YH9JSyRNbnZNloYvwTUzS8wdrZlZYg5aM7PEHLRmZok5aM3MEnPQmpkl5qA1M0vMQWtmlth/A2k71S5A3KGJAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "KZVoxk6OMfNW",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "5d859d9e-2126-4c2a-9a35-2b3487728113"
      },
      "source": [
        "#Calculate Accuracy and Print it\n",
        "accu_knn=(sum([1 for i, j in zip(preds1, y_test) if i == j])/len(y_test))*100\n",
        "print(\"Accuracy = \" ,accu_knn, \"%\")"
      ],
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy =  79.48717948717949 %\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HrJQWGXgbOHe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "7e129e78-7569-420f-fc10-8a9be0c49574"
      },
      "source": [
        "# calculating the cross validation for a set of predictions:\n",
        "results_knn= model_selection.cross_val_score(neigh, X_train, y_train,cv=None )*100\n",
        "results_knn"
      ],
      "execution_count": 104,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([66.66666667, 69.84126984, 70.96774194, 67.74193548, 74.19354839])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hU5rbZSr2HJm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "a89dbfb4-b525-4952-ad93-82810a8da464"
      },
      "source": [
        "acc_cross_validation['KNN']=results_knn\n",
        "acc_cross_validation"
      ],
      "execution_count": 105,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "      <th>KNN</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>85.714286</td>\n",
              "      <td>96.825397</td>\n",
              "      <td>66.666667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>93.650794</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>69.841270</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>83.870968</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>70.967742</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>85.483871</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>67.741935</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>91.935484</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>74.193548</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   NaiveBayes  logistic regression        KNN\n",
              "0   85.714286            96.825397  66.666667\n",
              "1   93.650794           100.000000  69.841270\n",
              "2   83.870968           100.000000  70.967742\n",
              "3   85.483871            96.774194  67.741935\n",
              "4   91.935484            96.774194  74.193548"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 105
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "S_LiZSJ95nf8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "outputId": "2eb01d26-1925-49d4-efa4-7b68bc47c625"
      },
      "source": [
        "df_result['KNN']=pd.Series({'Accuracy':accu_knn,'Cross_Validation':results_knn,'Confusion_Matrix':cm_knn,'Sensitivity':TPR_knn,'Specificity':TNR_knn,'Precision':percision_knn})\n",
        "df_result"
      ],
      "execution_count": 106,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "      <th>KNN</th>\n",
              "      <th>Perceptron</th>\n",
              "      <th>SVM</th>\n",
              "      <th>Combination</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Accuracy</th>\n",
              "      <td>89.7436</td>\n",
              "      <td>96.1538</td>\n",
              "      <td>79.4872</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>89.3617</td>\n",
              "      <td>97.7273</td>\n",
              "      <td>100</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Precision</th>\n",
              "      <td>84.8485</td>\n",
              "      <td>96.9697</td>\n",
              "      <td>100</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>90.3226</td>\n",
              "      <td>94.1176</td>\n",
              "      <td>67.3469</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            NaiveBayes logistic regression      KNN Perceptron  SVM Combination\n",
              "Accuracy       89.7436             96.1538  79.4872        NaN  NaN         NaN\n",
              "Specificity    89.3617             97.7273      100        NaN  NaN         NaN\n",
              "Precision      84.8485             96.9697      100        NaN  NaN         NaN\n",
              "Sensitivity    90.3226             94.1176  67.3469        NaN  NaN         NaN"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 106
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "tQTgc_bsRNa0",
        "colab": {}
      },
      "source": [
        "# 4- Perceptron"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "R7zfY1s4_R4e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 98
        },
        "outputId": "c7d68185-78d0-48d7-a685-e9bd8dcff00e"
      },
      "source": [
        "from sklearn.linear_model import Perceptron\n",
        "perp=Perceptron(max_iter=100, eta0=0.01,shuffle=True)\n",
        "perp.fit(X_train, y_train)\n",
        "Test_y=pd.Series(perp.predict(X_test), name='y')"
      ],
      "execution_count": 107,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Perceptron(alpha=0.0001, class_weight=None, early_stopping=False, eta0=0.01,\n",
              "           fit_intercept=True, max_iter=100, n_iter_no_change=5, n_jobs=None,\n",
              "           penalty=None, random_state=0, shuffle=True, tol=0.001,\n",
              "           validation_fraction=0.1, verbose=0, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 107
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "MSjJVaaoXwJa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 57
        },
        "outputId": "1a8cf810-4f81-44a6-aeb1-bb03970d9755"
      },
      "source": [
        "# confusion matrix\n",
        "os.chdir(home_path)\n",
        "from sklearn.metrics import confusion_matrix\n",
        "cm_per=confusion_matrix(y_test,Test_y)\n",
        "cm_per"
      ],
      "execution_count": 108,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[32,  1],\n",
              "       [ 0, 45]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LDGJw2TCKSba",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "32a1f3b7-1a31-4fbf-dfee-cd039bb032cb"
      },
      "source": [
        "#Sensitivity_(True positive rate TPR) \n",
        "TP_per= cm_per[0][0]\n",
        "FN_per= cm_per[1][0]\n",
        "TPR_per= TP_per/(TP_per+FN_per) * 100\n",
        "print(\"Sensitivity \"+str(TPR_per)+'%')"
      ],
      "execution_count": 109,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sensitivity 100.0%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BKeCvymBKSob",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "7ea0141f-c749-412f-e47e-b8c4f90b691b"
      },
      "source": [
        "#Specificity(Total Negative Result TNR)\n",
        "TN_per= cm_per[1][1]\n",
        "FP_per= cm_per[0][1] \n",
        "TNR_per = TN_per/(TN_per+FP_per) * 100\n",
        "print(\"Specificity \"+str(TNR_per)+'%')"
      ],
      "execution_count": 110,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Specificity 97.82608695652173%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FCHg3w1ngvj5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "1239b770-2c3f-4f18-8c51-33d34e0fbace"
      },
      "source": [
        "percision_per=(TP_per/(TP_per+FP_per))*100\n",
        "percision_per"
      ],
      "execution_count": 111,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "96.96969696969697"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 111
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "HY3q7HLOXwfX",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        },
        "outputId": "50ee2e56-2216-4c98-e7cc-23af2c4af52b"
      },
      "source": [
        "os.chdir(home_path)\n",
        "cm_picture4=sn.heatmap(pd.DataFrame(cm_per),cmap=\"Blues\", annot=True)\n",
        "figure4 = cm_picture4.get_figure()   \n",
        "plt.title(\"Confusion Matrix Perceptron\")\n",
        "figure4.savefig('/content/drive/My Drive/project/CGR_data/Confusion_Perceptron_4kmers.jpg', dpi=400)"
      ],
      "execution_count": 112,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Confusion Matrix Perceptron')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 112
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVoAAAEICAYAAAAeFzyKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAZjElEQVR4nO3de5gcZZ328e89M0k4BQgExpgEowThBVbCGhAEFBAUEDlsEAVEUHiHVbMXXLKCIIuAsguegAtZ3WEJREQOr7sRRAQjEgElQAIhBCIHAU1CDiIkHORgkt/7Rz0TO+PMdPdM13Slcn9y1ZWuqu6nfj3Tc88zT50UEZiZWX5aml2AmVnZOWjNzHLmoDUzy5mD1swsZw5aM7OcOWjNzHLmoC0ASRtK+qmkFZL+3wDaOU7SLxpZWzNI+rmkE5pdh1mjOGjrIOlYSbMkvSppcQqEvRvQ9FFAO7BlRHy8v41ExHUR8eEG1LMWSftKCknTui3fJS2fUWM750n6YbXnRcTBETG1H3WeKGlV+v68LGmOpEPrbSdP6es1vtl12OBy0NZI0heBS4F/JwvFbYD/BA5vQPPvAJ6MiJUNaCsvfwL2lLRlxbITgCcbtQFlBvqZvC8iNgE2B64CbpI0os462gZYQ781c9uWo4jwVGUCNgNeBT7ex3OGkQXx82m6FBiW1u0LLAROB5YBi4HPpHXnA28Bf03bOAk4D/hhRdvjgADa0vyJwDPAK8CzwHEVy++teN37gQeBFen/91esmwF8DfhNaucXwMhe3ltX/d8HvpCWtQKLgHOBGRXPvQxYALwMzAb2ScsP6vY+H6mo48JUx+vA+LTs5LT+e8D/VLR/MXAnoB7q7P7+N05ft4np+/Mt4I/A0vReNuz2/s4ElgDXpvd3NvD79PWZDYxNz98BmA68CDwBHF2xzWtS29PT634NvCOtuzvV81r6Gnyil23367PkqbhT0wtYF6YUEitJQdfLcy4AZgJbA1sBvwW+ltbtm15/ATAEOAT4CzAirT+PtYO1+/y49APalsLjZWD7tG4UsFN6vCZogC2Al4Dj0+uOSfNbpvUzUoi8G9gwzV/Uy3vr+uF+P3B/WnYIcAdwMmsH7aeALdM2T0/hsUFP76uijj8CO6XXDGHtoN2IrNd8IrAP8AIwppc6K99/G3AqWdhtBlwC3JK+LsOBnwL/0e37czFZyG0IfAl4FNgeELBLel8bk/0i+Uzaxq6pph1TW9ekbX4gtXUZa4d/AOO7fW27b7vfnyVPxZyaXsC6MAHHAUuqPOf3wCEV8x8BnkuP9yXrrbVVrF8G7JEerxVAPcyPY+2gXQ5MIvXIKp5XGTTHAw90W38fcGJ6PAM4p2Ld54Hbe3lv+wIL0+OnUvjckL4uawVtD699Cdilp/dVUccFPSw7uWL+fWS9xz8Ax/SxrRNTCC1P4TcTOIAsKF8Dtq147p7AsxXv7y3SL4S07Ang8B628Qngnm7L/gv4anp8DXBDxbpNgFX8rTfcU9B233a/P0ueijl5jLY2fwZGVhk/eztZEHT5Q1q2po1Yewz2L2Q/hHWJiNfIftj/GVgs6WeSdqihnq6aRlfML+lHPdcCk4H9gGndV0r6V0nz0xEUy8l6kyOrtLmgr5URcT/ZUImAm6q0NTMiNo+IkRGxR0T8kqxXuBEwW9LyVNftaXmXP0XEGxXzY8kCr7t3AO/raie1dRzwtp7eT0S8SvZL4u30rvu2B+WzZIPHQVub+4A3gSP6eM7zZD+EXbZJy/rjNbJg6FL5Q0xE3BERB5ING/wOuLKGerpqWtTPmrpcS9b7vS0i/lK5QtI+wBnA0WR/ym5ONj6srtJ7abPPS8hJ+gLZn9XPp/br9QJZL3CnFMKbR8Rmke00662GBcC2PbS1APh1RTubR8QmEfG5iueMrah9E7Lhir4+C9233cjPkhWAg7YGEbGCbKfPFZKOkLSRpCGSDpb0jfS064FzJG0laWR6ftVDmXoxB/iApG0kbQac1bVCUrukwyVtTBb+rwKre2jjNuDd6ZC0NkmfAHYEbu1nTQBExLPAB4Gv9LB6ONmf7n8C2iSdC2xasX4pMK6eIwskvRv4OtnY7/HAGZIm1FnzarJfRpdI2jq1O1rSR/p42X8DX5O0XToa4j3piItbyb6ux6fPwBBJu0n6PxWvPUTS3pKGku1wnBkRXb3cpcC7qpTcyM+SFYCDtkYR8W3gi8A5ZEGygOxP6J+kp3wdmAXMJduJ8lBa1p9tTQduTG3NZu1wbEl1PE/2J+kHgc/10MafgUPJdkj9mawneGhEvNCfmrq1fW9E9NTDuoPsT/Inyf7cfYO1hwW6Tsb4s6SHqm0nDdX8ELg4Ih6JiKfIjgS4VtKwOss+E3gamCnpZeCXZGPNvfkO2TDFL8h2Pl5FNib+CvBh4JNk34Ml/G1HVpcfAV8l+/68l+yXRJfzgKlp2OHoXrbdsM+SFYMifOFvs0aRdA3ZjsNzml2LFYd7tGZmOXPQmpn1QVKrpIcl3Zrmr5H0bDrFe04t+wx8up9ZA0XEic2uwRruVGA+a+/Y/VJE/LjWBtyjNTPrhaQxwEfJjkLpt9x7tDufM9172+zvzPy3A5pdghXQJsOk6s/q24a7Tq45c96Yc8UpQEfFos6I6KyYv5TsiJ3h3V56YTp88U7gyxHxZl/bcY/WzNZbEdEZERMrpjUhmy6xuSwiZnd72VlkFxbajexklDOrbcdBa2blopbap77tBRwm6Tmya3vsL+mHEbE4Mm8CVwO7V2vIQWtm5dLSWvvUh4g4KyLGRMQ4shNUfhURn5I0CrLrJ5Odlj+vWkk+6sDMymXgw7zVXCdpK7JreMwhu8BTnxy0ZlYuA75Jx9+LiBlkl+8kIvav9/UOWjMrl/x7tHVz0JpZueTQox0oB62ZlYt7tGZmOatyNEEzOGjNrFw8dGBmljMPHZiZ5cw9WjOznDlozcxy1uqdYWZm+fIYrZlZzjx0YGaWM/dozcxy5h6tmVnO3KM1M8tZAU/BLV4f28xsIBp3K5usOalV0sOSbk3z75R0v6SnJd0oaWi1Nhy0ZlYuUu1TbU4F5lfMXwxcEhHjgZeAk6o14KA1s3JpYI9W0hjgo8B/p3kB+wM/Tk+ZSnbfsD55jNbMyqWxRx1cCpwBDE/zWwLLI2Jlml8IjK7WiHu0ZlYuddwFV1KHpFkVU0dXM5IOBZZFxOyBluQerZmVSx2Hd0VEJ9DZy+q9gMMkHQJsAGwKXAZsLqkt9WrHAIuqbcc9WjMrlwaN0UbEWRExJiLGAZ8EfhURxwF3AUelp50A3FytJAetmZVL44866O5M4IuSniYbs72q2gs8dGBmpaIczgyLiBnAjPT4GWD3el7voDWzUskjaAfKQWtmpaIWB62ZWa7cozUzy5mD1swsZw5aM7O8FS9nHbRmVi7u0ZqZ5aylpXjnYTlozaxU3KM1M8tb8XLWQWtm5eIerZlZzhy0ZmY58ym4ZmY5c4/WzCxnDlozs5wVMWiLd2SvmdkASKp5qtLOBpIekPSIpMcknZ+WXyPpWUlz0jShWk3u0ZpZuTSuQ/smsH9EvCppCHCvpJ+ndV+KiB/X2pCD1sxKpVGn4EZEAK+m2SFpin7V1JCKzMwKop6hA0kdkmZVTB3d2mqVNAdYBkyPiPvTqgslzZV0iaRh1Wpyj9bMyqWOoYOI6AQ6+1i/CpggaXNgmqSdgbOAJcDQ9NozgQv62o6DNidD21qYevJEhra20Noipj+2lCt+9QwXfXxndnr7pqxcHcxbuILzb57PytX9+mvE1nHnn3s29/x6BltssSU3Tftps8spjZzugrtc0l3AQRHxrbT4TUlXA/9a7fUeOsjJWytX89kps5l0xUyOumIme203kveM2YyfPbKEj132W468/D6GDWll0sTRzS7VmuRjhx3J5d+7stlllE4DjzrYKvVkkbQhcCDwO0mj0jIBRwDzqtVUtUcraQfgcKArERYBt0TE/GqvXd+9/tYqANpaRVurCIJ7nnxhzfpHF66gfdOqwztWUv84cTeeX7Sw2WWUTgN7tKOAqZJayTqlN0XErZJ+JWkrskGKOcA/V2uoz6CVdCZwDHAD8EBaPAa4XtINEXHRAN5E6bUIbvr8HmyzxYZcf/8CHl348pp1bS3iYxNGcdHPnmhihWbl06hrHUTEXGDXHpbvX29b1Xq0JwE7RcRfKxdK+g7wGNBj0KY9dx0Aow4+lS3+8aP11lUKqwOOumImwzdo47Jjd2H81hvz9LLXADjnsB2Y/dxLPPSH5U2u0qxc1sUzw1YDb+9h+ai0rkcR0RkREyNi4voaspVeeWMlDzz7EntvNxKAz+33LkZsNJRv/PzJJldmVj6NGqNtpGo92tOAOyU9BSxIy7YBxgOT8yxsXTdioyGsXB288sZKhrW1sOe2WzDlnueY9N7R7LXdlpw0ZTbhgw3MGq6AHdq+gzYibpf0bmB31t4Z9mA6vsx6sdXwYVw4aSdaW7LfnHfMW8qvn3iBOed/iMUr3uC6U3YH4JePL+P7dz3T5GqtGc4+44vMmvUgy5e/xMEHfJBTPv8vHPFPRzW7rHVeEYcOFDl3q3Y+Z7r7bfZ3Zv7bAc0uwQpok2EDT8ntz7yj5sx54uKPDEoq+4QFMyuVAnZoHbRmVi4tvpWNmVm+3KM1M8tZEXeGOWjNrFQKmLMOWjMrl0Zd+LuRHLRmViru0ZqZ5cxjtGZmOStgzjpozaxcitijLd6osZnZAEi1T323ow0kPSDpEUmPSTo/LX+npPslPS3pRklDq9XkoDWzUmlpUc1TFW8C+0fELsAE4CBJewAXA5dExHjgJbLrdvdd0wDfk5lZoTTqerSReTXNDklTAPsDP07Lp5LdN6xPDlozK5V6hg4kdUiaVTF1rN2WWiXNAZYB04HfA8sjYmV6ykL+dgnZXnlnmJmVSj07wyKiE+jsY/0qYEK6G+40YIf+1OSgNbNSyeOgg4hYLukuYE9gc0ltqVc7huxmCH3y0IGZlUqjdoZJ2ir1ZJG0IXAgMB+4C+i6FcYJwM3VanKP1sxKpYHH0Y4CpkpqJeuU3hQRt0p6HLhB0teBh4GrqjXkoDWzUmlU0EbEXGDXHpY/Q3YfxZo5aM2sVAp4YpiD1szKpYin4DpozaxUCpizDlozKxffnNHMLGctBezSOmjNrFQKmLMOWjMrF+8MMzPLWQGHaB20ZlYu3hlmZpYz4aA1M8tVATu0DlozKxfvDDMzy1kBc9ZBa2bl4hMWzMxyVsSjDnyHBTMrlXpuzth3Oxor6S5Jj0t6TNKpafl5khZJmpOmQ6rV5B6tmZVKA4cOVgKnR8RDkoYDsyVNT+suiYhv1dqQg9bMSqVRMRsRi4HF6fErkuZTw63Fe+KhAzMrFUn1TB2SZlVMHb20OY7stjb3p0WTJc2VNEXSiGo1OWjNrFRaVPsUEZ0RMbFi6uzenqRNgP8BTouIl4HvAdsCE8h6vN+uVpOHDsysVBp51IGkIWQhe11E/C9ARCytWH8lcGu1dhy0ZlYqjTozTFlDVwHzI+I7FctHpfFbgCOBedXactCaWak0sEO7F3A88KikOWnZ2cAxkiYAATwHnFKtIQetmZVKo3q0EXEvPR/EcFu9bTlozaxUindemIPWzEqmtYCn4DpozaxUfJlEM7OcFTBnHbRmVi6+TKKZWc4KmLP5B+2s8w7MexO2Dhqx2+Rml2AF9PrD3x1wGx6jNTPLWauD1swsXwU8ustBa2bl4qA1M8uZx2jNzHLmHq2ZWc4K2KF10JpZubQVMGkdtGZWKgXMWd8zzMzKpUWqeeqLpLGS7pL0uKTHJJ2alm8habqkp9L/vjmjma1fpNqnKlYCp0fEjsAewBck7Qh8GbgzIrYD7kzzfXLQmlmp1HMX3L5ExOKIeCg9fgWYD4wGDgempqdNBY6oVpPHaM2sVOq58LekDqCjYlFnL7ccHwfsCtwPtFfcnHEJ0F5tOw5aMyuVeo6jTaH6d8FaSdImZLccPy0iXq48ISIiQlJUran2kszMik91/KvaljSELGSvi4j/TYuXShqV1o8CllVrx0FrZqXSqDFaZV3Xq4D5EfGdilW3ACekxycAN1eryUMHZlYqDTwFdy/geOBRSXPSsrOBi4CbJJ0E/AE4ulpDDlozK5VGXVQmIu6l97uXf6iethy0ZlYqrQUcEHXQmlmp+OaMZmY582USzcxyVsAOrYPWzMqlpYbjYwebg9bMSsU9WjOznLUVcJDWQWtmpeIerZlZznx4l5lZzgqYsw5aMyuXAp4Y5qA1s3Lx0IGZWc4ctGZmOStezDpozaxkCtihLeS4sZlZv0mqeaqhrSmSlkmaV7HsPEmLJM1J0yHV2nHQmlmptNQx1eAa4KAell8SERPSdFu1Rjx0YGal0sidYRFxd7rV+IC4R2tmpVLP0IGkDkmzKqaOGjczWdLcNLQwotqTHbRmVir1DB1ERGdETKyYOmvYxPeAbYEJwGLg29Ve4KEDMyuVRt2csTcRsbRiW1cCt1Z7jXu0ZlYqqmPqV/vSqIrZI4F5vT23i3u0ZlYqrQ3s0Uq6HtgXGClpIfBVYF9JE4AAngNOqdaOg9bMSqWRIwcRcUwPi6+qtx0HrZmVigp4Eq6D1sxKpYin4DpozaxUfBdcM7OcuUdrZpYzX4/WzCxnBbzbuIPWzMrFRx2YmeWsgCMHDtrB8pt77ubiiy5k9arVHDnp45z0f2u9SJCVTUuL+M11Z/D8shVMOvX7dJ7/KfZ573hWvPoGAB3nXsvcJxc1ucp1l3u066lVq1bx7xdewH9deTXt7e0c+4mj2He//dl2/Phml2ZNMPnY/Xji2aUM33iDNcvOvvQnTPvlnCZWVR5FHKP1RWUGwbxH5zJ27DsYM3YsQ4YO5aBDPsqMu+5sdlnWBKO33pyD9t6Jq6f9ttmllFaLVPM0aDUN2pbWY8uWLuVto962Zn7r9naWLl3axyusrL75pUl85bKfsHp1rLX8vC98jAduPItvnP5PDB3iPzQHIu+rd/VHv4NW0mf6WLfmquVXXVnLdXTNyu/gfXZm2Yuv8PD8BWstP/fyW9jlyK+x96e+yYjNNub0zxzQpArLoYg92oH86jwfuLqnFekq5Z0Ab6wkenrO+mTr9naWLF6yZn7Z0qW0t7c3sSJrhj0nvItDP/gPHLT3TgwbOoRNN96AKV//NJ895wcAvPXXlfzg5pmc9ukPNbnSdVsBh2j7DlpJc3tbBTgparTTzv/AH//4HAsXLqB963Zuv+1n/Mc3q979wkrm3Mtv4dzLbwFgn/dux2mf/hCfPecHvG3kpix54WUADtvvPTz+++ebWea6r4BJW61H2w58BHip23IBHs2vUVtbG2d95Vw+13Eyq1ev4ogjJzF+/HbNLssK4uoLT2DkiOFIMPeJhfzLhTc0u6R1WiOHBCRNAQ4FlkXEzmnZFsCNwDiyC38fHRHdM3LtdiJ6/8te0lXA1RFxbw/rfhQRx1Yr1EMH1pMRu01udglWQK8//N0Bp+SDz6yoOXN2e9dmfW5P0geAV4EfVATtN4AXI+IiSV8GRkTEmX210+fOsIg4qaeQTeuqhqyZ2aBr4GEHEXE38GK3xYcDU9PjqcAR1drx4V1mViqq51/FEVJpquWUzfaIWJweL6GG/VU+YM/MSqWeIdrKI6T6IyJCUtWhCvdozaxUBuGEhaVdtxxP/y+r9gIHrZmViqSap366BTghPT4BuLnaCxy0ZlYqUu1T9bZ0PXAfsL2khZJOAi4CDpT0FHBAmu+Tx2jNrFQaeb5CRBzTy6q6Tt9z0JpZuayDZ4aZma1TfOFvM7Oc+VY2ZmY5c9CameXMQwdmZjlzj9bMLGcFzFkHrZmVTAGT1kFrZqUymPcCq5WD1sxKpXgx66A1s7IpYNI6aM2sVHx4l5lZzgo4ROugNbNyKWDOOmjNrFwGcEHv3DhozaxUGpmzkp4DXgFWASsjYmJ/2nHQmlmp5NCf3S8iXhhIAw5aMyuX4o0c+J5hZlYuquNfDQL4haTZkjr6W5N7tGZWKvWM0abwrAzQzojorJjfOyIWSdoamC7pdxFxd701OWjNrFRa6gjaFKqdfaxflP5fJmkasDtQd9B66MDMSkZ1TH20Im0saXjXY+DDwLz+VOQerZmVSgMP72oHpqXjctuAH0XE7f1pyEFrZqXSqJyNiGeAXRrRloPWzEqlgCeGOWjNrFx8Cq6ZWc6KF7MOWjMrmQJ2aB20ZlYuvvC3mVneipezDlozK5cC5qyD1szKxbcbNzPLWQFz1tc6MDPLm3u0ZlYqRezROmjNrFR8eJeZWc7cozUzy5mD1swsZx46MDPLWRF7tD68y8xKpTE3skltSQdJekLS05K+3N+aHLRmVi4NSlpJrcAVwMHAjsAxknbsT0keOjCzUmngKbi7A0+nW9og6QbgcODxehvKPWg3aCvgyHSTSOrods/49dbrD3+32SUUhj8XjVVP5kjqADoqFnVWfC9GAwsq1i0E3tefmjx0MLg6qj/F1kP+XDRJRHRGxMSKKZdfeA5aM7OeLQLGVsyPScvq5qA1M+vZg8B2kt4paSjwSeCW/jTknWGDy+Nw1hN/LgooIlZKmgzcAbQCUyLisf60pYhoaHFmZrY2Dx2YmeXMQWtmljMH7SBp1Kl8Vh6SpkhaJmles2uxfDloB0EjT+WzUrkGOKjZRVj+HLSDY82pfBHxFtB1Kp+txyLibuDFZtdh+XPQDo6eTuUb3aRazGyQOWjNzHLmoB0cDTuVz8zWPQ7awdGwU/nMbN3joB0EEbES6DqVbz5wU39P5bPykHQ9cB+wvaSFkk5qdk2WD5+Ca2aWM/dozcxy5qA1M8uZg9bMLGcOWjOznDlozcxy5qA1M8uZg9bMLGf/H8bPkbCEXZPTAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qF5rDQ8dBiHc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 57
        },
        "outputId": "dc2eef3e-e0a9-4bed-de3f-c4e0a3722152"
      },
      "source": [
        "#Calculate Accuracy and Print it\n",
        "accu_per=(sum([1 for i, j in zip(Test_y, y_test) if i == j])/len(y_test))*100\n",
        "print(\"Accuracy = \" ,accu_per, \"%\")\n",
        "acc_per2=((TP_per+TN_per)/(TP_per+FP_per+FN_per+TN_per))*100 # another way\n",
        "acc_per2"
      ],
      "execution_count": 113,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy =  98.71794871794873 %\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "98.71794871794873"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 113
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "XR_e6qziHvFR",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 57
        },
        "outputId": "05512cab-73e6-4315-f9ae-f3f76c3a19d4"
      },
      "source": [
        "# calculating the cross validation for a set of predictions:\n",
        "results_per = model_selection.cross_val_score(perp, X_train, y_train,cv=None)*100\n",
        "results_per"
      ],
      "execution_count": 114,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 96.82539683, 100.        , 100.        ,  98.38709677,\n",
              "        96.77419355])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 114
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y5fjPIEN2TWJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "8be07f9a-52dd-4510-ddd5-613fa477b8e7"
      },
      "source": [
        "acc_cross_validation['Perceptron']=results_per\n",
        "acc_cross_validation"
      ],
      "execution_count": 115,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "      <th>KNN</th>\n",
              "      <th>Perceptron</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>85.714286</td>\n",
              "      <td>96.825397</td>\n",
              "      <td>66.666667</td>\n",
              "      <td>96.825397</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>93.650794</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>69.841270</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>83.870968</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>70.967742</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>85.483871</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>67.741935</td>\n",
              "      <td>98.387097</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>91.935484</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>74.193548</td>\n",
              "      <td>96.774194</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   NaiveBayes  logistic regression        KNN  Perceptron\n",
              "0   85.714286            96.825397  66.666667   96.825397\n",
              "1   93.650794           100.000000  69.841270  100.000000\n",
              "2   83.870968           100.000000  70.967742  100.000000\n",
              "3   85.483871            96.774194  67.741935   98.387097\n",
              "4   91.935484            96.774194  74.193548   96.774194"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 115
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9tWp80la6P5U",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "outputId": "e0c59c98-4f03-4d19-c6b0-0753b5f7953f"
      },
      "source": [
        "df_result['Perceptron']=pd.Series({'Accuracy':accu_per,'Cross_Validation':results_per,'Confusion_Matrix':cm_per,'Sensitivity':TPR_per,'Specificity':TNR_per,'Precision':percision_per})\n",
        "df_result"
      ],
      "execution_count": 116,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "      <th>KNN</th>\n",
              "      <th>Perceptron</th>\n",
              "      <th>SVM</th>\n",
              "      <th>Combination</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Accuracy</th>\n",
              "      <td>89.7436</td>\n",
              "      <td>96.1538</td>\n",
              "      <td>79.4872</td>\n",
              "      <td>98.7179</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>89.3617</td>\n",
              "      <td>97.7273</td>\n",
              "      <td>100</td>\n",
              "      <td>97.8261</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Precision</th>\n",
              "      <td>84.8485</td>\n",
              "      <td>96.9697</td>\n",
              "      <td>100</td>\n",
              "      <td>96.9697</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>90.3226</td>\n",
              "      <td>94.1176</td>\n",
              "      <td>67.3469</td>\n",
              "      <td>100</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            NaiveBayes logistic regression      KNN Perceptron  SVM Combination\n",
              "Accuracy       89.7436             96.1538  79.4872    98.7179  NaN         NaN\n",
              "Specificity    89.3617             97.7273      100    97.8261  NaN         NaN\n",
              "Precision      84.8485             96.9697      100    96.9697  NaN         NaN\n",
              "Sensitivity    90.3226             94.1176  67.3469        100  NaN         NaN"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 116
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a1TC1Zu0ovrX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 5- Support Vector Machine(SVM)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nvsHO3_qowBx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Create Linear SVM object\n",
        "from sklearn import svm\n",
        "support = svm.LinearSVC(random_state=20)"
      ],
      "execution_count": 117,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qdLL71lQowRx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 98
        },
        "outputId": "e95f5a12-ba4e-4605-d08f-dd81ea3cef20"
      },
      "source": [
        "# Train the model using the training sets and check score on test dataset\n",
        "from sklearn.metrics import accuracy_score\n",
        "support.fit(X_train, y_train)\n",
        "predicted= support.predict(X_test)"
      ],
      "execution_count": 118,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
              "          intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
              "          multi_class='ovr', penalty='l2', random_state=20, tol=0.0001,\n",
              "          verbose=0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VxF2p0AXpAbg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "d4f9c1d0-f9d9-401a-e129-03637c1c23d6"
      },
      "source": [
        "score_svm1=support.score(X_test, y_test) # method 1 to calculate accuracy of svm model\n",
        "score_svm2=accuracy_score(y_test,predicted)  # method 2 to calculate accuracy of svm model\n",
        "#Calculate Accuracy and Print it by method3\n",
        "accu_svm=(sum([1 for i, j in zip(predicted, y_test) if i == j])/len(y_test))*100\n",
        "print(\"Your Model Accuracy is\", score_svm1*100,score_svm2*100,accu_svm)"
      ],
      "execution_count": 119,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Your Model Accuracy is 96.15384615384616 96.15384615384616 96.15384615384616\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "621Bi85yx8zh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 57
        },
        "outputId": "4423d1bb-2e57-4678-d61b-661640603f63"
      },
      "source": [
        "# confusion matrix\n",
        "os.chdir(home_path)\n",
        "from sklearn.metrics import confusion_matrix\n",
        "cm_svm=confusion_matrix(y_test,predicted)\n",
        "cm_svm"
      ],
      "execution_count": 120,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[31,  2],\n",
              "       [ 1, 44]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 120
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "erkf24QPyRig",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "3699513b-aea1-4f75-c75e-a20eeec66510"
      },
      "source": [
        "#Sensitivity_(True positive rate TPR) \n",
        "TP_svm= cm_svm[0][0]\n",
        "FN_svm= cm_svm[1][0]\n",
        "TPR_svm= TP_svm/(TP_svm+FN_svm) * 100\n",
        "print(\"Sensitivity \"+str(TPR_svm)+'%')"
      ],
      "execution_count": 121,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sensitivity 96.875%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gQ-RIsJ8yRfD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "765c7951-1ab6-412b-9beb-3510f4d84dcf"
      },
      "source": [
        "#Specificity(Total Negative Result TNR)\n",
        "TN_svm= cm_svm[1][1]\n",
        "FP_svm= cm_svm[0][1] \n",
        "TNR_svm = TN_svm/(TN_svm+FP_svm) * 100\n",
        "print(\"Specificity \"+str(TNR_svm)+'%')"
      ],
      "execution_count": 122,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Specificity 95.65217391304348%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O93yuzuqegkh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "0029bd5c-1605-4125-e084-788cf2603d91"
      },
      "source": [
        "float32_epsilon = np.finfo(np.float32).eps\n",
        "percision_svm_=(TP_svm/(TP_svm+FP_svm)+float32_epsilon)*100\n",
        "percision_svm_"
      ],
      "execution_count": 123,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "93.93940586032289"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 123
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wa6WfvM8yRbn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        },
        "outputId": "37ad6b2a-b8a6-46b0-b347-8c578eef8d5f"
      },
      "source": [
        "os.chdir(home_path)\n",
        "cm_picture5=sn.heatmap(pd.DataFrame(cm_svm),cmap=\"Blues\", annot=True)\n",
        "figure5 = cm_picture5.get_figure()   \n",
        "plt.title(\"Confusion Matrix SVM\")\n",
        "figure5.savefig('/content/drive/My Drive/project/CGR_data/Confusion_SVM_4kmers.jpg', dpi=400)"
      ],
      "execution_count": 124,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Confusion Matrix SVM')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 124
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVoAAAEICAYAAAAeFzyKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAXPElEQVR4nO3deZRcdZnG8e/T3VmABAgQMBKQHWUZgiyCgCCKBsVJZJGdwDC2IlEYtohw2ERHHNlUxGlkiewIIpmALBNBDCAQSICEsMkyJCRptrBvgXf+qBssmu6uqu66Xbd/eT6ce9J1761fvR36PHn7dzdFBGZmlp+mRhdgZpY6B62ZWc4ctGZmOXPQmpnlzEFrZpYzB62ZWc4ctImRtJSk/5H0iqQ/9GKcfSXdXM/aGkHSnyWNa3QdtmRz0DaIpH0kTZP0uqR5WSBsW4ehdwdWAVaMiD16OkhEXBoRX6lDPR8haQdJIenaDus3ydbfVuU4J0m6pNJ+EbFzREzsQZ0DJZ0uaU72/+hpSWdl226UdEon7xkjab6kFkkXZd/PmA77nJmtP7DWmqz/ctA2gKQjgLOAn1IKxdWB3wBjuntflT4FPBYRi+owVl6eB7aWtGLZunHAY/X6AJX05uf7WGBzYEtgKLADcH+2bSKwnyR1eM/+wKVlf/ePAQeU1dQCfAv4Ry/qsv4oIrz04QIsB7wO7NHNPoMoBfFz2XIWMCjbtgMwBzgSaAfmAQdl204G3gXeyz7jYOAk4JKysdcAAmjJXh8IPAm8BjwF7Fu2fmrZ+z4P3Au8kv35+bJttwE/Bu7IxrkZWKmL721x/b8FDs3WNQNzgROA28r2PRt4FngVuA/YLls/usP3+UBZHT/J6ngLWCdb9+/Z9nOBa8rGPw2YAqiTOicDh3fxPSyV/T18oWzdMOBtYJPs9UXAL4AFwLBs3S7An4GpwIGN/ln00neLO9q+tzUwGLi2m32OA7YCRgGbUOqqji/b/glKgb0qpTA9R9KwiDiRUpd8ZUQMiYjzuytE0jLAL4GdI2IopTCd0cl+KwDXZ/uuCJwBXN+hI90HOAhYGRgIHNXdZwO/55/d3leBmZT+USl3L6W/gxWAy4A/SBocETd2+D43KXvP/kArpS70mQ7jHQlsLOlASdtR+rsbFxGdXYf+d+AISd+TtHF59xoRbwFXldUPpU71kYh4oGzd28B1wF7Z6wOy79uWMA7avrci8EJ0/6v9vsApEdEeEc9T6lT3L9v+Xrb9vYi4gVJXt34P6/kA2EjSUhExLyJmdbLP14HHI+LiiFgUEZcDjwDfKNvnwoh4rCyERnX3oRFxJ7CCpPXpIoAi4pKIeDH7zNMpdfqVvs+LImJW9p73Ooz3JqW/xzOAS4DvR8ScLsb5T0od777ANGBuh4NqE4HdJQ3OXh+Qrevo98ABkpYHtgf+VKF+S5CDtu+9CKyUzdd15ZN8tBt7Jlv34RgdgvpNYEithUTEG8CewHeBeZKul/TpKupZXNOqZa/n96Cei4HxwBfppMOXdJSk2dkZFAspdfErVRjz2e42RsTdlKZKROkfhK72ez8izomIbYDlKU1JXCDpM9n2qcALwFhJa1P6reOyTsaZCgyn9FvK5OwfIlvCOGj73l3AO8DYbvZ5jtJBrcVW5+O/VlfrDWDpstefKN8YETdFxE7ACEpd6nlV1LO4prk9rGmxi4HvATdk3eaHsl/tj6H0K/mwiFie0rzo4l/hu7rtXLe3o5N0KKXO+Lls/Ioi4q2IOAd4GdigbNPi6Y/9gJsiYkEXQ1xCadrC0wZLKAdtH4uIVygd9DlH0lhJS0saIGlnST/PdrscOF7ScEkrZftXPJWpCzOAL0haXdJylI6mAyBpleyUpGUohf/rlKYSOroBWC87Ja1F0p6UAmdyD2sCICKeovTr9HGdbB4KLKJ0hkKLpBOAZcu2LwDWqOXMAknrAadSCsb9gWMkdTrFIenw7FS0pbLveVxW0/Sy3X4PfBn4Np1PGyz2S2An4PZqa7W0OGgbIJtvPILSAa7nKf26O55/zt+dSmle8EHgIUqnFZ3aw8+6BbgyG+s+PhqOTVkdzwEvUQq9QzoZ40VKR8yPpDT1cQywS0S80JOaOow9NSI669ZvAm6kdIrUM5QOLJVPCyy+GONFSfdTQTZVcwlwWkQ8EBGPAz8CLpY0qJO3vAmcTmlK5AXgUGC3iHiyrPangTuBZYBJ3XyPL0XElC4OutkSQP5/b2aWL3e0ZmY5c9CameXMQWtmljMHrZlZzro7ab4uRp00xUfb7GOmHrtjo0uwAhoy6GM36qnZUpuOrzpz3pr+615/XjXc0ZqZ5Sz3jtbMrE/16u6Y+XDQmllampobXcHHOGjNLC29n+atOwetmaXFUwdmZjlzR2tmljN3tGZmOXNHa2aWM591YGaWM08dmJnlzFMHZmY5c0drZpYzB62ZWc6afTDMzCxfnqM1M8tZAacOileRmVlvSNUvVQ2nZknTJU3OXq8p6W5JT0i6UtLASmM4aM0sLWqqfqnOYcDsstenAWdGxDrAy8DBlQZw0JpZWurY0UoaCXwd+F32WsCOwNXZLhOBsZXG8RytmaWlvpfgngUcAwzNXq8ILIyIRdnrOcCqFUuqZ0VmZg1Xw9SBpFZJ08qW1g+HkXYB2iPivt6W5I7WzNJSw+ldEdEGtHWxeRvgXyV9DRgMLAucDSwvqSXrakcCcyt9jjtaM0tLnQ6GRcSxETEyItYA9gL+EhH7ArcCu2e7jQOuq1SSg9bM0lL/sw46mgAcIekJSnO251d6g6cOzCwtOdyPNiJuA27Lvn4S2LKW9ztozSwtvgTXzCxnBbwE10FrZmlxR2tmli85aM3M8uWgNTPLmZoctGZmuXJHa2aWMwetmVnOHLRmZnkrXs46aM0sLe5ozcxy1tTkK8PMzHLljtbMLG/Fy1kHrZmlxR2tmVnOHLRmZjkr4iW4xTs8Z2bWC5KqXiqMM1jSPZIekDRL0snZ+oskPSVpRraMqlSTO1ozS0odpw7eAXaMiNclDQCmSvpztu3oiLi62oEctGaWlHoFbUQE8Hr2ckC2RE/G8tSBmSWllqkDSa2SppUtrR3GapY0A2gHbomIu7NNP5H0oKQzJQ2qVJM7WjNLSw0NbUS0AW3dbH8fGCVpeeBaSRsBxwLzgYHZeycAp3T3Oe5ozSwpTU1NVS/VioiFwK3A6IiYFyXvABdSxaPHHbRmlpQ6nnUwPOtkkbQUsBPwiKQR2ToBY4GZlWry1IGZpaV+p9GOACZKaqbUlF4VEZMl/UXS8OyTZgDfrTSQgzYnA1uauOCgzzKguYmWJvG/D7dz7m1PseeWI9l3q9VYfYWl2eHnt7PwzfcaXao1yPz58zjhuAm89OKLSOKbu32LffY7oNFl9Xt1POvgQWDTTtbvWOtYDtqcvLvoA749cTpvvfs+LU3iwn/bjKlPvMiM/1vI3x57gd8d+NlGl2gN1tzczH8cOYHPbLAhb7zxOvvttRtbbf151lp7nUaX1q/1y0twJX0aGAOsmq2aC0yKiNl5FpaCt959H4CWZtHSLCLg0fmvV3iXLSmGD1+Z4cNXBmCZZYaw5ppr096+wEHbS/0uaCVNAPYGrgDuyVaPBC6XdEVE/Czn+vq1JsHl39mS1VZYiivvmcPMua82uiQrqOfmzuGRR2az0cabNLqUfq+I9zqo1NEeDGwYER+ZSJR0BjAL6DRos5N+WwFG7nI4K262Sx1K7X8+CNjzt/cwdHALZ+z5L6y98jL8o/2NRpdlBfPmm29w9BE/4KhjjmXIkCGNLqffK2JHW+n0rg+AT3ayfkS2rVMR0RYRm0fE5ktqyJZ77e1F3Pv0y2yzzoqNLsUK5r333uPoI37Azl//Bjt++SuNLicJ9Tq9q54qdbSHA1MkPQ48m61bHVgHGJ9nYf3dsKUHsOiD4LW3FzGopYmt1lqBC+94utFlWYFEBD8+8XjWXHNt9jvgoEaXk4wCNrTdB21E3ChpPUpXPpQfDLs3uzTNurDS0EH8eOwGNDVBk8TNs9r522MvsvfnRnLgNp9ixSEDueqQzzH18Rc4ZdIjjS7XGmDG9Pu5fvJ1rLPueuy9x1gADv3Bf7Dtdts3uLL+rYhTByrdoCY/o06aku8HWL809diaT0W0JcCQQb1PyfUn3FR15jx62lf7JJV9Hq2ZJaWADa2D1szS0tQPT+8yM+tX3NGameWsiAfDHLRmlpQC5qyD1szSUssNvfuKg9bMkuKO1swsZ0Wcoy1ej21m1gtS9Uv342iwpHskPSBplqSTs/VrSrpb0hOSrpQ0sFJNDlozS0odbyrzDrBjRGwCjAJGS9oKOA04MyLWAV6mdJfDbjlozSwp9eposyfdLr5T/4BsCWBH4Ops/URKD2jsloPWzJLS1KSqF0mtkqaVLa3lY0lqljQDaAduAf4BLIyIRdkuc/jnDbe65INhZpaUWg6GRUQb0NbN9veBUdljx68FPt2Tmhy0ZpaUPE46iIiFkm4FtgaWl9SSdbUjKd06tlueOjCzpNTrYJik4Vkni6SlgJ2A2cCtwO7ZbuOA6yrV5I7WzJJSx452BDBRUjOlpvSqiJgs6WHgCkmnAtOB8ysN5KA1s6TU6zaJEfEgsGkn65+k9NSZqjlozSwpRbwyzEFrZklx0JqZ5ayAOeugNbO0uKM1M8tZAXPWQWtmafHDGc3MctZUwJbWQWtmSSlgzjpozSwtPhhmZpazAk7ROmjNLC0+GGZmljPhoDUzy1UBG1oHrZmlxQfDzMxyVsCcddCaWVqKeMGCH2VjZkmp5Sm43ZG0mqRbJT0saZakw7L1J0maK2lGtnytUk3uaM0sKXVsaBcBR0bE/ZKGAvdJuiXbdmZE/KLagRy0ZpaUek0dRMQ8YF729WuSZgOr9qimulRkZlYQqmWRWiVNK1taOx1TWoPS88PuzlaNl/SgpAskDatUk4PWzJJSy+PGI6ItIjYvW9o6GW8IcA1weES8CpwLrA2MotTxnl6pJk8dmFlS6nnBgqQBlEL20oj4I0BELCjbfh4wudI4DlozS0q97nWg0pUP5wOzI+KMsvUjsvlbgG8CMyuN5aA1s6TU8cqwbYD9gYckzcjW/QjYW9IoIICnge9UGshBa2ZJqdfUQURMhU7vUHNDrWM5aM0sKb7XgZlZzooXsw5aM0tMcwHvk+igNbOkeOrAzCxnBcxZB62ZpaWIt0l00JpZUgqYs/kH7d+P/1LeH2H90LAtxje6BCugt6b/utdjeI7WzCxnzQ5aM7N8FfDsLgetmaXFQWtmljPP0ZqZ5cwdrZlZzgrY0DpozSwtLQVMWgetmSWlgDnroDWztBTxElw/BdfMkiJVv3Q/jlaTdKukhyXNknRYtn4FSbdIejz7048bN7MlS5OqXypYBBwZERsAWwGHStoA+CEwJSLWBaZkr7vlqQMzS0q9bvydPel2Xvb1a5JmA6sCY4Adst0mArcBE7obyx2tmSWllo5WUqukaWVLa2djSloD2BS4G1il7HHj84FVKtXkjtbMkqIanhoWEW1AW7fjSUOAa4DDI+LV8ivPIiIkRaXPcdCaWVLqeWWYpAGUQvbSiPhjtnqBpBERMU/SCKC9Yk31K8nMrPHqdTBMpdb1fGB2RJxRtmkSMC77ehxwXaWa3NGaWVLqeFOZbYD9gYckzcjW/Qj4GXCVpIOBZ4BvVRrIQWtmSWmu0+/pETEVupzwrenRMQ5aM0tKEa8Mc9CaWVJ8m0Qzs5wVsKF10JpZWppqOI+2rzhozSwp7mjNzHLWUsBJWgetmSXFHa2ZWc58epeZWc4KmLMOWjNLSxFv4OKgNbOkeOrAzCxnDlozs5wVL2YdtGaWmAI2tA5aM0tLHe9HWzcOWjNLShHPOihiTWZmPdYkVb1UIukCSe2SZpatO0nSXEkzsuVrFWvq5fdkZlYokqpeqnARMLqT9WdGxKhsuaHSIJ46MLOk1LN7jIjbJa3R23Hc0ZpZUmrpaCW1SppWtrRW+THjJT2YTS0Mq7Szg9bMkqIalohoi4jNy5a2Kj7iXGBtYBQwDzi90hs8dWBmSWnO+fSuiFiw+GtJ5wGTK73HHa2ZJUWqfunZ+BpR9vKbwMyu9l3MHa2ZJUV1vAhX0uXADsBKkuYAJwI7SBoFBPA08J1K4zhozSwp9Zw5iIi9O1l9fq3jOGjNLCl+Cq6ZWc4KeKsDB62ZpcX3ozUzy1kBnzbuoDWztNTzrIN6cdCaWVIKOHPgCxb6wgnHH8sO223NrmN2aXQpVgBNTeKuyydwzdnf/cj604/ZnefvqHg1p1WgGv7rKw7aPjBm7K6c+9+/a3QZVhDj9/kijz614CPrPrvB6iw/dOkGVZSWJlW/9FlNffdRS67NNt+CZZdbrtFlWAGsuvLyjN52Qy689s4P1zU1iZ8ePpbjzv5TAytLRz1v/F23mvrsk8yM/zp6N447+0988EF8uO6QPbfn+r8+xPwXXm1gZemo5e5dfaXHQSvpoG62fXiPx/PPq+auY2bp23m7jWh/6TWmz372w3Ujhi/Hrjttym+u+GsDK0tLETva3px1cDJwYWcbsns6tgG8vYjobB+zJc3Wo9Zil+03ZvS2GzJo4ACWXWYw9119HO+8u4hZk04EYOnBA5h53YlsNObkBlfbfxXwpIPug1bSg11tAlapfzlm6TrhV5M44VeTANhus3U5/IAvsdthv/3IPs/fcbpDtrcKmLSVOtpVgK8CL3dYL+DOj+9unZlw1BFMu/ceFi58mZ12/AKHHPp9dt1tj0aXZZakIl6Cq4iuf7OXdD5wYURM7WTbZRGxT6UP8NSBdWbYFuMbXYIV0FvTf93rlLz3yVeqzpwt1lquT1K52442Ig7uZlvFkDUz63PFa2h9epeZpaWeV4ZlT7ltlzSzbN0Kkm6R9Hj2p5+Ca2ZLljo/M+wiYHSHdT8EpkTEusCU7HW3HLRmlpR6XrAQEbcDL3VYPQaYmH09ERhbaRwHrZklRVIty4cXV2VLaxUfsUpEzMu+nk8Vp7r6NolmlpRazu4qv7iqJyIiJFU8y8EdrZklpQ/udbBA0giA7M/2Sm9w0JpZWvJP2knAuOzrccB1ld7goDWzpNT59K7LgbuA9SXNkXQw8DNgJ0mPA1/OXnfLc7RmlpR6XoEbEXt3selLtYzjoDWzpBTwVgcOWjNLi5+Ca2aWM3e0ZmY5K2DOOmjNLDEFTFoHrZklpYg3/nbQmllSihezDlozS00Bk9ZBa2ZJ8eldZmY5K+AUrYPWzNJSwJx10JpZWlTAltZBa2ZJKWDOOmjNLC0FzFkHrZklpoBJ66A1s6T49C4zs5zVc45W0tPAa8D7wKKI2Lwn4zhozSwpTfVvaL8YES/0ZgAHrZklpnhTB344o5klRap+qUIAN0u6T1JrT2tyR2tmSamln83CszxA2yKirez1thExV9LKwC2SHomI22utyUFrZkmp5WBYFqpt3Wyfm/3ZLulaYEug5qD11IGZJUVS1UuFcZaRNHTx18BXgJk9qckdrZklpY6HwlYBrs0CuQW4LCJu7MlADlozS0q9zqONiCeBTeoxloPWzJLiK8PMzPJWvJx10JpZWgqYsw5aM0uLHzduZpazAuasz6M1M8ubO1ozS0oRO1oHrZklxad3mZnlzB2tmVnOHLRmZjnz1IGZWc7c0ZqZ5ayAOeugNbPEFDBpHbRmlpQiXoKriGh0DUsMSa0dnkdk5p+LJYAvwe1bPX6KpiXNPxeJc9CameXMQWtmljMHbd/yPJx1xj8XifPBMDOznLmjNTPLmYPWzCxnDto+Imm0pEclPSHph42uxxpP0gWS2iXNbHQtli8HbR+Q1AycA+wMbADsLWmDxlZlBXARMLrRRVj+HLR9Y0vgiYh4MiLeBa4AxjS4JmuwiLgdeKnRdVj+HLR9Y1Xg2bLXc7J1ZrYEcNCameXMQds35gKrlb0ema0zsyWAg7Zv3AusK2lNSQOBvYBJDa7JzPqIg7YPRMQiYDxwEzAbuCoiZjW2Kms0SZcDdwHrS5oj6eBG12T58CW4ZmY5c0drZpYzB62ZWc4ctGZmOXPQmpnlzEFrZpYzB62ZWc4ctGZmOft/AoW0z4yJugoAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2opSoiX1yie8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        },
        "outputId": "1e240b0e-337d-47a7-c9e6-72564b2fd598"
      },
      "source": [
        "# calculating the mean R^2 for a set of predictions:\n",
        "results_svm = model_selection.cross_val_score(support,X_train, y_train,cv=None)*100\n",
        "results_svm"
      ],
      "execution_count": 125,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/svm/_base.py:947: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
            "  \"the number of iterations.\", ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([100.        , 100.        , 100.        ,  96.77419355,\n",
              "        96.77419355])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 125
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mlBvsSoU2759",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "d2edc260-1ea0-49f4-80ca-1ef645f2099d"
      },
      "source": [
        "acc_cross_validation['SVM']=results_svm\n",
        "acc_cross_validation"
      ],
      "execution_count": 126,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "      <th>KNN</th>\n",
              "      <th>Perceptron</th>\n",
              "      <th>SVM</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>85.714286</td>\n",
              "      <td>96.825397</td>\n",
              "      <td>66.666667</td>\n",
              "      <td>96.825397</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>93.650794</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>69.841270</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>83.870968</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>70.967742</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>85.483871</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>67.741935</td>\n",
              "      <td>98.387097</td>\n",
              "      <td>96.774194</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>91.935484</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>74.193548</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>96.774194</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   NaiveBayes  logistic regression        KNN  Perceptron         SVM\n",
              "0   85.714286            96.825397  66.666667   96.825397  100.000000\n",
              "1   93.650794           100.000000  69.841270  100.000000  100.000000\n",
              "2   83.870968           100.000000  70.967742  100.000000  100.000000\n",
              "3   85.483871            96.774194  67.741935   98.387097   96.774194\n",
              "4   91.935484            96.774194  74.193548   96.774194   96.774194"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 126
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SnzrKsbXzD-S",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "outputId": "33d09d0d-b86e-4b4f-8cdc-102c9d1acf4e"
      },
      "source": [
        "df_result['SVM']=pd.Series({'Accuracy':accu_svm,'Cross_Validation':results_svm,'Confusion_Matrix':cm_svm,'Sensitivity':TPR_svm,'Specificity':TNR_svm,'Precision':percision_svm_})\n",
        "df_result"
      ],
      "execution_count": 127,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "      <th>KNN</th>\n",
              "      <th>Perceptron</th>\n",
              "      <th>SVM</th>\n",
              "      <th>Combination</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Accuracy</th>\n",
              "      <td>89.7436</td>\n",
              "      <td>96.1538</td>\n",
              "      <td>79.4872</td>\n",
              "      <td>98.7179</td>\n",
              "      <td>96.1538</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>89.3617</td>\n",
              "      <td>97.7273</td>\n",
              "      <td>100</td>\n",
              "      <td>97.8261</td>\n",
              "      <td>95.6522</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Precision</th>\n",
              "      <td>84.8485</td>\n",
              "      <td>96.9697</td>\n",
              "      <td>100</td>\n",
              "      <td>96.9697</td>\n",
              "      <td>93.9394</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>90.3226</td>\n",
              "      <td>94.1176</td>\n",
              "      <td>67.3469</td>\n",
              "      <td>100</td>\n",
              "      <td>96.875</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            NaiveBayes logistic regression  ...      SVM Combination\n",
              "Accuracy       89.7436             96.1538  ...  96.1538         NaN\n",
              "Specificity    89.3617             97.7273  ...  95.6522         NaN\n",
              "Precision      84.8485             96.9697  ...  93.9394         NaN\n",
              "Sensitivity    90.3226             94.1176  ...   96.875         NaN\n",
              "\n",
              "[4 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 127
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Et-fcch72wZX"
      },
      "source": [
        "## Model Selection"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "dRqa1LH_7jHf"
      },
      "source": [
        "In This study we try to select best model for coding/noncoding DNA detection by two methods:\n",
        "1. from built in outsource function from (https://scipy-lectures.org/packages/scikit-learn/index.html)\n",
        "2. aggergate above result from each model in new data frame called **df_result**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Ky_aecXZ7iOx",
        "colab": {}
      },
      "source": [
        "# 1. outsourcing code"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "HsEG_dpF2vEd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        },
        "outputId": "62019cc8-df3b-495f-b058-0b3e6d583ab0"
      },
      "source": [
        "from sklearn import model_selection\n",
        "from sklearn import metrics\n",
        "for Model in [GaussianNB, LogisticRegression,KNeighborsClassifier,Perceptron,svm.LinearSVC]:\n",
        "    clfsa = Model().fit(X_train, y_train)\n",
        "    y_pred_model = clfsa.predict(X_test)\n",
        "    print('%s: %s' % \n",
        "          (Model.__name__, metrics.f1_score(y_pred_model, y_test, average=\"macro\"))) "
      ],
      "execution_count": 128,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GaussianNB: 0.8940217391304348\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "LogisticRegression: 0.9607580077142379\n",
            "KNeighborsClassifier: 0.9095277547638774\n",
            "Perceptron: 0.9868131868131869\n",
            "LinearSVC: 0.9604395604395605\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "SLGIwNwK5Kjj",
        "colab": {}
      },
      "source": [
        "# 2. Create df_result to collect result on it:"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "WCY1pWBU9Dn8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "outputId": "d5101386-b8c8-4fd4-ed49-a0cf18ebcace"
      },
      "source": [
        "df_result"
      ],
      "execution_count": 129,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "      <th>KNN</th>\n",
              "      <th>Perceptron</th>\n",
              "      <th>SVM</th>\n",
              "      <th>Combination</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Accuracy</th>\n",
              "      <td>89.7436</td>\n",
              "      <td>96.1538</td>\n",
              "      <td>79.4872</td>\n",
              "      <td>98.7179</td>\n",
              "      <td>96.1538</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>89.3617</td>\n",
              "      <td>97.7273</td>\n",
              "      <td>100</td>\n",
              "      <td>97.8261</td>\n",
              "      <td>95.6522</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Precision</th>\n",
              "      <td>84.8485</td>\n",
              "      <td>96.9697</td>\n",
              "      <td>100</td>\n",
              "      <td>96.9697</td>\n",
              "      <td>93.9394</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>90.3226</td>\n",
              "      <td>94.1176</td>\n",
              "      <td>67.3469</td>\n",
              "      <td>100</td>\n",
              "      <td>96.875</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            NaiveBayes logistic regression  ...      SVM Combination\n",
              "Accuracy       89.7436             96.1538  ...  96.1538         NaN\n",
              "Specificity    89.3617             97.7273  ...  95.6522         NaN\n",
              "Precision      84.8485             96.9697  ...  93.9394         NaN\n",
              "Sensitivity    90.3226             94.1176  ...   96.875         NaN\n",
              "\n",
              "[4 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 129
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "izPFG5od9Tk7",
        "colab_type": "text"
      },
      "source": [
        "## Merging the three best together(SVM,Logistic regression and Perceptron):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MCcLjAwu9Sxu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.ensemble import VotingClassifier\n",
        "combo = VotingClassifier([('lsvc', svm.LinearSVC()),\n",
        "                            ('lr', linear_model.LogisticRegression()),\n",
        "                            ('per', Perceptron())])"
      ],
      "execution_count": 130,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZztljCjv_wJO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 703
        },
        "outputId": "cdbc3ffd-adb2-43fb-ac9a-61d457871f4a"
      },
      "source": [
        "combo.fit(X_train, y_train)\n",
        "y_pre_combo= combo.predict(X_test)"
      ],
      "execution_count": 131,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "VotingClassifier(estimators=[('lsvc',\n",
              "                              LinearSVC(C=1.0, class_weight=None, dual=True,\n",
              "                                        fit_intercept=True, intercept_scaling=1,\n",
              "                                        loss='squared_hinge', max_iter=1000,\n",
              "                                        multi_class='ovr', penalty='l2',\n",
              "                                        random_state=None, tol=0.0001,\n",
              "                                        verbose=0)),\n",
              "                             ('lr',\n",
              "                              LogisticRegression(C=1.0, class_weight=None,\n",
              "                                                 dual=False, fit_intercept=True,\n",
              "                                                 intercept_scaling=1,\n",
              "                                                 l1_ratio=None, max_iter=1...\n",
              "                                                 solver='lbfgs', tol=0.0001,\n",
              "                                                 verbose=0, warm_start=False)),\n",
              "                             ('per',\n",
              "                              Perceptron(alpha=0.0001, class_weight=None,\n",
              "                                         early_stopping=False, eta0=1.0,\n",
              "                                         fit_intercept=True, max_iter=1000,\n",
              "                                         n_iter_no_change=5, n_jobs=None,\n",
              "                                         penalty=None, random_state=0,\n",
              "                                         shuffle=True, tol=0.001,\n",
              "                                         validation_fraction=0.1, verbose=0,\n",
              "                                         warm_start=False))],\n",
              "                 flatten_transform=True, n_jobs=None, voting='hard',\n",
              "                 weights=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 131
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RRUXWs1_A4az",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "25735363-86a5-46fb-acdd-4c4cfc62f696"
      },
      "source": [
        "#Calculate Accuracy and Print it by method3\n",
        "accu_combo=(sum([1 for i, j in zip(y_pre_combo, y_test) if i == j])/len(y_test))*100\n",
        "print(\"Your Model Accuracy is\", accu_combo)"
      ],
      "execution_count": 132,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Your Model Accuracy is 97.43589743589743\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wl2qoHV9A5Es",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 57
        },
        "outputId": "4a1ae146-f8a5-4a43-e9f5-2ca0f837a58d"
      },
      "source": [
        "# confusion matrix\n",
        "os.chdir(home_path)\n",
        "from sklearn.metrics import confusion_matrix\n",
        "cm_combo=confusion_matrix(y_test,y_pre_combo)\n",
        "cm_combo"
      ],
      "execution_count": 133,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[32,  1],\n",
              "       [ 1, 44]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 133
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eTa-s76wA5Ap",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "c6265602-7421-4514-e501-1a03a2ab07e6"
      },
      "source": [
        "#Sensitivity_(True positive rate TPR) \n",
        "TP_combo= cm_combo[0][0]\n",
        "FN_combo= cm_combo[1][0]\n",
        "TPR_combo= TP_combo/(TP_combo+FN_combo) * 100\n",
        "print(\"Sensitivity \"+str(TPR_combo)+'%')"
      ],
      "execution_count": 134,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sensitivity 96.96969696969697%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pe3KSPr8BNR1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "e340f51a-96c0-4f32-a0a4-079cf679ccd1"
      },
      "source": [
        "#Specificity(Total Negative Result TNR)\n",
        "TN_combo= cm_combo[1][1]\n",
        "FP_combo= cm_combo[0][1] \n",
        "TNR_combo = TN_combo/(TN_combo+FP_combo) * 100\n",
        "print(\"Specificity \"+str(TNR_combo)+'%')"
      ],
      "execution_count": 135,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Specificity 97.77777777777777%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kBD5fg00A48o",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "290d8593-60de-43e9-9cd5-f881bb9f9ed4"
      },
      "source": [
        "percision_combo=(TP_combo/(TP_combo+FP_svm))*100\n",
        "percision_combo"
      ],
      "execution_count": 136,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "94.11764705882352"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 136
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uzx8qWqQBaM5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        },
        "outputId": "8996e9bb-1ed2-4fb3-adc6-307ee2951ee1"
      },
      "source": [
        "os.chdir(home_path)\n",
        "cm_picture6=sn.heatmap(pd.DataFrame(cm_combo),cmap=\"Blues\", annot=True)\n",
        "figure6 = cm_picture6.get_figure()   \n",
        "plt.title(\"Confusion Matrix COMBO (SVM+Logistic+Perceptron)\")\n",
        "figure6.savefig('/content/drive/My Drive/project/CGR_data/Confusion_combo.jpg', dpi=400)"
      ],
      "execution_count": 137,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Confusion Matrix COMBO (SVM+Logistic+Perceptron)')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 137
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWMAAAEICAYAAACK8ZV4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAdxUlEQVR4nO3de7wVZb3H8c93s7mYooiaImpWXkq7YKlHS9MwzWsQ2kU9ZmWHrOilL7M0LdOyjnZSs+xYFClp3srrITOVvLxQU1FJUVLJKEEEUTHMK/I7fzzPtsVy7XVhL/Yame+b17zY65mZZz2z1sxvfvPMZSkiMDOzzurqdAPMzMzB2MysEByMzcwKwMHYzKwAHIzNzArAwdjMrAA6GowlrSbp/yQ9I+k3fajnYEnXtrNtnSDp95IO7XQ7OknSLZK26XQ7+ouknSU9uILzln59aSdJ60uaJWlwRxoQEQ0H4CBgOvAsMB/4PbBTM/M2qPcQ4A6gu691rYwB2BUI4PKq8nfn8hubrOdE4PyV3NYRwKT8/SwB/gKcBKyexwv4KvAw8DzwD+C/gcEVdZybl2tMVd1n5PJP59efBl7J68OzwCPAF6rm2Qj4NfAk8K/8Pe/bYBn2A66peD0M+CXweF6mh4Bj87i/AJ+tUccRwPT894253e+umubyXL5ri5/xSv8eO7wOzcnrxrPAgrw+rNGJ5e3gZ/C/wJc7sXwNM2NJRwE/BL4HrA9skhs8ptG8TXgT8FBELG1DXSvLE8COktapKDuUFBjaQskKH6VIGg7cBqwG7BgRQ4HdScHsrXmyHwHjgU8BQ4G9gN2AS6qqeyhP01N3N/Bx4K9V090WEWtExBrA/sD3ezLa3J5pwEvA1sC6pIB+gaQD6izK4cB5Fa/PANYA3g6sBXwEmJ3HTa5sZ4VD8rjelmcdYEfS9/oakjaVNKdOG1/XJJ0o6cQ6k+yXv9P3ANsC32ix/j6ty33Rpvf+NfD5drSnZQ32EmuR9pIfqzPNYFKwfiwPPyRnW6TMci7wFWAhKWv7TB53EmljfTm/x2FU7fmATUkZTHd+/WlSFrYE+BtwcEX5tIr53gfcCTyT/39fxbgbge8At+R6rgXW7WXZetr/U+BLuWwAMA84gYrMGDgTeBT4J3AXsHMu37NqOf9c0Y7v5nY8D2yWyz6Xx58NXFpR/6nAVEA12nkycB/Q1ctybE7KZLevKt8YeBEYnV+fC/yAlBWtncv2JR0JTWP5zHhaVV13AAflv78DzKxuD3AM8PdelmFQ/hw2qiibCYztZZk2ApYCb6oo2yp/1utWfMYn5O9wQC6bkD/budTIjEnr3Jxe3vNEesnMSDuK+4HF+X3fXjHuPcA9eX37DXAxcHLlOlb1Gc3L0z5I2mHWW4c+VzHvfwGz8rwPAO/pZRlO7GUZ5gAfqnj9P8CU/PcOwK15+f5c+dlRe13eGrgOeCqvT8flabuAY0k79ydJycDwqu19PCmWzAeOXoHtaIW3f6AbeI6K9aq/hkZ7kR2BIaTDut4cn7+oUaTD9+1Zfm+6ASmojyQF3J9IWjsivkXKti+OlGFNqtcQSauTsru9ImV+7wNm1JhuOPC7PO06wOnA76oy24OAzwBvJAWBo+u9N/Ar/p1dfZgUJB6rmuZO0mcwHLgA+I2kIRFxTdVyvrtinkNIK95QUpCq9BXgnZI+LWln0md3aOQ1psqHgMsiYlkv7d+NtMHfUVkYEY8CfyJl0T1eAK4EPplffyovf68kbQdsQerKItd3aY32XEI6stqiRjWbA8siYm5F2Z+A70r6jKTNq9o+F7iB9Bn2OAS4OiIWVZQ9RgpMezS7PK2StAVwIXAksB5wNfB/kgZJGkTafs4lrRsXAh/tpZ4tSTuL7fI6/mHSjqHeOtQz78dIgfZTwJqkncOTfVimjYG9gXskjSRtUyfnZTgauFTSehWzVK7LC4DrgWuADUkBcmqe7svAWGCXPO5p4CdVb/9B0vqwB3CMpA+1sB0toQ/bf6Sj9NmkWNavGgXjdYBFUb8b4WDg2xGxMCKeIGW8lRvIy3n8yxFxNWmvtuUKtncZ8A5Jq0XE/Ii4v8Y0+wAPR8R5EbE0Ii4k9S/uVzHNORHxUEQ8TwoQo+q9aUTcCgzPG0vNjTkizo+IJ/N7nkY6Ymi0nOdGxP15nper6nuO9DmeDpxP6seaW6sS0vc0v877rFtn/Pw8vtKvgE9JGkbaaK6oMd8OkhZLWkLKis8j9UfXe7/5FeOrDSNtSJW+TDpsnAA8IGm2pL0qxk8mr2v58PRglu+iqF6etwHDIuK2GtP0xSeA30XEdfl7/AGpy+h9pESlG/hR3gYuI31etbxCWm+2kjQwIuZERHX3UG8+B3w/Iu6MZHZEVO/gm3GFpMWkI6GbSAHwP0k7uasjYllEXEfa8e5dMd+r6zLpaOrxiDgtIl6IiCURcXue7nDg+IiYGxEvknYgB+TusB4nRcS/IuI+4BzgwAZtrnzvPej79r+EtD72q0bB+Elg3aoPqtqGLJ/V/T2XvVpHVTB/jtQP2JKI+BdppT8cmC/pd3njatSenjaNrHj9+Aq05zxSUPggNY4UJB2dz8Q+k1fmtagddCo9Wm9kXoEfIZ18q+7brfQk6QRebxbVGT8ij69832mkDO940mHq8zXm+1NEDMsZ3Aakw9LvNXi/ERXjqz1Nymwq2/F8RHwvIt5L2uFcQjriGJ4nuQwYIWkH0uH+G0hZUbXLgNGk7++86pGSDso7lsXAvcAmPa/zsEmNOistt87lI4JHSevchsC8qiOamt97RMwmZdcnAgslXSRpw1rT1rAxr+3XB0DSlIrlOxY4tmLZplRNPjZ/r2+KiC/m7/5NwMcqPxNgJ5b/jiuXqde25Lour6hnFmkntH4vdVXHk1oqp2/H9j+U1B3TrxoF49tIfYpj60zzGOkD7rEJrz2Eb9a/SBtUjw0qR0bEHyJid9JK8Bfg5020p6dN81awTT3OA75IyhCeqxyRuxG+RjrRtXZEDCP1V6mn6b3UWfeReZK+RMqUHsv19+Z64KN1Tl78EdhY0vZV9W9Mytym1pjnfFJXScND+ohYAFzKv7OP64FxNdrzcdKGU+vk5+zUJI2sMY6I+Ccp2K8OvDmXPQf8lnS0cghwUUS8VGPe50j93l+gRjCOiAtyABoGvAv4R8/rPPyj7gdQtc5JEikgzSMdDYzMZT027q2i3Jadcn1BOlcADdYV0uf61lojImLfiuU7BTilYtn2bVBvT93nVX0mq0fEKZVvUzX9W+rUtVdVXUMionL7rPx8KuNJM9tRn7b/nHhuRuoX71d1g3FEPEM6AfITSWMlvUHSQEl7Sfp+nuxC4BuS1pO0bp7+/BVszwzgA5I2kbQW8PWeEfkawDG57/hFUndHrT7Sq4EtcrbTLekTpBM71RlASyLib6RD9uNrjB5KOpn0BNAt6QRSv12PBcCmrZzpzf2QJ5MOEQ8Bviapt+6U0/P7TZb0pjz/SEmnS3pXRDxEOgn5a0k7SBogaWtSAL0+Iq6vUeePSH2/NzfR1nVI/aA93UZnkI4MJknaQNIQSQeSPruvVmWJAOQgej3pM+6p95uStst9r0NIl60tJp3Y6jGZdMS0P7W7KHocB+wSEXMaLU8DXXl5eobBpIx9H0m7SRpI2om9SDrhdRsp85uQ18cxpPMqryFpS0mjc50vkE5I9azjjdahXwBHS3pvvqpgs551oQ3OB/aT9OG87gyRtKukjXqZfgrpiOVISYMlDZX0H3ncT0nnAXrW0/XyZ1LpmznWbE3q2704lzezHfV1+9+e1E+/Il08fdIwOOT+z6NIJ+WeIO3ZJvDvfsSTSf1H95LO6N+dy1qW+6IuznXdxfIfYFdux2OkM7S7kDKd6jqeJPVZfYV0+P410vWttQ6NW23ftIiolfX/gXSy4iHSIdELLH/o1HNDy5OS7m70PnnvfD5wakT8OSIeJgWT81TjgvSIeIrUP/kycHvux51Kys57LgWbQNpgzyftyK4hnVnev5dlfSoiptYKnNmOkp6V9CzpUPMJUh9vz3ewE+nk7wOk7+Eo4JCIuLiX+gB+xvLnG4LUZ7iI9L3vDuwTEc9WTHNzXs65EXFnbxVHxGO5+6WvDiQFyZ7hrxHxIGmn+ePc1v1Il4i9lHcy40gnYBfn6aaQgnW1waTMdRHpUPqN/DshqbsORcRvSFcVXEDq87yCdLKtzyKd6B1DWgd7YsBX6SV+RMQS0ne1X16Oh0nde5CuOroKuDavp38C/qOqiptI6+1U4AcR0XNDV8PtqA3b/8GkHUa/U+/bmln/k3QLMCEi7ul0W1YWSbcDP42IczrdliKRtCnpktWB0YF7DyS9kbQj2CYiXujv9693Ys6s30XE+zvdhnaTtAupa2URKfN6F+nIxAokIhaSbjDqCAdjs5VvS1K/8uqkq2MOiIh6lyJaCbmbwsysAPwITTOzAihEN8VWx13r9Nxe4+5v79F4IiudId2o8VT1rbbNhKZjzvP3nNXn92uGM2MzswIoRGZsZtavOvOUz7ocjM2sfLoGdLoFr+FgbGblo37pBm6Jg7GZlY+7KczMCsCZsZlZATgzNjMrAGfGZmYF4KspzMwKwN0UZmYF4G4KM7MCcGZsZlYADsZmZgUwwCfwzMw6z33GZmYFUMBuiuK1yMxsZZOaH5qqTgMk3SNpSn79Zkm3S5ot6WJJgxrV4WBsZuWjruaH5hwBzKp4fSpwRkRsBjwNHNaoAgdjMyufNmbGkjYC9gF+kV8LGA38Nk8yGRjbqB73GZtZ+bT3dugfAl8DhubX6wCLI2Jpfj0XGNmwSe1skZnZ60IL3RSSxkuaXjGMf7UaaV9gYUTc1dcmOTM2s/Jp4dK2iJgITOxl9PuBj0jaGxgCrAmcCQyT1J2z442AeY3ex5mxmZVPm07gRcTXI2KjiNgU+CTwx4g4GLgBOCBPdihwZaMmORibWfm0/2qKascAR0maTepDntRoBndTmFn5rITnGUfEjcCN+e9HgO1bmd/B2MzKx7dDm5kVQAFvh3YwNrPycWZsZtZ5cjA2M+s8B2MzswJQl4OxmVnHOTM2MysAB2MzswJwMDYzK4LixWIHYzMrH2fGZmYF0NXlO/DMzDrOmbGZWREULxY7GJtZ+TgzNjMrAAdjM7MCKOLt0MU7pWhmtpJJanpooq4hku6Q9GdJ90s6KZefK+lvkmbkYVS9epwZm1nptLmb4kVgdEQ8K2kgME3S7/O4r0bEb5upxMHYzEqnncE4IgJ4Nr8cmIdotR53U5hZ6bTSTSFpvKTpFcP4GvUNkDQDWAhcFxG351HflXSvpDMkDa7XJmfGZlY+LSTGETERmNhgmleAUZKGAZdLegfwdeBxYFCe/xjg273V4czYzEqnq6ur6aEVEbEYuAHYMyLmR/IicA6wfd02rfDSmJm9TrX5aor1ckaMpNWA3YG/SBqRywSMBWbWq8fdFGZWPu29zHgEMFnSAFKCe0lETJH0R0nr5XebARxerxIH4w4a1N3Fr/5rOwZ1d9HdJa6duYCzpv6V73/8nWw9ck2WLgvue/QZTrziAZYua/nkrK0CTvjG17n5phsZPnwdLrtySqebs8po89UU9wLb1Cgf3Uo97qbooJeWLuOzk6Yz7se3Me7Ht7HTFuvyro3XYsqM+exzxi2MOfNWBg/sYv/tRna6qdYhY8aO4+yf/aLTzVjltLObol3akhlLehswBuiJGvOAqyJiVjvqX5U999IrAHQPEN1dgoCbH1r06vj75j7DBmsO6VTzrMPeu+12zJs3t9PNWOUU8dkUfc6MJR0DXETqF7kjDwIulHRsX+tf1XUJLpuwA9OO25VbZz/JvXOfeXVcd5f4yKgNmfbwojo1mFmr1KWmh/7Sjsz4MGDriHi5slDS6cD9wCm1ZsoXTo8H2GDPI1h7m73b0JTXn2UB4876E0OHdPOj/xzFZuuvwewF6Waeb455O9PnPM1dcxZ3uJVmq5ZVMjMGlgEb1igfkcfVFBETI2LbiNi2rIG40pIXlnLHI0+x8+brAPDF0W9h+OqDOPXqBzvcMrNVz6raZ3wkMFXSw8CjuWwTYDNgQhvqX2WtvfpAlr4SLHlhKYO7u3jfZuvwi5v+xv7bjuT9m6/LZydNJ3wRhVnbFTAx7nswjohrJG1Buruk8gTenfkWQevFekMH898HvIMuia4ucc19j3PTg4u49zsf4rHFL3Dh4emGneseWMjZf3ykw621Tjjm6KOYfucdLF78NLuP/gBf+NKXGbf/xzrdrNe9InZTKAqQem113LWdb4QVzt3f3qPTTbACGtLd91s2tjzmD03HnAdP/XC/RG7f9GFmpVPAxNjB2MzKp6uAP7vkYGxmpePM2MysAIp4As/B2MxKp4Cx2MHYzMqn1YfG9wcHYzMrHWfGZmYFUMQ+4+Ll6mZmK5nU/NC4Lg2RdIekP0u6X9JJufzNkm6XNFvSxZIG1avHwdjMSqfNDwp6ERgdEe8GRgF7StoBOBU4IyI2A54mPeGyVw7GZlY67cyM8y9AP5tfDsxDAKOB3+byyaQfJe2Vg7GZlU5Xl5oeJI2XNL1iGF9dn6QBkmYAC4HrgL8CiyNiaZ5kLv9+kFpNPoFnZqXTygm8iJgITGwwzSvAKEnDgMuBt7XaJgdjMyudlXUxRUQslnQDsCMwTFJ3zo43Ij1auFfupjCz0mnnCTxJ6+WMGEmrAbsDs4AbgAPyZIcCV9arx5mxmZVOmzPjEcBkSQNICe4lETFF0gPARZJOBu4BJtWrxMHYzEqnnY/QjIh7gW1qlD9C+gWkpjgYm1npFPEOPAdjMysdB2MzswIoYCx2MDaz8nFmbGZWAAWMxQ7GZlY+/kFSM7MC6CpgauxgbGalU8BY7GBsZuXjE3hmZgVQwC5jB2MzKx+fwDMzKwDhYGxm1nEFTIwdjM2sfHwCz8ysAAoYix2Mzax8injTh392ycxKp5Vfh25E0saSbpD0gKT7JR2Ry0+UNE/SjDzsXa8eZ8ZmVjptToyXAl+JiLslDQXuknRdHndGRPygmUocjM2sdNrZTRER84H5+e8lkmYBI1tuU9taZGb2OqFWBmm8pOkVw/he65U2Jf0e3u25aIKkeyX9UtLa9drkYGxmpSOp6SEiJkbEthXDxF7qXAO4FDgyIv4JnA28FRhFypxPq9cmd1OYWem0+6YPSQNJgfjXEXEZQEQsqBj/c2BKvTocjM2sdNr5bAqlO0gmAbMi4vSK8hG5Pxngo8DMevU4GJtZ6bT5Drz3A4cA90makcuOAw6UNAoIYA7w+XqVOBibWem0s5siIqZBzScPXd1KPQ7GZlY6fjaFmVkBFC8UOxibWQkNKOAzNB2Mzax03E1hZlYABYzFDsZmVj5FfISmg7GZlU4BY3ExgvHd396j002wAlp7uwmdboIV0PP3nNXnOtxnbGZWAAMcjM3MOq+AV7Y5GJtZ+TgYm5kVgPuMzcwKwJmxmVkBFDAxdjA2s/LpLmA0djA2s9IpYCx2MDaz8ini7dD+dWgzKx2p+aFxXdpY0g2SHpB0v6QjcvlwSddJejj/v3a9ehyMzax0utT80ISlwFciYitgB+BLkrYCjgWmRsTmwNT8ulfupjCz0mnnw+XzL0DPz38vkTQLGAmMAXbNk00GbgSO6a0eZ8ZmVjqtZMaSxkuaXjGM761eSZsC2wC3A+vnQA3wOLB+vTY5Mzaz0lELv4IXEROBiQ3rlNYALgWOjIh/Vt7lFxEhKerN72BsZqXT7jvwJA0kBeJfR8RluXiBpBERMV/SCGBh3Ta1t0lmZsXXzhN4SinwJGBWRJxeMeoq4ND896HAlfXqcWZsZqXT5gcFvR84BLhP0oxcdhxwCnCJpMOAvwMfr1eJg7GZlc6ANvYJRMQ06LUTerdm63EwNrPSKeIdeA7GZlY6foSmmVkBFDAxdjA2s/LpauE64/7iYGxmpePM2MysALoL2GnsYGxmpePM2MysAHxpm5lZARQwFjsYm1n5FPGhPA7GZlY67qYwMysAB2MzswIoXih2MDazEipgYuxgbGbl0+bnGbeFg7GZlU4Rr6YoYpvMzFaqLqnpoRFJv5S0UNLMirITJc2TNCMPezdsUx+XyczsdUdS00MTzgX2rFF+RkSMysPVjSpxN4WZlU47s9CIuFnSpn2tx5mxmZVOK5mxpPGSplcM45t8mwmS7s3dGGs3mtjB2MxKRy0METExIratGCY28RZnA28FRgHzgdMazeBuCjMrnQEr+dK2iFjQ87eknwNTGs3jzNjMSkdqflix+jWi4uVHgZm9TdvDmbGZlY7aeEO0pAuBXYF1Jc0FvgXsKmkUEMAc4PON6nEwNrPSaWcvRUQcWKN4Uqv1OBibWen416HNzAqggI+mcDA2s/Lx84zNzAqgq3ix2MHYzMqnnVdTtIuDsZmVTgF7KXzTR1Gc8I2vs+vOOzJuzL6dbooVRFeXuO3CY7j0zMOXKz/tawfwxC0N7661OtTCv/7iYFwQY8aO4+yf/aLTzbACmXDQB3nwbwuWK3vPVpswbOgbOtSiVUeXmh/6rU3991ZWz3u33Y4111qr082wghj5xmHsudPWnHP5ra+WdXWJ7x05luPPvKKDLVs1tPPh8m1rU7+9k5k17X++uj/Hn3kFy5bFq2Vf+MQu/O6m+3h80T872LJVQytPbesvKz0YS/pML+WvPiN00s+beSKdWTnstfM7WPjUEu6Z9eirZSPWW4txu2/D/150UwdbtuooYmbcH1dTnAScU12Ynwk6EeCFpUT1eLOy2nHUW9h3l3ey505bM3jQQNZcfQh3/fZ4XnxpKfdf9S0A3jBkIDOv/BbvGHNSh1v7+lTAiynaE4wl3dvbKGD9dryHWVmc8OOrOOHHVwGw83s358hP7cb+R/x0uWmeuOU0B+K+KGA0bldmvD7wYeDpqnIBt752cqt2zNFHMf3OO1i8+Gl2H/0BvvClLzNu/491ullmq6Qi3g6tiL73EEiaBJwTEdNqjLsgIg6qN7+7KayWtbeb0OkmWAE9f89ZfY6kdz7yTNMxZ7u3rNUvkbstmXFEHFZnXN1AbGbW74qXGPvSNjMrn3begZd//XmhpJkVZcMlXSfp4fy/fx3azKxam38D71xgz6qyY4GpEbE5MDW/rsvB2MxKp503fUTEzcBTVcVjgMn578nA2Eb1OBibWelIamV49Qa1PIxv4i3Wj4j5+e/HaeISXz9C08xKp5Ur2ypvUFsRERGSGl694czYzEqnH55NsUDSCID8/8JGMzgYm1n5rPxofBVwaP77UODKRjM4GJtZ6bT50rYLgduALSXNlXQYcAqwu6SHgQ/l13W5z9jMSqedd0NHxIG9jNqtlXocjM2sdAr4aAoHYzMrH/86tJlZATgzNjMrgALGYgdjMyuhAkZjB2MzK50iPlzewdjMSqd4odjB2MzKqIDR2MHYzErHl7aZmRVAAbuMHYzNrHwKGIsdjM2sfFTA1NjB2MxKp4Cx2MHYzMqngLHYwdjMSqiA0djB2MxKx5e2mZkVQLv7jCXNAZYArwBLI2LbVutwMDaz0ulaOYnxByNi0YrO7GBsZiVUvG4K/yCpmZWO1PzQpACulXSXpPEr0iZnxmZWOq3kxTm4VgbYiRExsWqynSJinqQ3AtdJ+ktE3NxKmxyMzax0WjmBlwNvdfCtnmZe/n+hpMuB7YGWgrG7KcysdCQ1PTRR1+qShvb8DewBzGy1Tc6Mzax02nz6bn3g8hy4u4ELIuKaVitxMDaz0mnndcYR8Qjw7r7W42BsZqXjO/DMzIqgeLHYwdjMyqeAsdjB2MzKp6uADzR2MDaz0ilgLPZ1xmZmReDM2MxKp4iZsYOxmZWOL20zMysAZ8ZmZgXgYGxmVgDupjAzKwBnxmZmBVDAWOxgbGYlVMBo7GBsZqVTxNuhFRGdboNVkDS+xu9rWcl5vVj1+Xbo4lmhX5a1VZ7Xi1Wcg7GZWQE4GJuZFYCDcfG4X9Bq8XqxivMJPDOzAnBmbGZWAA7GZmYF4GBcIJL2lPSgpNmSju10e6zzJP1S0kJJMzvdFlu5HIwLQtIA4CfAXsBWwIGStupsq6wAzgX27HQjbOVzMC6O7YHZEfFIRLwEXASM6XCbrMMi4mbgqU63w1Y+B+PiGAk8WvF6bi4zsxJwMDYzKwAH4+KYB2xc8XqjXGZmJeBgXBx3AptLerOkQcAngas63CYz6ycOxgUREUuBCcAfgFnAJRFxf2dbZZ0m6ULgNmBLSXMlHdbpNtnK4duhzcwKwJmxmVkBOBibmRWAg7GZWQE4GJuZFYCDsZlZATgYm5kVgIOxmVkB/D+yeIVXJB8XLgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CPJtBAnbBlRB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 602
        },
        "outputId": "66247409-e4e4-4ea6-ae76-d683b8b37e4c"
      },
      "source": [
        "# calculating the mean R^2 for a set of predictions:\n",
        "results_combo = model_selection.cross_val_score(combo, X_train, y_train,cv=None)*100\n",
        "results_combo"
      ],
      "execution_count": 138,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/svm/_base.py:947: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
            "  \"the number of iterations.\", ConvergenceWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 96.82539683, 100.        , 100.        ,  96.77419355,\n",
              "        96.77419355])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 138
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2offpAx23OJm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "5a7c80ea-627b-4c04-861d-988a5a4d8b52"
      },
      "source": [
        "acc_cross_validation['Combination']=results_svm\n",
        "acc_cross_validation"
      ],
      "execution_count": 139,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "      <th>KNN</th>\n",
              "      <th>Perceptron</th>\n",
              "      <th>SVM</th>\n",
              "      <th>Combination</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>85.714286</td>\n",
              "      <td>96.825397</td>\n",
              "      <td>66.666667</td>\n",
              "      <td>96.825397</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>93.650794</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>69.841270</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>83.870968</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>70.967742</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>85.483871</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>67.741935</td>\n",
              "      <td>98.387097</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>96.774194</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>91.935484</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>74.193548</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>96.774194</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   NaiveBayes  logistic regression  ...         SVM  Combination\n",
              "0   85.714286            96.825397  ...  100.000000   100.000000\n",
              "1   93.650794           100.000000  ...  100.000000   100.000000\n",
              "2   83.870968           100.000000  ...  100.000000   100.000000\n",
              "3   85.483871            96.774194  ...   96.774194    96.774194\n",
              "4   91.935484            96.774194  ...   96.774194    96.774194\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 139
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gWuotYkLDKXP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "outputId": "c852ad50-afb2-4f06-d536-07bfc83f5e6f"
      },
      "source": [
        "df_result['Combination']=pd.Series({'Accuracy':accu_combo,'Cross_Validation':results_combo,'Confusion_Matrix':cm_combo,'Sensitivity':TPR_combo,'Specificity':TNR_combo,'Precision':percision_combo})\n",
        "df_result"
      ],
      "execution_count": 140,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "      <th>KNN</th>\n",
              "      <th>Perceptron</th>\n",
              "      <th>SVM</th>\n",
              "      <th>Combination</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Accuracy</th>\n",
              "      <td>89.7436</td>\n",
              "      <td>96.1538</td>\n",
              "      <td>79.4872</td>\n",
              "      <td>98.7179</td>\n",
              "      <td>96.1538</td>\n",
              "      <td>97.4359</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>89.3617</td>\n",
              "      <td>97.7273</td>\n",
              "      <td>100</td>\n",
              "      <td>97.8261</td>\n",
              "      <td>95.6522</td>\n",
              "      <td>97.7778</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Precision</th>\n",
              "      <td>84.8485</td>\n",
              "      <td>96.9697</td>\n",
              "      <td>100</td>\n",
              "      <td>96.9697</td>\n",
              "      <td>93.9394</td>\n",
              "      <td>94.1176</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>90.3226</td>\n",
              "      <td>94.1176</td>\n",
              "      <td>67.3469</td>\n",
              "      <td>100</td>\n",
              "      <td>96.875</td>\n",
              "      <td>96.9697</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            NaiveBayes logistic regression  ...      SVM Combination\n",
              "Accuracy       89.7436             96.1538  ...  96.1538     97.4359\n",
              "Specificity    89.3617             97.7273  ...  95.6522     97.7778\n",
              "Precision      84.8485             96.9697  ...  93.9394     94.1176\n",
              "Sensitivity    90.3226             94.1176  ...   96.875     96.9697\n",
              "\n",
              "[4 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 140
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xHOZZ3MxLgHg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 490
        },
        "outputId": "33b383db-9f9c-4718-e6b2-4e92357b2c1c"
      },
      "source": [
        "# bar plot of cross validation for each Models\n",
        "index=['first fold', 'second fold', 'third fold', 'fourth fold', 'fifth fold']\n",
        "df_cross=acc_cross_validation.copy()\n",
        "df_cross.index=index\n",
        "df_cross\n",
        "axx = df_cross.plot.bar(rot=0)\n",
        "figure7 = axx.get_figure()   \n",
        "plt.title(\"Cross Validatio summary for data without FS\")\n",
        "figure7.savefig('/content/drive/My Drive/project/CGR_data/Cross_Validatio_summary_Without.jpg', dpi=400)"
      ],
      "execution_count": 148,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NaiveBayes</th>\n",
              "      <th>logistic regression</th>\n",
              "      <th>KNN</th>\n",
              "      <th>Perceptron</th>\n",
              "      <th>SVM</th>\n",
              "      <th>Combination</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>first fold</th>\n",
              "      <td>85.714286</td>\n",
              "      <td>96.825397</td>\n",
              "      <td>66.666667</td>\n",
              "      <td>96.825397</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>second fold</th>\n",
              "      <td>93.650794</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>69.841270</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>third fold</th>\n",
              "      <td>83.870968</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>70.967742</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>fourth fold</th>\n",
              "      <td>85.483871</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>67.741935</td>\n",
              "      <td>98.387097</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>96.774194</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>fifth fold</th>\n",
              "      <td>91.935484</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>74.193548</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>96.774194</td>\n",
              "      <td>96.774194</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             NaiveBayes  logistic regression  ...         SVM  Combination\n",
              "first fold    85.714286            96.825397  ...  100.000000   100.000000\n",
              "second fold   93.650794           100.000000  ...  100.000000   100.000000\n",
              "third fold    83.870968           100.000000  ...  100.000000   100.000000\n",
              "fourth fold   85.483871            96.774194  ...   96.774194    96.774194\n",
              "fifth fold    91.935484            96.774194  ...   96.774194    96.774194\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 148
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Cross Validatio summary for data without FS')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 148
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3gU5fbA8e8hhACC1KAoKEVAkECA0BUDSFGRXm2AhauCitiwXVD5qVyxIVxRBEHNpRcRBRUVkG7oVQVEiYUmNYCS5Pz+mMmyJJuwyQYCw/k8T55Medu8O3t29p3ZGVFVjDHGeEue3G6AMcaYnGfB3RhjPMiCuzHGeJAFd2OM8SAL7sYY40EW3I0xxoMsuHuYiKiIXOVOjxKR54JJm416bhORL7PbzguNiFQRkTUiclhEHjoD5ceKSEJOlxuK0+0j52Kbz3cW3NMQkVtFJF5EjojIHyIyR0SuzaW2zBWRFwIsbycif4pI3mDLUtX7VPXFHGhTOfeDwFe3qsapastQy76APAF8q6qFVXV4bjZERHqJyKIzXU/afSSUg4msEpH5InJPJutT9+kjfn9r3XVlRGSaiOwVkYMiskFEep2NdofKgrsfERkAvAm8BFwCXAH8F2iXQfqgg2s2jQduFxFJs/wOIE5Vk85w/eY0srkPXAlsPIv1meAUVdVC7l9Nd9lHwE6c16wEzntvV241MEtU1f6cX+kWAY4AXTJJMxiYCnwMHALuAS4DZgF/AVuBe/3S1wPi3bS7gNfd5fndMvYBB4DvgUsC1FcAOAg08VtWDDgO1HTLX+qW8QcwAsjnl1aBq9zpccAQv3WPu3l+B+5Kk/ZmYLXb7p3AYL98v7ppj7h/DYFewCK/NI3cbTro/m+USZ8+CfwGHAZ+AJpn0N5YIMFvfoe7DeuARGAMzgfyHLeseUAxN205t8293e3ZD9wH1HXzHwBG+JVdEfjGfX32AnE4b3z/up908/7ttmNamu0aDrwVYHu/AZLd1/AIUBln3/sQ2AP8AjwL5HHT9wIWA2+47RkSoMwCbn/tBza57fHvq4HANrdfNgEd3OVV3XYku205cLrXP0DdC4BO7nRjt59vduebA2v8tmORO73QTZfo1tst9fUFHgV24+ybvdO8PzPqo8HAx35pU1/vvMD/penvEQG2wZc+wLojQHRux6dsxbTcbsC58ge0BpICvcB+aQYDJ4D2ON96Crg76n9xAna0u/M1c9MvBe5wpwsBDdzpfwGfAgWBMKAOcHEGdY4G3veb/5ffG6YO0MDdicsBm4H+fmkDBnd3W3cB1YGLgP+lSRsLRLnbWMNN295dl+6NkOaNWxwnyNzhtquHO18iwLZVwQkel/mVXTFte/3alDa4L8MJ6JfjBIRVQC33tfgGGJSmzaPcdS1x3uwzgVJ++a93018FtAAigEj3NX4zTd1rgLLuPlAaJ1AVddfndcurk8FrOh+4x2/+Q+AToLDb1h+Bu/36Ngl40C23QIDyXgG+c/u+LLAhTV91wTkIyYMTSBOB0mlfuzR9HfD1D1D3C8Db7vTTOB8iQ/3WvRWoHvz2N786k9w84cBNwFFOfkBn1keDySC4B+rvANtwSvo06+bhfLh2B67I7TiVlT8bljmpBLBXTz/UsVRVZ6pqClAS52jlSVU9rqprgPeBO920J4CrRKSkqh5R1WV+y0vg7NzJqrpSVQ9lUN94oLOI5Hfn73SX4eZbpqpJqroDeBe4Poht7Qp8oKobVDUR583ho6rzVXW9qqao6jpgQpDlgnPU95OqfuS2awKwBbglQNpknABaTUTCVXWHqm4Lsh5wgsouVf0NJ7gtV9XVqnocmIET6P296L5OX+IEuAmqutsvfy13+7eq6leq+req7gFeD7D9w1V1p6oeU9U/cD4AurjrWuPsSytPtwEiEoYTOJ5S1cPu6/gazodjqt9V9W23P48FKKYr8H+q+peq7sT51uCjqlNU9Xf39ZwE/ITzrS+gLL7+C/zWNQFe9pu/3l0frBPAC6p6QlU/xzlqrhJkH+WEvSJywP17zF3WBWffeA742T0RXjeH6z0jLLiftA8oGcSY5k6/6cuAv1T1sN+yX3COBAHuxvnavUVEvheRNu7yj4AvgIki8ruI/EdEwgNVpqqLcIYG2otIRZw35f8ARKSyiMx2T64ewjlXUDKIbb0szXb84r9SROqLyLciskdEDuIMYQRTbmrZv6RZ5t8nPqq6FeiP8+GyW0QmishlQdYDp459HgswXyg76UXkErctv7n9+jHpt39nmvnxwO3u9O04r3EwSuIcqfr3Wdr+SltXWqd7Pe90g9IBETmA840tw9czi6//UqCyiFyC8831Q6CsiJTE2VcXnqbt/valObg6ivOaBNNHOaGkqhZ1/4YBqOp+VR2oqtfgfEtcA8wMcB7snGPB/aSlOOOn7U+Tzv82mr8DxUWksN+yK3DGkFHVn1S1B85X/6HAVBG5yD0yeV5Vq+GMT7fh5NF+IB+6628HvlDV1KD0Ds5RcSVVvRjna3EwO90fOF/f/dvs73845xHKqmoRnOGM1HJPdxvR33FOPvnz9Ulaqvo/Vb3WzaM4/QTOkXVBv6SXnqbenPSS25Yot19vJ32/pu2HmUANEamO83rGBVnXXpwjVv8+S9tfp+vzDF9PEbkSZ2ivH87QWFGcYZvMXs/MXv9TqOpRYCXwMLBBVf8BlgADgG2quvc0bQ/G6frodPtKjtz61t2WYTgfpsVzoswzyYK7S1UPAv8GRopIexEpKCLhInKjiPwngzw7cXbkl0Ukv4jUwDla/xhARG4XkUh3COeAmy1FRJqKSJT7dfMQzo6bkknzPgRuAO7FHZJxFXbzHxGRq4H7g9zcyUAvEakmIgWBQWnWF8b5RnJcROoBt/qt2+O2tUIGZX+OcyR3q4jkFZFuQDVgdtqE7vXezUQkAmcM/Bgn+2ENcJOIFBeRS3GO8M+WwjhDAgdF5HKcE5SZcoeCpuIExhWq+mswFalqMs7r8X8iUtgNxgNw96EgTQaeEpFiIlIGZ3w+1UU4wW0PgIj0xjlyT7ULKCMi+fyWZfb6B7IA58MjdQhmfpr5QHaR8T50iiD6aA3QRESuEJEiwFPZrSstERkqItXdfbkwzntsq6ruy055Z5MFdz+q+hrOTvMszpthJ85OOjOTbD1wTsj8jjPOO0hV57nrWgMbReQI8BbQ3R0zvRQnEBzCOQm6gEy+xrtjjEtw3qiz/FY9hvPGO4xzdDYpyO2cg3PJ5zc4V/h8kybJA8ALInIY5wNvsl/eozhXICx2v+Y3SFP2Ppwj10dxhrqeANpkcAQXgXMycC/wJ843nNQ35kfAWpyTl18Gu2055HmgNs7VPp8B04PMNx7nRGSwQzKpHsQ5+twOLML5gBibhfzP4wxT/IzTV776VXUTzvj0UpwgF4VzgjDVNziXZf4pIqmvUYavfwYW4HwgLMxgPpDBwHh3H+p6mvIhkz5S1a9w9o91ON8i0h5IvIVz3mq/iGT1dwUFcd7XB9y6rwTaZrGMXCGq9rAOY3KCiFyBM0x2aSYnyI05K+zI3ZgcICJ5cL71TbTAbs4F9ms3Y0IkIhfhDHn8gjMUZ0yus2EZY4zxIBuWMcYYDzonhmVKliyp5cqVy+1mGGPMeWXlypV7VTUy0LpzIriXK1eO+Pj43G6GMcacV0Qk7a/BfWxYxhhjPMiCuzHGeJAFd2OM8aBzYszdGBOcEydOkJCQwPHjx3O7KeYsyp8/P2XKlCE8PODNYwOy4G7MeSQhIYHChQtTrlw5zoO7zpocoKrs27ePhIQEypcvH3Q+G5Yx5jxy/PhxSpQoYYH9AiIilChRIsvf1k4b3EVkrIjsFpENfsuKi8hXIvKT+7+Yu1xEZLiIbBWRdSJSO8tbYozJlAX2C092XvNgjtzHkf5+GQOBr1W1EvC1Ow9wI1DJ/euD8zAJY4wxZ9lpx9xVdaGIlEuzuB3OA23BuYf1fJynwbcDPlTnhjXLRKSoiJR2nzFpjMlh5QZ+lqPl7Xjl5tOmEREGDBjAa6+9BsCwYcM4cuQIgwcPzjDPrFmz2LRpEwMHDswwTUYGDx7M6NGjiYyM5Pjx4zRt2pSRI0eSJ4+NKmcmuydUL/EL2H/iPFsQnGca+j/LMcFdli64i0gfnKN7rrgi7VPezh0j70v7HAvH8f2vp1v26KR0Dxs6dwwuksHygwEXb766arpl38SODJjWy30RqB/AQ32RDREREUyfPp2nnnqKkiWDe7Ru27Ztads2+8+4eOSRR3jsscdISUmhSZMmLFiwgKZNm2a7vAtByB997lF6lm8tqarvqWqMqsZERga8NYIx5hyUN29e+vTpwxtvvJFu3aeffkr9+vWpVasWN9xwA7t2OY/7HTduHP369ePgwYNceeWVpKQ4T1NMTEykbNmynDhxgm3bttG6dWvq1KnDddddx5YtW9KV/88//3D8+HGKFSsGwOjRo6lbty41a9akU6dOHD16lMOHD1O+fHlOnDgBwKFDh3zzGdUxZcoUqlevTs2aNWnSpMkZ6bezLbvBfZeIlAZw/+92l//GqQ/qLUMGD0Y2xpy/+vbtS1xcHAcPnvpt59prr2XZsmWsXr2a7t2785//nPr44SJFihAdHc2CBc7jVWfPnk2rVq0IDw+nT58+vP3226xcuZJhw4bxwAMP+PK98cYbREdHU7p0aSpXrkx0dDQAHTt25Pvvv2ft2rVUrVqVMWPGULhwYWJjY/nsM2fIauLEiXTs2DHTOl544QW++OIL1q5dy6xZs/CC7Ab3WUBPd7on8Inf8jvdq2YaAAdtvN0Ybzi2YQPHNmyAlBTCf/2VHq1b89ozz5ySJiEhgVatWhEVFcWrr77Kxo0b05XTrVs3Jk1yHok7ceJEunXrxpEjR1iyZAldunQhOjqaf/3rX/zxx8nQ8cgjj7BmzRp2795NYmIiEydOBGDDhg1cd911REVFERcX56vvnnvu4YMPPgDggw8+oHfv3pnW0bhxY3r16sXo0aNJTk7O+c7LBcFcCjkB5+G6VUQkQUTuxnmocQsR+Qm4wZ0H58n323Eeujwa50G7xhgP6nfHHYyfMYPExETfsgcffJB+/fqxfv163n333YDXZrdt25a5c+fy119/sXLlSpo1a0ZKSgpFixZlzZo1vr/NmzenyxseHk7r1q1ZuNB59navXr0YMWIE69evZ9CgQb76GjduzI4dO5g/fz7JyclUr1490zpGjRrFkCFD2LlzJ3Xq1GHfvn1nosvOqtMGd1XtoaqlVTVcVcuo6hhV3aeqzVW1kqreoKp/uWlVVfuqakVVjVJVu4+vMR5VvEgROrVqxZgxY3zLDh48yOWXXw7A+PHjA+YrVKgQdevW5eGHH6ZNmzaEhYVx8cUXU758eaZMmQI4v8pcu3ZturyqyuLFi6lYsSIAhw8fpnTp0pw4cYK4uLhT0t55553ceuut9O7dGyDTOrZt20b9+vV54YUXiIyMZOfOnZzvvH37gRy4QoQMroow5lwQzKWLmfp9deD5y2oFlf2hO+9k1ISJJB74m92/HKL/A0/QsUMnihYpSuMGMfxz7Bh/bvuJg7t3cfTgAV++bt260aVLF+bPn+9bFhcXx/3338+QIUM4ceIE3bt3p2bNmoAz5v7xxx9z4sQJatSo4Rsrf/HFF6lfvz6RkZHUr1+fw4cP+8q77bbbePbZZ+nRo8dp63j88cf5afN6VJXm19ajZmRKpn1xbMOGdMsADhcOfOVfStKudMsurVgpg17NGd4O7saYHLdnxQrf9CUlS7Jjy5+++Rtb3syNLZ0PHP+A1q1TR7p16uib79y5M2mf31y+fHnmzp2brr7BgwdneA39/fffz/333x9w3aJFi+jcuTNFixY9bR3Tp09P/0F3nrPgbozxnAcffJA5c+bw+eef53ZTco0Fd2OM57z99tu53YRcZ7/fNcYYD7LgbowxHmTB3RhjPMjG3D0o0J0Cd+TPhYYYY3KNBXdjzmcZ/ZYju/rMP22SyHr1TrkcMivuueceBgwYQLVq1QKuHzduHC1btuSyyy4LKv25bu5Xn/PjT1vo1+e2s163BXdjzFnz/vvvZ7p+3LhxVK9e3RfcT5c+I0lJSeTNG1p4S05OJiwsLKQyWre4idYtbgr4I6YzzcbcjTHZoqo8/dprNGnZgOtbNWTmp9MASElJ4clnB3Bty1Z069mL2+6+h9lznB8OxcbGEh8fT3JyMr169aJ69epERUXxxhtvMHXqVOLj47ntttuIjo7m2LFjvvQAc+fOpXbt2tSsWZPmzZuna8+4ceNo27YtzZo1o3nz5iQmJnLXXXdRr149atWqxSefOPc3PHr0KK1u6UDFylfTrHUbomrFMOGzbwEoVKkxjz7/OjVv6MbSlev4eNpn1KtXz3ejseTkZJKTk+nzzDPEdOhA3Q4dePvDDwH4b1wc191Qj9jWjejTz7nlwcQpcTz178cA2JmQQOfb76TZzbfQ5Y47+fXXXwHn/jgPPfQQjRo1okKFCkydOjVHXh87cjfGZMsn8+axbssWvp2zmH1/7aN1u6Y0rN+YFfHL2JnwKwvnzmHvvn00aXUjPTp3PiXvmjVr+O2339jg/oz/wIEDFC1alBEjRjBs2DBiYmJOSb9nzx7uvfdeFi5cSPny5fnrr78CtmnVqlWsW7eO4sWL8/TTT9OsWTPGjh3LgQMHqFevHjfccAPvvPMOhYsUYcY3y/hpyya6tT55//bEo8eoX6s6rw0awOaftjN05DgWL15MeHg4DzzwAHFxcVxzzTX8vns38TNmOG0/dAiAYWPGsGLRRiIiIjjod6uFVM88/yJdO7ana8eOTJgylYceeoiZM2cC8Mcff7Bo0SK2bNlC27Zt6Zymv7LDjtyNMdmyZNUqutx0E2FhYZSKLEXD+o1ZvXYVy79fxi03tSdPnjyUioykUYP66fJWqFCB7du38+CDDzJ37lwuvvjiTOtatmwZTZo0oXz58gAUL148YLoWLVr41n355Ze88sorREdHExsby/Hjx/n1119ZtGgRrdt2AqDS1dWoVPUaX/6wsDA63ex8K/h60QpWrt9M3bp1iY6O5uuvv2b79u1UqFCBnxMSGPDSS3y5aBEXFyoEQPXKlXmg/z1MnTEp4JBQ/JrVdLjlFgA6t2/HokWLfOvat3f6q1q1ar4HnITKjtyNMWddsWLFWLt2LV988QWjRo1i8uTJjB07NuRyL7roIt+0qjJt2jSqVKkSdP78Efl84+yq0LPLLbz8dvp2LZ82jXmLF/P+5MlM++IL3n3xRWaMHMmXm3by5ddzeXPEMOZ/sTToeiMiIk5pd06wI3djTLY0qlOHaXPnkpyczN59e1m2Ygm1a9ahXkx9Zs+dRUpKCnv27mXp8vRX1uzdu5eUlBQ6derEkCFDWLVqFQCFCxc+5c6OqRo0aMDChQv5+eefATIclvHXqlUr3n77bV+wXL3auTFY48aN+fJTZ0hl249b2LplU8D8za+tx9TZ89i9e7evzl9++cXX9vYtWjDowQdZs3kzKSkpJPz5J9c2asJzA5/n0OFDJCYeOaW8urVqM3O2c5ny9Fmfct111512G0JhR+7GnM8yuH110EK4E2K75s1ZsXYtTW9sjIjw3MDnKVXqEtrc2I7vFi+gSesbubx0aaKuqUbhwoVPyfvbb7/Ru3dv37NUX375ZcA5uXjfffdRoEABli49eeQbGRnJe++9R8eOHUlJSaFUqVJ89dVXmbbvueeeo3///tSoUYOUlBTKly/P7NmzeeCBB2jf9VY6NGtAuYqVqFj5agoFGBaqVrkCQ554gJYtW5KSkkJ4eDgjR46kQIEC9LzrLl/bX3j4YZKTk7nrqafYf/RvVJV7ev2LIkWKnlLekEHP8ciTT/HO+2MoUbwYcRMnZb3Ts0By6itAKGJiYjT1jHiOyoH7uZ+PT7kP/COmWwMn9nhfBJSF/SLgPf7Jvb7YvHkzVasGblO2ZBTcQ7yHeWLiEQpEJPLX/v3c1LEzsyZPpFRk5Bm/h3kwkpOTWfXzHiLy52fnjp/pc2t7Zs3/njr5M3jc8zlyP/dAr72IrFTVmEDp7cjdGJPjbr+rGwcP7eWff07wSL++lIqMzO0m+Rw9epReHW8kKekEqsoz/zeM8Hz5crtZOc6CuzEmx82Y9Fmu/HAnGIULF2bC59/mdjPOODuhaowxHmTB3RhjPMiCuzHGeJAFd2OM8SDPnFC1e5ibC1HU+KgcLW99iw9Pm8b/lr9zFy7ksWGvM/mjmUycEsfId9/i+0XriSzpXB1TsUY029atAaD0VZUZMGAAr732GgDDhg3jyJEjDB48OEe3wTjsyN0Yky3fLlvGY6+8woRxUylbxrm+u3jxEowaHfjh1BH58jF9+nT27t17Npt5wbLgbozJskXx8fQdPJhpI0ZQ7soKvuU9utzOJ7NnsP9A+tsDhOXNS58+fXjjjTfOZlMvWBbcjTFZ8veJE3R7+GEmvfUWVSpUOGXdRRddRI+utzN67KiAefv27UtcXBwHD4Z42wRzWp4ZczcmEDsXk/PC8+alfnQ042fMYNjAgenW39PrXzS/6Truu6tbunUXX3wxd955J8OHD6dAgQJno7kXLDtyN8ZkSR4RPh42jPj16/nP6NHp1hcpUpQO7TrzwcdxAfP379+fMWPGkJiYeKabekGz4G6MybKCBQow/b//ZdJnnxE3Kf0VNvfd04+PJk4iOSkp3brixYvTtWtXxowZczaaesGyYRljzmPre64POu26hPSPfquR5+ds1128SBE+GTWK5r3vpkTxkqesK1G8BDe2uIH3PhgXMO+jjz7KiBEjsl23OT0L7saYLEm9xh2gzKWXEr/I+YBp3eKmU9I9/8zTPP/M07751OvdAS655BKOHj16hlt6YQtpWEZEHhGRjSKyQUQmiEh+ESkvIstFZKuITBIR791L0xhjznHZDu4icjnwEBCjqtWBMKA7MBR4Q1WvAvYDd+dEQ40xxgQv1BOqeYECIpIXKAj8ATQDprrrxwPtQ6zDGGNMFmU7uKvqb8Aw4FecoH4QWAkcUNXUU+QJwOWB8otIHxGJF5H4PXv2ZLcZxhhjAghlWKYY0A4oD1wGXAS0Dja/qr6nqjGqGhN5Dj2CyxhjvCCUYZkbgJ9VdY+qngCmA42Bou4wDUAZIIOnzhpjjDlTQrkU8leggYgUBI4BzYF44FugMzAR6Al8EmojjTGBbb66atBpwwPlTzNf9Zv/nbacQjVrck2lSiQnJ1OlfHleH/4hBQsUDLodOWHmzJlUrlyZatWqndV6zyehjLkvxzlxugpY75b1HvAkMEBEtgIlAPsZmjEeUiAiguVTpxI/Ywb5wsP5MG5sUPmSAvxaNbtmzpzJpk2bzng957OQrpZR1UGqerWqVlfVO1T1b1Xdrqr1VPUqVe2iqn/nVGONMeeWRnXq8POO7SQeTeThx/vSql1Tmt90LXO+dG7YNmnadHr2uY/Ot99Jlzt6cuTIEXr37k1UVBQ1atRg2rRpAHz55Zc0bNiQ2rVr06VLF44cOQJAuXLleOKJJ4iKiqJevXps3bqVJUuWMGvWLB5//HGio6PZtm0bsbGx9O/fn5iYGN566y2+/vpratWqRVRUFHfddRd///23r7xBgwbR7cbr6XRDI37e+mPudNxZYPeWMcZkS1JSEl9+9x1Vq1TjzRHDuK5RE7745FumT5jNCy8/5/sF6vqNGxk9YjgzJsTx4osvUqRIEdavX8+6deto1qwZe/fuZciQIcybN49Vq1YRExPD66+/7qsnNX2/fv3o378/jRo1om3btrz66qusWbOGihUrAvDPP/8QHx9P37596dWrF5MmTWL9+vUkJSXxzjvv+MorWbIkk+YsoOsddzH+Xe/eAsGCuzEmS479/Tf1O3emcffulC1dmlu73cmC775h+Dtv0OzGa+nQvQ1///03Cb//DkCTaxtTrGhRAObNm0ffvn19ZRUrVoxly5axadMmGjduTHR0NOPHj+eXX37xpenRo4fv/9KlSzNsV7duzi2Gf/jhB8qXL0/lypUB6NmzJwsXLvSl69ixIwBVa0Tze8KvOdEl5yS7t4wxJktSx9xTHc6XD1Vl7DsfcVXFSr7lKUm7WL12HQVPc992VaVFixZMmDAh4HoRCTid1kUXXRRU+yMiIuA4hOUJC3jXSq+wI3djTMhimzRnzPh3UVUA1m9YGzBdixYtGDlypG9+//79NGjQgMWLF7N161YAEhMT+fHHk2PhkyZN8v1v2LAhAIULF+bw4cMB66hSpQo7duzwlffRRx9x/fXXh7iF5x87cjfmPFZ1S9qLGTOW07f89TfgoSd47vmBxLZuhKakULbslXz0XvoHZT/77LP07duX6tWrExYWxqBBg+jYsSPjxo2jR48evhOfQ4YM8Q2r7N+/nxo1ahAREeE7uu/evTv33nsvw4cPZ6rftwiA/Pnz88EHH9ClSxeSkpKoW7cu9913X45s5/nEgrsxJkv8b/mbqkD+Agx7+a1TlqUk7aJbp45069TRt6xQoUKMHz8+Xf5mzZrx/fffB6zv8ccfZ+jQoacsa9y48SmXQs6fP/+U9c2bN2f16tXpytqxYwcAvycc4JqatRgzZXbAOr3AgrsxxmRi496N6ZZVCJDuXGPB3Rhzzko90jZZZydUjTHGgyy4G2OMB1lwN8YYD7LgbowxHmQnVI05j42875uQ8n+XZr7vv4sFlW/oe+8x+fPPCcuTB82bn5tateH438d59snBvjQbNm3i/kcG8N0Xc6l7fVMuK30py7+P962Pjo4mKSmJDRs2hLQNJjAL7sa4osZHpVs2ORfaca5bvmYNcxYsYMnkyUTky8eOExfx409beOix+08J7p/M/owObdr45hMTE9m5cydly5Zl8+bgf3xlsseGZYwxWfLH3r2UKFaMiHz5AChRvAQN6zemaJGirFx98sh81pw5tL/lZHC/5cYbfbcSmDBhgu+GYObMsOB+gYsaHxXwz5iM3NCoEQl//kmNNm14eMgQlixbBECHtp2Z+alzf/b4Vd9TtEhRKpQr58t3c+tWTJ8+HYBPP/2UW2655ay3/UJiwX1w9swAABQ+SURBVN0YkyWFChZkyaRJjBg0iJLFitGnX28mTomjXZuOzJ7zCSkpKcz8dBod2tx8Sr5iRYtSrFgxJk6cSNWqVSlY8Ow+mu9CY2PuxpgsCwsLo0ndujSpW5eKNRoyadoEune5jSvKXsmSZYv4bO4sPp2c/ha+3bp1o2/fvowbN+7sN/oCY8HdGJMlP/78M3ny5OGqK68EYMOm9ZS9vCwAHW7pxL9ffJory5bjstKXpsvboUMH/vjjD1q1asXv7sM8zJlhwd2Y81jfUc2CTptTt/w9cvQoj778MgcPHyZvWBhXVKjCsJeHA3DLzR145vkneWnwfwLmLVy4ME8++WSW6zRZZ8HdGJNOZndCrH3NNXz78ce+5YcLX+GbLlG8BL9t3Qc4t/xN9f2Cb9OVV65cObvG/QyyE6rGGONBF+SRe0aX+tkPVowxXmFH7sYY40EW3I0xxoMsuBtjjAdZcDfGGA+6IE+oGuMVr3Vrc/pEmfgqzfyjb7x42jx/7t3LE0OHsnLDBooULkyJS8rw4r9foWKFq7Ldjl69etGmTRs6d+58yvL4+Hg+/PBDhg8fnq1yX3rpJZ5++mnffKNGjViyZEm223k+sSN3Y0zQVJXuDz/MdXXrsnHOHJZMnswzTwxiz97dZ6S+mJiYbAd2cIK7vwslsIMFd2NMFixYsYLwvHm5t2tX37JrqkVRv25Dnn/pWZq0bMD1rRr67g65ZNlyOvS4jV7/up/6TZsxcOBA4uLiqFevHlFRUWzbts1Xzrx584iJiaFy5crMnj0bgPnz59PGvSf84MGDueuuu4iNjaVChQqnBP327dtTp04drrnmGt577z0ABg4cyLFjx4iOjua2224DoFChQoDzIfX6kOfo2LwhnW5oxNxZzt0q5y+JJ7bzvXS+93GubtKR2/o9g6qeqe48o2xYxhgTtE1bt1KrWrV0yz+bO4sNm9bz7ZzF7PtrH63bNaV+nYlOni1bWPjFHIoWKUqjFi255557WLFiBW+99RZvv/02b775JgA7duxgxYoVbNu2jaZNm7J169Z09WzZsoVvv/2Ww4cPU6VKFe6//37Cw8MZO3YsxYsX59ixY9StW5dOnTrxyiuvMGLECNasWZOunK/nfMoPmzYw5ctFHPhrH7e2acYdDccCsHrDD2z8ZgqXXRpJ43a9Wb18NbUb1M7Jbjwr7MjdGBOy5d8vo0PbzoSFhVEqshQN6zdmzbr1AETXiOKSUqWIiMhHxYoVadmyJQBRUVHs2LHDV0bXrl3JkycPlSpVokKFCmzZsiVdPTfffDMRERGULFmSUqVKsWuXc4uD4cOHU7NmTRo0aMDOnTv56aefMm3v6hXLaN22E2FhYZSILEWdBo35fu0mAOpFX0OZyy4hT548RF9Thd92/pYTXXTWhRTcRaSoiEwVkS0isllEGopIcRH5SkR+cv8H91BGY8w5r2rFiqzetClLefK5T2wCyJMnDxEREb7ppKQk3zoROSVf2nnAlxec2w4nJSUxf/585s2bx9KlS1m7di21atXi+PHjWWrjKXXkC/erIw/JScnZLis3hXrk/hYwV1WvBmoCm4GBwNeqWgn42p03xnhAbP36/H3iBGOmTPEt27h5A0UuLsInn04nOTmZvfv2smzFEmrVrJGlsqdMmUJKSgrbtm1j+/btVKlSJah8Bw8epFixYhQsWJAtW7awbNky37rw8HBOnDiRLk+t+g354tMZJCcn89e+vaxavoR60ddkqb3numyPuYtIEaAJ0AtAVf8B/hGRdkCsm2w8MB+we3wacwY8Oml20Glz4pa/IsLEN9/kiaFDeX3sWPJHRHD5FRV58d+vkHj0CE1vbIyI8NzA5ykVGcnWbduDLvuKK66gXr16HDp0iFGjRpE/f/6g8rVu3ZpRo0ZRtWpVqlSpQoMGDXzr+vTpQ40aNahduzZxcXG+5c1bt2HdyhV0aXktIkL/p5/n0lIl2bJ1R9DtPdeFckK1PLAH+EBEagIrgYeBS1T1DzfNn8AloTXRGHMuuaxUKT5+7TXffOotfwc9PYRBTw/xLU9J2kWjBvVp1KC+b9n8+fN907GxscTGxgJk+GQm/zSDBw8+ZZ3/7YLnzJkTMP/QoUMZOnSob/7IkSOA8yE14NkXGfCs/3X9PxPbKIbYRjG+JSP+byAb/YaVziehDMvkBWoD76hqLSCRNEMw6lxDFPA6IhHpIyLxIhK/Z8+eEJphjDEmrVCCewKQoKrL3fmpOMF+l4iUBnD/B/x1g6q+p6oxqhoTGRkZQjOMMcakle3grqp/AjtFJPWsR3NgEzAL6Oku6wl8ElILjTGnOF9/VGOyLzuveag/YnoQiBORfMB2oDfOB8ZkEbkb+AXomkl+Y0wW5M+fn3379lGiRImAlwoa71FV9u3bF/QJ5lQhBXdVXQPEBFjVPJRyjTGBlSlThoSEBLJznmrX/mPplm2WwOX8mTd9aEg+FLjc4wfSX2oIoCnpM+z/JylAyrPvfOuL/PnzU6ZMmaDTg91+wJjzSnh4OOXLl89W3hsHfpZu2Y78twZM27X8FemWTX45cDD6LnZkwOXH97+ebllWLt08ky6EvrDbDxhjjAdZcDfGGA+y4G6MMR5kwd0YYzzIgrsxxniQBXdjjPEgC+7GGONBFtyNMcaDLLgbY4wHWXA3xhgPsuBujDEeZMHdGGM8yIK7McZ4kAV3Y4zxIAvuxhjjQRbcjTHGgyy4G2OMB1lwN8YYD7LgbowxHmTB3RhjPMiCuzHGeJAFd2OM8SAL7sYY40EW3I0xxoMsuBtjjAdZcDfGGA+y4G6MMR5kwd0YYzzIgrsxxniQBXdjjPEgC+7GGONBFtyNMcaDQg7uIhImIqtFZLY7X15ElovIVhGZJCL5Qm+mMcaYrMiJI/eHgc1+80OBN1T1KmA/cHcO1GGMMSYLQgruIlIGuBl4350XoBkw1U0yHmgfSh3GGGOyLtQj9zeBJ4AUd74EcEBVk9z5BODyQBlFpI+IxItI/J49e0JshjHGGH/ZDu4i0gbYraors5NfVd9T1RhVjYmMjMxuM4wxxgSQN4S8jYG2InITkB+4GHgLKCoied2j9zLAb6E30xhjTFZk+8hdVZ9S1TKqWg7oDnyjqrcB3wKd3WQ9gU9CbqUxxpgsORPXuT8JDBCRrThj8GPOQB3GGGMyEcqwjI+qzgfmu9PbgXo5Ua4xxpjssV+oGmOMB1lwN8YYD7LgbowxHmTB3RhjPMiCuzHGeJAFd2OM8SAL7sYY40EW3I0xxoMsuBtjjAdZcDfGGA+y4G6MMR5kwd0YYzzIgrsxxniQBXdjjPEgC+7GGONBFtyNMcaDLLgbY4wHWXA3xhgPsuBujDEeZMHdGGM8yIK7McZ4kAV3Y4zxIAvuxhjjQRbcjTHGgyy4G2OMB1lwN8YYD7LgbowxHmTB3RhjPMiCuzHGeJAFd2OM8SAL7sYY40EW3I0xxoOyHdxFpKyIfCsim0Rko4g87C4vLiJfichP7v9iOddcY4wxwQjlyD0JeFRVqwENgL4iUg0YCHytqpWAr915Y4wxZ1G2g7uq/qGqq9zpw8Bm4HKgHTDeTTYeaB9qI40xxmRNjoy5i0g5oBawHLhEVf9wV/0JXJJBnj4iEi8i8Xv27MmJZhhjjHGFHNxFpBAwDeivqof816mqAhoon6q+p6oxqhoTGRkZajOMMcb4CSm4i0g4TmCPU9Xp7uJdIlLaXV8a2B1aE40xxmRVKFfLCDAG2Kyqr/utmgX0dKd7Ap9kv3nGGGOyI28IeRsDdwDrRWSNu+xp4BVgsojcDfwCdA2ticYYY7Iq28FdVRcBksHq5tkt1xhjTOjsF6rGGONBFtyNMcaDLLgbY4wHWXA3xhgPsuBujDEeZMHdGGM8yIK7McZ4kAV3Y4zxIAvuxhjjQRbcjTHGgyy4G2OMB1lwN8YYD7LgbowxHmTB3RhjPMiCuzHGeJAFd2OM8SAL7sYY40EW3I0xxoMsuBtjjAdZcDfGGA+y4G6MMR5kwd0YYzzIgrsxxniQBXdjjPEgC+7GGONBFtyNMcaDLLgbY4wHWXA3xhgPsuBujDEeZMHdGGM8yIK7McZ4kAV3Y4zxIAvuxhjjQWckuItIaxH5QUS2isjAM1GHMcaYjOV4cBeRMGAkcCNQDeghItVyuh5jjDEZOxNH7vWAraq6XVX/ASYC7c5APcYYYzIgqpqzBYp0Blqr6j3u/B1AfVXtlyZdH6CPO1sF+CFHG5I9JYG9ud2Ic4T1hcP64STri5POlb64UlUjA63Ie7ZbkkpV3wPey636AxGReFWNye12nAusLxzWDydZX5x0PvTFmRiW+Q0o6zdfxl1mjDHmLDkTwf17oJKIlBeRfEB3YNYZqMcYY0wGcnxYRlWTRKQf8AUQBoxV1Y05Xc8Zck4NE+Uy6wuH9cNJ1hcnnfN9keMnVI0xxuQ++4WqMcZ4kAV3Y4zxoPM6uIvIQyKyWUTiRKRtVm51ICLlROTWTNa/KiIbReTVTNIMFpHHMih7Q7BtOR+ISKyIzM5g3QQRWScij2SSf5z7G4igy81i+4qKyANBtvf9YH41LSK9RGREgOURIjJPRNaISLdM8s8XkXSXy2VUbij83ws5VN7TftNB7c8iEikiy0VktYhcl0m6HSJSMsDygO+nnJBRrEjb5mD3oTRlX+3uC6tFpGIm6Y5ksDzgeyNUuXadew55ALhBVRPc+XRX5YhIXlVNCpC3HHAr8L8Myu4DFFfV5JxoqFeJyKVAXVW9KpebUhRnf/jv6RKm/sAuLREJC/L1ruWWE52lFp5Zad8L2SIiAgjwNPBSFrM3B9Zn1L+5LKNY4WuziJQD3iGIfSiN9sBUVR2SEw3NKeftkbuIjAIqAHNE5BH/oyH3k3CUiCwH/iMi17ufrKmfroWBV4Dr3GWPpCl7FlAIWCki3dwjl2/co9OvReSKAO2pIyJrRWQt0DeHt/UiEfnMLX9D6tGiW+cCEVkpIl+ISGl3+VXukeVaEVklIhXF8aqbf71fGbHuEeZUEdniHtmIu661u2wV0DGD5n0JXO7243UiEi0iy9y+miEixQJsTzDlZtUrQEW3HanftgplsF2+I2oROSIir7mvW0MR6S0iP4rICqBxgLaXAj4G6rp1VRSR5u5+tV5ExopIRIB8mZYbigDvheIiMtN9DZaJSA033SlHxu6+UM79+0FEPgQ2AGOAAu72pX4TCBOR0eJ8m/1SRAqkaUM08B+gnZuvgIj0cPtkg4gMzaDtz7j9sgjnl+o5LqNYkbbNwFCC3If8yr4J6A/cLyLfussGuNu8QUT6B2iPuPX/ICLzgFJnYrtR1fP2D9gBlHSnewEj3OlxwGwgzJ3/FGjsThfC+cYSC8zOpOwjftOfAj3d6buAme70YOAxd3od0MSdfhXYkIPb2QkY7TdfBAgHlgCR7rJuOJedAiwHOrjT+YGCbhlf4VyeegnwK1Da7YeDOD82ywMsBa518+0EKuEcyU0O1F8434A2+M2vA653p18A3vR7TToHW242+ihtOwJul7tuPhDjTivQ1Z0u7fZLJJAPWJy6T6Wpy7fv+G1PZXf+Q6C/fz3BlpuD74W3gUHudDNgTdr91Z3f4PZbOSAFaJDB/l8OSAKi3fnJwO0B2tCLk+/By/y2OS/wDdDev61AHWA9zv55MbDVv31nsH/82+k/HfQ+lKZsX7/6bdNFOLFmI1DLv09xDmhS34uXAQeAzjm9zeftkXsQpujJr9iLgddF5CGgqAYepslMQ04O33yEE/x8RKSoW+5CvzQ5aT3QQkSGish1qnoQ5yinOvCVe9TxLFBGnG8ll6vqDABVPa6qR902T1DVZFXdBSwA6rrlr1DVBFVNAdbg7ORXAz+r6k/q7JEfn66RIlIEpx8WuIvGA03SJMtyuSEItF1pJQPT3On6wHxV3aPOTe8mBVFHFZzt+dGdD7TN2Sk3FNfi7oOq+g1QQkQuPk2eX1R1WSbrf1bVNe70SgL3pb+6nNzmJCCO9P1yHTBDVY+q6iHOzR87BrMP+bsWZ5sSVfUIMB1nO/014eR78XecD74cd76PuWcmMXVCVV8Rkc+Am4DFItIq95qVdar6o4jUxmn/EBH5GpgBbFTVhv5p3eCeVX/7TSfjnf0imO06rhfOeZUkTh2Kze83nUjm0vZlgYwSesx5+97w8pG7j4hUVNX1qjoU5/YIVwOHgWAD4RKc2ygA3AZ8579SVQ8AB0TkWr80OUZELgOOqurHOEM+tXHuohkpIg3dNOEico2qHgYSRKS9uzxCRAq6be4mImEiEolz9LAik2q3AOXk5Nn/Hqdrp/uNYr+cvFLiDpxvCCGVG6SsvJ4ZWQ5cLyIlRCQc6BJEnh9wtif1hHKgbc5OuaH4DncfFJFYYK97ZLwDZ9/BPVgon0kZJ9y2ZtcKnG0uKc4zHnqQvl8WAu3d8fnCwC0h1JcTcmIf+g5nmwqKyEVAB9LEC5ztTn0vlgaahlhnQOfNp1CI+otIU5xxxY3AHHc62T2RNk5V38gk/4PAByLyOLAH6B0gTW9grIgozknGnBQFvCoiKcAJ4H5V/Uecy6eGu8MheYE3cbbvDuBdEXnBTd8F50i/IbAWZ5z5CVX9U0SuDlShqh4X57bMn4nIUZwdNJgdvycwyv1A2U6avgqh3Eyp6j4RWSzOJXtzgM+yUcYfIjIYZ2z1AM7X8NPlOS4ivYEpIpIX5+BhVKjlhmgwzr64DjiK85qAM/x0p4hsxPnA+TFwdsD5ef06cU56P5PVBrjbPBD4Fufcymeq+kmaNKtEZBLOPrkbp+9yTQ7tQ6tEZBwnD5zeV9XVaZLNwDkXsgnnvMTS7Lc6Y3b7AWOM8aALYljGGGMuNBbcjTHGgyy4G2OMB1lwN8YYD7LgbowxHmTB3RhjPMiCuzHGeND/A700+GqxfLpwAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vTbvuVQZzRD9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "5cdbaae5-9b76-49aa-925c-ff91aee6bd70"
      },
      "source": [
        "fig = plt.figure()\n",
        "fig.suptitle('Model Comparison For data without FS')\n",
        "ax = fig.add_subplot(111)\n",
        "plt.boxplot(acc_cross_validation)\n",
        "ax.set_xticklabels(df_resultcol,rotation=10)\n",
        "plt.show()\n",
        "fig.savefig('/content/drive/My Drive/project/CGR_data/Model_Comparison__WITHOUT.jpg', dpi=400)"
      ],
      "execution_count": 147,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 0.98, 'Model Comparison For data without FS')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 147
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'boxes': [<matplotlib.lines.Line2D at 0x7fa0842669b0>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa08427f208>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa08428b978>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841ba0f0>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841c5828>],\n",
              " 'caps': [<matplotlib.lines.Line2D at 0x7fa084274400>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa084274780>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa08427fc50>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa08427ff98>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0842963c8>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa084296710>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841bab00>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841bae48>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841d2278>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841d25c0>],\n",
              " 'fliers': [<matplotlib.lines.Line2D at 0x7fa084274e80>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa08428b668>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa084296da0>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841c5518>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841d2c50>],\n",
              " 'means': [],\n",
              " 'medians': [<matplotlib.lines.Line2D at 0x7fa084274b00>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa08428b320>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa084296a58>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841c51d0>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841d2908>],\n",
              " 'whiskers': [<matplotlib.lines.Line2D at 0x7fa084266cc0>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa084274080>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa08427f5c0>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa08427f908>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa08428bcf8>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa084296080>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841ba470>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841ba7b8>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841c5ba8>,\n",
              "  <matplotlib.lines.Line2D at 0x7fa0841c5ef0>]}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 147
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Text(0, 0, 'NaiveBayes'),\n",
              " Text(0, 0, 'logistic regression'),\n",
              " Text(0, 0, 'KNN'),\n",
              " Text(0, 0, 'Perceptron'),\n",
              " Text(0, 0, 'SVM')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 147
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAElCAYAAAAfhqICAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3debwcVZn/8c83C4Q9gQQIS4wsaiQK6tURBWUTBRdw+SHBBSSKzkhwH9E4AzoGUXEYZXABg6BIFFEEHEUQAxgVJCxCAJVFwhZCgIRVTEie3x/P6aTS3Jvc3O7c26n7fb9e93W7q6qrTlVXPXXqOaeqFRGYmVm9DBnoApiZWfs5uJuZ1ZCDu5lZDTm4m5nVkIO7mVkNObibmdWQg3sHkjReUkga1otpj5A0qz/K1d8kfVvSfwx0OdaGTvzeJH1W0ndXMb7jymw9c3BvkaS7JC2WNLpp+PUlQI8fmJItL8d6ko6XdJukJ0t5zxjocvVGRHwoIv6rv5ZXOak+Ufn7c38tfxXlOl7S2Wt7ORFxQkS8vyyz1xWMdij75X6rGL+XpGVN381FZdwuki6R9IikRZKulXRgf5S7kzm4t8ffgUmNN5JeBGw4cMVZyXnAW4DDgM2AXYFrgX0HslCrI2noAC5+ZERsXP52XdMP91dAHITur3wvG0fEm8vwi4BLga2BLYFjgMcGqpCdwsG9PX4AvLfy/nDg+9UJJG0m6fuSFkiaK+lzkoaUcUMlnSTpIUl3Am/s5rPTJc2TdJ+kL/Ym+JWa0OuAgyLimoh4JiIejYhTI2J6mWYbSReWWs/tkj5Q+fzxkn4i6WxJj0u6SdLzJH1G0oOS7pG0f2X6yyV9SdKfJD0m6QJJm1fG/0TSA5IelXSlpF0q486U9C1Jv5T0JLB3GfbFMn60pF+Umtkjkn5X2X4TyrIXSbpZ0lua5nuqpP8r63C1pB1Xt+262Zar207nle30GHBEN5/fonz+MUl/AnZsGv/1sj0fKzXPPcvwNwCfBd5ZvZKQ9D5Jt5Z1ulPSB1dR9rmSXlZev6vUyHcp7ydL+nllPRpXCFeW/4vKcnevzO8kSQsl/V3SAb3cRsu/y/J+L0n3ltc/AMYBF5Vl/XuPX8Sz12008Fzg9IhYXP5+HxGDPn3k4N4eVwGbliAzFDgUaL6MPoWsOe8AvJY8GbyvjPsA8CbgJUAX8I6mz54JPAPsVKbZH3h/L8q1H/CniLhnFdP8CLgX2KYs9wRJ+1TGv5k8eY0Crgd+Te432wJfAL7TNL/3AkcCY0uZv1EZ9ytgZ7J2dR3ww6bPHgZMAzYBmg/OT5RyjgG2IgNeSBpO1twuKfOdAvxQ0vMrnz0U+HxZh9vLMtbU6rbTQeRV0shu1gvgVOBpcrscWf6qrgF2AzYHzgF+ImlERFwMnAD8uOlK4kFyn9mU3I9OlvTSHsp+BbBXef1a4E7gNZX3V3Tzmcb4xlXMH8v7fwH+CowGvgJMl6QybnXbqFsR8R7gbuDNZVlfWd1nKh4mv9OzJR0saas1+Gy9RYT/WvgD7iKD6OeALwFvIC8RhwEBjAeGAouBF1Y+90Hg8vL6t8CHKuP2L58dRgayfwIbVMZPAmaW10cAs3oo2+nAj1ZR9u2BpcAmlWFfAs4sr48HLq2MezPwBDC0vN+klHNkeX85cGJl+heW9R7azbJHls9uVt6fCXy/aZozgS+W118ALgB2appmT+ABYEhl2Azg+Mo8vlsZdyDwlx62x/hSpkWVv0/2cjtduYrtPBRYArygMuyEnr63Mn4hsGtl/mevZj/8OfCRHsZNBi4sr28lKwY/Ku/nAi9tXk5lWwyrzOcI4PbK+w3LNFv3Yhst/y7L+72Ae5uPo1Ws317Asqbv5pAybjvgf4E7yjRXAjv3x/HfyX+uubfPD8ia5xE0pWTIWs5w8kBqmEvWfiFrOvc0jWt4TvnsvJJ2WETWlrfsRZkeJmuKPdkGeCQiHu+hXADzK6//ATwUEUsr7wE2rkzTvB7DgdHK1NOJku4oqYu7yjSje/hss6+SNbRLShri2Mo63BMRy1axDg9UXj/VVN7ujI6IkeXvJHq3nVZV9jHkibqn7xhJnyxplkfLd7wZK28bmqY/QNJVJQWyiDxp9TT9FcCeksaSJ5pzgVcrG9U3A25YRdmbLd+WEfFUebkxvdtGrbq/8r2MjIhzSznujYijI2JH8nh5kmcfg4OOg3ubRMRcsmH1QOBnTaMfImtuz6kMGwfcV17PI2s+1XEN95A192rA2TQidmH1fgO8QtJ2PYy/H9hc0iY9lKsvmtdjCbn+h5Gpi/3IgDK+TKPK9D0+ojQiHo+IT0TEDmQD8ccl7VvWYftG/r1N69CsN9tpVY9XXUCmqLr9jkt+/d+BQ4BRETESeJQV22aleUtaH/gpcBKwVZn+l6y8LVcULOJ28qQ2hbzCeIwM0keRVw/LuvvYKtanO6vbRk+ycieDrVtcXrciU5CnAhPbMb91mYN7e00G9omIJ6sDS033XGCapE0kPQf4OCvy8ucCx0jaTtIo4NjKZ+eR+eSvSdpU0hBJO0p67eoKExG/IVNE50t6maRhZfkfknRkORD+AHxJ0ghJLy7r0Eq3u3dLeqGkDclUynll/TchT1IPkwf5CWsyU0lvkrRTye8+SqYAlgFXk4Hr3yUNl7QXmT76UQvrsJJWt1NZ/58Bx0vaUNILyUb3hk3I4L8AGCbpP8lcesN8YHzlBLYesH6Z/pnSqLk/q3YFcDQr8uuXN71vtoDcvjusdgXp1Ta6AThQ0uaStgY+2jSL+b1dVpWkUZI+X/aNIaWB9UiyHWxQc3Bvo4i4IyJm9zB6Cll7uZNsLDwHOKOMO51sqPwz2dDYXPN/L3lA30LmYs9j1emWqneQtbofk0FxDtlo+5syfhJZi74fOB84rpwU+uoHZH71AWAE2S0N8jJ5LlmTu4U1P/h2LmV+Avgj8M2ImBkRi8lgfgB5hfBN4L0R8ZcW1qE7rW6no8n0xQPk9vleZdyvgYuBv5Hb6GlWTuH8pPx/WNJ1JfVxDFkpWEheFV24muVfQZ5Eruzh/UpKymUa8PuSDnzl6ldxldvoB+T+fRdZWflx02e/BHyuLOuTvVhWw+KyzN+Q3R/nkJWII9ZgHrWk0iBh1jJJl5MNcj3e5Whm/cM1dzOzGnJwNzOrIadlzMxqyDV3M7MacnA3M6shB3czsxpycDczqyEHdzOzGnJwNzOrIQd3M7MacnA3M6shB3czsxpycDczqyEHdzOzGnJwNzOrIQd3M7MacnA3M6shB3czsxoaNtAFABg9enSMHz9+oIthZrZOufbaax+KiDHdjeuI4D5+/Hhmz+7pd6XNzKw7kub2NM5pGTOzGnJwNzOrIQd3M7MacnA3M6uh1QZ3SWdIelDSnMqwzSVdKum28n9UGS5J35B0u6QbJb10bRbezMy615ua+5nAG5qGHQtcFhE7A5eV9wAHADuXv6OAb7WnmGZmtiZWG9wj4krgkabBBwFnlddnAQdXhn8/0lXASElj21VYMzPrnb7m3LeKiHnl9QPAVuX1tsA9lenuLcPMzKwftXwTU0SEpFjTz0k6ikzdMG7cuFaL0ZvltWU+EWu8qv3v+M0GugTp+EcHugTeFjZo9TW4z5c0NiLmlbTLg2X4fcD2lem2K8OeJSJOA04D6OrqWusRszdBWdK6EbxXQ59/bMDXQxJx/IAWAfC2sMGrr2mZC4HDy+vDgQsqw99bes28Eni0kr4xM7N+stqau6QZwF7AaEn3AscBJwLnSpoMzAUOKZP/EjgQuB14CnjfWiizmZmtxmqDe0RM6mHUvt1MG8CHWy1UX2y++eYsXLiw5fm0kpsfNWoUjzzS3LHIzKz/dcRTIdth4cKFHZFbNTPrBH78gJlZDTm4m5nVkIO7mVkN1SbnbmatGVQ3+g0CDu5mBqw+KNflJr/BwmkZM7MacnA3M6shB3czsxpycDczqyEHdzOzGnJwNzOrIQd3M7Maqk0/9zhu0wH/1Z04btMBXb6ZWYM64aaErq6umD17dkvz6IQbLDqhDJ1Sjk4oQ6eUoxPKAAx45We5TvjJwZpsC0nXRkRXt+M6YadzcK9fOTqhDJ1Sjk4oQ6eUoxPK0CnlaEcZVhXcnXM3M6shB3czsxpycDczqyEHdzOzGnJwNzOroZaCu6SPSJoj6WZJHy3Djpd0n6Qbyt+B7SmqmZn1Vp9vYpI0EfgA8ApgMXCxpF+U0SdHxEltKJ+ZmfVBK3eoTgCujoinACRdAbytLaWylrXrJ9P6atSoUQO6fLPBrpW0zBxgT0lbSNoQOBDYvow7WtKNks6Q1O1RLukoSbMlzV6wYEELxbBmEdHyX6vzeeSRRwZ4K5gNbn0O7hFxK/Bl4BLgYuAGYCnwLWBHYDdgHvC1Hj5/WkR0RUTXmDFj+loMMzPrRksNqhExPSJeFhGvARYCf4uI+RGxNCKWAaeTOXkzM+tHrfaW2bL8H0fm28+RNLYyyVvJ9I2ZmfWjVh/5+1NJWwBLgA9HxCJJp0jaDQjgLuCDLS7DzMzWUEvBPSL27GbYe1qZp5mZtc53qJqZ1ZCDu5lZDdXmZ/as93p7g9PqphvoHzsws545uA9CDspm9ee0jJlZDbnmbmaDUt2fv+TgbmaDTht+mLrj05tOy5iZ1ZCDu5lZDTm4m5nVkIO7mVkNObibmdWQe8uYmTXpTTfJ3kwzkD1qHNzNzJp0ejfH3nBaxsyshhzczcxqyMHdzKyGHNzNzGrIwd3MrIYc3M3Maqil4C7pI5LmSLpZ0kfLsM0lXSrptvJ/7T7X0szMnqXPwV3SROADwCuAXYE3SdoJOBa4LCJ2Bi4r783MrB+1UnOfAFwdEU9FxDPAFcDbgIOAs8o0ZwEHt1ZEMzNbU60E9znAnpK2kLQhcCCwPbBVRMwr0zwAbNViGc3MbA31+fEDEXGrpC8DlwBPAjcAS5umCUnd3scr6SjgKIBx48b1tRhmZtaNlhpUI2J6RLwsIl4DLAT+BsyXNBag/H+wh8+eFhFdEdE1ZsyYVophZmZNWu0ts2X5P47Mt58DXAgcXiY5HLiglWWYmdmaa/WpkD+VtAWwBPhwRCySdCJwrqTJwFzgkFYLaWZma6al4B4Re3Yz7GFg31bma2ZmrfEdqmZmNeTgbmZWQw7uZmY15OBuZlZDtfoN1d78YO3aNGqUn5FmZp2hNsG9HT9oK6kWP4xr1hNXgAaP2gR3M1u1VisurvysWxzcrfZcW7XByMHdas21VRus3FvGzKyGHNzNzGrIwd3MrIYc3M3MasjB3cyshhzczcxqyMHdzKyGHNzNzGrIwd3MrIYc3M3MasjB3cyshgbNs2V6+/Co1U3n54yY2bqgpZq7pI9JulnSHEkzJI2QdKakv0u6ofzt1q7CtiIi2vJnZrYu6HPNXdK2wDHACyPiH5LOBQ4toz8VEee1o4Bm1j96c3Xbm2lcCeoMraZlhgEbSFoCbAjc33qRzGwgOCjXS5/TMhFxH3AScDcwD3g0Ii4po6dJulHSyZLWb0M5zcxsDfQ5uEsaBRwEPBfYBthI0ruBzwAvAF4ObA58uofPHyVptqTZCxYs6GsxzMysG600qO4H/D0iFkTEEuBnwKsiYl6kfwLfA17R3Ycj4rSI6IqIrjFjxrRQDDMza9ZKcL8beKWkDZWtLPsCt0oaC1CGHQzMab2YZma2JvrcoBoRV0s6D7gOeAa4HjgN+JWkMYCAG4APtaOgZmbWey31lomI44Djmgbv08o8zcysdX78gJlZDTm4m5nVkIO7mVkNObibmdWQg7uZWQ05uJuZ1ZCDu5lZDTm4m5nVkIO7mVkNObibmdWQg7uZWQ05uJuZ1ZCDu5lZDTm4m5nVkIO7mVkNObibmdWQg7uZWQ05uJuZ1ZCDu5lZDTm4m5nVUEvBXdLHJN0saY6kGZJGSHqupKsl3S7px5LWa1dhzcysd/oc3CVtCxwDdEXERGAocCjwZeDkiNgJWAhMbkdBzcys91pNywwDNpA0DNgQmAfsA5xXxp8FHNziMszMbA31ObhHxH3AScDdZFB/FLgWWBQRz5TJ7gW2bbWQZma2ZlpJy4wCDgKeC2wDbAS8YQ0+f5Sk2ZJmL1iwoK/FMDOzbrSSltkP+HtELIiIJcDPgFcDI0uaBmA74L7uPhwRp0VEV0R0jRkzpoVimJlZs1aC+93AKyVtKEnAvsAtwEzgHWWaw4ELWiuimZmtqVZy7leTDafXATeVeZ0GfBr4uKTbgS2A6W0op5mZrYFhq5+kZxFxHHBc0+A7gVe0Ml8zM2uN71A1M6shB3czsxpycDczqyEHdzOzGnJwNzOrIQd3M7MacnA3M6shB3czsxpycDczqyEHdzOzGnJwNzOrIQd3M7MacnA3M6shB3czsxpycDczqyEHdzOzGnJwNzOrIQd3M7MacnA3M6shB3czsxpycDczq6Fhff2gpOcDP64M2gH4T2Ak8AFgQRn+2Yj4ZZ9LaGZma6zPwT0i/grsBiBpKHAfcD7wPuDkiDipLSU0M7M11q60zL7AHRExt03zMzOzFrQruB8KzKi8P1rSjZLOkDSqTcswM7Neajm4S1oPeAvwkzLoW8COZMpmHvC1Hj53lKTZkmYvWLCgu0nMzKyP2lFzPwC4LiLmA0TE/IhYGhHLgNOBV3T3oYg4LSK6IqJrzJgxbSiGmZk1tCO4T6KSkpE0tjLurcCcNizDzGzAzZgxg4kTJzJ06FAmTpzIjBkzVv+hAdLn3jIAkjYCXgd8sDL4K5J2AwK4q2mcmdk6acaMGUydOpXp06ezxx57MGvWLCZPngzApEmTBrh0z6aIGOgy0NXVFbNnzx7oYpg9iyQ64RixgTdx4kROOeUU9t577+XDZs6cyZQpU5gzZ2ASFJKujYiubsd1wo7r4G6dysHdGoYOHcrTTz/N8OHDlw9bsmQJI0aMYOnSpQNSplUFdz9+wMysFyZMmMCsWbNWGjZr1iwmTJgwQCVaNQd3M7NemDp1KpMnT2bmzJksWbKEmTNnMnnyZKZOnTrQRetWSw2qZmaDRaPRdMqUKdx6661MmDCBadOmdWRjKjjnbrZKzrlbJ3PO3cxskHFwNzOrIQd3M7MacnA3M6shB3czsxpycDczqyEHdzOzGnJwNzOrIQd3M7MacnA3M6shB3czsxpycDczqyEHdzOzGnJwNzOrIT/P3QY1SW2Zxo8Ftk7j4G6DmoOy1VWf0zKSni/phsrfY5I+KmlzSZdKuq38H9XOApuZ2er1ObhHxF8jYreI2A14GfAUcD5wLHBZROwMXFbe2zpixowZTJw4kaFDhzJx4kRmzJgx0EUysz5oV1pmX+COiJgr6SBgrzL8LOBy4NNtWo6tRTNmzGDq1KlMnz6dPfbYg1mzZjF58mSAjv2dSDPrXrt6yxwKNKp4W0XEvPL6AWCrNi3D1rJp06Yxffp09t57b4YPH87ee+/N9OnTmTZt2kAXzczWUMs/kC1pPeB+YJeImC9pUUSMrIxfGBHPyrtLOgo4CmDcuHEvmzt3bkvlsNYNHTqUp59+muHDhy8ftmTJEkaMGMHSpUsHsGRm1p21/QPZBwDXRcT88n6+pLFlwWOBB7v7UEScFhFdEdE1ZsyYNhTDWjVhwgRmzZq10rBZs2YxYcKEASqRmfVVO4L7JFakZAAuBA4vrw8HLmjDMqwfTJ06lcmTJzNz5kyWLFnCzJkzmTx5MlOnTh3oopnZGmqpQVXSRsDrgA9WBp8InCtpMjAXOKSVZVj/aTSaTpkyhVtvvZUJEyYwbdo0N6aarYNazrm3Q1dXV8yePXugi2Fmtk5Z2zl3MzPrMA7uZmY15OBuZlZDDu5mZjXk4G5mVkMO7ma2Sn6Y3LrJz3M3sx75YXLrLvdzN7MeTZw4kVNOOYW99957+bCZM2cyZcoU5syZM4AlM1h1P3cHdzPrkR8m19l8E5OZ9YkfJrfucnA3sx75YXLrLgd33BvArCeTJk1i2rRpTJkyhREjRjBlyhQ/TG4dMeh7y7g3gNmqTZo0ycfCOmjQN6i6N4CZravcW2YV3BvAzNZV7i2zCu4NYGZ1NOiDu3sDmFkdDfoGVf+0nJnV0aDPuZuZraucczczG2RaCu6SRko6T9JfJN0qaXdJx0u6T9IN5e/AdhXWzMx6p9Wc+9eBiyPiHZLWAzYEXg+cHBEntVw6MzPrkz4Hd0mbAa8BjgCIiMXAYkntKZmZmfVZK2mZ5wILgO9Jul7SdyVtVMYdLelGSWdIGtV6Mc3MbE30ubeMpC7gKuDVEXG1pK8DjwH/CzwEBPBfwNiIOLKbzx8FHFXePh/4a58K0l6jybKbt0WDt8MK3hYrdMq2eE5EjOluRCvBfWvgqogYX97vCRwbEW+sTDMe+EVETOzTQvqZpNk9dSsabLwtkrfDCt4WK6wL26LPaZmIeAC4R9Lzy6B9gVskja1M9lbAT98yM+tnrfaWmQL8sPSUuRN4H/ANSbuRaZm7gA+2uAwzM1tDLQX3iLgBaL40eU8r8xxgpw10ATqIt0XydljB22KFjt8WHfH4ATMzay8/fsDMrIYc3PuJ0hBJQwe6LHWhAb5jTtJQSdtI8nFky0laT9Iukoavfuq1xzvlWtBdEI+0LCL8804tqAbS6OecoqQ3SDqkcrPeIcAXgPH9WY66KRWfdbLSI2lrSftL2qQy+HTgJuClA1QswMG9LZprbt0FcUnbSvo3Sd+U9KHuPjfYSdq40rW225p5RCwr49aX9BJJW7W5DGPKozUa78+RdER5+xngR8A+5f084HHyhhZbjVLpedY+Xyo+HV3pad4XJf2o3Mj5SeBiYL/K6AfIu/d37u6z/WXQ/1hHb5QvR43AUh3eqJFXhm0AvAF4ObAwIr4qaWPgU4DIM/ocWBGoBqtyoIs8vpcBuwNbSvpb2a5RptsOmB8RSyS9H9ie3L7PAa6VdHxE3LmGyx7aCCiSPk0G6puAy4GfSjomIp4AnizlOhP4OfAq4OWSHiTvqh4CbNvKdqijShBf/j02HSdDImKZpG3Jk+O7gYeBrw50oJf0QmAP4MXAQmApuV9cXsY/n+zq/QRwA/Ao8AJJzyX3h6eB88o8zqbs4/25DuCa+3IlfzpR0kGSnleGDYEVKZXqMEmbVYLPTpJ+Xma1H7A3cDuwSNJ3gMXAAWSt76cRsfKPttZc47K7m1TVsohY2ti2EXEp8Bvy6aKUx0ffBFwEfETSMGAocCDwhYjYBdgM+H+N5fSyPPsA3yqvDyBPxlcATwEzgd3IkzHALGDX8vp2MqBfDpxIHvhPAtUb9wYdSZtJepmkLRrDyne7rHKMDJP0OkkflvTGEtifD9wKvB34BzAB+NyArEQh6RXAfwMvA+aS+8WWrNzFe3tgTET8BZgP/L38fYjch3YHfgC8sP9K/myDtubeqHVL2pzMm76VPHCfIoPypyJiXrlB60XkAb44Is6WtANwuaQdI2IJmXN9usz6OOAEYGPy8v1NwIeBbwDHAA+VM/wnIqITnqfTVpXa+LJKjS3I2k91umHAW8hHRG9Bbrcgv4uTJf0D2IEM5AuB75I3xP2BrBE9Xmb1R+AV5fWQ5uX04HYykAD8G3BmRMwtd1ffBPwFeJWkXciDu/H46hvIg/q3kt4LfJS8/N6CQaKcoJd/t8VQ8gR7jqRHgBFkBWcjMiheQB5XHyC34RvLcfVz4EHghoj4maSXAydI2iAi/tFvK7Wy1wC3R8TRjQGSFgH/LWm3cm/PM2SlAnJfGQH8jjzOLyLTMg8CI2HgrtAHRc29pzxfeflPsmb9xYjYJyLeBEwExpXxhwOfJQPNqyW9s6QAfkwGa4C3AT8trzcF/hXYEfg/4CUR8UxEnBoRkyJiCnlSfVO717O/ldp4t+0N1YNf0lhJn5L0a0lnKp859C/AYcCvyRtC7iQDwF3kwfJcYHRE3FPSI+eT2+wu8uTRaNS8inzwHPT+0vcB4Pkll/5QRJxVhi8o83gQ+Bownay9rVcCzj3AiFIhmAK8C9gJ2EDSpr1c9jqhXG11d9ws/25LugzyyuW9wE+Aa4AxZAXnEOCrZErry8CREfFx4AwyDTOEfPjW/DKf+8jA+by1s1arVq78HgWeI+kt5W+LiJhNnog+USZ9HnBFqSDeA6xfhp8PfJ8M9A8BT5d9fUDUJrhLGi1p1/J6pSuSbnLlW0raQ9LwiHiSPKg3k/RCSZPI3N8dZfKfAO8HLiRzq++UNAaYBuwn6UXA5uROCRkMzoiI4yJiBvn8nVGSxkl6n7IxdWlj/gPV2LImGmmV5uHVlEq5NB8h6QDlr3OdL+ngMulryZ4DxwInRsRd5NXODmQt+vpSU1tE1sh3Ji/Xd6os7j5gVJkmymuAP1Muf3tbQ4r87QGRQeb0yvBnyvxHRsTFwD1kgFrKikvse4GuiHgcOJlMt+1BqaWtSxr7nrKhc6X9sCkVWW1g/p9yor4GuFjSu8htdiUZxHePiLvJ7+W2iPgdsB5wIyu+syvJ42U74BZW9Cp5gmz7aFxV9aty0joLWEKm5Q4BfivpSHI/2b1cdR8I/KFSgZlHXt2fTAb4qyPiMfJq/uUwMMf5OpmWKRtqCCsa4iDTJpsAfy4HaXX6ScClEfGQpLNY8Sz6aySdCtxGBuu3k4F9MXnp+GNgAzIHN5QMOOPI2vglkmYB/04GigVlcWcDb1E+X2c0Was5jrxC2A+4GziF0jjT3935VqcE8ZUaiatplXI5vUVJWe1OHgQPk7WXbwN7AV8kr2AOk7S4fHYCuS0bj0n9FRkwjwI2ljQyIg6WtIBsKP0r8LikN5M1oQOBGSWVNhzYStKwiFgo6T/6cCl/aSnPsZJ+Bny/rPODwDbKp57+K3A8GbhfBFxLHsivAy6JiDMk7UieqJ551hIGSCPl2NP7huaGTkkbAhtFxALl7zB8g7yKXSDpixFxJZlPnk/uyyOBc8lAfjFZ+RlJHgsLyO+vEdQeBF4JXA9sXYYtIdMarySPicVkCu5FZPtUvysn/rcpu7s+h6yYvI1sC/oemWbauZS14S7gtRFxsaTjyHYYyKv7RWW+/X+cR0RH/gHrVV6/jsxnDV/F9MPIoL0euZOMkvUAAAsoSURBVKO8pDLuejIv+35gchn2YrKW/Vby7HoqsGMZtzd5Bn8JcDTw4zJ8G/JLPqy8H0OmFOY0leVFZIA7sixnyEBvzx62mXo53eZkA+SVwCXAn8jAuDuwDNirTPd2sob2GeAX5IH7frL94Utlm1wAfKdp/huRJ4htyNztDLJR9UXk5fD1ZBDZskw/FhhaXvdp25KX2IeSgeUKspcGwKvLur6mvH89WQP7dnl/APCGgf7uulmfkaXsB/Ry+u3ImukGZNvHLLKS8zGysvJh4NDK93oB2ZD4VeB/KvM5g+wOuCfwTWCHMvxI8mS/cXn/IfKEfhiZujixsj2/VZnfKFZxnPfTthxSeb0N2fPlVWSe/Wdk+rCrMs0hwAcHeh9o/uvImruki4Ehkg6LiIfIM+gEsiY8T9KryIaPHYHvRsTVZA73k8CnyY3/Dkk3l8/NIi//DyRrau8r01wEXEd+ac+U5dxBBqXhZO3zb8Dhkl5PnmSGkkGHyBrOjyg530YNKSJuIhvmmteruetf23RXO6tc4RDddC9rTC9pS2CriLhJ0hfJk9s/gK9HxEXktt6SrMHsQrYljCVrbM9Q8o/kyXUeWVs5gbyKelLZ7e0zZVnjgPOVXeC6yt/WZM1vEbn9ZwIblPK8KzJ1Vl3PeZV1WFa2azSv/2r8AfhURLytpMo+K+lXZG31LlY0kP+W/MGZhWU9frUGy+gXki4CXgDcDIyRtA1wbkQ8rrwP4KXAAxFxfbmiPJr8ru4lf2DnjcDXIuL8Mr+hrOjy+X7yu7+Z3CfuKuMa/kKm1xpXuWPJ9pM7gcnkMfQE2X5xB5mbv5pMc1K25/JtGhEL27lt1pSk9YGXKjtNjCGvUG6KiD+U8SeTlZxbGp+JiHMHoqyr05HBnTy4v0Q2oJ1JBthdgS3KZf7+5OXbH4GDyxdyM5nyGE/W3D9J7rTzgXER8Yik+8hc2T7VhZUDYAnZV/Uhsqayfpn/UDLf9gnyUv5fKamF0qD0TrKGQjW4aDX9fNulpEmWRcQzkl5A1phOiIhFZbnd9h4p7RIHkTXlMcCdks4kL5/fTJ6Ebi7tGPsDF5YT7RWSZgM7RcSdkh4la2t3SPoreZI9M0qKRNLOwJPKLoe7k3n0CyPiPklvJLfvb4CZEfEUebK9rnxWjcC+qgDex+16G6WXS0TcKumDwHYR8TdWDjZLyH1trXx/bXITmbI6p2zTScDV5UT6cfKE+4SkS4DLyLTC7yLic5IOJCs3/1eZ36bksbQI+FhkqgIASdcCn5K0P9nzZQfyBHg/8HvgXEl/Aj5CBvTHYPl2vLT8rUSVew46wGKysvH/yNTgWeSVHQCRbQi/G5iirZlODe7XkGf3saVWdR4ZbLaPiDnKn/R7ORmE9iF3wt+RO9IukfnwmeQO9llyZ4WswR8t6Q1kA93uZMA4vdRWPk7W7u8gLz0bO/VpdP+Iz0PJlMVlzSPWUiAXuR0UEUsljSR7bHyXrEHfAXy+EhA3IS+zXwwMi4hGAyflZLABmSPfNSLmS5pOXnrvR14lLSLbMTYBnpK0YQnAj5d5XkK2Q7wcuCMiLlL+IteZyt4jzyUv279N1oT/BHyHTLMQEd0+NrWRp60G8nZvz3Kiem3l/VNkJWJd9DvyWDiHDLLPkGmSTYH/JK+wvkt23fwjGYQfLZ/9J9m9s5pDfgyYDXw6IhaXytO/kDX3OWTceB2ZorkBOL/sj2cBP4iIf5b5nNFcUK1omK92le2UwN7Y584vf+u0Tg3ut5E72IVkC/SNZE52+zL+EDIAXUpe/u0ZEaG8a7Bxt+AlwBHkFcBFsLyGdgTZwLkpeVna6AZ3bER8rKcCVXfKMq+IiJN6mr4dlHe2fg44NbJLYFDp7hcRi5T9rZE0gvz92q9IOjciriDznGPIK4/7u0nd3EdeYo4kr3A2JAPcd8m+vk+Uec8ia+/XkSeQzcg0AGQNb19WNIBNJS/bhwE3RkSjofmHPaxjTw241ntzgM+XfeAg8tj5Axmg3l7e30YG26fI77rRqDkbeEzSxyLiZGXXvQ3IHPsLJP2ebBN5DJgaEU9IWgicExGNG72A5bXz6j0kQ5pPyp0UyOuuU4P7fGBiRNws6RwyQP8M2Lxcnh8DvD4i7pX0H5Xhj5DdlTYqud5TyLTOnxozjojrgYOblkejttFTXnxt75RacTdsdZlPSDo1si8typtsDiRryreQ+c4tyJPY7Ij4Z0kx7UheSr6dPGld3rSsRpB/lOzVsB15CTqbbD94sCy7izyZnU6eDC8oaat/kg3XkMF8ea2vHODLL2O7WceVUis+2NviAbLt4kLyZPtNct94DHhbRNxbnVjZI2mCpPUi4lFJU4HPSLqBrDx8HvhrREyT9L2IuL9pecvIk/v1Jb23tOk7XWtpSOu9jgzu5VJwmLJ725mSDgPeAVwT2YB2I/DJcvW+MVnzHEvW8CGDMxHxW1bcjLSSVQTxtZFO2TSy32u3QbwE2+r7rcgGxbvIbpUPRsRPyJPaVmQ3ytsj4g+SzidTIt8uH/8Lmc56AXn5vV6Z53qNS+/KgfgwmWJpXBF9nzwhnK28UUdkg/V1JRV2ZmQj8jspJ8iImNvDOj+rgdcH+9oR+cydBRGxf3W4pF8C00pbynbk1dcxZLvKYvK4eTAibpH0EeCJpvQMjcBe9tuh5eT9TjL9QzR1O7bO0ZHBvbiKfL7DLPIW8VlkD5pRZKPmMWQj6KlkT4DHyTTDSr1UGnnqbi4P+zPQzJW0V0T8uSmIrx8R/yyXsK8ib71fn7xD8wbydu3nAFtLupTstXAvebn9RJnNncBLJI0ueeRbyN4uT1BqbmSf7MWSNi418kbgfYzchlvD8t4/3yFzuA9WUiqQJ9Hpyv7fS8k7DBvr0V0gd2qlf10laY+ImFU5kU8lH9nwCfKq9o/k931ZRPym+uGIeARWe7wsK9/1X/phfaxFHfsze5I+QT5V8Yzy/kXAw91cIjZ/7ll5voEm6bdkQ9P3So36M2Rj3m+A0yPiakknAU9GxHHK7oh7kH2s3wQcFBHvlfQmMgWzgMx7n0N2TTuRvPxeLOkl5A1Z7yb76J5INoxuC9wbEe+pBuNGwF9F2ZdvT+WzP26LiEVt3UDWsurx0t3J1gafTq65/3ep0Vb7ji9XahhDaHqIUacF9uJ6spHxe+RNU09GxHhJHydznZ8ja+uN2tT5ZJfO8WRvlA9I2joifgH8Qnk7+NvJ7p77kY1kn5f0FPA/ZE+XrSJ7Fn2ArMnfUubV3Aulx8BexldTVte0shFsrWocL8+q3FSOFbdxDCIdG9yrgb0xrPq+/F9XdtSryG5okGmWK8vrn5PdBXcl89uN/OXfyf7DY8kuocOB0SXvuRdZC+8ic+DPlFr/EeSl91PAPo2gHXnDzzlrcd2sA1SOl2dVbtaxY8XapGODOzw7b7sOX2rOJoM45DZv3IV3NxnEzyYD+eFk7X0TsgFs+4i4XNkffVsyvbIHmZb5EeVmm4j4E5UeQazIxwPd9xu3+vH3a1UdHdzrIiL+LmlD5c0g1wBHlgbSIWRe/AGyYfgbykcm3E0+96bx0KV3AveVWtm/dbeMRu+f7i67fdCbDT4O7v3nceCVEfEd5S35V5CB/QQycD9TuqMpso/+D8n+5ETp5w49PhGzU9sazGyAOLj3n1+TXRkB/gvYMCoPwCr2JPslb1ymf9ZDqpw/NbPe6NiukHWlpockVfsVl14wG0fEfQNXQjOrAwf3ftQc2M3M1hYHdzOzGqrNb6iamdkKDu5mZjXk4G5mVkMO7mZmNeTgbmZWQw7uZmY19P8B4k9u3f40w7sAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X7i3Lu2ZLgbP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 317
        },
        "outputId": "2ec2791d-6b58-4181-aaa0-b37eb561d5aa"
      },
      "source": [
        "Accuracy=df_result.loc['Accuracy']\n",
        "Sensitivity=df_result.loc['Sensitivity'] # Recall\n",
        "Specificity=df_result.loc['Specificity']\n",
        "Precision=df_result.loc['Precision']\n",
        "index2=['NaiveBayes','logistic regression', 'KNN', 'Perceptron', 'SVM','Combination']\n",
        "df2 = pd.DataFrame({'Accuracy': Accuracy,'Precision':Precision,'Specificity': Specificity,'Sensitivity': Sensitivity}, index=index2)\n",
        "ax2 = df2.plot.bar(rot=10)\n",
        "figure8 = ax2.get_figure()   \n",
        "plt.title(\"Summary For withot FS\")\n",
        "figure8.savefig('/content/drive/My Drive/project/CGR_data/Summary_WITHOUT.jpg', dpi=400)"
      ],
      "execution_count": 149,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Summary For withot FS')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 149
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEYCAYAAAC9Xlb/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5wV1fnH8c+XJh0RUFGCQESlIwKKJRAFNKCCEsUSQIMliaixYzQWLNEU6w8VbGBDECJYI6KCBaSKFEFBRAVRVpQu/fn9cc5d7i67sLC77MI879drX3vvtHtm7swz5zxzZq7MDOecc3u3EkVdAOecc4XPg71zziWAB3vnnEsAD/bOOZcAHuydcy4BPNg751wCeLB3rohJOl/S6O2Mbydp0e4sk9v7eLB3mSQdL2m8pBWSfpL0kaRWRV2uwiZprKR1klan/bXZXZ9vZs+bWce08pikQwti2ZIukPThDqbJdf0l/U3SV3HYIklDC6JcbvcrVdQFcMWDpMrAa8CfgWFAGeAEYH1RlmtnSRIgM9uyk7P2MbMn8vG5pcxs067OXwxss/6SegE9gPZm9qWkA4HTi6R0Lt+8Zu9SDgMwsyFmttnMfjGz0WY2A0DSbZKeS00sqU6sgZaK78dKujO2DFZLelVSNUnPS1opabKkOmnzm6S/SJonaZWkOyT9Os6/UtIwSWXitFUlvSYpQ9LP8XWttGWNlXSXpI+AtcA1kqamr5ykqyWN2pkNIqmEpJslfS1pqaRnJFXJtv69JX0DvJvD/OMkdYuvj4vTd47vT5I0Pb7OrH1Lej/O/mncjt3TlndNLMcSSRemDa8Sy5YRy3pzLHsD4DGgTVzW8p1Zf6AV8JaZfQlgZt+b2cCdXIYrJjzYu5QvgM2SBkv6naSqu7CMcwg1wYOBXwMTgKeB/YA5wK3Zpj8ZOAo4BrgeGAj8AfgV0Bg4N05XIi7nEKA28Avwf9mW1QO4BKgEPATUjcEuffwzO7k+F8S/3wL1gIo5fG5boEFcl+zGAe3SplsA/Cbt/bjsM5hZanwzM6toZqm0yYFAFcK27Q30T/uOHo7j6sXl9gQuNLM5wJ+ACXFZ++ZlpdN8DPSUdJ2klpJK7uT8rhjxYO8AMLOVwPGAAY8DGZJekXTATizmaTP70sxWAG8CX5rZmJjeeAk4Mtv0/zSzlWY2G5gFjDazBWnzHxnLtszMRpjZWjNbBdxFCGrpBpnZbDPbZGbrgaGEEweSGgF1CGmq3DwkaXn8mxaHnQ/cF8u0GrgROCfVmoluM7M1ZvZLDsscl1bO3wD/SHufY7Dfjo1APzPbaGZvAKuBw2MAPge40cxWmdlC4D+Ek9vO2Gb9zew54HLCiWwcsFTSDTu5XFdMeLB3mcxsjpldYGa1CDXrg4AHdmIRP6S9/iWH9xV3ZXpJ5SUNiCmKlcD7wL7ZaprfZlv2YOC8mMPvAQyLJ4HcXGFm+8a/FnHYQcDXadN8TbjOlX4CzP656SYAh8UTZnNCy+JXkqoDreN65NWybNcE1hK2T3WgdA7lPHgnlg05r3/q4nF7YF9CK+EOSTm1Ylwx58He5cjM5gKDCEEfYA1QPm2SA3djca4BDgeONrPKbE2FKG2aLI9vNbOPgQ2Ei8znAc/uwud+R0gdpdQGNpH1pJTrY2PNbC0wFbgSmGVmG4DxwNWEVs+Pu1Cm7H4k1Pqzl3Pxjsq3M2KL4iVgBlv3CbcH8WDvAJB0RLwAWCu+/xUhZ/5xnGQ68BtJteNFyht3Y/EqEWr6yyXtx7a5/9w8Q8ixbzSz7XY/zMUQ4CpJdSVVBO4Ghu5kr5txQB+2pmzGZnufkx8I+fcdMrPNhN5Td0mqJOkQwskkdTH9B6BW6mL3zogXjjvH5ZaQ9DugETBxZ5flip4He5eyCjgamChpDSHIzyLUqjGztwl58BmE2ur28t8F7QGgHKEW+zHwvzzO9yyhFvrcjibMxVNxGe8DXwHrCDnsnTGOcLJ6P5f3ObkNGBzz52fn4TMuJ7S8FgAfAi/EskPoJTQb+F7SzrYkVgJ/A74BlgP/BP68iydOV8TkP17i9laSygFLgRZmNq+oy+NcUfKavdub/RmY7IHeOb+D1u2lJC0kXMDtWsRFca5Y8DSOc84lgKdxnHMuATzYO+dcAhSLnH316tWtTp06RV0M55zbo0ydOvVHM6uRl2mLRbCvU6cOU6ZMKepiOOfcHkXS1zueKvA0jnPOJYAHe+ecSwAP9s45lwDFImfvkmPjxo0sWrSIdevWFXVR9kply5alVq1alC5duqiL4ooZD/Zut1q0aBGVKlWiTp06hEfNu4JiZixbtoxFixZRt27doi6OK2Z2mMaR9FT83ctZacP2k/R2/P3Qt1M/j6bgIUnzJc2Q1CL3JbskWrduHdWqVfNAXwgkUa1aNW81uRzlJWc/CDgl27C+wDtmVh94J74H+B1QP/5dAjxaMMV0exMP9IXHt63LzQ6DvZm9D/yUbXAXws++Ef93TRv+jAUfE346rmZBFda5gjJy5EgkMXfu3KIuinO7xa7m7A8wsyXx9fds/U3Og8n6m5yL4rAlZCPpEkLtn9q1a+9iMdyerk7f1wt0eQvv6Zyn6YYMGcLxxx/PkCFDuP322wu0DCmbN2+mZMmSO54wj36ZNSvXceUa+y8Fuu3Ld9dLC4/N3OlHZ5rZQDNraWYta9TI092+zhWI1atX8+GHH/Lkk0/y4osvAiEwX3vttTRu3JimTZvy8MMPAzB58mSOPfZYmjVrRuvWrVm1ahWDBg2iT58+mcs79dRTGTt2LAAVK1bkmmuuoVmzZkyYMIF+/frRqlUrGjduzCWXXELqKbPz58+nffv2NGvWjBYtWvDll1/Ss2dPRo4cmbnc888/n1GjRu2mreL2drtas/9BUk0zWxLTNEvj8MXAr9Kmq8XWHz52rlgYNWoUp5xyCocddhjVqlVj6tSpTJo0iYULFzJ9+nRKlSrFTz/9xIYNG+jevTtDhw6lVatWrFy5knLlym132WvWrOHoo4/mP//5DwANGzbklltuAaBHjx689tprnHbaaZx//vn07duXM844g3Xr1rFlyxZ69+7N/fffT9euXVmxYgXjx49n8ODB2/s45/JsV2v2rwC94utewKi04T1jr5xjgBVp6R7nioUhQ4ZwzjnnAHDOOecwZMgQxowZw6WXXkqpUqH+s99++/H5559Ts2ZNWrVqBUDlypUzx+emZMmSdOvWjRmLljNj0XIGDX+NJke2pP4RDXnr7XcYM34Kq1atYvHixZxxxhlA6Btfvnx52rZty7x588jIyGDIkCF069Zth5/nXF7tcE+SNARoB1SXtAi4FbgHGCapN/A1kPpR5DeATsB8YC1wYSGU2bld9tNPP/Huu+8yc+ZMJLF582YkZQb0vChVqhRbtmzJfJ/e1bFs2bKZefr169Zx103XMuT1dznwoFo8et89bFi/frvL7tmzJ8899xwvvvgiTz/99E6unXO522GwN7Nzcxl1Ug7TGnBZfgvl8qfJ4Ca5jpvZa+ZuLEnxM3z4cHr06MGAAQMyh7Vt25ZmzZoxYMAAfvvb32amcQ4//HCWLFnC5MmTadWqFatWraJcuXLUqVOHRx55hC1btrB48WImTZqU42etj4F936rVWLtmNW+/PooOnbtQqVIlatWqxciRI+natSvr169n8+bNlC9fnmNOO4ZzTz6X6vtXx/Y3Zv84O3N59Qp307g02+s4sL1OAMX52PM24p7qtiq5j6vrvZtyM2TIEG644YYsw7p168acOXOoXbs2TZs2pXTp0lx88cX06dOHoUOHcvnll/PLL79Qrlw5xowZw3HHHUfdunVp2LAhDRo0oEWLnO8drFylCt3O7Um39sdSff/9adQsTvfdJzz77xu59IY7ueVv11G6VCleGvBP6h1Si+r7V6de/Xqc2OnEwt4Ubjebc0SDXMc1mDun0D/fg70rUrnWkr77JNd5Zpcps8uf9957720z7Iorrsh8fd9992UZ16pVKz7++ONt5nn++edzXP7q1auzvO9z/c30uf7mbFN9Rf16tXn3pYHbzP/L2l/4+quv6XRmp9xWYa9V1MFwb+fBvhjbblOy7G4siNstxrw/kZ7X9aPnpT2pVLlSURdnu3Y1zeGKjgd754qJ9r85mrc/ebuoi1GotpfTHrYby5FE/jx755xLAA/2zjmXAJ7GSRi/COZcPu2hPeE82DvnCtYeGgz3dp7GcYlTsmRJmjdvTuPGjTnrrLNYu3Ztvpd5yy23MGbMmFzHD3v2KV4d/mK+P8e5XeU1e1e0tlcLzEWj7S5vxQ7nL1euHNOnTwfCkyUfe+wxrr766szxmzZt2uln0vTr12+748/u8cedWp5zBc1r9i7RTjjhBObPn8/YsWM54YQTOP3002nYsCGbN2/muuuuo1WrVjRt2jTL4xXuvfdemjRpQrNmzejbN/xI2wUXXMDw4cMBwtMsTzyG33c4jv/c8XcAHr3vHgY/Fh6bPH3W5xxzak+atj+bM3pfw8/LVwLQ7vcXc1+/+zin4zl0ProzUydM3Z2bwu3lvGbvEmvTpk28+eabnHJK+NXNadOmMWvWLOrWrcvAgQOpUqUKkydPZv369Rx33HF07NiRuXPnMmrUKCZOnEj58uX56aesP+K2bNkyXn75Zf77zsdIYuWKbVsaPf/6dx6+4wbatjmKW/71KLffN4AH+l0HwOZNm3lx9Iu8//b7PPrvR3lixBOFvyFcInjN3iXOL7/8QvPmzWnZsiW1a9emd+/eALRu3Zq6desCMHr0aJ555hmaN2/O0UcfzbJly5g3bx5jxozhwgsvpHz58kB4FHK6KlWqULZsWW699nLGvPnqNs+/X7VyBctXrKZtm6MA6HXWqbw/ceujIU7qHJ4v2LBZQxZ/4z8F4QqO1+xd4qTn7NNVqFAh87WZ8fDDD3PyySdnmeatt97a7rJLlSrFpEmTeGLoKN5+/RVeHPQ4Twx9Jc9lK7NPeO5PyZIl2bx5c57nc25HvGafzZwjGuT655Lj5JNP5tFHH2Xjxo0AfPHFF6xZs4YOHTrw9NNPZ/bgyZ7GWb16NStWrOCEEzty3a138cVnWX83tlLlKlStUokPJk4D4NkRr9P2mJyfmulcQdqja/b+MCZXWC666CIWLlxIixYtMDNq1KjByJEjOeWUU5g+fTotW7akTJkydOrUibvvvjtzvlWrVtGlSxeWr1qDmXHtLXdts+zBD/TjT33vYu26ddSrXYun77ttN66ZS6o9OtjvKn8YUzGSW1fJXXzE8Xa7ZUbZH0MM0K5dO9q1a5f5vkSJEtx9991ZAnlK3759M3vhpAwaNCjz9aRJk5ixaHmW8X++euv0zRsfzsevPbPNcscOfzxz3apWq8roaaPzsDbO5Y2ncZxzLgE82DvnXAJ4sHfOuQTwYO+ccwngwd455xLAg71zziWAB3uXSHfddReNGjWiadOmNG/enIkTJxbYsjt16pT5TJznnxpA198ezY2XX8zY0W/wZP/7tzvv+Z3OB2DxN4t5fUTu95E4t7MS2c/eFR/bu+dhV8zsNXOH00yYMIHXXnuNadOmsc8++/Djjz+yYcOGAivDG2+8kdnPftgzTzJwyMscUPNgANp17AR8leu8z7/xPACLvw3BvnM3vznQFYy9t2Z/W5Xc/1yiLVmyhOrVq7PPPvsAUL16dQ466CDq1KnD9ddfT5MmTWjdujXz588HICMjg27dutGqVStatWrFRx99BISbsy688EKaNGlC06ZNGTFiBAB16tTh55+WcceNV7Hom4X8pcdZPPv4I4wa9gJ33xyebvlDxjLO6H0Nzdp3p1n77oyf/CkArQ5pBcADdzzAtI+n0a1dN5557Bl6ndaLT+fOzVyHk3r2ZMbnn++eDeb2CntvsHcuFx07duTbb7/lsMMO4y9/+Qvjxo3LHFelShVmzpxJnz59+Otf/wrAlVdeyVVXXcXkyZMZMWIEF110EQB33HFH5vQzZszgxBNPzPI5f//H/ex/wIE8MexVelz8lyzjrvj7P2l7TAs+HTOUaW+9QKPD62UZ/9e//5UWx7RgxNgR9PxTT848/0yeGzUKgHkLF7Ju/XqaHn54gW8bt/fyYO8Sp2LFikydOpWBAwdSo0YNunfvnvm4g3PPPTfz/4QJEwAYM2YMffr0oXnz5px++umsXLmS1atXM2bMGC677LLM5VatWjXPZXj3o8n8uedZQHjCZZXKlbY7fcfTO/LmuHFs3LiRZ15+mT906bIzq+yc5+xdMpUsWTLzeThNmjRh8ODBAEjKnCb1esuWLXz88ceULVu2SMoKUK58OU5s04bX3nuPEW+9xUdDhxZZWdyeyWv2LnE+//xz5s2bl/l++vTpHHLIIQAMjUF06NChtGnTBghpn4cffjjL9AAdOnSgf//+mcN//vnnPJfhpONb8+gzLwGwefNmVqxclWV8hYoVWLs66w+hX3DmmVxzzz0c1bgxVav4tSe3czzYu8RZvXo1vXr1omHDhjRt2pTPPvuM2267DQgBu2nTpjz44IPcf3/oJvnQQw8xZcoUmjZtSsOGDXnssccAuPnmm/n5559p3LgxzZo147333stzGR7sdx3vjZ9Ck5PO5qhTzuezLxZkGX9Yw8MoUbIEZ7Y7k2ceC0/IbNGoEZUrVKBH164FsBVc0ngaxxWpXLtK7uIjjvPiqKOOYvz48TmOu+6667j33nuzDKtevXpmjT9dxYoVM9M/6RYuXJjZ9fLNCTMyh3c5+zy6nH0e8BUH1KjGqKe37XM/+evJAJQuXZqnXn4qy7jvli5lixntjz12+yvoXA7yVbOXdJWk2ZJmSRoiqaykupImSpovaaik/B2ZzjlGDR1F2/PO47bLL6dECW+Qu523y3uNpIOBK4CWZtYYKAmcA9wL3G9mhwI/A70LoqDOFbaFCxdSvXr1oi5Gjrp078K8MWM4M9tv4jqXV/mtIpQCykkqBZQHlgAnAsPj+MGAJxidc66I7XKwN7PFwL+BbwhBfgUwFVhuZpviZIuAg/NbSOecc/mTnzROVaALUBc4CKgAnLIT818iaYqkKRkZGbtaDOecc3mQn9447YGvzCwDQNJ/geOAfSWVirX7WsDinGY2s4HAQICWLVtaPsrh9lDZf5Q7XVO/BulcgcrPIfUNcIyk8gq3Gp4EfAa8B/w+TtMLGJW/IjpX8ArzEcfHxq6Ri7/9hjdefilz+OxPP+GeW27Y7rxDBw1l1NBwyIwcMpKl3y8tsHK5ZNvlmr2ZTZQ0HJgGbAI+IdTUXwdelHRnHPZkQRTU7Z1Kt2+T4/A525lnuzWUudubMyjsRxyPHz+eGYuW892ib3hj1HA6nRGegdOo2ZE0anYk23vEcfcLume+HvniSA5tcCj7H7h/gZXNJVe+GstmdquZHWFmjc2sh5mtN7MFZtbazA41s7PMbH1BFda5gpDbI46nTp1K27ZtOeqoozj55JNZsmQJAO3ateOGG26gdevWHHbYYXzwwQcAzJ49m9atW9O8eXOaNm2a+QiGihUrAvDgP27nk0kTOPvkE3j28UeYPOFD+lzQnS1btlDn6M4sX7H1EQn1j+vCDxnL6P/P/jzd/2lGvzKa2dNn0/dPfenWrhvjRo/j7CuuyJz+nfHj6X7llbtle7m9g2dGXeLk9IjjjRs3cvnllzN8+HCmTp3KH//4R2666abMeTZt2sSkSZN44IEHuP322wF47LHHuPLKK5k+fTpTpkyhVq1aWT7nyhtv5cjWbRj21gdZHnFcokQJupzcjpf/9y4AE6fN5JBaNTmgRrWtZTy9I42aN+Kex+5hxNgR/KbDb/jiq6/I+OknAJ4dOZKeZ5xRaNvI7X082LvEyekRxwMGDGDWrFl06NCB5s2bc+edd7Jo0aLMec4880wgPGph4cKFALRp04a7776be++9l6+//ppy5crluQzdT+vI0FdGA/DiqLfofnrH7U4viXNPO40XX3uN5StXMnHGDE4+/vidXHOXZP5sHJdI2R9x3L9/fxo1apT5DPvsUimfkiVLsmlTuI3kvPPO4+ijj+b111+nU6dODBgwYJsfMMlNm5ZNmb/wWzKW/czIt8Zy85UX7XCeHl278vs+fdhnn304s2NHSpXyw9flndfsXeLk9IjjBg0akJGRkRnsN27cyOzZs7e7nAULFlCvXj2uuOIKunTpwowZM7KMr1ChImtXr85xXkmcccpvufq2/9Cgfl2q7bfvNtNkf8zxQfvvT8399+fegQP9yZdup3mwd4mT0yOO+/Xrx/Dhw7nhhhto1qwZzZs3z/XJmCnDhg2jcePGNG/enFmzZtGzZ88s4+s3aESJkiU5q+PxPPv4I9vM3/30k3nuv2/Q/bScUzhdz+lKv2v70a1dN9b9sg6Aczp3ptaBB3JEvXo5zuNcbrwd6IrUxjE5p02alsi9e2JhPeK4evXqvP/++9sMHzt2bJZpUjn7vn370rdv322mX716NTMWLad06dI8MfSVLONatTmeVNfLls0aYounZRl/2fVbf+aww2kd6HBah60jVxjjp03jwm7ddrSKzm3Da/bO7SGOPftsZn3xBeeeempRF8Xtgbxm79weYvywYUVdBLcH85q9c84lgAd7t9uZ+XPvCotvW5cbD/ZutypbtizLli3zoFQIzIxly5ZRtmzZoi6KK4Y8Z+92q1q1arFo0SIyMjL44edfcp1ujnL/jYPvt3MzUYmM4lF/KYx127wy988rXbIkEE6m2R/b4Bx4sHe7WenSpalbty4Av+v7eq7TLSx7Xq7jzq5bO9dxM3vN3PXCFaDCWLdh/9iU67gGeXjap0u24lENcs45V6g82DvnXAJ4sHfOuQTwYO+ccwngwd455xLAg71zziWAB3vnnEsAD/bOOZcAflOV26vMOaJBruP8xiOXZF6zd865BPBg75xzCeDB3jnnEsCDvXPOJYAHe+ecSwAP9s45lwAe7J1zLgE82DvnXAJ4sHfOuQTwYO+ccwmQr2AvaV9JwyXNlTRHUhtJ+0l6W9K8+L9qQRXWOefcrslvzf5B4H9mdgTQDJgD9AXeMbP6wDvxvXPOuSK0y8FeUhXgN8CTAGa2wcyWA12AwXGywUDX/BbSOedc/uSnZl8XyACelvSJpCckVQAOMLMlcZrvgQPyW0jnnHP5k59gXwpoATxqZkcCa8iWsjEzAyynmSVdImmKpCkZGRn5KIZzzrkdyU+wXwQsMrOJ8f1wQvD/QVJNgPh/aU4zm9lAM2tpZi1r1KiRj2I455zbkV0O9mb2PfCtpMPjoJOAz4BXgF5xWC9gVL5K6JxzLt/y+0tVlwPPSyoDLAAuJJxAhknqDXwNnJ3Pz3DOOZdP+Qr2ZjYdaJnDqJPys1znnHMFy++gdc65BPBg75xzCeDB3jnnEsCDvXPOJYAHe+ecSwAP9s45lwAe7J1zLgE82DvnXAJ4sHfOuQTwYO+ccwngwd455xLAg71zziWAB3vnnEsAD/bOOZcAHuydcy4BPNg751wCeLB3zrkE8GDvnHMJ4MHeOecSwIO9c84lgAd755xLAA/2zjmXAB7snXMuATzYO+dcAniwd865BPBg75xzCeDB3jnnEsCDvXPOJYAHe+ecSwAP9s45lwAe7J1zLgE82DvnXAJ4sHfOuQTId7CXVFLSJ5Jei+/rSpooab6koZLK5L+Yzjnn8qMgavZXAnPS3t8L3G9mhwI/A70L4DOcc87lQ76CvaRaQGfgifhewInA8DjJYKBrfj7DOedc/uW3Zv8AcD2wJb6vBiw3s03x/SLg4JxmlHSJpCmSpmRkZOSzGM4557Znl4O9pFOBpWY2dVfmN7OBZtbSzFrWqFFjV4vhnHMuD0rlY97jgNMldQLKApWBB4F9JZWKtftawOL8F9M551x+7HLN3sxuNLNaZlYHOAd418zOB94Dfh8n6wWMyncpnXPO5Uth9LO/Abha0nxCDv/JQvgM55xzOyE/aZxMZjYWGBtfLwBaF8RynXPOFQy/g9Y55xLAg71zziWAB3vnnEsAD/bOOZcAHuydcy4BPNg751wCeLB3zrkE8GDvnHMJ4MHeOecSwIO9c84lgAd755xLAA/2zjmXAB7snXMuATzYO+dcAniwd865BPBg75xzCeDB3jnnEsCDvXPOJYAHe+ecSwAP9s45lwAe7J1zLgE82DvnXAJ4sHfOuQTwYO+ccwngwd455xLAg71zziWAB3vnnEsAD/bOOZcAHuydcy4BPNg751wCeLB3zrkE8GDvnHMJsMvBXtKvJL0n6TNJsyVdGYfvJ+ltSfPi/6oFV1znnHO7Ij81+03ANWbWEDgGuExSQ6Av8I6Z1Qfeie+dc84VoV0O9ma2xMymxdergDnAwUAXYHCcbDDQNb+FdM45lz8FkrOXVAc4EpgIHGBmS+Ko74EDCuIznHPO7bp8B3tJFYERwF/NbGX6ODMzwHKZ7xJJUyRNycjIyG8xnHPObUe+gr2k0oRA/7yZ/TcO/kFSzTi+JrA0p3nNbKCZtTSzljVq1MhPMZxzzu1AfnrjCHgSmGNm96WNegXoFV/3AkbtevGcc84VhFL5mPc4oAcwU9L0OOxvwD3AMEm9ga+Bs/NXROecc/m1y8HezD4ElMvok3Z1uc455wqe30HrnHMJ4MHeOecSwIO9c84lgAd755xLAA/2zjmXAB7snXMuATzYO+dcAniwd865BPBg75xzCeDB3jnnEsCDvXPOJYAHe+ecSwAP9s45lwAe7J1zLgE82DvnXAJ4sHfOuQTwYO+ccwngwd455xLAg71zziWAB3vnnEsAD/bOOZcAHuydcy4BPNg751wCeLB3zrkE8GDvnHMJ4MHeOecSwIO9c84lgAd755xLAA/2zjmXAB7snXMuATzYO+dcAniwd865BPBg75xzCVAowV7SKZI+lzRfUt/C+AznnHN5V+DBXlJJoD/wO6AhcK6khgX9Oc455/KuMGr2rYH5ZrbAzDYALwJdCuFznHPO5ZHMrGAXKP0eOMXMLorvewBHm1mfbNNdAlwS3x4OfF6gBdm+6sCPu/Hzdjdfvz3X3rxu4OtX0A4xsxp5mbBUYZckN2Y2EBhYFJ8taduJE5cAABaoSURBVIqZtSyKz94dfP32XHvzuoGvX1EqjDTOYuBXae9rxWHOOeeKSGEE+8lAfUl1JZUBzgFeKYTPcc45l0cFnsYxs02S+gBvASWBp8xsdkF/Tj4VSfpoN/L123PtzesGvn5FpsAv0DrnnCt+/A5a55xLAA/2rkhIUiEtt6SkgyT5vr2HkVRGUiNJpYu6LHsjPyAKgYIS8W5iF6UHYCug/GF8NMfZkirEQWcD/YA6BbH8vUXcJ4vN/ijpQEkdJVVKG/w4MBNoUUTF2qPsbIXJg30+5RTULdhiZpuLqly7m6SKkg5Pe7/NjmhmW+K4fSQdKemAPC67hqQqae9fkHRBfHsj4S7tE+P7JcAqws0tiRP3x22O67hP7vb9Mft+IOlFSS2Ba4H/Ae3TRn8PZAD1c5o3Kba33unjUhWmvG6nIrupak8lqUQqaMHWAJZtmoMJj4hoDMwws8eyz7eniwFFhH1uC9AG2F/SFzGwpHbEWsAPZrZR0kWEezBaAYcAUyXdZmYL4rQlUwFJ0g2EwD0TGAuMkHSFma0G1sTPGwSMBI4FWklaSrgTuwRw8O7YDkUpLahnbu/0fSy1z8X9sTrwB2AZ8K/CCPzxGVjHA02Bn4HNhO9ubBx/OGDAamA6sAI4QlJdwne2Dhgel/Eccf8q6HIWV7GV08LMxsUK5JbsLeC046oi0ATYaGZT8rJ8r9lnk0rB5DQctjmYykk6Q9Ldkq6LwyoC1xEeATETmJV9vj1JqvmfQ+tli5ltTq2Xmb0NjAHKx/lukzQTeBW4UlIpQlfcTkA/M2sEVAHOitOfCDwaX/8OOAUYB6wF3gOaE7YrwIdAs/h6PiHAjwXuIQSZNUDNgt0SRUtSFUlHSaqWGha/gy1pAaCUpA6SLpPUOQb6w4E5QDfgF6ABcHMhlK81cB9wFPA14bvbH+iRNtmvgBpmNhf4Afgq/v2J8D23AZ4lPEBxrxdbw8enDdoP6AgQjy2Lx97hqXSXpBaSRgBPAXcB/yfpsLx8XiKDfdyAjSV1SW2oVIBPpWDSh0mqknZAHSppZFxUe+C3hICzXNIAYAPhiZ8vAiPM7MPduW75kUpJZW8qxh1vc9p0pSSdKWmApOEKF9UaEp522lxSC6AeIbCfALQELgUmAXMJaRaACYQH5wEsIAQigL8Ag8zsa2Al4aQ5GjhWUiNCIPl1nHY6IYC8SwgyfyWkAjKD4p4m+3cQlSScGA+OJ+BykjpJOkvSK5J6x/E3ErbNZZLOAL4AlgLTzewW4P+A4yWVK+Bi/4bwAMRLzexfZjaGEJDqS2oep9lEOMFD2A/KAh8QWma/JqRxlgL7wp5bQcpJTum12Ep9KO3918ALqe9e0mXAVEIr535JNQitn98CD5rZicAMoLOk8jsqQyKCfdrG20/S/wHfEDbyxcAtkmrGWlCZWHv6o6Q/xGH1gJna2kOgDmGDA9xKqFFuAtoC3YEtcdlXAH+X9JrSctnFRQwo2Xe+VG3d0qarKek6SW9JGiSpDnA0cB7hxrmBhEC9FlhIOIDrAtXN7Nu4Q78MnBrHC0hdTP2Y0AKCkLI5XCEX/6OZDY7DMwhN+aXAf4AnCUG9jKRyZvYtUFbSfsDlwPnAoUA5SZXzvaEKUW6tyPTvQCENBqGl0hN4iXCXeg3gbsIF6X8RUlr3An80s6sJgfYPhGP8R0JNGsKjSzYBeaoN5nU9CCmZQySdHv+qxfTCSOCaOOlhwDhJit/bPnH4y8AzhMD/I7Au7md7lNT3Gf+ynKxTrbA4Xan4/0igiaSPJL0eJ50CHB2D9+nABWbWivDYme6E42wcsQVNqEDVIA+Vm70u2Ody8KSC13pCzftOMzvRzE4l5NVrx/G9gL8RaqXHSeoe88lDCcEb4ExgRHxdGfgzoVbyOnCkmW0ys/5mdq6ZXU64LnJqQa9nXqXSMNmHp6dgYoqgrKTfxZr6y5K6xknbEnpH9AXuMbOFhBNePUKL5hMz+wVYTqix1yekDQ5N+7jFQNU4jcXXAJ8Sm+xmtp5wIniK0CsjVc5Ncb59zex/wLeEALeZrc39RUBLM1sF3E9oWR1PrCEWpbSKRk4BIL0VmX4B+oF4gp0M/E/S+YRt8D4hqLcxs28I22+emX0AlCHU8lLb9n1CUK8FfMbWHi6rCSfWVCsq3+LxNRjYSEi1nQ28K+mPhO+yjUJevhMwPu14XELIO99PCPgTzWwloTLVKm6LYneRNr1MkhpKekhS2dT3mZ5aS5uuh6SRkuYC/5ZUnfD9LABeNbPOcdIPCMdWA+BLwvcFYfseQEj1zCNexCYcDxXIw76+RwZ7SdUlNYuvs1xkzt70k7S/pOMllTazNYSaYpX4JZ1LuGD1ZZz8JeAiwrN8Dga6x6bTXUB7SU0IG3tTnP5rwuMgbjWzIcC3kqpKqi3pQkl/IgSlL2NZCnXHzaW2bmkXPctIqhlft5H0X0mPAw8TgkE74E7CwXeKpE6x/A2AcoRcOMCb8e8S4F+SRprZcsK2PYSQQ18l6TRJ+xIO8iHxACgNHCCplJn9TGj9pFIKbwOjgL6SLkhbl6VAeUkHEk6uJxB27iZx/BKgQ1zfp4AhhIMg9T0VuOzfZW7fbfqF05iDLR/3KeK+8qykT4CXJP0mztaGsI7tgc6EikZVQu+VEmw9sDMI2zn12UuBY+LrA+P/jYSUSWr4BsJ1jdS2KxBmtsHMziRca7kbeIxQMaoKPE3I3dePn5+yEGgbt9GthHQdhMrV8rjcIrtAmy2oX63Q4i+RrUxzzewKM1un0J20g6Q7Jc2W9JS2/nDTIuBuMzuC0KLpa2ap9GR6oJ5GaDmvJFSMUuNWEC58zyO00lJpzMWEVl8tdqDY98aJG7wEW3t9QLg4Vwn4NNb80qc/F3jbzH6UNJiQUsgAJkvqT9hYdxEuWC0j7HydJQ0lBLT7CPnROYQa/5FmNlrSh8D1hNpnRvy454DTFXKS1Qkb/VZCC6I9IV30MLE3QkHuuLEpnP1K/TY9LBTSG3cBjQg1pn0ltY2juwInmtlYSd0INeLlwHGEmvkUwrWHFoSA3UTS92Z2KXBTXH4F4BtJBxG2ywmEnfkC4A5Cn/d5wAPxM68ClprZ5njg3JVW3KmEnXchISXRiFBT/CqW6TAze1/Sq4ST8jGEmu5/SOu1YWY35W0r7px44moEVDazN9PH5fTdKqRgjiVcpO5A2H8OAB6R9AAhFfa6mfWI2/8aSV8RauWlzWwFsELSbEIQnUi4xlGJsK0/i+8rmNlqSZOALpJWxOm/MrPvFC6U143lXCfpTrbWGAtM/D7XAJ9JWg6cRAhC/0cI+HUJefmUV4gtkZjuI77uX9Bl25Gc4ky277Ri/DtYUm1gnZlNBSpJ+hdhXZYAtxFSK60IseQaoDdh3/6zpH8CRxCOLQgpuYvSPmcCcJ2ZzZO0Brg4VgYaAovNbJWkZYTrYBAqkk8Dn+xwJc2syP+AMmmvOwCXEXb23KYvRdhxyhCC6ZFp4z4hHAAXAb3jsKaEWvgZ8UvoD/w6jvstoYl0JNAHGBqHH0ToXXJefF+DkJ+ela0sTQi14T/GzymxC+uvnIYRTjoldzDv/kCT+PpO4KNY7tPisK6EtFN1QkpmNaGZWJ5wolP8u5aQg/8zIUBViPOXSPus2oSdNtW19A5CM/35uLwWhNp+tTh9hTysZ4nUcEKN9r/xdQNCz4w3CSehG4DWcVxpoGr28hXyPvoq4aQ1Mm7j3kClOO6AWMYj4/vmwBPAO3HfOgUYAJyRtryScbt9Eb+vGYQWSXXC/v982rTXA48QerM8DRwXh7eLZTkobbt0iNutD1B7Nx7D+8Tv73zCRfLXgH+kjT8hDi+/u8qU27EWt/02+2IO+2UT4Ij4/qz4XQ6L2/wV4OI47p9x3SoTWiWpuNMZeCu+bk9I9VaJ+/aHcb9pCCxJK1tNQpdLCK2zAYSY9t/U90molO70fl/kNXtJ/wNKSDrPzH4kpAEaEHb6JZKOJVzp/zXwhJlNJDRzriUEgLXA72PtJ7UR5xNqon0lXRineZXQRKpCaN4fQjgrziUcJJUJB14vSScTDpqSxOaumWVIepF4QTFVs7bQFEs1P9PXK3s/9JzWvQzhi90k6QjCCeNuM1tu4VvNsS90TF11IeyANYAFkgYRmvGnxc+dHVNdHYFX4rYdJ2kKcKiZLYg1wHpm9qWkz+N2HWQhB4+k+sAaha6QbQi1/VfMbLGkznH7jAHeM7O1cftOS9s+a9K2hcV1yiLbtplHvNBkZnMkXQrUMrMvCEE/Nc9GQioi+/yFaSYhFfVCXPdzgYmxlnc1oVa3WtJoQpCvD3xgZjfHdFgVwsGeUpnQ8lsOXGXhJzwBkDQVuE5SR0Jvo3rAu8B3hEAzLNbiryRcsF4Jmdvl7fiXhdLuYSgkGwjB6SxCGm8w4UIisWwfEPLRu4VCF9XTCTXggwjB8oW4DTZnm7YiocXWjhBsR8f5/gB8IukWQiXnKkLwPjseEzdJGkaIGy0IFanFbE1VfQpUj63rQwitgRWSDiUca0dY6FM/MdbevzCz7pL6K1wD+F5Sn/i9ZkodnzuryIM9oQ/1PwgXMQcRNlwzoJqkDYRg9TOhedNV0j7AbMKBUodQs7+WcBb9gXD2+0nSYsLFoBPTP0zhrs2NhJs5fiSkc/aJyy9JqKleQzhg/kz8ibHYJO9OyEWSHri0g5tb4jSpGrQspDD2JdSCniCcfL4Ebk8LkJUIO1dToJSZpS6YEk8O5Qg7ZzMz+0HSk4SaX3vCiXI5oblfCVgrqXwMyKviMkcTUlWtgC/N7FVJJwCDFHqx1CVcLH2MkP6ZxNZaBhZ+aWwbqTxn+vbJa0COJ6S2ae/XEvaH4uADwon0BULQ3USorVYGbiEc2E8QangTCEF5RZx3PaF7aHq+eiWhKX+DmW2I+/XRhL7wswjHZgdCT5vpwMtxvxkMPGvhgjaE7ygLbb0gvyVtfyzUu2fj57wc/4pMWnrzAkIF5X+Eip4B9SRtIRzzmwktoB8JXX0vBR4EfiLs89eY2WGSriLcB3A9Ie+eSpdMIHy/tQjXhzpZ6L23nHj3tpktisdpTUKcO1fSNELsGkbYLyC0hkuZ2XdxvstT65M90OdHcQj2kwm5yJoKFzSHE4Lir8xslqQHCQHpNMIt8csJB95KoJGFfPp7hFrO3wgHH4Qafh9JpxDOtm0I++Tj8WC4mlD7/xJ4IO1AzO3nEs8hXJx9J/uInIJZrC3cDPS30AXRyJpXXi6pZ5y2LCGv+U9Jw8xsHGEHq0E4+XyXQ45+MSG3uy/hJFeeEBifIPR3Xh2X/SHhhDktrmsVQs4Qwk53EiEvDyEPfyxhv5hhZqlrE8/nsD1SQSVLyyWn2vteYhZwe/yuuhD2v/GE4NYtvp9HCL5rCd9J6iLpFGClpKvM7H6FboXlCBejj5D0EaFHxUrgJgv5958JNdHUjWTA1oM/tT8ohzuzCzuwF2dxm7QltLxOtNC7BwjdiAktoc8JJ9VhhP3/W2CTmT0Upzsd+DJWXKYD7SzcAb6S+J3G47choSK0CKgQK2grgbqSqsfKyxxCWvN9SX8ldM2em+2YWVqY2ySlOAT7eYQN9AqhF8gMwoXT1E8bnk2orb5N6KZ0QvxCl7L1lvjRhDP5Pwhn8VQa4ALCBdPKhC8k1Xe7r5ldlVuB0mtGcVlmZv/ezvSpG7LSv8DVkvpb6E+c2tE6EU5cnxHSR9ViuaeY2frY6vg1ofnbLZZzbLbPSgX9FYSLdLUIO+8UQoppafzslrH8j8f1HxVbMusJ1zogBPfM2mYMJJlN7xzWMUsqJmFB5XtC0/4VwknyEcJ3uBI408wWpU8sKQNoIKlMbLrfBNwoaTrhpH878LmZ3SXp6VStLs0Wwkn5k5i225xt2+fYgnRASNt8aWYrFXrhpWrHHYCVqWNf0huEY/Jr4AtJFWLLeimhJWbxe5RC19gvgIskzY+fkXrkwwZCi2B/QlqzMvECuJl1i58lM5u1O1Y+N8Uh2P8ANDaz2ZJeIATs/wL7xQBzBXBybBL9PW34T4T+uxXMbI2khwlpoEmpBZvZJ4QLlFmkmsDKJa9uWe8WrUzMieYU1OOXmP7+AKCchf7op0taamYvxfU4gNAzZ76ZjZf0MmGnfCzOPpfQwjmCkAYoE5dZJtXySDvglxFSMqmT4jOEE8RzMUcowjWOabF1NMjCdYfuqW1i4Y69beTQikh8UIk1uwwz65g+PAaMuxSumdQitKKuIASMDYSW11Iz+0zSlcDqbOkcUoE+7l8lY3DqTkgXYdl6nLkdqkToBl3WzNalDa8PjJFUOdb4pxJSLAsJlaBfEyqb3xGuk0DIJJQlVKR+IFROexBq+Pda6PUnMzt3ewUqDi3eIg/2FvKVpRTuhhwk6Tzg98DkmAObAVwbU8EVCQdPTcKXAiGoYeF2+drbfsJ2g3peAtjXktqZ2afZgvo+ZrY+nv2PJXQx3IewY0wn3J17CHCgpLcJZ/1FhGZ/qpvZAuDItCbfZ4TeQauJNUZgdNxGFWONPRWIVxKuPaSalRkKj2v4gBBcUimY1HZ7UqGf+mbChafUeuQU2It8xyymPpZ0vJl9mHYCvomQ772GUAGZQPhe3rHwyIBMZvYTbL1+k0P6ZQuwJX4nc3fD+uytFhC65e5HSIGWs3BR8zvCTZRvEI6f+oTrgQsI12CaEeLKUsLd3KlK5SOElt2nwDN76vFSLH6WUNIowpP4PlS4Uv0hIb98KqHJewXhABoOfG/hTsmclpPjQZTPsr1LuCD2dKxx30i4iDgGeNzMJkr6N7DGzG5V6MN8PHByLH8XM+sp6VRCyiaDkDd/gVCjuIeQBtigcPv0XYRgfFAct5yQrlpkoT92ZnBOnQC2U/bMfK6kVoS7LZcX1LZJGknXAD+b2VM5nSRd8aDQ+eFxYIKZ3Zc2vBshLZxBCOKNgb8TausXAwvN7PU4/7psrYI9XnEJ9pkHUXzfBFiWQx4z+3yF/thgSf8h3ERzsaQbCReO/yLpakKX0JsJF1NfM7P/STqK0IvmDkLvnvsIz7f4Pi6vCiHd0pNwLeIFwoltLeHGo3eAXhauOdQk1PQ/A+bY1h4YrgikAnxO+12saKTSfEm6llEsKVxk7UFIvVQldhsm3AjZndDafd1CV+5EKPI0TnRfPIjS+65nSjuQsjxzYjflkT8mdKeDkJZ5P74eSeie2IyQIkrlVb8i5PtqEnoZlSb0tS1B6Cp5MOFC3yALXSj/Tajx/0QI+CemautmtoRwMnDFQNo+us1+F/dLD/LFhJm9onAvwnmEmvwUQi8YI9wElTjFItinB/rUsPT3RXwgTSHeak7YXj/H198QgvpzhMDei5DaqUS4UPcrC48hKEcI8MsJ6Z0MQlfHNwHMbBJpF5XJdht7PNHtMXnBvZ1/D3uO2Jq+b4cTJkSxCPaw7UFUXA4qM/tK4eFV+xDuCfhjvOBagpBX/57w+IWHFO7i/YbQlSv1EKXuhGdabCHcvLGN1AXknJr/xWU7OOf2bMUm2Bdzq4BjzGyAwiMExhEC/d2EQL4pdquThW6gzxPvjrPYzx5yfahb4rs1OucKnwf7vHmL0HUSwoXX8jGfnu4EQn/rinH6N7ONL+p0lHMuwYpFb5w9hbI9TCq9q2fsZVPRzBYXXQmdcy5nHuzzKHugd865PYkHe+ecS4A98mcJnXPO7RwP9s45lwAe7J1zLgE82DvnXAJ4sHfOuQTwYO+ccwnw/1akUWF05XcrAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K6Jr89pSOemV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "9dcb51e1-cfdd-4fa1-9bc9-c8f63908700b"
      },
      "source": [
        "print('Duration: {} seconds'.format(time.time() - start))"
      ],
      "execution_count": 144,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Duration: 1342.0150032043457 seconds\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}