rf_upsampling_and_shap_vals_ml_tabular_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "rf_upsampling_and_shap_vals_ml_tabular_data.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "-1) Importing the data"
      ],
      "metadata": {
        "id": "IWjtLhrY92mL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#basic imports\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from numpy import mean\n",
        "from numpy import std\n",
        "\n",
        "#sklearn imports\n",
        "from sklearn.datasets import make_classification\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.model_selection import RepeatedStratifiedKFold\n",
        "from sklearn.feature_selection import SelectFromModel\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.utils import resample\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import confusion_matrix"
      ],
      "metadata": {
        "id": "FWSVJfyg92Ts"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#importing the data\n",
        "data = pd.read_csv(\"/content/drive/MyDrive/clean_zebra_data.csv\")\n",
        "\n",
        "#separating labels\n",
        "X = data.drop(\"labels\", axis = 1)\n",
        "y=data['labels']"
      ],
      "metadata": {
        "id": "-Z8KTR8e-ENP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#the data is quite concentrated on snorts and whinies\n",
        "datacount= data[['labels']].groupby(data['labels']).agg('count')\n",
        "datacount.plot(kind = 'bar', legend= False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 331
        },
        "id": "NN3xQkk--jsX",
        "outputId": "a19ba462-3173-4527-b576-680e074f314f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f19ca779bd0>"
            ]
          },
          "metadata": {},
          "execution_count": 43
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEpCAYAAABoRGJ5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAATRklEQVR4nO3debBkZX3G8e+joCgwCjKO1AiOWrjgAsK4LyVSBsGUYMqgiIqGZJKIZdSYkpgYNYkGjJoEE1AMlCCuCRpRiUtNEbe4zaCySokjBhCZMRrBGBfglz/OudCz0ff2Xc70y/dTdau739PL756aefq97znve1JVSJLacqehC5AkLTzDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQTsNXQDAXnvtVatWrRq6DEmaKuvXr/9RVS3f1rYdItxXrVrFunXrhi5DkqZKku9vb5vDMpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QG7RCTmBbDqhM/OXQJs3LVSc8cugRJDbLnLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAaNDfck+yS5IMllSS5N8kd9+55JPpvkO/3tHn17kpyS5MokFyU5aLF/CUnS5mbTc78J+OOq2h94HHBCkv2BE4G1VbUfsLZ/DHA4sF//swY4bcGrliTdrrHhXlXXVdWF/f0bgcuBlcCRwFn9084CjurvHwmcXZ2vAPdMsveCVy5J2q45jbknWQU8CvgqsKKqrus3/RBY0d9fCVw98rJr+jZJ0hKZdbgn2Q04F3hFVd0wuq2qCqi5fHCSNUnWJVm3adOmubxUkjTGrMI9yc50wf6+qvpI33z9zHBLf7uxb78W2Gfk5fft2zZTVadX1eqqWr18+fJJ65ckbcNszpYJcAZweVW9fWTTecBx/f3jgI+NtL+oP2vmccBPR4ZvJElLYKdZPOeJwAuBi5N8s297LXAS8OEkxwPfB47ut50PHAFcCfwceMmCVixJGmtsuFfVF4FsZ/Oh23h+ASfMsy5J0jw4Q1WSGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWrQ2HBPcmaSjUkuGWl7Q5Jrk3yz/zliZNufJrkyyRVJDluswiVJ2zebnvt7gGdso/3vqurA/ud8gCT7A88DHta/5tQkd16oYiVJszM23Kvq88CPZ/l+RwIfrKpfVtX3gCuBx8yjPknSBOYz5v6yJBf1wzZ79G0rgatHnnNN37aVJGuSrEuybtOmTfMoQ5K0pUnD/TTggcCBwHXA2+b6BlV1elWtrqrVy5cvn7AMSdK2TBTuVXV9Vd1cVbcA7+a2oZdrgX1Gnnrfvk2StIQmCvcke488fDYwcybNecDzktw1yf2B/YCvza9ESdJc7TTuCUk+ADwV2CvJNcDrgacmORAo4Crg9wGq6tIkHwYuA24CTqiqmxendEnS9owN96o6ZhvNZ9zO898EvGk+RUmS5scZqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSgsVdikqQd2aoTPzl0CbNy1UnPXNLPs+cuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWrQ2HBPcmaSjUkuGWnbM8lnk3ynv92jb0+SU5JcmeSiJActZvGSpG2bTc/9PcAztmg7EVhbVfsBa/vHAIcD+/U/a4DTFqZMSdJcjA33qvo88OMtmo8EzurvnwUcNdJ+dnW+Atwzyd4LVawkaXYmHXNfUVXX9fd/CKzo768Erh553jV9myRpCc37gGpVFVBzfV2SNUnWJVm3adOm+ZYhSRoxabhfPzPc0t9u7NuvBfYZed59+7atVNXpVbW6qlYvX758wjIkSdsyabifBxzX3z8O+NhI+4v6s2YeB/x0ZPhGkrREdhr3hCQfAJ4K7JXkGuD1wEnAh5McD3wfOLp/+vnAEcCVwM+BlyxCzZKkMcaGe1Uds51Nh27juQWcMN+iJEnz4wxVSWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQTvN58VJrgJuBG4Gbqqq1Un2BD4ErAKuAo6uqp/Mr0xJ0lwsRM/9kKo6sKpW949PBNZW1X7A2v6xJGkJLcawzJHAWf39s4CjFuEzJEm3Y77hXsBnkqxPsqZvW1FV1/X3fwismOdnSJLmaF5j7sCTquraJPcGPpvk26Mbq6qS1LZe2H8ZrAHYd99951mGJGnUvHruVXVtf7sR+CjwGOD6JHsD9Lcbt/Pa06tqdVWtXr58+XzKkCRtYeJwT7Jrkt1n7gO/AVwCnAcc1z/tOOBj8y1SkjQ38xmWWQF8NMnM+7y/qj6V5OvAh5McD3wfOHr+ZUqS5mLicK+qDcAB22j/b+DQ+RQlSZofZ6hKUoMMd0lqkOEuSQ0y3CWpQYa7JDVovjNUJU1g1YmfHLqEsa466ZlDl6B5sOcuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGeYFszco0XNAZvKizNMOeuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBixbuSZ6R5IokVyY5cbE+R5K0tUUJ9yR3Bv4JOBzYHzgmyf6L8VmSpK0tVs/9McCVVbWhqn4FfBA4cpE+S5K0hcUK95XA1SOPr+nbJElLYLCFw5KsAdb0D3+W5IqhapmDvYAfLeQb5uSFfLep4/5cOO7LhTUt+/N+29uwWOF+LbDPyOP79m23qqrTgdMX6fMXRZJ1VbV66Dpa4f5cOO7LhdXC/lysYZmvA/sluX+SuwDPA85bpM+SJG1hUXruVXVTkpcBnwbuDJxZVZcuxmdJkra2aGPuVXU+cP5ivf9ApmoYaQq4PxeO+3JhTf3+TFUNXYMkaYG5/IAkNchwl6QGGe5jJPnt2bRpvCR3nU2bpPlzzH2MJBdW1UHj2jSe+3JhJLnd/VVVFy5VLS1JEuBY4AFV9ZdJ9gXuU1VfG7i0iQw2Q3VHl+Rw4AhgZZJTRjYtA24apqrplOQ+dMtP3C3Jo4D0m5YBdx+ssOn1ttvZVsDTlqqQxpwK3EK3//4SuBE4F3j0kEVNynDfvh8A64BnAetH2m8EXjlIRdPrMODFdDOV38Zt4X4D8NqBappaVXXI0DU06rFVdVCSbwBU1U/6SZhTyXDfjqr6VpJLgMOq6qyh65lmVXVWkvcCx1TV+4aupyVJHk63rPYuM21VdfZwFU21X/fLlRdAkuV0Pfmp5AHV21FVNwP7TPO3946iqm7Bv3gWVJLXA+/ofw4B3kL3l6YmcwrwUeDeSd4EfBF487AlTc4DqmMkORt4KN3aOP87015Vbx+sqCmV5CS6lfY+xOb78seDFTXFklwMHAB8o6oOSLICOKeqnj5waVMryUOAQ+mGDtdW1eUDlzQxh2XG+27/cydg94FrmXbP7W9PGGkr4AED1NKC/6uqW5LclGQZsJHNV2PVLCRZVlU3JNmTbh9+YGTbntPa+TDcx6iqNwIk2a1//LNhK5peVXX/oWtozLok9wTeTXfQ/2fAl4ctaSq9H/hNun1YdL320dup7Hw4LDNGf8DqvcCefdOPgBe5yuXcJdkZ+EPgKX3TfwDvqqpfD1ZUI5KsApZV1UUDl6IdhOE+RpL/BP6sqi7oHz8VeHNVPWHQwqZQkn8GdgZmzj56IXBzVf3ucFVNr9Ym3ewIkqyku7rRraMaVfX54SqanOE+RpJvVdUB49o0nvtyYSU5jX7STVU9NMkewGeqaion3Qwtycl0x4UuA27um6uqpvIMJMfcx9uQ5HV0QzMALwA2DFjPNLs5yQOr6rsASR7Abf+JNHdNTbrZARwFPLiqfjl0IQvBcB/vd4A3Ah/pH3+hb9Pc/QlwQZINdAer7ge8ZNiSplpTk252ABvohg2bCHeHZbSk+lUgH9w/vKKVXtIQkhxLN4xwEN1xjOcAf15V/zJoYVMqybl08wbWMhLwVfXywYqaB8N9jCQPAl4NrGLzgywuzjSBJE9g633pdPkJtTTpZmhJjttW+7QuP2K4j5HkW8A76c6BvXV8uKrWb/dF2qZ+fZkHAt9k8wNWU9kzGlp/dsxWquq/lroW7XgM9zGSrK+qg4euowVJLgf2L//RLYh++YGZyTa7APenG+p62KCFTakkTwTewG2nQoau8zGVk5g8oDrex5O8lG5BodFxuKmckjywS4D7ANcNXUgLquoRo4/7i3i8dKByWnAG3eJ2m/2VPq3suY+R5HsjD2/dWdP6bT6kJBcABwJfY/Mvyqk8j3hHlOTiLUNfs5Pkq1X12KHrWCj23Md7DfCpfmGh19GdmfBXA9c0rd4wdAEtSfKqkYd3ovu3+YOBymnBBUn+lu6059HOx1RettCe+xhJLqqqRyZ5El2ovxX4i5a+4ZdKkl25bSXDBwEPAf7dtWUm06/nPuMm4Crg3Kr6xTAVTbf+L8st1bSeGWe4j5HkG1X1qCR/A1xcVe+faRu6tmmTZD3wZGAP4EvA14FfVdWxgxYmNchhmfGuTfIu4OnAyf0kHK9gNZlU1c+THA+cWlVv6U811QSSfJyR40Bb8ljG7CR5QVWds8Uw162m9cI8hvt4RwPPAN5aVf+TZG+6afSauyR5PN1Khsf3bX5RTm4D3dlH5/SPjwGuB/5tsIqm0679bVMX43FYRksmyVPoZvt+qapO7hcOe4WTmCaTZF1VrR7Xpjsme+5aSitGhwqqakOSLwxZ0JTbNckDqmoD3LrK5q5jXqPt6Bde+z22Xh5jKhcKtOeuJZPkwqo6aFybZifJYXSX2JtZgnoVsKaqPjNYUVOsvzDPF9h6qZFzBytqHuy5a9ElORw4AliZ5JSRTcvoTuHTZJYBD6dbduBZwBPoLgOpydy9ql4zdBELxYNZWgo/BtYBv6DrFc38nAccNmBd0+51VXUD3YHApwH/CJw2bElT7RNJjhi6iIXisIwW3czQS5L3V9Xzh66nFc7BWBhJbuS2U0p3o5udOvMXZVXVskEKmyeHZbQU7pLk+cBjk/zWlhur6iPbeI3Gcw7GAqiq3QGSnAN8HvhCC+vi23PXouuXbjiWbs7AeVtsrmk9G2FoSe5ONwfj4qr6Tj8H4xEeUJ1MkkPoZlA/me66AxfSBf0/DFrYhAx3LZkkx1fVGUPXIW1Pf03aRwOHAH9AtxbSQ4atajKGu5ZMkrvQ/Yd5St/0OeCdLhymHUGStXTzBL5Md0rkF6tq47BVTc7xOS2lU4GD+9tT6Zao9ewO7SguAn5Fd3rpI4GHJ7nbsCVNzp67lkySb1XVAePapCEl2R14Md1SGfepqrsOW9FkPFtGS+nmJA+squ/CrdPlp/5yZmpDkpfRHUw9mG5t/DPphmemkuGupfRquqvdjE6Xf8lw5Uib2QV4O7C+qqZ+5rThrqV0L7rxzFXAUcDjgZ8OWZA0o6reOnQNC8kDqlpKM9Pll9GdauZ0eWmRGO5aSjPj688E3l1VnwTuMmA9UrMMdy2lmenyzwXOd7q8tHg8FVJLxuny0tIx3CWpQf5JLEkNMtwlqUGGu+4wkvxszPZVSS6Z43u+J8lz5leZtPAMd0lqkOGuO5wkuyVZm+TCJBcnOXJk805J3pfk8iT/2p/hQ5KDk3wuyfokn+7P9NnyfU9KclmSi5I0NdtR08dw1x3RL4BnV9VBdDNl35Yk/bYHA6dW1UOBG4CXJtkZeAfwnKo6mG5BqTeNvmGSewHPBh5WVY8E/nppfhVp21xbRndEAd6c5CnALcBKYEW/7eqq+lJ//xzg5cCn6NbE+Wz/HXBn4Lot3vOndF8aZyT5BPCJRf0NpDEMd90RHQssBw6uql8nuYpuRUCALSd+FN2XwaVV9fjtvWFV3ZTkMcChwHOAlwFPW+jCpdlyWEZ3RPcANvbBfghwv5Ft+yaZCfHnA18ErgCWz7Qn2TnJw0bfMMluwD2q6nzglYAXINGg7Lnrjuh9wMeTXAysA749su0K4IQkZwKXAadV1a/60x1PSXIPuv83fw9cOvK63YGPJdmFrqf/qiX4PaTtcvkBSWqQwzKS1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBv0/fbxCw3aIAVMAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "0) First, divide into a part of the data that I will use, and a part of the data that I will not use"
      ],
      "metadata": {
        "id": "DYQzLxEv-Xgm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_train,df_test = train_test_split(data, test_size = 0.3,stratify = data['labels'])\n"
      ],
      "metadata": {
        "id": "foVyYNQtEtEH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now we will split df_train into minority classes df and majority classes df. With those two datasets, we upsample the minority dataset and merge with the original majority dataset"
      ],
      "metadata": {
        "id": "fop4WFBIFLNj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "filter_list = ['whinnie', 'snort']\n",
        "minor_filter_list = ['softsnort', 'squeal']\n",
        "df_train_major = df_train[df_train.labels.isin(filter_list)]\n",
        "df_train_minor = df_train[df_train.labels.isin(minor_filter_list)]"
      ],
      "metadata": {
        "id": "hyEJDnNLE-dk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#actually doing the upscaling\n",
        "df_train_minor_upsampled = resample(df_train_minor, \n",
        "                                    replace=True,    # sample with replacement\n",
        "                                    n_samples= 252 , # to match majority class\n",
        "                                    random_state=42)\n",
        "df_train_new = df_train_minor_upsampled.append(df_train_major)"
      ],
      "metadata": {
        "id": "d9H-r6wVHZ86"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#splitting the df_train into train and test\n",
        "y_train = df_train_new['labels']\n",
        "X_train = df_train_new.drop(\"labels\",axis = 1)"
      ],
      "metadata": {
        "id": "Iib3qvDJIvPa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#creating X train, test and y train and test\n",
        "X_test = df_test.drop('labels',axis =1)\n",
        "y_test =  df_test['labels']"
      ],
      "metadata": {
        "id": "g5qQjUhDfkgy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Best Random Forest model with upsampling"
      ],
      "metadata": {
        "id": "QizCVgz7MAS_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#defining the rf model\n",
        "best_r_model = RandomForestClassifier(max_depth = 4, min_samples_split = 12, n_estimators = 461)"
      ],
      "metadata": {
        "id": "4VQoQDl5Md4v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#fitting it - upsampling\n",
        "best_r_model.fit(X_train,y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ssMhR_8JMqr7",
        "outputId": "96e7c3a3-08e8-404b-cd1c-b80e01c52d6f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomForestClassifier(max_depth=4, min_samples_split=12, n_estimators=461)"
            ]
          },
          "metadata": {},
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#scores\n",
        "best_r_model.score(X_train,y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "15KkrhLZNDP2",
        "outputId": "08644fd0-4292-4359-e6a0-90adbc7cd1db"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.998015873015873"
            ]
          },
          "metadata": {},
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "best_r_model.score(X_test,y_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mbWP_lgVNS3c",
        "outputId": "f1f4b232-7167-477a-8520-2e475f4d6b9f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9919354838709677"
            ]
          },
          "metadata": {},
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#creating a y pred\n",
        "y_pred = best_r_model.predict(X_test)"
      ],
      "metadata": {
        "id": "f_BLykeZdTkM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "classification_report(y_test,y_pred)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "id": "16DQTkZLctr7",
        "outputId": "9c4ad67f-683f-4af3-a9e0-4595e615949f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'              precision    recall  f1-score   support\\n\\n       snort       0.99      1.00      0.99        71\\n   softsnort       1.00      0.75      0.86         4\\n      squeal       1.00      1.00      1.00        12\\n     whinnie       1.00      1.00      1.00        37\\n\\n    accuracy                           0.99       124\\n   macro avg       1.00      0.94      0.96       124\\nweighted avg       0.99      0.99      0.99       124\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "confusion_matrix(y_test,y_pred)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LJuaq6T0dnrY",
        "outputId": "fda6c79b-e941-4b44-d35f-0f1b219a8cd8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[71,  0,  0,  0],\n",
              "       [ 1,  3,  0,  0],\n",
              "       [ 0,  0, 12,  0],\n",
              "       [ 0,  0,  0, 37]])"
            ]
          },
          "metadata": {},
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# RFE with Random Forest"
      ],
      "metadata": {
        "id": "vB-GT3CfUt1r"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# define dataset\n",
        "X = data.drop('labels',axis = 1)\n",
        "y = data['labels']"
      ],
      "metadata": {
        "id": "1ZqdJNaiVGMJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# messing up with the data to test the selector\n",
        "#import random\n",
        "#X['random'] = random.sample(range(0,10000),413)"
      ],
      "metadata": {
        "id": "-3nK7KhgZ-kM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Selecting from model\n",
        "model = RandomForestClassifier(max_depth = 5, min_samples_split = 15, n_estimators = 15)\n",
        "estimator = SelectFromModel(model)\n",
        "\n",
        "estimator.fit(X_train,y_train)\n",
        "estimator.get_support()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GQUpzKskUx0l",
        "outputId": "3ea6857d-0f9d-48a4-f635-d1519d2a461f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ True,  True, False,  True, False, False, False, False, False])"
            ]
          },
          "metadata": {},
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Now selecting the features based on this response:\n",
        "X_train.columns[estimator.get_support()]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qx7tRpPsV2JB",
        "outputId": "9805d9d9-d3a7-40d6-ff71-b09eac55f39b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['q25', 'q50', 'fpeak'], dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 77
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "best hyperparams: {'max_depth': 5, 'min_samples_split': 22, 'n_estimators': 30}\n",
        "the score for training was 0.91\n",
        "the score for test was 0.92\n",
        "\n",
        "gonna run a confusion matrix now"
      ],
      "metadata": {
        "id": "HYUJ8YkHu3iH"
      }
    }
  ]
}