Machine Learning Medicare Fraud 2014.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#some of these we do not need\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm_notebook, tqdm\n",
    "import glob, os, csv\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "import plotly.figure_factory as ff\n",
    "import plotly\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The data has a size of (904999, 215)\n"
     ]
    }
   ],
   "source": [
    "#import final dataset\n",
    "\n",
    "new_main_2017 = pd.read_csv('data/new_main_2017.csv', low_memory=False)\n",
    "print(\"The data has a size of {}\".format(new_main_2017.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_main_2017.drop(['Unnamed: 0'], axis=1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "101"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new_main_2017[new_main_2017['is_excluded'] == 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropIndex = new_main_2017[new_main_2017['is_excluded'] == 1].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "904898"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new_main_2017[new_main_2017['is_excluded'] == 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "only_excluded = new_main_2017[new_main_2017['is_excluded'] == 1]\n",
    "new_main_2017.drop(dropIndex, inplace = True)\n",
    "new_main_2017 = new_main_2017.sample(frac=1)\n",
    "random_sample = new_main_2017.iloc[0:85000]\n",
    "new_df = pd.concat([only_excluded, random_sample])\n",
    "new_main_2017 = new_main_2017.drop(new_main_2017.iloc[0:85000].index, axis = 0)\n",
    "new_df = new_df.sample(frac=1)\n",
    "#part_d_df.iloc[0:85000].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t_cost_per_claim_excluded</th>\n",
       "      <th>t_claim_count_schedule2</th>\n",
       "      <th>t_claim_per_bene</th>\n",
       "      <th>%b2bg_claim_count</th>\n",
       "      <th>%b2bg_drug_cost</th>\n",
       "      <th>%b2t_claim_count</th>\n",
       "      <th>%b2t_drug_cost</th>\n",
       "      <th>t_cost_per_claim</th>\n",
       "      <th>t_cost_per_claim_ge65</th>\n",
       "      <th>t_claim_per_bene_ge65</th>\n",
       "      <th>...</th>\n",
       "      <th>x0_Technician</th>\n",
       "      <th>x0_Technician/Technologist</th>\n",
       "      <th>x0_Thoracic Surgery</th>\n",
       "      <th>x0_Thoracic Surgery (Cardiothoracic Vascular Surgery)</th>\n",
       "      <th>x0_Undefined Physician type</th>\n",
       "      <th>x0_Unknown Supplier/Provider Specialty</th>\n",
       "      <th>x0_Urology</th>\n",
       "      <th>x0_Vascular Surgery</th>\n",
       "      <th>x0_Veterinarian</th>\n",
       "      <th>x0_Voluntary Health or Charitable Agency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>257108</th>\n",
       "      <td>3300.27</td>\n",
       "      <td>12</td>\n",
       "      <td>4.9</td>\n",
       "      <td>50</td>\n",
       "      <td>93</td>\n",
       "      <td>50.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>251.17</td>\n",
       "      <td>276.77</td>\n",
       "      <td>4.78</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352146</th>\n",
       "      <td>3331.30</td>\n",
       "      <td>4</td>\n",
       "      <td>5.8</td>\n",
       "      <td>50</td>\n",
       "      <td>94</td>\n",
       "      <td>50.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>118.75</td>\n",
       "      <td>118.79</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198656</th>\n",
       "      <td>350.76</td>\n",
       "      <td>2</td>\n",
       "      <td>2.6</td>\n",
       "      <td>30</td>\n",
       "      <td>85</td>\n",
       "      <td>30.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>213.88</td>\n",
       "      <td>107.62</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>554609</th>\n",
       "      <td>2620.84</td>\n",
       "      <td>38</td>\n",
       "      <td>14.9</td>\n",
       "      <td>20</td>\n",
       "      <td>88</td>\n",
       "      <td>20.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>96.76</td>\n",
       "      <td>105.53</td>\n",
       "      <td>16.32</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661994</th>\n",
       "      <td>156.01</td>\n",
       "      <td>6</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.62</td>\n",
       "      <td>8.74</td>\n",
       "      <td>1.95</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 214 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        t_cost_per_claim_excluded  t_claim_count_schedule2  t_claim_per_bene  \\\n",
       "257108                    3300.27                       12               4.9   \n",
       "352146                    3331.30                        4               5.8   \n",
       "198656                     350.76                        2               2.6   \n",
       "554609                    2620.84                       38              14.9   \n",
       "661994                     156.01                        6               1.9   \n",
       "\n",
       "        %b2bg_claim_count  %b2bg_drug_cost  %b2t_claim_count  %b2t_drug_cost  \\\n",
       "257108                 50               93              50.0            93.0   \n",
       "352146                 50               94              50.0            94.0   \n",
       "198656                 30               85              30.0            85.0   \n",
       "554609                 20               88              20.0            87.0   \n",
       "661994                  0                0               0.0             0.0   \n",
       "\n",
       "        t_cost_per_claim  t_cost_per_claim_ge65  t_claim_per_bene_ge65  ...  \\\n",
       "257108            251.17                 276.77                   4.78  ...   \n",
       "352146            118.75                 118.79                   0.00  ...   \n",
       "198656            213.88                 107.62                   0.00  ...   \n",
       "554609             96.76                 105.53                  16.32  ...   \n",
       "661994              8.62                   8.74                   1.95  ...   \n",
       "\n",
       "        x0_Technician  x0_Technician/Technologist  x0_Thoracic Surgery  \\\n",
       "257108            0.0                         0.0                  0.0   \n",
       "352146            0.0                         0.0                  0.0   \n",
       "198656            0.0                         0.0                  0.0   \n",
       "554609            0.0                         0.0                  0.0   \n",
       "661994            0.0                         0.0                  0.0   \n",
       "\n",
       "        x0_Thoracic Surgery (Cardiothoracic Vascular Surgery)  \\\n",
       "257108                                                0.0       \n",
       "352146                                                0.0       \n",
       "198656                                                0.0       \n",
       "554609                                                0.0       \n",
       "661994                                                0.0       \n",
       "\n",
       "        x0_Undefined Physician type  x0_Unknown Supplier/Provider Specialty  \\\n",
       "257108                          0.0                                     0.0   \n",
       "352146                          0.0                                     0.0   \n",
       "198656                          0.0                                     0.0   \n",
       "554609                          0.0                                     0.0   \n",
       "661994                          0.0                                     0.0   \n",
       "\n",
       "        x0_Urology  x0_Vascular Surgery  x0_Veterinarian  \\\n",
       "257108         0.0                  0.0              0.0   \n",
       "352146         0.0                  0.0              0.0   \n",
       "198656         0.0                  0.0              0.0   \n",
       "554609         0.0                  0.0              0.0   \n",
       "661994         0.0                  0.0              0.0   \n",
       "\n",
       "        x0_Voluntary Health or Charitable Agency  \n",
       "257108                                       0.0  \n",
       "352146                                       0.0  \n",
       "198656                                       0.0  \n",
       "554609                                       0.0  \n",
       "661994                                       0.0  \n",
       "\n",
       "[5 rows x 214 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#new_df.drop(['Unnamed: 0'], axis=1, inplace = True)\n",
    "new_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t_cost_per_claim_excluded</th>\n",
       "      <th>t_claim_count_schedule2</th>\n",
       "      <th>t_claim_per_bene</th>\n",
       "      <th>%b2bg_claim_count</th>\n",
       "      <th>%b2bg_drug_cost</th>\n",
       "      <th>%b2t_claim_count</th>\n",
       "      <th>%b2t_drug_cost</th>\n",
       "      <th>t_cost_per_claim</th>\n",
       "      <th>t_cost_per_claim_ge65</th>\n",
       "      <th>t_claim_per_bene_ge65</th>\n",
       "      <th>...</th>\n",
       "      <th>x0_Technician</th>\n",
       "      <th>x0_Technician/Technologist</th>\n",
       "      <th>x0_Thoracic Surgery</th>\n",
       "      <th>x0_Thoracic Surgery (Cardiothoracic Vascular Surgery)</th>\n",
       "      <th>x0_Undefined Physician type</th>\n",
       "      <th>x0_Unknown Supplier/Provider Specialty</th>\n",
       "      <th>x0_Urology</th>\n",
       "      <th>x0_Vascular Surgery</th>\n",
       "      <th>x0_Veterinarian</th>\n",
       "      <th>x0_Voluntary Health or Charitable Agency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>8.198980e+05</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.0</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.000000</td>\n",
       "      <td>819898.0</td>\n",
       "      <td>819898.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.208188e+03</td>\n",
       "      <td>25.550653</td>\n",
       "      <td>6.391945</td>\n",
       "      <td>12.572100</td>\n",
       "      <td>38.918228</td>\n",
       "      <td>12.351305</td>\n",
       "      <td>38.859981</td>\n",
       "      <td>125.702657</td>\n",
       "      <td>102.939560</td>\n",
       "      <td>4.793268</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001271</td>\n",
       "      <td>0.000073</td>\n",
       "      <td>0.000335</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.010807</td>\n",
       "      <td>0.002899</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.618513e+03</td>\n",
       "      <td>40.597077</td>\n",
       "      <td>7.124494</td>\n",
       "      <td>16.525934</td>\n",
       "      <td>37.134218</td>\n",
       "      <td>15.888406</td>\n",
       "      <td>36.909156</td>\n",
       "      <td>555.607738</td>\n",
       "      <td>370.703944</td>\n",
       "      <td>6.868071</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001104</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.035627</td>\n",
       "      <td>0.008554</td>\n",
       "      <td>0.018311</td>\n",
       "      <td>0.001913</td>\n",
       "      <td>0.103396</td>\n",
       "      <td>0.053766</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.671800e+02</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.900000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.530000</td>\n",
       "      <td>13.970000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>6.744900e+02</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>3.600000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>55.060000</td>\n",
       "      <td>46.540000</td>\n",
       "      <td>2.220000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.370932e+03</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>8.300000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>93.300000</td>\n",
       "      <td>79.940000</td>\n",
       "      <td>6.520000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.713038e+06</td>\n",
       "      <td>394.000000</td>\n",
       "      <td>307.100000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>171279.730000</td>\n",
       "      <td>76223.890000</td>\n",
       "      <td>270.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 214 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       t_cost_per_claim_excluded  t_claim_count_schedule2  t_claim_per_bene  \\\n",
       "count               8.198980e+05            819898.000000     819898.000000   \n",
       "mean                2.208188e+03                25.550653          6.391945   \n",
       "std                 5.618513e+03                40.597077          7.124494   \n",
       "min                 0.000000e+00                 0.000000          0.000000   \n",
       "25%                 1.671800e+02                 2.000000          1.900000   \n",
       "50%                 6.744900e+02                 7.000000          3.600000   \n",
       "75%                 2.370932e+03                29.000000          8.300000   \n",
       "max                 1.713038e+06               394.000000        307.100000   \n",
       "\n",
       "       %b2bg_claim_count  %b2bg_drug_cost  %b2t_claim_count  %b2t_drug_cost  \\\n",
       "count      819898.000000    819898.000000     819898.000000   819898.000000   \n",
       "mean           12.572100        38.918228         12.351305       38.859981   \n",
       "std            16.525934        37.134218         15.888406       36.909156   \n",
       "min             0.000000         0.000000          0.000000        0.000000   \n",
       "25%             0.000000         0.000000          0.000000        0.000000   \n",
       "50%            10.000000        42.000000         10.000000       43.000000   \n",
       "75%            20.000000        73.000000         20.000000       73.000000   \n",
       "max           100.000000       100.000000        100.000000      100.000000   \n",
       "\n",
       "       t_cost_per_claim  t_cost_per_claim_ge65  t_claim_per_bene_ge65  ...  \\\n",
       "count     819898.000000          819898.000000          819898.000000  ...   \n",
       "mean         125.702657             102.939560               4.793268  ...   \n",
       "std          555.607738             370.703944               6.868071  ...   \n",
       "min            0.000000               0.000000               0.000000  ...   \n",
       "25%           23.530000              13.970000               0.000000  ...   \n",
       "50%           55.060000              46.540000               2.220000  ...   \n",
       "75%           93.300000              79.940000               6.520000  ...   \n",
       "max       171279.730000           76223.890000             270.000000  ...   \n",
       "\n",
       "       x0_Technician  x0_Technician/Technologist  x0_Thoracic Surgery  \\\n",
       "count  819898.000000                    819898.0        819898.000000   \n",
       "mean        0.000001                         0.0             0.001271   \n",
       "std         0.001104                         0.0             0.035627   \n",
       "min         0.000000                         0.0             0.000000   \n",
       "25%         0.000000                         0.0             0.000000   \n",
       "50%         0.000000                         0.0             0.000000   \n",
       "75%         0.000000                         0.0             0.000000   \n",
       "max         1.000000                         0.0             1.000000   \n",
       "\n",
       "       x0_Thoracic Surgery (Cardiothoracic Vascular Surgery)  \\\n",
       "count                                      819898.000000       \n",
       "mean                                            0.000073       \n",
       "std                                             0.008554       \n",
       "min                                             0.000000       \n",
       "25%                                             0.000000       \n",
       "50%                                             0.000000       \n",
       "75%                                             0.000000       \n",
       "max                                             1.000000       \n",
       "\n",
       "       x0_Undefined Physician type  x0_Unknown Supplier/Provider Specialty  \\\n",
       "count                819898.000000                           819898.000000   \n",
       "mean                      0.000335                                0.000004   \n",
       "std                       0.018311                                0.001913   \n",
       "min                       0.000000                                0.000000   \n",
       "25%                       0.000000                                0.000000   \n",
       "50%                       0.000000                                0.000000   \n",
       "75%                       0.000000                                0.000000   \n",
       "max                       1.000000                                1.000000   \n",
       "\n",
       "          x0_Urology  x0_Vascular Surgery  x0_Veterinarian  \\\n",
       "count  819898.000000        819898.000000         819898.0   \n",
       "mean        0.010807             0.002899              0.0   \n",
       "std         0.103396             0.053766              0.0   \n",
       "min         0.000000             0.000000              0.0   \n",
       "25%         0.000000             0.000000              0.0   \n",
       "50%         0.000000             0.000000              0.0   \n",
       "75%         0.000000             0.000000              0.0   \n",
       "max         1.000000             1.000000              0.0   \n",
       "\n",
       "       x0_Voluntary Health or Charitable Agency  \n",
       "count                             819898.000000  \n",
       "mean                                   0.000001  \n",
       "std                                    0.001104  \n",
       "min                                    0.000000  \n",
       "25%                                    0.000000  \n",
       "50%                                    0.000000  \n",
       "75%                                    0.000000  \n",
       "max                                    1.000000  \n",
       "\n",
       "[8 rows x 214 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_main_2017.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import decomposition\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix, roc_curve, roc_auc_score, auc\n",
    "from sklearn.utils import resample\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.svm import SVC\n",
    "from scipy.stats import pointbiserialr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t_cost_per_claim_excluded</th>\n",
       "      <th>t_claim_count_schedule2</th>\n",
       "      <th>t_claim_per_bene</th>\n",
       "      <th>%b2bg_claim_count</th>\n",
       "      <th>%b2bg_drug_cost</th>\n",
       "      <th>%b2t_claim_count</th>\n",
       "      <th>%b2t_drug_cost</th>\n",
       "      <th>t_cost_per_claim</th>\n",
       "      <th>t_cost_per_claim_ge65</th>\n",
       "      <th>t_claim_per_bene_ge65</th>\n",
       "      <th>...</th>\n",
       "      <th>x0_Technician</th>\n",
       "      <th>x0_Technician/Technologist</th>\n",
       "      <th>x0_Thoracic Surgery</th>\n",
       "      <th>x0_Thoracic Surgery (Cardiothoracic Vascular Surgery)</th>\n",
       "      <th>x0_Undefined Physician type</th>\n",
       "      <th>x0_Unknown Supplier/Provider Specialty</th>\n",
       "      <th>x0_Urology</th>\n",
       "      <th>x0_Vascular Surgery</th>\n",
       "      <th>x0_Veterinarian</th>\n",
       "      <th>x0_Voluntary Health or Charitable Agency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1000000.000000</td>\n",
       "      <td>1000000.000000</td>\n",
       "      <td>1000000.000000</td>\n",
       "      <td>1000000.000000</td>\n",
       "      <td>1000000.000000</td>\n",
       "      <td>1000000.000000</td>\n",
       "      <td>1000000.000000</td>\n",
       "      <td>1000000.000000</td>\n",
       "      <td>1000000.000000</td>\n",
       "      <td>1000000.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1000000.0</td>\n",
       "      <td>1000000.0</td>\n",
       "      <td>1000000.0</td>\n",
       "      <td>1000000.0</td>\n",
       "      <td>1000000.0</td>\n",
       "      <td>1000000.0</td>\n",
       "      <td>1000000.0</td>\n",
       "      <td>1000000.0</td>\n",
       "      <td>1000000.0</td>\n",
       "      <td>1000000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>5194.967630</td>\n",
       "      <td>38.609067</td>\n",
       "      <td>9.485671</td>\n",
       "      <td>11.374050</td>\n",
       "      <td>36.234950</td>\n",
       "      <td>10.876360</td>\n",
       "      <td>36.026284</td>\n",
       "      <td>105.511338</td>\n",
       "      <td>89.082136</td>\n",
       "      <td>7.510315</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>28249.745002</td>\n",
       "      <td>55.813583</td>\n",
       "      <td>9.620971</td>\n",
       "      <td>14.980311</td>\n",
       "      <td>36.168503</td>\n",
       "      <td>12.989995</td>\n",
       "      <td>35.937057</td>\n",
       "      <td>175.287247</td>\n",
       "      <td>164.542628</td>\n",
       "      <td>9.325134</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>40.990000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.940000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>293.230000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.200000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>44.060000</td>\n",
       "      <td>28.760000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1206.820000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>5.800000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>72.230000</td>\n",
       "      <td>53.450000</td>\n",
       "      <td>3.780000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>3338.240000</td>\n",
       "      <td>54.000000</td>\n",
       "      <td>13.800000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>71.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>104.190000</td>\n",
       "      <td>79.820000</td>\n",
       "      <td>11.630000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>286886.380000</td>\n",
       "      <td>299.000000</td>\n",
       "      <td>46.100000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>98.000000</td>\n",
       "      <td>1419.570000</td>\n",
       "      <td>1258.960000</td>\n",
       "      <td>43.550000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 214 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       t_cost_per_claim_excluded  t_claim_count_schedule2  t_claim_per_bene  \\\n",
       "count             1000000.000000           1000000.000000    1000000.000000   \n",
       "mean                 5194.967630                38.609067          9.485671   \n",
       "std                 28249.745002                55.813583          9.620971   \n",
       "min                    40.990000                 1.000000          0.000000   \n",
       "25%                   293.230000                 3.000000          2.200000   \n",
       "50%                  1206.820000                11.000000          5.800000   \n",
       "75%                  3338.240000                54.000000         13.800000   \n",
       "max                286886.380000               299.000000         46.100000   \n",
       "\n",
       "       %b2bg_claim_count  %b2bg_drug_cost  %b2t_claim_count  %b2t_drug_cost  \\\n",
       "count     1000000.000000   1000000.000000    1000000.000000  1000000.000000   \n",
       "mean           11.374050        36.234950         10.876360       36.026284   \n",
       "std            14.980311        36.168503         12.989995       35.937057   \n",
       "min             0.000000         0.000000          0.000000        0.000000   \n",
       "25%             0.000000         0.000000          0.000000        0.000000   \n",
       "50%            10.000000        43.000000         10.000000       43.000000   \n",
       "75%            20.000000        71.000000         20.000000       70.000000   \n",
       "max           100.000000       100.000000         60.000000       98.000000   \n",
       "\n",
       "       t_cost_per_claim  t_cost_per_claim_ge65  t_claim_per_bene_ge65  ...  \\\n",
       "count    1000000.000000         1000000.000000         1000000.000000  ...   \n",
       "mean         105.511338              89.082136               7.510315  ...   \n",
       "std          175.287247             164.542628               9.325134  ...   \n",
       "min            2.940000               0.000000               0.000000  ...   \n",
       "25%           44.060000              28.760000               0.000000  ...   \n",
       "50%           72.230000              53.450000               3.780000  ...   \n",
       "75%          104.190000              79.820000              11.630000  ...   \n",
       "max         1419.570000            1258.960000              43.550000  ...   \n",
       "\n",
       "       x0_Technician  x0_Technician/Technologist  x0_Thoracic Surgery  \\\n",
       "count      1000000.0                   1000000.0            1000000.0   \n",
       "mean             0.0                         0.0                  0.0   \n",
       "std              0.0                         0.0                  0.0   \n",
       "min              0.0                         0.0                  0.0   \n",
       "25%              0.0                         0.0                  0.0   \n",
       "50%              0.0                         0.0                  0.0   \n",
       "75%              0.0                         0.0                  0.0   \n",
       "max              0.0                         0.0                  0.0   \n",
       "\n",
       "       x0_Thoracic Surgery (Cardiothoracic Vascular Surgery)  \\\n",
       "count                                          1000000.0       \n",
       "mean                                                 0.0       \n",
       "std                                                  0.0       \n",
       "min                                                  0.0       \n",
       "25%                                                  0.0       \n",
       "50%                                                  0.0       \n",
       "75%                                                  0.0       \n",
       "max                                                  0.0       \n",
       "\n",
       "       x0_Undefined Physician type  x0_Unknown Supplier/Provider Specialty  \\\n",
       "count                    1000000.0                               1000000.0   \n",
       "mean                           0.0                                     0.0   \n",
       "std                            0.0                                     0.0   \n",
       "min                            0.0                                     0.0   \n",
       "25%                            0.0                                     0.0   \n",
       "50%                            0.0                                     0.0   \n",
       "75%                            0.0                                     0.0   \n",
       "max                            0.0                                     0.0   \n",
       "\n",
       "       x0_Urology  x0_Vascular Surgery  x0_Veterinarian  \\\n",
       "count   1000000.0            1000000.0        1000000.0   \n",
       "mean          0.0                  0.0              0.0   \n",
       "std           0.0                  0.0              0.0   \n",
       "min           0.0                  0.0              0.0   \n",
       "25%           0.0                  0.0              0.0   \n",
       "50%           0.0                  0.0              0.0   \n",
       "75%           0.0                  0.0              0.0   \n",
       "max           0.0                  0.0              0.0   \n",
       "\n",
       "       x0_Voluntary Health or Charitable Agency  \n",
       "count                                 1000000.0  \n",
       "mean                                        0.0  \n",
       "std                                         0.0  \n",
       "min                                         0.0  \n",
       "25%                                         0.0  \n",
       "50%                                         0.0  \n",
       "75%                                         0.0  \n",
       "max                                         0.0  \n",
       "\n",
       "[8 rows x 214 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#part_d_excluded = new_df[new_df[\"is_excluded\"]==1]\n",
    "\n",
    "fudge = resample(only_excluded, replace=True, n_samples=1000000, random_state = 0)\n",
    "#fudge.drop(['Unnamed: 0'], axis=1, inplace = True)\n",
    "fudge.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "std = fudge.describe().loc[\"std\"]\n",
    "std = std[0:26]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "rtstd = std.map(lambda x : math.sqrt(x)/2) #use half of square root of standard deviation as the range for gaussian normal\n",
    "rtstd = rtstd.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key in rtstd:\n",
    "    #fudge[key] = fudge[key].map(lambda x: (np.random.normal(x, rtstd[key])))\n",
    "    fudge[key] = fudge[key].map(lambda x: (np.random.normal(x, 2)))\n",
    "fudge = fudge[fudge>=0]\n",
    "fudge.dropna(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'t_cost_per_claim_excluded': 84.03830228283725,\n",
       " 't_claim_count_schedule2': 3.735424432090368,\n",
       " 't_claim_per_bene': 1.5508844893809823,\n",
       " '%b2bg_claim_count': 1.9352203219559672,\n",
       " '%b2bg_drug_cost': 3.007012758064355,\n",
       " '%b2t_claim_count': 1.802081760350314,\n",
       " '%b2t_drug_cost': 2.99737621069871,\n",
       " 't_cost_per_claim': 6.619804500549423,\n",
       " 't_cost_per_claim_ge65': 6.413708518062818,\n",
       " 't_claim_per_bene_ge65': 1.5268541048291235,\n",
       " 'mapd_cost_per_claim': 6.461408506432181,\n",
       " 'pdp_cost_per_claim': 6.725735816482665,\n",
       " 'lis_cost_per_claim': 6.822596052308591,\n",
       " 'nonlis_cost_per_claim': 6.207153722441226,\n",
       " 'opioid_cost_per_claim': 4.650466862010797,\n",
       " 'opioid_claim_per_bene': 0.884740725108428,\n",
       " 'antibiotic_cost_per_claim': 2.981750867470343,\n",
       " 'antibiotic_claim_per_bene': 0.47019428185377854,\n",
       " 'antipsych_cost_per_claim_ge65': 4.626473486567583,\n",
       " 'antipsych_claim_per_bene_ge65': 0.6299808071079215,\n",
       " 'total_30_day_fill_count': 37.12751686106159,\n",
       " 'total_day_supply': 197.95167898077315,\n",
       " 'total_30_day_fill_count_ge65': 32.32204101305794,\n",
       " 'total_day_supply_ge65': 172.3850331940279,\n",
       " 'opioid_day_supply': 75.1628609755071,\n",
       " 'opioid_prescriber_rate': 1.9270085656978702}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rtstd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t_cost_per_claim_excluded</th>\n",
       "      <th>t_claim_count_schedule2</th>\n",
       "      <th>t_claim_per_bene</th>\n",
       "      <th>%b2bg_claim_count</th>\n",
       "      <th>%b2bg_drug_cost</th>\n",
       "      <th>%b2t_claim_count</th>\n",
       "      <th>%b2t_drug_cost</th>\n",
       "      <th>t_cost_per_claim</th>\n",
       "      <th>t_cost_per_claim_ge65</th>\n",
       "      <th>t_claim_per_bene_ge65</th>\n",
       "      <th>...</th>\n",
       "      <th>x0_Technician</th>\n",
       "      <th>x0_Technician/Technologist</th>\n",
       "      <th>x0_Thoracic Surgery</th>\n",
       "      <th>x0_Thoracic Surgery (Cardiothoracic Vascular Surgery)</th>\n",
       "      <th>x0_Undefined Physician type</th>\n",
       "      <th>x0_Unknown Supplier/Provider Specialty</th>\n",
       "      <th>x0_Urology</th>\n",
       "      <th>x0_Vascular Surgery</th>\n",
       "      <th>x0_Veterinarian</th>\n",
       "      <th>x0_Voluntary Health or Charitable Agency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>88779.000000</td>\n",
       "      <td>88779.000000</td>\n",
       "      <td>88779.000000</td>\n",
       "      <td>88779.000000</td>\n",
       "      <td>88779.000000</td>\n",
       "      <td>88779.000000</td>\n",
       "      <td>88779.000000</td>\n",
       "      <td>88779.000000</td>\n",
       "      <td>88779.000000</td>\n",
       "      <td>88779.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>88779.0</td>\n",
       "      <td>88779.0</td>\n",
       "      <td>88779.0</td>\n",
       "      <td>88779.0</td>\n",
       "      <td>88779.0</td>\n",
       "      <td>88779.0</td>\n",
       "      <td>88779.0</td>\n",
       "      <td>88779.0</td>\n",
       "      <td>88779.0</td>\n",
       "      <td>88779.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3749.685021</td>\n",
       "      <td>125.597206</td>\n",
       "      <td>21.617403</td>\n",
       "      <td>17.842973</td>\n",
       "      <td>66.867958</td>\n",
       "      <td>17.839472</td>\n",
       "      <td>66.379500</td>\n",
       "      <td>83.527709</td>\n",
       "      <td>76.457404</td>\n",
       "      <td>20.724264</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3261.616662</td>\n",
       "      <td>77.628251</td>\n",
       "      <td>10.629778</td>\n",
       "      <td>6.805507</td>\n",
       "      <td>12.416309</td>\n",
       "      <td>6.811781</td>\n",
       "      <td>12.445927</td>\n",
       "      <td>55.878955</td>\n",
       "      <td>48.829895</td>\n",
       "      <td>10.849723</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>40.015461</td>\n",
       "      <td>0.009173</td>\n",
       "      <td>0.006240</td>\n",
       "      <td>0.001424</td>\n",
       "      <td>0.002893</td>\n",
       "      <td>0.001596</td>\n",
       "      <td>0.004005</td>\n",
       "      <td>1.826792</td>\n",
       "      <td>0.430153</td>\n",
       "      <td>0.003615</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1820.021566</td>\n",
       "      <td>63.542963</td>\n",
       "      <td>14.536125</td>\n",
       "      <td>11.658953</td>\n",
       "      <td>64.410213</td>\n",
       "      <td>11.658504</td>\n",
       "      <td>64.080420</td>\n",
       "      <td>52.473698</td>\n",
       "      <td>46.780928</td>\n",
       "      <td>13.662859</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3007.673760</td>\n",
       "      <td>119.805679</td>\n",
       "      <td>19.829182</td>\n",
       "      <td>19.085133</td>\n",
       "      <td>69.166635</td>\n",
       "      <td>19.077220</td>\n",
       "      <td>68.904377</td>\n",
       "      <td>66.019853</td>\n",
       "      <td>58.487771</td>\n",
       "      <td>18.782942</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>4194.117101</td>\n",
       "      <td>153.953130</td>\n",
       "      <td>25.210635</td>\n",
       "      <td>21.323705</td>\n",
       "      <td>73.538681</td>\n",
       "      <td>21.318104</td>\n",
       "      <td>72.986823</td>\n",
       "      <td>102.536191</td>\n",
       "      <td>95.223265</td>\n",
       "      <td>25.005930</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>286884.635255</td>\n",
       "      <td>305.883382</td>\n",
       "      <td>53.409089</td>\n",
       "      <td>100.000097</td>\n",
       "      <td>102.671602</td>\n",
       "      <td>64.732898</td>\n",
       "      <td>101.556704</td>\n",
       "      <td>1422.028460</td>\n",
       "      <td>1259.433704</td>\n",
       "      <td>51.720256</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 214 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       t_cost_per_claim_excluded  t_claim_count_schedule2  t_claim_per_bene  \\\n",
       "count               88779.000000             88779.000000      88779.000000   \n",
       "mean                 3749.685021               125.597206         21.617403   \n",
       "std                  3261.616662                77.628251         10.629778   \n",
       "min                    40.015461                 0.009173          0.006240   \n",
       "25%                  1820.021566                63.542963         14.536125   \n",
       "50%                  3007.673760               119.805679         19.829182   \n",
       "75%                  4194.117101               153.953130         25.210635   \n",
       "max                286884.635255               305.883382         53.409089   \n",
       "\n",
       "       %b2bg_claim_count  %b2bg_drug_cost  %b2t_claim_count  %b2t_drug_cost  \\\n",
       "count       88779.000000     88779.000000      88779.000000    88779.000000   \n",
       "mean           17.842973        66.867958         17.839472       66.379500   \n",
       "std             6.805507        12.416309          6.811781       12.445927   \n",
       "min             0.001424         0.002893          0.001596        0.004005   \n",
       "25%            11.658953        64.410213         11.658504       64.080420   \n",
       "50%            19.085133        69.166635         19.077220       68.904377   \n",
       "75%            21.323705        73.538681         21.318104       72.986823   \n",
       "max           100.000097       102.671602         64.732898      101.556704   \n",
       "\n",
       "       t_cost_per_claim  t_cost_per_claim_ge65  t_claim_per_bene_ge65  ...  \\\n",
       "count      88779.000000           88779.000000           88779.000000  ...   \n",
       "mean          83.527709              76.457404              20.724264  ...   \n",
       "std           55.878955              48.829895              10.849723  ...   \n",
       "min            1.826792               0.430153               0.003615  ...   \n",
       "25%           52.473698              46.780928              13.662859  ...   \n",
       "50%           66.019853              58.487771              18.782942  ...   \n",
       "75%          102.536191              95.223265              25.005930  ...   \n",
       "max         1422.028460            1259.433704              51.720256  ...   \n",
       "\n",
       "       x0_Technician  x0_Technician/Technologist  x0_Thoracic Surgery  \\\n",
       "count        88779.0                     88779.0              88779.0   \n",
       "mean             0.0                         0.0                  0.0   \n",
       "std              0.0                         0.0                  0.0   \n",
       "min              0.0                         0.0                  0.0   \n",
       "25%              0.0                         0.0                  0.0   \n",
       "50%              0.0                         0.0                  0.0   \n",
       "75%              0.0                         0.0                  0.0   \n",
       "max              0.0                         0.0                  0.0   \n",
       "\n",
       "       x0_Thoracic Surgery (Cardiothoracic Vascular Surgery)  \\\n",
       "count                                            88779.0       \n",
       "mean                                                 0.0       \n",
       "std                                                  0.0       \n",
       "min                                                  0.0       \n",
       "25%                                                  0.0       \n",
       "50%                                                  0.0       \n",
       "75%                                                  0.0       \n",
       "max                                                  0.0       \n",
       "\n",
       "       x0_Undefined Physician type  x0_Unknown Supplier/Provider Specialty  \\\n",
       "count                      88779.0                                 88779.0   \n",
       "mean                           0.0                                     0.0   \n",
       "std                            0.0                                     0.0   \n",
       "min                            0.0                                     0.0   \n",
       "25%                            0.0                                     0.0   \n",
       "50%                            0.0                                     0.0   \n",
       "75%                            0.0                                     0.0   \n",
       "max                            0.0                                     0.0   \n",
       "\n",
       "       x0_Urology  x0_Vascular Surgery  x0_Veterinarian  \\\n",
       "count     88779.0              88779.0          88779.0   \n",
       "mean          0.0                  0.0              0.0   \n",
       "std           0.0                  0.0              0.0   \n",
       "min           0.0                  0.0              0.0   \n",
       "25%           0.0                  0.0              0.0   \n",
       "50%           0.0                  0.0              0.0   \n",
       "75%           0.0                  0.0              0.0   \n",
       "max           0.0                  0.0              0.0   \n",
       "\n",
       "       x0_Voluntary Health or Charitable Agency  \n",
       "count                                   88779.0  \n",
       "mean                                        0.0  \n",
       "std                                         0.0  \n",
       "min                                         0.0  \n",
       "25%                                         0.0  \n",
       "50%                                         0.0  \n",
       "75%                                         0.0  \n",
       "max                                         0.0  \n",
       "\n",
       "[8 rows x 214 columns]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fudge.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t_cost_per_claim_excluded</th>\n",
       "      <th>t_claim_count_schedule2</th>\n",
       "      <th>t_claim_per_bene</th>\n",
       "      <th>%b2bg_claim_count</th>\n",
       "      <th>%b2bg_drug_cost</th>\n",
       "      <th>%b2t_claim_count</th>\n",
       "      <th>%b2t_drug_cost</th>\n",
       "      <th>t_cost_per_claim</th>\n",
       "      <th>t_cost_per_claim_ge65</th>\n",
       "      <th>t_claim_per_bene_ge65</th>\n",
       "      <th>...</th>\n",
       "      <th>x0_Technician</th>\n",
       "      <th>x0_Technician/Technologist</th>\n",
       "      <th>x0_Thoracic Surgery</th>\n",
       "      <th>x0_Thoracic Surgery (Cardiothoracic Vascular Surgery)</th>\n",
       "      <th>x0_Undefined Physician type</th>\n",
       "      <th>x0_Unknown Supplier/Provider Specialty</th>\n",
       "      <th>x0_Urology</th>\n",
       "      <th>x0_Vascular Surgery</th>\n",
       "      <th>x0_Veterinarian</th>\n",
       "      <th>x0_Voluntary Health or Charitable Agency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>101.000000</td>\n",
       "      <td>101.000000</td>\n",
       "      <td>101.000000</td>\n",
       "      <td>101.000000</td>\n",
       "      <td>101.000000</td>\n",
       "      <td>101.000000</td>\n",
       "      <td>101.000000</td>\n",
       "      <td>101.000000</td>\n",
       "      <td>101.000000</td>\n",
       "      <td>101.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>101.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>101.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>5221.393465</td>\n",
       "      <td>38.673267</td>\n",
       "      <td>9.499010</td>\n",
       "      <td>11.386139</td>\n",
       "      <td>36.306931</td>\n",
       "      <td>10.891089</td>\n",
       "      <td>36.099010</td>\n",
       "      <td>105.625248</td>\n",
       "      <td>89.195347</td>\n",
       "      <td>7.519802</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>28518.834547</td>\n",
       "      <td>56.116684</td>\n",
       "      <td>9.667021</td>\n",
       "      <td>15.035272</td>\n",
       "      <td>36.333935</td>\n",
       "      <td>13.045996</td>\n",
       "      <td>36.102771</td>\n",
       "      <td>176.536412</td>\n",
       "      <td>165.641181</td>\n",
       "      <td>9.375172</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>40.990000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.940000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>293.230000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.200000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>44.060000</td>\n",
       "      <td>28.760000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1206.820000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>5.800000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>72.230000</td>\n",
       "      <td>53.450000</td>\n",
       "      <td>3.780000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>3338.240000</td>\n",
       "      <td>54.000000</td>\n",
       "      <td>13.800000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>71.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>104.190000</td>\n",
       "      <td>79.820000</td>\n",
       "      <td>11.630000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>286886.380000</td>\n",
       "      <td>299.000000</td>\n",
       "      <td>46.100000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>98.000000</td>\n",
       "      <td>1419.570000</td>\n",
       "      <td>1258.960000</td>\n",
       "      <td>43.550000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 214 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       t_cost_per_claim_excluded  t_claim_count_schedule2  t_claim_per_bene  \\\n",
       "count                 101.000000               101.000000        101.000000   \n",
       "mean                 5221.393465                38.673267          9.499010   \n",
       "std                 28518.834547                56.116684          9.667021   \n",
       "min                    40.990000                 1.000000          0.000000   \n",
       "25%                   293.230000                 3.000000          2.200000   \n",
       "50%                  1206.820000                11.000000          5.800000   \n",
       "75%                  3338.240000                54.000000         13.800000   \n",
       "max                286886.380000               299.000000         46.100000   \n",
       "\n",
       "       %b2bg_claim_count  %b2bg_drug_cost  %b2t_claim_count  %b2t_drug_cost  \\\n",
       "count         101.000000       101.000000        101.000000      101.000000   \n",
       "mean           11.386139        36.306931         10.891089       36.099010   \n",
       "std            15.035272        36.333935         13.045996       36.102771   \n",
       "min             0.000000         0.000000          0.000000        0.000000   \n",
       "25%             0.000000         0.000000          0.000000        0.000000   \n",
       "50%            10.000000        43.000000         10.000000       43.000000   \n",
       "75%            20.000000        71.000000         20.000000       70.000000   \n",
       "max           100.000000       100.000000         60.000000       98.000000   \n",
       "\n",
       "       t_cost_per_claim  t_cost_per_claim_ge65  t_claim_per_bene_ge65  ...  \\\n",
       "count        101.000000             101.000000             101.000000  ...   \n",
       "mean         105.625248              89.195347               7.519802  ...   \n",
       "std          176.536412             165.641181               9.375172  ...   \n",
       "min            2.940000               0.000000               0.000000  ...   \n",
       "25%           44.060000              28.760000               0.000000  ...   \n",
       "50%           72.230000              53.450000               3.780000  ...   \n",
       "75%          104.190000              79.820000              11.630000  ...   \n",
       "max         1419.570000            1258.960000              43.550000  ...   \n",
       "\n",
       "       x0_Technician  x0_Technician/Technologist  x0_Thoracic Surgery  \\\n",
       "count          101.0                       101.0                101.0   \n",
       "mean             0.0                         0.0                  0.0   \n",
       "std              0.0                         0.0                  0.0   \n",
       "min              0.0                         0.0                  0.0   \n",
       "25%              0.0                         0.0                  0.0   \n",
       "50%              0.0                         0.0                  0.0   \n",
       "75%              0.0                         0.0                  0.0   \n",
       "max              0.0                         0.0                  0.0   \n",
       "\n",
       "       x0_Thoracic Surgery (Cardiothoracic Vascular Surgery)  \\\n",
       "count                                              101.0       \n",
       "mean                                                 0.0       \n",
       "std                                                  0.0       \n",
       "min                                                  0.0       \n",
       "25%                                                  0.0       \n",
       "50%                                                  0.0       \n",
       "75%                                                  0.0       \n",
       "max                                                  0.0       \n",
       "\n",
       "       x0_Undefined Physician type  x0_Unknown Supplier/Provider Specialty  \\\n",
       "count                        101.0                                   101.0   \n",
       "mean                           0.0                                     0.0   \n",
       "std                            0.0                                     0.0   \n",
       "min                            0.0                                     0.0   \n",
       "25%                            0.0                                     0.0   \n",
       "50%                            0.0                                     0.0   \n",
       "75%                            0.0                                     0.0   \n",
       "max                            0.0                                     0.0   \n",
       "\n",
       "       x0_Urology  x0_Vascular Surgery  x0_Veterinarian  \\\n",
       "count       101.0                101.0            101.0   \n",
       "mean          0.0                  0.0              0.0   \n",
       "std           0.0                  0.0              0.0   \n",
       "min           0.0                  0.0              0.0   \n",
       "25%           0.0                  0.0              0.0   \n",
       "50%           0.0                  0.0              0.0   \n",
       "75%           0.0                  0.0              0.0   \n",
       "max           0.0                  0.0              0.0   \n",
       "\n",
       "       x0_Voluntary Health or Charitable Agency  \n",
       "count                                     101.0  \n",
       "mean                                        0.0  \n",
       "std                                         0.0  \n",
       "min                                         0.0  \n",
       "25%                                         0.0  \n",
       "50%                                         0.0  \n",
       "75%                                         0.0  \n",
       "max                                         0.0  \n",
       "\n",
       "[8 rows x 214 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "only_excluded.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(88779, 214)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fudge.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t_cost_per_claim_excluded</th>\n",
       "      <th>t_claim_count_schedule2</th>\n",
       "      <th>t_claim_per_bene</th>\n",
       "      <th>%b2bg_claim_count</th>\n",
       "      <th>%b2bg_drug_cost</th>\n",
       "      <th>%b2t_claim_count</th>\n",
       "      <th>%b2t_drug_cost</th>\n",
       "      <th>t_cost_per_claim</th>\n",
       "      <th>t_cost_per_claim_ge65</th>\n",
       "      <th>t_claim_per_bene_ge65</th>\n",
       "      <th>...</th>\n",
       "      <th>x0_Technician</th>\n",
       "      <th>x0_Technician/Technologist</th>\n",
       "      <th>x0_Thoracic Surgery</th>\n",
       "      <th>x0_Thoracic Surgery (Cardiothoracic Vascular Surgery)</th>\n",
       "      <th>x0_Undefined Physician type</th>\n",
       "      <th>x0_Unknown Supplier/Provider Specialty</th>\n",
       "      <th>x0_Urology</th>\n",
       "      <th>x0_Vascular Surgery</th>\n",
       "      <th>x0_Veterinarian</th>\n",
       "      <th>x0_Voluntary Health or Charitable Agency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>82944</th>\n",
       "      <td>3483.532868</td>\n",
       "      <td>215.766340</td>\n",
       "      <td>19.990503</td>\n",
       "      <td>16.875812</td>\n",
       "      <td>72.228089</td>\n",
       "      <td>18.189547</td>\n",
       "      <td>71.406809</td>\n",
       "      <td>78.657694</td>\n",
       "      <td>79.125992</td>\n",
       "      <td>16.751167</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>644968</th>\n",
       "      <td>2602.044279</td>\n",
       "      <td>151.919238</td>\n",
       "      <td>17.371792</td>\n",
       "      <td>8.933958</td>\n",
       "      <td>65.903505</td>\n",
       "      <td>13.323704</td>\n",
       "      <td>62.558383</td>\n",
       "      <td>80.924757</td>\n",
       "      <td>81.645599</td>\n",
       "      <td>16.988584</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82944</th>\n",
       "      <td>3483.597118</td>\n",
       "      <td>216.526370</td>\n",
       "      <td>17.203577</td>\n",
       "      <td>17.319448</td>\n",
       "      <td>73.481025</td>\n",
       "      <td>19.161120</td>\n",
       "      <td>73.470003</td>\n",
       "      <td>80.516639</td>\n",
       "      <td>77.133769</td>\n",
       "      <td>15.651839</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>683678</th>\n",
       "      <td>4413.369417</td>\n",
       "      <td>22.356673</td>\n",
       "      <td>14.528631</td>\n",
       "      <td>9.103700</td>\n",
       "      <td>74.959101</td>\n",
       "      <td>10.424137</td>\n",
       "      <td>77.060729</td>\n",
       "      <td>103.023594</td>\n",
       "      <td>65.076639</td>\n",
       "      <td>10.200439</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>557956</th>\n",
       "      <td>1819.334509</td>\n",
       "      <td>113.185578</td>\n",
       "      <td>20.094973</td>\n",
       "      <td>9.314014</td>\n",
       "      <td>52.879933</td>\n",
       "      <td>12.912846</td>\n",
       "      <td>56.969737</td>\n",
       "      <td>31.527946</td>\n",
       "      <td>35.228795</td>\n",
       "      <td>23.035270</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 214 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        t_cost_per_claim_excluded  t_claim_count_schedule2  t_claim_per_bene  \\\n",
       "82944                 3483.532868               215.766340         19.990503   \n",
       "644968                2602.044279               151.919238         17.371792   \n",
       "82944                 3483.597118               216.526370         17.203577   \n",
       "683678                4413.369417                22.356673         14.528631   \n",
       "557956                1819.334509               113.185578         20.094973   \n",
       "\n",
       "        %b2bg_claim_count  %b2bg_drug_cost  %b2t_claim_count  %b2t_drug_cost  \\\n",
       "82944           16.875812        72.228089         18.189547       71.406809   \n",
       "644968           8.933958        65.903505         13.323704       62.558383   \n",
       "82944           17.319448        73.481025         19.161120       73.470003   \n",
       "683678           9.103700        74.959101         10.424137       77.060729   \n",
       "557956           9.314014        52.879933         12.912846       56.969737   \n",
       "\n",
       "        t_cost_per_claim  t_cost_per_claim_ge65  t_claim_per_bene_ge65  ...  \\\n",
       "82944          78.657694              79.125992              16.751167  ...   \n",
       "644968         80.924757              81.645599              16.988584  ...   \n",
       "82944          80.516639              77.133769              15.651839  ...   \n",
       "683678        103.023594              65.076639              10.200439  ...   \n",
       "557956         31.527946              35.228795              23.035270  ...   \n",
       "\n",
       "        x0_Technician  x0_Technician/Technologist  x0_Thoracic Surgery  \\\n",
       "82944             0.0                         0.0                  0.0   \n",
       "644968            0.0                         0.0                  0.0   \n",
       "82944             0.0                         0.0                  0.0   \n",
       "683678            0.0                         0.0                  0.0   \n",
       "557956            0.0                         0.0                  0.0   \n",
       "\n",
       "        x0_Thoracic Surgery (Cardiothoracic Vascular Surgery)  \\\n",
       "82944                                                 0.0       \n",
       "644968                                                0.0       \n",
       "82944                                                 0.0       \n",
       "683678                                                0.0       \n",
       "557956                                                0.0       \n",
       "\n",
       "        x0_Undefined Physician type  x0_Unknown Supplier/Provider Specialty  \\\n",
       "82944                           0.0                                     0.0   \n",
       "644968                          0.0                                     0.0   \n",
       "82944                           0.0                                     0.0   \n",
       "683678                          0.0                                     0.0   \n",
       "557956                          0.0                                     0.0   \n",
       "\n",
       "        x0_Urology  x0_Vascular Surgery  x0_Veterinarian  \\\n",
       "82944          0.0                  0.0              0.0   \n",
       "644968         0.0                  0.0              0.0   \n",
       "82944          0.0                  0.0              0.0   \n",
       "683678         0.0                  0.0              0.0   \n",
       "557956         0.0                  0.0              0.0   \n",
       "\n",
       "        x0_Voluntary Health or Charitable Agency  \n",
       "82944                                        0.0  \n",
       "644968                                       0.0  \n",
       "82944                                        0.0  \n",
       "683678                                       0.0  \n",
       "557956                                       0.0  \n",
       "\n",
       "[5 rows x 214 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fudge=fudge.sample(n=10000)\n",
    "fudge.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "ag_2017 = pd.concat([new_main_2017, fudge])\n",
    "ag_2017.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 214)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ag_2017[ag_2017[\"is_excluded\"]==1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(829898, 214)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ag_2017.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Applying PCA, 2 components and 3 components for variance visualization\n",
    "\n",
    "pca = decomposition.PCA(n_components='mle')\n",
    "pca.fit(part_d_df)\n",
    "npi_drug_df_transformed = pca.transform(part_d_df)\n",
    "plt.plot(np.cumsum(pca.explained_variance_ratio_), color='r')\n",
    "\n",
    "pca = decomposition.PCA(n_components='mle')\n",
    "pca.fit(part_d_df)\n",
    "npi_drug_cross_df_transformed = pca.transform(part_d_df)\n",
    "plt.plot(np.cumsum(pca.explained_variance_ratio_))\n",
    "plt.xlabel('number of components')\n",
    "plt.ylabel('cumulative explained variance')\n",
    "\n",
    "plt.vlines(1, color='grey', ymax=0.1, ymin=0, linestyle=':')\n",
    "plt.vlines(2, color='grey', ymax=0.1, ymin=0, linestyle=':')\n",
    "\n",
    "plt.annotate('1 components', xy=(0.6, 0.05), xytext=(0.9, 0.05), fontsize=15, rotation=90)\n",
    "plt.annotate('2 components', xy=(1.6, 0.05), xytext=(1.9, 0.05), fontsize=15, rotation=90)\n",
    "\n",
    "plt.savefig('VarExp.jpeg', dpi=1000)\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training without Upsampling or Downsampling\n",
    "#fudged data (ag_2017) only for train, regular data (new_df) only for test\n",
    "\n",
    "X_train = ag_2017.drop(['is_excluded'], axis=1)\n",
    "y_train = ag_2017['is_excluded']\n",
    "X_test = new_df.drop(['is_excluded'], axis =1)\n",
    "y_test = new_df[\"is_excluded\"]\n",
    "#X = new_main_2017.drop(['is_excluded'], axis=1)\n",
    "#y = new_main_2017['is_excluded']\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = decomposition.PCA(n_components='mle')\n",
    "X_train = pca.fit_transform(X_train)\n",
    "X_test = pca.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[9.90249356e-01 7.51255316e-03 1.50476425e-03 6.95328820e-04\n",
      " 2.12595268e-05 8.98263318e-06 3.62458289e-06 1.55292210e-06\n",
      " 1.00527749e-06 5.96445922e-07 3.89277866e-07 2.83384443e-07\n",
      " 1.26738997e-07 7.02058786e-08 5.72225979e-08 2.98159283e-08\n",
      " 8.34500393e-09 6.25277760e-09 3.84033728e-09 6.41433707e-10\n",
      " 1.97901431e-10 1.92922228e-10 8.04623155e-11 3.82528391e-11\n",
      " 2.81908638e-11]\n"
     ]
    }
   ],
   "source": [
    "print(pca.explained_variance_ratio_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Labels = ag_2017.drop([\"is_excluded\"],axis = 1).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['total_day_supply',\n",
       " 'total_day_supply_ge65',\n",
       " 'opioid_day_supply',\n",
       " 't_cost_per_claim_excluded',\n",
       " 'lis_cost_per_claim',\n",
       " 'total_30_day_fill_count',\n",
       " 'mapd_cost_per_claim',\n",
       " 'mapd_cost_per_claim',\n",
       " 'lis_cost_per_claim',\n",
       " 't_cost_per_claim_ge65',\n",
       " 't_cost_per_claim',\n",
       " 'antibiotic_cost_per_claim',\n",
       " 'antipsych_cost_per_claim_ge65',\n",
       " '%b2bg_drug_cost',\n",
       " 'total_30_day_fill_count_ge65',\n",
       " 'opioid_cost_per_claim',\n",
       " 'opioid_prescriber_rate',\n",
       " 't_claim_count_schedule2',\n",
       " '%b2bg_claim_count',\n",
       " 't_claim_per_bene',\n",
       " 't_claim_per_bene_ge65',\n",
       " '%b2t_drug_cost',\n",
       " '%b2t_claim_count',\n",
       " 'antipsych_claim_per_bene_ge65',\n",
       " 'opioid_claim_per_bene']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#new Labels after PCA\n",
    "most_important = [np.abs(pca.components_[i]).argmax() for i in range(len(pca.components_))]\n",
    "most_important_names = [Labels[most_important[i]] for i in range(len(pca.components_))]\n",
    "dic = {'PC{}'.format(i+1): most_important_names[i] for i in range(len(pca.components_))}\n",
    "newLabels = list(dic.values())\n",
    "newLabels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=25, n_jobs=None,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = RandomForestClassifier(n_estimators=25)\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "101\n"
     ]
    }
   ],
   "source": [
    "#excluded samples in test data\n",
    "count = 0\n",
    "for x in y_test:\n",
    "    if int(x)== 1:\n",
    "        count +=1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "101010\n"
     ]
    }
   ],
   "source": [
    "#excluded samples in train data\n",
    "count2 = 0\n",
    "for x in y_train:\n",
    "    if int(x)== 1:\n",
    "        count2 +=1\n",
    "print(count2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([6.13077826e-02, 4.03535326e-02, 4.31563815e-02, 1.16046544e-02,\n",
       "       2.34645164e-02, 1.06847241e-02, 2.32142326e-02, 5.90239257e-02,\n",
       "       4.10690646e-02, 3.76783461e-02, 4.29818179e-02, 4.28378182e-02,\n",
       "       4.42484529e-02, 4.61165115e-02, 3.63436501e-02, 2.29182419e-02,\n",
       "       3.52418866e-02, 2.51572743e-02, 1.75878640e-02, 7.33856296e-03,\n",
       "       5.66234338e-02, 5.47119874e-02, 4.49337166e-02, 4.68286357e-02,\n",
       "       2.89219381e-02, 2.49363649e-02, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       9.62283124e-07, 3.05309343e-06, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 4.95499374e-07, 1.71603924e-07, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 4.68950350e-06, 0.00000000e+00,\n",
       "       1.79363267e-03, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       2.95605070e-06, 4.55016767e-04, 0.00000000e+00, 2.64815919e-03,\n",
       "       2.30464528e-05, 2.12624365e-07, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 2.25887598e-05, 1.56030646e-05,\n",
       "       0.00000000e+00, 0.00000000e+00, 3.97665091e-07, 0.00000000e+00,\n",
       "       1.36841502e-06, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 1.12957105e-06, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 1.65181458e-03, 0.00000000e+00,\n",
       "       3.80128647e-05, 0.00000000e+00, 4.42686244e-03, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 4.36446353e-04, 2.56185293e-05,\n",
       "       0.00000000e+00, 5.80376866e-06, 7.29575028e-03, 1.23487483e-03,\n",
       "       1.07589141e-06, 5.46535187e-03, 1.13520881e-03, 2.09637787e-03,\n",
       "       7.99714596e-06, 0.00000000e+00, 1.11959280e-06, 0.00000000e+00,\n",
       "       3.72659789e-07, 0.00000000e+00, 7.13738209e-06, 1.22535192e-03,\n",
       "       0.00000000e+00, 0.00000000e+00, 2.78684891e-05, 0.00000000e+00,\n",
       "       2.64699093e-05, 0.00000000e+00, 2.69177256e-05, 6.70654803e-03,\n",
       "       7.04960828e-04, 3.95118127e-03, 3.39488182e-07, 3.91946151e-07,\n",
       "       1.16753002e-07, 1.16078606e-07, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 2.21088626e-07, 0.00000000e+00,\n",
       "       0.00000000e+00, 1.76925053e-05, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 9.78825160e-08, 6.75677528e-05,\n",
       "       4.57210976e-06, 2.24570857e-03, 2.05881889e-05, 5.40815455e-06,\n",
       "       0.00000000e+00, 2.46580452e-03, 0.00000000e+00, 0.00000000e+00,\n",
       "       1.59768057e-03, 0.00000000e+00, 0.00000000e+00, 2.62327307e-05,\n",
       "       0.00000000e+00, 2.72539031e-04, 8.97018700e-06, 0.00000000e+00,\n",
       "       1.36646982e-05, 4.15561413e-03, 1.12397965e-04, 4.80290187e-06,\n",
       "       0.00000000e+00, 1.43820481e-05, 5.41668482e-05, 6.38246362e-08,\n",
       "       1.89260545e-03, 0.00000000e+00, 7.09574197e-06, 0.00000000e+00,\n",
       "       9.91679047e-05, 2.09733618e-07, 0.00000000e+00, 0.00000000e+00,\n",
       "       1.74886800e-04, 0.00000000e+00, 0.00000000e+00, 1.65213573e-03,\n",
       "       2.97842545e-07, 1.88469914e-03, 9.10750713e-03, 0.00000000e+00,\n",
       "       0.00000000e+00, 7.52370260e-06, 0.00000000e+00, 5.64346323e-04,\n",
       "       2.37641513e-04, 1.80814096e-06, 1.31717209e-07, 0.00000000e+00,\n",
       "       1.00773002e-05, 2.08806347e-05, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 7.54094271e-06, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 9.74648124e-05, 0.00000000e+00,\n",
       "       2.20456201e-07, 0.00000000e+00, 0.00000000e+00, 1.97971044e-03,\n",
       "       0.00000000e+00, 2.26837987e-06, 4.21290591e-07, 3.84556588e-04,\n",
       "       0.00000000e+00, 2.59383555e-10, 2.60404380e-07, 0.00000000e+00,\n",
       "       0.00000000e+00, 2.64785551e-07, 0.00000000e+00, 6.63476175e-07,\n",
       "       0.00000000e+00, 1.54127793e-05, 3.13673195e-06, 0.00000000e+00,\n",
       "       0.00000000e+00])"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.feature_importances_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "coef_df = pd.DataFrame(clf.feature_importances_, Labels, columns = [\"Importances\"])\n",
    "coef_df.sort_values(\"Importances\", ascending = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "check for contamination by not dropping the npis and seeing if any of them overlap between training and testing\n",
    "train_nps = set(X_train['npi'])\n",
    "test_nps = set(X_test['npi'])\n",
    "print(len(train_nps.intersection(test_nps))) # should be zero"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 0.]\n",
      " [1. 0.]\n",
      " [1. 0.]\n",
      " ...\n",
      " [0. 1.]\n",
      " [0. 1.]\n",
      " [0. 1.]]\n",
      "1.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[819898,      0],\n",
       "       [     0,  10000]], dtype=int64)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_train)\n",
    "y_pred_prob = clf.predict_proba(X_train)\n",
    "print(y_pred_prob)\n",
    "print(roc_auc_score(y_train, y_pred_prob[:,1]))\n",
    "confusion_matrix(y_train, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 0.]\n",
      " [1. 0.]\n",
      " [1. 0.]\n",
      " ...\n",
      " [1. 0.]\n",
      " [1. 0.]\n",
      " [1. 0.]]\n",
      "0.7105610949330227\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[85000,     0],\n",
       "       [   95,     6]], dtype=int64)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test)\n",
    "y_pred_prob = clf.predict_proba(X_test)\n",
    "print(y_pred_prob)\n",
    "print(roc_auc_score(y_test, y_pred_prob[:,1]))\n",
    "confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    819898\n",
       "1    100000\n",
       "Name: is_excluded, dtype: int64"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Upsampling only\n",
    "\n",
    "ag_2017_majority = ag_2017[ag_2017['is_excluded'] == 0]\n",
    "ag_2017_minority = ag_2017[ag_2017['is_excluded'] > 0]\n",
    "\n",
    "# Increase by 10 times\n",
    "ag_2017_minority_upsampled = resample(ag_2017_minority, replace=True, n_samples=100000, random_state=42) \n",
    "ag_2017_upsampled = pd.concat([ag_2017_majority, ag_2017_minority_upsampled])  \n",
    "ag_2017_upsampled['is_excluded'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#fudged data (ag_2017) only for train, regular data (new_df) only for test\n",
    "X_train = ag_2017_upsampled.drop(['is_excluded'], axis=1)\n",
    "y_train = ag_2017_upsampled['is_excluded']\n",
    "X_test = new_df.drop(['is_excluded'], axis =1)\n",
    "y_test = new_df[\"is_excluded\"]\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = decomposition.PCA(n_components='mle')\n",
    "X_train = pca.fit_transform(X_train)\n",
    "X_test = pca.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(pca.explained_variance_ratio_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Labels = df.drop([\"is_excluded\"],axis = 1).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#new Labels after PCA\n",
    "most_important = [np.abs(pca.components_[i]).argmax() for i in range(len(pca.components_))]\n",
    "most_important_names = [Labels[most_important[i]] for i in range(len(pca.components_))]\n",
    "dic = {'PC{}'.format(i+1): most_important_names[i] for i in range(len(pca.components_))}\n",
    "newLabels = list(dic.values())\n",
    "newLabels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=25, n_jobs=None,\n",
       "            oob_score=False, random_state=42, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = RandomForestClassifier(n_estimators=25, random_state=42)\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#excluded samples in test data\n",
    "count = 0\n",
    "for x in y_test:\n",
    "    if int(x)== 1:\n",
    "        count +=1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#excluded samples in train data\n",
    "count2 = 0\n",
    "for x in y_train:\n",
    "    if int(x)== 1:\n",
    "        count2 +=1\n",
    "print(count2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.feature_importances_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#check for contamination by not dropping the npis and seeing if any of them overlap between training and testing\n",
    "train_nps = set(X_train['npi'])\n",
    "test_nps = set(X_test['npi'])\n",
    "print(len(train_nps.intersection(test_nps))) # should be zero"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[819898,      0],\n",
       "       [     0, 100000]], dtype=int64)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_train)\n",
    "y_pred_prob = clf.predict_proba(X_train)\n",
    "print(roc_auc_score(y_train, y_pred_prob[:,1]))\n",
    "confusion_matrix(y_train, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7012737332556784\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[85000,     0],\n",
       "       [   96,     5]], dtype=int64)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test)\n",
    "y_pred_prob = clf.predict_proba(X_test)\n",
    "print(roc_auc_score(y_test, y_pred_prob[:,1]))\n",
    "confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    81980\n",
       "1    10000\n",
       "Name: is_excluded, dtype: int64"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Training on downsampled only and testing on regular\n",
    "\n",
    "ag_2017_majority = ag_2017[ag_2017['is_excluded'] == 0]\n",
    "ag_2017_minority = ag_2017[ag_2017['is_excluded'] > 0]\n",
    "\n",
    "# Reduce by 10 times\n",
    "ag_2017_majority_downsampled = resample(ag_2017_majority, replace=True, n_samples=81980, random_state=42) \n",
    "ag_2017_downsampled = pd.concat([ag_2017_majority_downsampled, ag_2017_minority]) \n",
    "ag_2017_downsampled['is_excluded'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fudged data (ag_2017) only for train, regular data (new_df) only for test\n",
    "X_train = ag_2017_downsampled.drop(['is_excluded'], axis=1)\n",
    "y_train = ag_2017_downsampled['is_excluded']\n",
    "X_test = new_df.drop(['is_excluded'], axis =1)\n",
    "y_test = new_df[\"is_excluded\"]\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = decomposition.PCA(n_components='mle')\n",
    "X_train = pca.fit_transform(X_train)\n",
    "X_test = pca.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(pca.explained_variance_ratio_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Labels = df.drop([\"is_excluded\"],axis = 1).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#new Labels after PCA\n",
    "most_important = [np.abs(pca.components_[i]).argmax() for i in range(len(pca.components_))]\n",
    "most_important_names = [Labels[most_important[i]] for i in range(len(pca.components_))]\n",
    "dic = {'PC{}'.format(i+1): most_important_names[i] for i in range(len(pca.components_))}\n",
    "newLabels = list(dic.values())\n",
    "newLabels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=25, n_jobs=None,\n",
       "            oob_score=False, random_state=42, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = RandomForestClassifier(n_estimators=25, random_state=42)\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#excluded samples in test data\n",
    "count = 0\n",
    "for x in y_test:\n",
    "    if int(x) == 1:\n",
    "        count +=1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#excluded samples in train data\n",
    "count2 = 0\n",
    "for x in y_train:\n",
    "    if int(x)== 1:\n",
    "        count2 +=1\n",
    "print(count2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.feature_importances_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#check for contamination by not dropping the npis and seeing if any of them overlap between training and testing\n",
    "train_nps = set(X_train['npi'])\n",
    "test_nps = set(X_test['npi'])\n",
    "print(len(train_nps.intersection(test_nps))) # should be zero"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[81980,     0],\n",
       "       [    0, 10000]], dtype=int64)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_train)\n",
    "y_pred_prob = clf.predict_proba(X_train)\n",
    "print(roc_auc_score(y_train, y_pred_prob[:,1]))\n",
    "confusion_matrix(y_train, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7267889924286547\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[85000,     0],\n",
       "       [   96,     5]], dtype=int64)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test)\n",
    "y_pred_prob = clf.predict_proba(X_test)\n",
    "print(roc_auc_score(y_test, y_pred_prob[:,1]))\n",
    "confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    100000\n",
       "0     81980\n",
       "Name: is_excluded, dtype: int64"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Upsampling/downsampling on training and testing on regular\n",
    "\n",
    "ag_2017_majority = ag_2017[ag_2017['is_excluded'] == 0]\n",
    "ag_2017_minority = ag_2017[ag_2017['is_excluded'] > 0]\n",
    "\n",
    "ag_2017_updownsampled = pd.concat([ag_2017_majority_downsampled, ag_2017_minority_upsampled]) \n",
    "ag_2017_updownsampled['is_excluded'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fudged data (ag_2017) only for train, regular data (new_df) only for test\n",
    "\n",
    "X_train = ag_2017_updownsampled.drop(['is_excluded'], axis=1)\n",
    "y_train = ag_2017_updownsampled['is_excluded']\n",
    "X_test = new_df.drop(['is_excluded'], axis =1)\n",
    "y_test = new_df[\"is_excluded\"]\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = decomposition.PCA(n_components='mle')\n",
    "X_train = pca.fit_transform(X_train)\n",
    "X_test = pca.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(pca.explained_variance_ratio_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Labels = df.drop([\"is_excluded\"],axis = 1).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#new Labels after PCA\n",
    "most_important = [np.abs(pca.components_[i]).argmax() for i in range(len(pca.components_))]\n",
    "most_important_names = [Labels[most_important[i]] for i in range(len(pca.components_))]\n",
    "dic = {'PC{}'.format(i+1): most_important_names[i] for i in range(len(pca.components_))}\n",
    "newLabels = list(dic.values())\n",
    "newLabels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=25, n_jobs=None,\n",
       "            oob_score=False, random_state=42, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = RandomForestClassifier(n_estimators=25, random_state=42)\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#excluded samples in test data\n",
    "count = 0\n",
    "for x in y_test:\n",
    "    if int(x)== 1:\n",
    "        count +=1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#excluded samples in train data\n",
    "count2 = 0\n",
    "for x in y_train:\n",
    "    if int(x)== 1:\n",
    "        count2 +=1\n",
    "print(count2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.feature_importances_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#check for contamination by not dropping the npis and seeing if any of them overlap between training and testing\n",
    "train_nps = set(X_train['npi'])\n",
    "test_nps = set(X_test['npi'])\n",
    "print(len(train_nps.intersection(test_nps))) # should be zero"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 81980,      0],\n",
       "       [     0, 100000]], dtype=int64)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_train)\n",
    "y_pred_prob = clf.predict_proba(X_train)\n",
    "print(roc_auc_score(y_train, y_pred_prob[:,1]))\n",
    "confusion_matrix(y_train, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7080670355270821\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[84999,     1],\n",
       "       [   96,     5]], dtype=int64)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test)\n",
    "y_pred_prob = clf.predict_proba(X_test)\n",
    "print(roc_auc_score(y_test, y_pred_prob[:,1]))\n",
    "confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In conclusion, results from downsampling is the best."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#comparison.iloc[].plot(kind=’bar’, figsize(10,8))\n",
    "#plt.grid(which='major', linestyle='-', linewidth='0.5', color='green')\n",
    "#plt.grid(which='minor', linestyle='-', linewidth='0.5', color='black')\n",
    "#plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plt.scatter(comparison.iloc[][\"Actual\"], .iloc[][\"Predicted\"])\n",
    "#plt.xlabel(\"Actual\", fontsize = 16)\n",
    "#plt.ylabel(\"Predicted\", fontsize = 16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Classification Model Comparisons\n",
    "\n",
    "def warn(*args, **kwargs):\n",
    "    pass\n",
    "\n",
    "import warnings\n",
    "warnings.warn = warn\n",
    "\n",
    "from scipy import interp\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import cycle\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.gaussian_process import GaussianProcessClassifier\n",
    "from sklearn.gaussian_process.kernels import RBF\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
    "from sklearn.model_selection import cross_validate, GridSearchCV, StratifiedKFold\n",
    "from sklearn.metrics import roc_curve, roc_auc_score, confusion_matrix, auc, recall_score, precision_score, accuracy_score, f1_score\n",
    "\n",
    "seed = 42"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Identifying the best model algorithm\n",
    "colors = [\"#08306b\", \"#08519c\", \"#1361a9\", \"#4292c6\", \"#6baed6\", 'rgb(76,000,153)', 'rgb(153,000,153)']\n",
    "\n",
    "scoring = {'AUC': 'roc_auc', 'Accuracy': 'accuracy', 'F1': 'f1'}\n",
    "\n",
    "names = [\"Naive Bayes\", \"Nearest Neighbors\", \"RBF SVM\", \n",
    "         \"AdaBoost\", \"GradientBoost\", \"ExtraTrees\", \"Random Forest\"]\n",
    "\n",
    "classifiers = [\n",
    "            GaussianNB(),\n",
    "            KNeighborsClassifier(n_neighbors=5),\n",
    "            SVC(kernel=\"rbf\", C=1, probability=True),\n",
    "            AdaBoostClassifier(n_estimators=25, random_state=seed),\n",
    "            GradientBoostingClassifier(n_estimators=1000, random_state=seed),\n",
    "            ExtraTreesClassifier(n_estimators=25, random_state=seed),\n",
    "            RandomForestClassifier(n_estimators=25, random_state=seed),]\n",
    "\n",
    "trace_dict = {'x_vals':[], 'y_vals':[], 'y_error':[], 'colors':[], 'names':[]}\n",
    "                                                                                         \n",
    "#iterate over classifiers\n",
    "counter = 0\n",
    "for color, name, clf in tqdm_notebook(zip(colors, names, classifiers), total=len(names)):\n",
    "    cv_results = cross_validate(clf, X, y, scoring=scoring, cv=5, return_train_score=True)\n",
    "    print('########################### \\nFor {}'.format(name))\n",
    "    for score in scoring:\n",
    "        print('{0:s}: Mean-{1:.4f}  STD-{2:.4f}'.format(score, cv_results['test_'+score].mean(),\\\n",
    "                                                        cv_results['test_'+score].std()))\n",
    "    \n",
    "    trace_dict['x_vals'].append(counter)\n",
    "    counter += 1\n",
    "    trace_dict['y_vals'].append(cv_results['test_F1'].mean())\n",
    "    trace_dict['y_error'].append()\n",
    "    trace_dict['colors'].append(color)\n",
    "    trace_dict['names'].append(name)\n",
    "    trace_dict['y_error'].append(np.std(cv_results['test_F1']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Identifying the best model algorithm\n",
    "colors = [\"#08306b\", \"#08519c\", \"#1361a9\", \"#4292c6\", \"#6baed6\", 'rgb(76,000,153)', 'rgb(153,000,153)']\n",
    "\n",
    "names = [\"GradientBoost\", \"ExtraTrees\", \"Random Forest\", \"RBF SVM\", \"Naive Bayes\", \"Nearest Neighbors\", \"AdaBoost\"]\n",
    "\n",
    "classifiers = [\n",
    "            GradientBoostingClassifier(n_estimators=50, random_state=seed),\n",
    "            ExtraTreesClassifier(n_estimators=25, random_state=seed),\n",
    "            RandomForestClassifier(n_estimators=25, random_state=seed),\n",
    "            SVC(kernel=\"rbf\", C=1, probability=True),        \n",
    "            GaussianNB(),\n",
    "            KNeighborsClassifier(n_neighbors=5),\n",
    "            AdaBoostClassifier(n_estimators=25, random_state=seed)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06dd9f55dd054fa9a327f36aa72ab8d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5966101339545719\n",
      "########################### \n",
      "For GradientBoost\n",
      "0.7547958066394874\n",
      "########################### \n",
      "For ExtraTrees\n",
      "0.6909806639487477\n",
      "########################### \n",
      "For Random Forest\n"
     ]
    }
   ],
   "source": [
    "trace_dict = {'x_vals':[], 'y_vals':[], 'colors':[], 'names':[]}\n",
    "                                                                                         \n",
    "# iterate over classifiers\n",
    "counter = 0\n",
    "for color, name, clf in tqdm_notebook(zip(colors, names, classifiers), total=len(names)):\n",
    "    X_train = ag_2017.drop(['is_excluded'], axis=1)\n",
    "    y_train = ag_2017['is_excluded']\n",
    "    X_test = new_df.drop(['is_excluded'], axis =1)\n",
    "    y_test = new_df[\"is_excluded\"]\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred_prob = clf.predict_proba(X_test)\n",
    "    cv_results = roc_auc_score(y_test, y_pred_prob[:,1])\n",
    "    print(c########################### \\nFor {}'.format(name))\n",
    "    \n",
    "    trace_dict['x_vals'].append(counter)\n",
    "    counterv_results)\n",
    "    print(' += 1\n",
    "    trace_dict['y_vals'].append(cv_results.mean())\n",
    "    trace_dict['colors'].append(color)\n",
    "    trace_dict['names'].append(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar(x=trace_dict['x_vals'], height=trace_dict['y_vals'], color=trace_dict['colors'], yerr=trace_dict['y_error'])\n",
    "plt.xticks(trace_dict['x_vals'], trace_dict['names'])\n",
    "print(trace_dict['y_error'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plotting ROC curves of the models\n",
    "\n",
    "colors = [\"#08306b\", \"#08519c\", \"#1361a9\", \"#4292c6\", \"#6baed6\", 'rgb(76,000,153)', 'rgb(153,000,153)']\n",
    "names = [\"Naive Bayes\", \"Nearest Neighbors\", \"RBF SVM\", \n",
    "         \"AdaBoost\", \"GradientBoost\", \"ExtraTrees\", \"Random Forest\"]\n",
    "\n",
    "X = df_upsampled.drop(['is_excluded'], axis=1)\n",
    "y = df_upsampled['is_excluded']\n",
    "\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X.values, y.values, test_size=0.33, random_state=42)\n",
    "n_classes = 1\n",
    "\n",
    "fpr = dict()\n",
    "tpr = dict()\n",
    "roc_auc = dict()\n",
    "\n",
    "plt.figure()\n",
    "lw = 2\n",
    "trace_dict = {}\n",
    "\n",
    "classifiers = [\n",
    "    GaussianNB(),\n",
    "    KNeighborsClassifier(n_neighbors=5),\n",
    "    SVC(kernel=\"rbf\", C=1, probability=True),\n",
    "    AdaBoostClassifier(n_estimators=50, random_state=seed),\n",
    "    GradientBoostingClassifier(n_estimators=1000, random_state=seed),\n",
    "    ExtraTreesClassifier(n_estimators=50, random_state=seed),\n",
    "    RandomForestClassifier(n_estimators=50, random_state=seed)]\n",
    "\n",
    "for color, name, clf in tqdm_notebook(zip(colors, names, classifiers), total=len(names)):\n",
    "    y_score = clf.fit(X_train, y_train).predict_proba(X_test)\n",
    "    fpr[name], tpr[name], _ = roc_curve(y_test[:], y_score[:, 1])\n",
    "    roc_auc[name] = auc(fpr[name], tpr[name])\n",
    "    trace_dict.update({'trace' + name: go.Scatter(x=fpr[name], y=tpr[name], mode='lines', \n",
    "                                       line=dict(color=color, width=lw),\n",
    "                                       showlegend=True,\n",
    "                                       name='{0:s} AUC: {1:.2f}'.format(name, roc_auc[name]))})\n",
    "\n",
    "trace_guess = go.Scatter(x=[0, 1], y=[0, 1], \n",
    "                mode='lines', \n",
    "                line=dict(color='navy', width=lw, dash='dash'),\n",
    "                showlegend=False)\n",
    "\n",
    "data = [trace_dict[trace] for trace in trace_dict] + [trace_guess]\n",
    "\n",
    "layout = go.Layout(title='Receiver Operating Characteristics',\n",
    "                   xaxis=dict(title='False Positive Rate'),\n",
    "                   yaxis=dict(title='True Positive Rate'),\n",
    "                   margin=go.Margin(l=50, r=50, b=30, t=65, pad=4),\n",
    "                   legend=dict(x=0.7, y=0.4, font=dict(size=18))\n",
    "                  )\n",
    "\n",
    "fig = go.Figure(data=data, layout=layout)\n",
    "py.iplot(fig, filename='ROCs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Optimizing PCA\n",
    "\n",
    "npi_drug_df = pd.read_pickle('npi_drug_df.pickle')\n",
    "npi_group = pd.read_pickle('npi_group.pickle')\n",
    "\n",
    "pcas = [1, 2, 3, 5] #20, 50, 100]\n",
    "F1_avg = []\n",
    "F1_std = []\n",
    "prec_avg = []\n",
    "prec_std = []\n",
    "recall_avg = []\n",
    "recall_std = []\n",
    "\n",
    "scoring = {'prec': 'precision', 'recall': 'recall', 'F1': 'f1'}\n",
    "\n",
    "clf = ExtraTreesClassifier(bootstrap=False, class_weight=None, criterion='gini',\n",
    "           max_depth=None, max_features=None, max_leaf_nodes=None,\n",
    "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "           min_samples_leaf=1, min_samples_split=2,\n",
    "           min_weight_fraction_leaf=0.0, n_estimators=200, n_jobs=1,\n",
    "           oob_score=False, random_state=None, verbose=0, warm_start=False)\n",
    "\n",
    "for pca in tqdm_notebook(pcas):\n",
    "    pca = decomposition.PCA(n_components=pca)\n",
    "    pca.fit(npi_drug_df)\n",
    "    npi_drug_df_transformed = pca.transform(npi_drug_df)\n",
    "    npi_label = pd.DataFrame(npi_group['is_excluded'].apply(lambda x: 1 if x > 0  else 0))\n",
    "    df = pd.concat([pd.DataFrame(npi_drug_df_transformed, index=npi_label.index), npi_label], axis=1)\n",
    "\n",
    "    df_majority = df[df['is_excluded'] == 0]\n",
    "    df_minority = df[df['is_excluded'] > 0]\n",
    "\n",
    "    df_minority_upsampled = resample(df_minority, replace=True, n_samples=5480, random_state=42) # Increase by 10 times\n",
    "    df_upsampled = pd.concat([df_majority, df_minority_upsampled])  \n",
    "    df_upsampled['is_excluded'].value_counts()\n",
    "    X = df_upsampled.drop(['is_excluded'], axis=1)\n",
    "    y = df_upsampled['is_excluded']\n",
    "\n",
    "    cv_results = cross_validate(clf, X, y, scoring=scoring, cv=3, return_train_score=False)\n",
    "    F1_avg.append(cv_results['test_F1'].mean())\n",
    "    F1_std.append(np.std(cv_results['test_F1']))\n",
    "    prec_avg.append(cv_results['test_prec'].mean())\n",
    "    prec_std.append(np.std(cv_results['test_prec']))\n",
    "    recall_avg.append(cv_results['test_recall'].mean())\n",
    "    recall_std.append(np.std(cv_results['test_recall']))\n",
    "    \n",
    "# Plotting\n",
    "data = [go.Scatter(x=pcas, y=F1_avg, error_y=dict(type='data', array=F1_std, visible=True, color=\"#6baed6\", thickness=1.5),\n",
    "                  marker=dict(color=\"#9ecae1\",size=8))]\n",
    "\n",
    "layout = go.Layout(\n",
    "    title='Performance of Random Forest with Increasing Components',\n",
    "    #width=500,\n",
    "    xaxis=dict(autorange=True, showgrid=True, zeroline=False, showline=False, autotick=True, ticks='',\n",
    "               showticklabels=True, title='Number of Components'),\n",
    "    yaxis=dict(title='F1 Score', zeroline=False))\n",
    "\n",
    "fig= go.Figure(data=data, layout=layout)\n",
    "py.iplot(fig, filename='PCAEffects')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trying out Pipeline Below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "from sklearn import decomposition, datasets\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "part_d_df_majority = part_d_df[part_d_df['is_excluded'] == 0]\n",
    "part_d_df_minority = part_d_df[part_d_df['is_excluded'] > 0]\n",
    "part_d_df_updownsampled = pd.concat([part_d_df_majority_downsampled, part_d_df_minority_upsampled]) \n",
    "part_d_df_updownsampled['is_excluded'].value_counts()\n",
    "\n",
    "X = part_d_df.drop(['is_excluded', 'npi', 'year', 'specialty_description', 'nppes_provider_state'], axis=1)\n",
    "y = part_d_df['is_excluded']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)\n",
    "\n",
    "X_train = part_d_df_updownsampled.drop(['is_excluded', 'npi', 'year', 'specialty_description', 'nppes_provider_state'], axis=1)\n",
    "y_train = part_d_df_updownsampled['is_excluded']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "variance_thresh = VarianceThreshold()\n",
    "select_k = SelectKBest(k=5)\n",
    "random_forest = RandomForestClassifier(n_estimators=50)\n",
    "\n",
    "pipeline_steps = [(\"remove_zero_variance\", variance_thresh), \n",
    "                  (\"select_features\", select_k), \n",
    "                  (\"random_forest\", random_forest)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('remove_zero_variance', VarianceThreshold(threshold=0.0)), ('select_features', SelectKBest(k=5, score_func=<function f_classif at 0x00000229409D21E0>)), ('random_forest', RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='aut...obs=None,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False))])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline_obj = Pipeline(pipeline_steps)\n",
    "pipeline_obj.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=int64)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_predict = pipeline_obj.predict(X_test)\n",
    "y_predict[0: 100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "report = classification_report(y_predict, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    226132\n",
      "           1       1.00      0.25      0.41       118\n",
      "\n",
      "   micro avg       1.00      1.00      1.00    226250\n",
      "   macro avg       1.00      0.63      0.70    226250\n",
      "weighted avg       1.00      1.00      1.00    226250\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.  , 0.  ],\n",
       "       [1.  , 0.  ],\n",
       "       [1.  , 0.  ],\n",
       "       [1.  , 0.  ],\n",
       "       [1.  , 0.  ],\n",
       "       [1.  , 0.  ],\n",
       "       [1.  , 0.  ],\n",
       "       [1.  , 0.  ],\n",
       "       [0.98, 0.02],\n",
       "       [1.  , 0.  ]])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_p = pipeline_obj.predict_proba(X_test)\n",
    "y_test_p[0: 10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x229412ebd68>]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD7CAYAAACVMATUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XtQU3feP/A3JIRbIBFBBDFIUVQE5e6Fu4pY8G691I5Pd8ZO2+3sdmbbmW6nf2wdx3Ht7uzMM+12nOll7Gxndle7+zxOL8+vIhIIN0GQoCiCoqKiAso1JCQkOb8/dNm6bQlg4JCT9+svk0PI+5vE9xySnM/xEARBABERSYKn2AGIiMh5WOpERBLCUicikhCWOhGRhLDUiYgkhKVORCQhLHUiIglhqRMRSQhLnYhIQuTTfYd2ux022+QOYpXJPCZ9W1fFNbsHrtk9PMuavbxk4/q5aS91m01AX59xUrdVq/0mfVtXxTW7B67ZPTzLmkNCAsb1c3z7hYhIQljqREQSwlInIpIQljoRkYSw1ImIJGRcpd7Y2Ij9+/f/6PqSkhLs3LkTe/bswcmTJ50ejoiIJsbhVxo//fRTfP311/D19X3q+pGREfz+97/HP/7xD/j6+uLFF19Ebm4uQkJCpiwsERGNzWGpazQafPTRR3jnnXeeur6trQ0ajQYqlQoAkJycjLq6Ojz//PNTEvShwYwvG+5hyGiZkt8/U/n4eGF4eETsGNOKa3YP7rRmwWaFfLAD/1WQAW+P8R1ENFkOSz0/Px9379790fUGgwEBAf/+Mry/vz8MBoPDO5TJPKBW+00wJvA/lzvxYcl1AICHx4RvTkQkirmeA1gjb0egpxlXb0UhO3HJlN7fpI8oVSqVGBoaGr08NDT0VMn/nMkeUfqvPXTdm+nwHefhslLAo+7cA9csPWbzMKqqdGhuboVKpUZOzhYsW7Z4yo8onXSpR0dHo729HX19ffDz80NdXR0OHDgw2V9HRCQZN2+2QacrhtFoRGJiClJTV0Mu95qW+55wqX/zzTcwGo3Ys2cP3n33XRw4cACCIGDnzp0IDQ2dioxERC7BaDSiokKL69dbMHt2MJ5/fivmzJk7rRnGVeoRERGjX1ncvHnz6PVr167F2rVrpyYZEZGLEAQBra3NqKgoxcjICNLS1iAxMRUy2fS/VTztUxqJiKRkcHAQZWXFuH37JkJDw5CbuwFBQbNFy8NSJyKaBEEQcPnyRVRXl0MQ7EhPz0F8fAI8PcU9UJ+lTkQ0QX19vSgtLcK9ex2IiNAgJycPgYEqsWMBYKkTEY2b3W5HY2M9amurIJfLkZu7AUuWLIPHDDp4hqVORDQODx92Q6stQnd3J6KiFiIray38/ZVix/oRljoR0RhsNivq6mrQ0HAe3t4+yM/fhOeeWzSj9s5/iKVORPQzHjy4B622CL29PVi8OBbp6dnw8fF1fEMRsdSJiP7DyIgFNTWVuHixAUplADZt2g6NJkrsWOPCUici+oE7d9pRWnoGg4MDiItbgVWrMqFQKMSONW4sdSIiAMPDw6iqKsPVq5ehVs/Ctm27ER4eIXasCWOpE5Hbu3HjGnS6EphMRiQlpSElZRXkctesR9dMTUTkBEbjEMrLS9DWdg3BwSEoLNyGkBDXHkzIUicityMIAlpamlFZqYXVasXKlRlISEgWZQCXs7HUicitDA4OoLT0DO7cacfcueHIzd2AWbOCxI7lNCx1InILgiCgqakR586VQxCAzMxcxMUlzNiDiCaLpU5Ektfb24PS0jO4f78D8+dHIicnDwEBgWLHmhIsdSKSLJvNBr2+HnV11ZDL5Vi7Nh+LF8dKbu/8h1jqRCRJ3d1d0GqL8PBhF6KjFyEzcy38/PzFjjXlWOpEJClWqxV1defQ0HAePj6+yM/fjOjoRWLHmjYsdSKSjPv3O6DVFqGvrxdLlizDmjXZ8PHxETvWtGKpE5HLs1gsqKmpwKVLegQEBGLTph3QaBaIHUsULHUicmm3b99CaekZGAyDiI9PxKpV6fDycp0BXM7GUicilzQ8bEJlZRlaWq5ArQ7C9u17ERYWLnYs0bHUicjltLW1Qqcrgdk8jOTklUhOXumyA7icjY8CEbmMoSEDystLcOPGdYSEzMHmzTsQHDxH7FgzCkudiGY8QRBw9eplVFWVwWq1YtWqDCQkpMDT01PsaDMOS52IZrSBgX6Ulhbj7t12hIXNQ27uBqjVs8SONWOx1IloRrLb7Whq0uPcuUp4eABZWeuwbNlySR/i7wwsdSKacXp6HkGrLUJn531oNAuQnb1esgO4nI2lTkQzhs1mQ0NDHerqzsHLywvr1m1ETMxS7p1PAEudiGaErq5OaLWn8ejRQ0RHxzwZwOUndiyXw1InIlFZrSM4f/4c9Po6+Pr6YePGLXjuuYVix3JZDkvdbrfj4MGDaGlpgUKhwOHDhxEZGTm6/fPPP8d3330HDw8PvP7668jLy5vSwEQkHffu3YVWW4T+/j4sXRqHNWuy4O3tXgO4nM1hqRcXF8NiseDEiRPQ6/U4evQojh07BgAYGBjAl19+iaKiIphMJmzbto2lTkQOWSxmfP99GS5cqEdgoApbtryAiAiN2LEkwWGp19fXIzMzEwCQkJCApqam0W2+vr4IDw+HyWSCyWTihxlE5FB7+w2UlZ2FwTCIFSuSkJaWDi8vL7FjSYbDUjcYDFAqlaOXZTIZrFbr6JyFsLAwFBYWwmaz4bXXXnN4hzKZB9TqiX/44eP7eOqaSuULP4X7fBQgk3lO6vFyZVyzNBmNRhQXF6GpqQnBwcHYtWuX2w3gmo7n2WE7KpVKDA0NjV622+2jha7T6dDV1YWzZ88CAA4cOICkpCQsX778Z3+fzSagr8844aDDJgsAoL/fBIuXbMK3d1Vqtd+kHi9XxjVLiyAIaGtrRXl5CcxmM1JSViE5OQ2zZwdKds0/51me55CQgHH9nMNST0pKglarRUFBAfR6PWJiYka3qVQq+Pj4QKFQwMPDAwEBARgYGJhUYCKSnqEhA3S6s7h5sw0hIaHYvHkDgoNDxI4laQ5LPS8vD5WVldi7dy8EQcCRI0dw/PhxaDQarFu3DlVVVdi9ezc8PT2RlJSE9PT06chNRDOYIAhobm5CVZUONpsVq1dnYcWKJA7gmgYOS93T0xOHDh166rro6OjRf7/55pt48803nZ+MiFxSf38fSkvPoKPjDsLDI5CbmweVigO4pov7fOJIRFPKbrfj0qUG1NRUwsPDE9nZ6xEbG89vxU0zljoRPbOenodPBnA9QGRkFLKz10OpHN8He+RcLHUimjSbzYYLF2pRX18DhcIbeXkFWLhwMffORcRSJ6JJ6ex8AK22CD09D7Fo0WJkZOTC11fa37V3BSx1IpqQkZERnD9fhcbGC/Dz80dBwVYsWBDt+IY0LVjqRDRuHR13oNUWYWCgH7Gxy7F6dSa8vb3FjkU/wFInIofMZjOqq3W4cuUSAgNV2Lr1BcybxwFcMxFLnYjGdOvWDZSVFcNoHEJCQjJSU9dwANcMxlInop9kMhlRUaHFtWstCAqajY0bNyM0NEzsWOQAS52IniIIAq5fb0F5uRYWixmpqauRlJQGmcx9Bum5MpY6EY0yGAZRVnYW7e03MGfOXOTmbsDs2cFix6IJYKkTEQRBwJUrl1BdrYPdbkd6ejbi4xM5gMsFsdSJ3Fx/fy+02jO4d+8u5s2bj5ycPKhUarFj0SSx1InclN1uR2PjBdTWVkImkyEnJw9Ll8bxEH8Xx1InckOPHnVDqy1CV1cnFiyIRlbWWg7gkgiWOpEbsdmsqK+vxYULtfD29kZeXiEWLozh3rmEsNSJ3ERn5/0nA7geISZmKdLTc+Dr6yt2LHIyljqRxI2MjKC2thKNjRfg769EYeE2REY+J3YsmiIsdSIJu3v3NkpLz2BgoB/Llq3A6tUZUCg4gEvKWOpEEmQ2D6OqSofm5iaoVGps27Yb4eERYseiacBSJ5KYmzevo6zsLEwmIxITU5CauhpyOQdwuQuWOpFEGI2PB3Bdv96C2bODUVCwFXPmzBU7Fk0zljqRixMEAa2tzaioKMXIyAjS0tYgMTGVA7jcFEudyIUNDg6grKwYt2/fQmhoGHJzNyAoaLbYsUhELHUiFyQIAi5fvojqah0EQUBGRg7i4hI4gItY6kSupq+vF1ptEe7f70BEhAY5OXkIDFSJHYtmCJY6kYt4PICrHrW1VZDL5cjN3YAlS5bxEH96CkudyAU8fNgNrfY0uru7EBW1EFlZa+HvrxQ7Fs1ALHWiGcxms6KurgYNDefh7e2D/PxNeO65Rdw7p5/FUieaoe7fvwettgh9fT1YvDgW6enZ8PHhAC4aG0udaIYZGbHg3LlKXLrUAKUyAJs27YBGs0DsWOQiHJa63W7HwYMH0dLSAoVCgcOHDyMyMnJ0e1lZGT7++GMAQGxsLN5//33+aUg0SXfutKO09AwGBwcQH5+AlSszoFAoxI5FLsRhqRcXF8NiseDEiRPQ6/U4evQojh07BgAwGAz44x//iL/85S8ICgrCp59+it7eXgQFBU15cCIpMZlMKCk5jatXL0OtnoXt2/cgLGye2LHIBTks9fr6emRmZgIAEhIS0NTUNLqtoaEBMTEx+OCDD3Dnzh3s2rWLhU40QTduXEN5uRZG4xCSktKQkrIKcjnfGaXJcfjKMRgMUCr//dUpmUwGq9UKuVyO3t5e1NTU4NSpU/Dz88NLL72EhIQEREVF/ezvk8k8oFb7TTioj+/jP0FVKl/4KdznBS+TeU7q8XJl7rJmg8GAoqLTuHq1GaGhc7Fnzx7MnRsmdqxp4y7P8w9Nx5odtqNSqcTQ0NDoZbvdProXoVarER8fj5CQEABASkoKmpubxyx1m01AX59xwkGHTRYAQH+/CRYv9xlUpFb7TerxcmVSX7MgCGhpuYLKylJYrVasXJmBnJxMDA6aJb3u/yT15/mnPMuaQ0LGd2Jwh4MikpKSoNPpAAB6vR4xMTGj2+Li4tDa2oqenh5YrVY0NjZi4cKFkwpM5A4GBwfw7bf/g5KS05g1azZ2796P5OQ0TlQkp3G4p56Xl4fKykrs3bsXgiDgyJEjOH78ODQaDdatW4e3334br7zyCgBg48aNT5U+ET0mCAKamvSorq4AAGRm5iIuLoHfFCOnc1jqnp6eOHTo0FPXRUdHj/67sLAQhYWFzk9GJBG9vT3Qaovw4ME9zJ8fiZycPAQEBIodiyTKfT5xJJpmNpsNen096uqqIZfLsXZtPhYvjuXeOU0pljrRFOju7oJWexoPH3YjOnoRMjPXws/PX+xY5AZY6kROZLVacf58NfT6Ovj6+iE/fzOioxeJHYvcCEudyEnu3+94MoCrF0uWLMOaNdnw8fEROxa5GZY60TOyWCw4d64cTU2NCAgIxObNOzF/fqTjGxJNAZY60TO4ffsWSkvPwGAYRHx8IlatSoeXFwdwkXhY6kSTMDxsQmVlGVparkCtDsL27XsRFhYudiwiljrRRAiCgBs3rkGnK4HZPIzk5JVITl7JAVw0Y/CVSDROQ0MG6HQluHnzOkJC5mDz5h0IDp4jdiyip7DUiRwQBAFXr15GVVUZrFYrVq/OxIoVyfD0dDg6iWjasdSJxjAw0I/S0jO4e/c2wsLmITd3A9TqWWLHIvpZLHWin2C329HUpMe5cxXw8PBAVtY6LFu2nIf404zHUif6Dz09j6DVFqGz8z40mgXIzl7PAVzkMljqRE/YbDY0NJxHXV0NvLy8sG7dRsTELOXeObkUljoRgK6uTmi1p/Ho0UMsXLgYGRm58PNzr1OtkTSw1MmtWa0jTwZw1cPX1w/PP78FUVE8exe5LpY6ua179+5Cqy1Cf38fli6Nw5o1WfD25gAucm0sdXI7FosZ1dUVuHy5EYGBKmzZ8gIiIjRixyJyCpY6uZX29hsoLS3G0JABK1YkIS0tHV5eXmLHInIaljq5BZPJhIoKLa5du4pZs2YjP38T5s7lAC6SHpY6SZogCLh+vRUVFSUwm81ISVmF5OQ0yGR86ZM08ZVNkjU0ZEBZ2VncutWGkJBQbNmyAbNnh4gdi2hKsdRJcgRBQHNzE6qqdLDZrFizJgvLlydxABe5BZY6SUp/fx9KS8+go+MOwsMjkJubB5WKA7jIfbDUSRLsdjsuXWpATU0lPDw8kZ29HrGx8TzEn9wOS51c3qNHD6HVFqGr6wEiI6OQnb0eSmWA2LGIRMFSJ5dls9lw4UIt6utroFB4Iy+vAAsXLubeObk1ljq5pM7OB9BqT6On5xEWLVqCjIwc+PpyABcRS51cysjICGprq3Dx4gX4+fmjoGArFiyIFjsW0YzBUieX0dFxB1ptEQYG+hEbuxyrV2fC29tb7FhEMwpLnWY8s9mM6modrly5hMBAFbZufQHz5nEAF9FPYanTjHbrVhvKyophNBqRkJCM1NQ1HMBFNAaHpW6323Hw4EG0tLRAoVDg8OHDiIyM/NHPvPrqq1i3bh1efPHFKQtL7sNkMj4ZwNWCoKBgbNy4FaGhc8WORTTjOSz14uJiWCwWnDhxAnq9HkePHsWxY8ee+pn//u//Rn9//5SFJPchCAIuX27C6dPfw2KxIDV1NZKS0iCTycSORuQSHJZ6fX09MjMzAQAJCQloamp6avv3338PDw8PZGVlTU1CchsGwyDKys6ivf0GQkPnIjd3A4KCgsWOReRSHJa6wWCAUqkcvSyTyWC1WiGXy9Ha2opvv/0WH374IT7++ONx3aFM5gG1euLfJ/bxVQAAVCpf+Cnc56MAmcxzUo+XKxEEAQ0NF1BSchZ2ux15eRuQnJziVgO43OF5/k9c89Rw2I5KpRJDQ0Ojl+12O+Tyxzc7deoUOjs78fLLL6OjowNeXl6YN2/emHvtNpuAvj7jhIMOmywAgP5+Eyxe7vOnuFrtN6nHy1X09/dCqz2De/fuYt68+cjJyUNkZLik1/xTpP48/xSueWJCQsY3+sJhqSclJUGr1aKgoAB6vR4xMTGj2955553Rf3/00UcIDg7m2zA0Lna7HY2NF1BbWwmZTIacnDwsXRrHQ/yJnpHDUs/Ly0NlZSX27t0LQRBw5MgRHD9+HBqNBuvWrZuOjCQxDx92Q6stQnd3JxYsiEZW1loO4CJyEoel7unpiUOHDj11XXT0jw/L/vWvf+28VCRJNpsV9fW1uHChFt7e3tiwoRDR0THcOydyIvf5xJFE9eDBPWi1Z9Db+wgxMUuRkZEDHx9fsWMRSQ5LnabU4wFclWhsvAB/fyUKC7chMvI5sWMRSRZLnabM3bu3UVp6BgMD/Vi2bAVWr86AQsEBXERTiaVOTmc2D6OqSofm5iaoVGps27Yb4eERYscicgssdXKqmzevo6zsLEwmIxITU5GaugpyOQdwEU0Xljo5hdE4hPJyLdraWjF7dggKCrZhzpxQsWMRuR2WOj0TQRDQ2tqMiopSjIyMIC0tHYmJKRzARSQSljpN2uDgAMrKinH79i2EhoY9GcA1W+xYRG6NpU4T9ng8biOqq8shCAIyMnIQF5fgVgO4iGYqljpNSF9fL7TaIty/34GICA1ycvIQGKgSOxYRPcFSp3Gx2+3Q6+tw/nw15HI5cnM3YMmSZTzEn2iGYamTQ48HcJ1Gd3cXoqIWIitrLfz9lY5vSETTjqVOP8tqtaK+vgYNDefh7e2D/PxNiI6OcXxDIhINS51+0v3796DVFqGvrweLF8ciPT2bA7iIXABLnZ4yMmLBuXOVuHSpAUplADZt2gGNZoHYsYhonFjqNOr27VsoKyvG4OAA4uMTsHJlBhQKhdixiGgCWOqE4eFhVFWV4erVy1CrZ2H79j0IC5sndiwimgSWuptra7uG8vISmExGJCWlISVl1eiJxYnI9fB/r5syGoeg05Xgxo1rCA4OQWHhdoSEzBE7FhE9I5a6mxEEAS0tV1BZWQqr1YqVKzOQkJDMAVxEEsFSdyMDA/0oKyvGnTvtmDs3HLm5GzBrVpDYsYjIiVjqbkAQBFy6pMe5cxUAgMzMXMTFJfAQfyIJYqlLXG9vD7TaIjx4cA/z50ciJycPAQGBYscioinCUpcom832ZADXOXh5ybF27UYsXryUe+dEEsdSl6Du7i5otafx8GE3oqMXITNzLfz8/MWORUTTgKUuIVbrCM6fPwe9vg6+vn7Iz9+M6OhFYsciomnEUpeI+/c7ngzg6sWSJcuwZk02fHx8xI5FRNOMpe7iLBYLzp0rR1NTIwICArF5807Mnx8pdiwiEglL3YXdvn0TpaXFMBgGER+fiFWr0uHlxQFcRO6Mpe6ChodNqKwsQ0vLFajVQdi+fS/CwsLFjkVEMwBL3YUIgoAbN65BpyuB2TyM5OSVSElZCZmMTyMRPeawDex2Ow4ePIiWlhYoFAocPnwYkZH/fs/2iy++wHfffQcAyM7Oxq9+9aupS+vGhoYM0OlKcPPmdYSEhGLz5p0IDg4ROxYRzTAOS724uBgWiwUnTpyAXq/H0aNHcezYMQDAnTt38PXXX+Orr76Ch4cH9u3bh/Xr12PJkiVTHtxdCIKA5uYmVFaWwWazYvXqTKxYkQxPT0+xoxHRDOSw1Ovr65GZmQkASEhIQFNT0+i2uXPn4rPPPhud8Ge1WuHt7T1FUd3PwEA//u///he3bt1EWNg85OZugFo9S+xYRDSDOSx1g8EApVI5elkmk8FqtUIul8PLywtBQUEQBAF/+MMfEBsbi6ioqDF/n0zmAbXab8JBfXwff6tDpfKFn0La7yHb7XbU1dWhrEwLDw8P5Oc/j6SkJLc5xF8m85zUa8SVcc3uYTrW7LAdlUolhoaGRi/b7fanzoxjNpvx3nvvwd/fH++//77DO7TZBPT1GSccdNhkAQD095tg8ZLu7O+enkfQaovQ2XkfGs0CbN68GYLghf5+k9jRpo1a7Tep14gr45rdw7OsOSQkYFw/57DUk5KSoNVqUVBQAL1ej5iYmNFtgiDgjTfewMqVK/Hqq69OKig9ZrPZ0NBwHnV1NVAovLB+/fNYtGgJVCp/t3vhE9HkOSz1vLw8VFZWYu/evRAEAUeOHMHx48eh0Whgt9tRW1sLi8WC8vJyAMBbb72FxMTEKQ8uJV1dD6DVFuHRo4dYuHAxMjJy4efnXn+WEpFzOCx1T09PHDp06KnroqOjR/996dIl56dyE1brCGprq9HYWA9fXz88//wWREUtFDsWEbkwaX/iOIN1dNxBaekZ9Pf3YenSOKxZkwVvbw7gIqJnw1KfZhaLGdXV5bh8+SICA1XYsuUFRERoxI5FRBLBUp9G7e03UFpaDKNxCCtWJCMtbQ28vLzEjkVEEsJSnwYmkwkVFVpcu3YVs2bNxsaNmxEaGiZ2LCKSIJb6FBIEAdevt6KiogRmsxkpKauQnJzGAVxENGXYLlPEYBiETleCW7faEBISii1bNmD2bA7gIqKpxVJ3sscDuC6hqkoHm82GNWuysHx5EgdwEdG0YKk7UX9/H0pLz6Cj4w7CwyOQm5sHlYoDuIho+rDUncBut+PixQbU1lbC09MT2dnrERsb7zYDuIho5mCpP6NHjx5Cqy1CV9cDREY+h+zsdVAqxzd4h4jI2Vjqk2Sz2XDhQi3q62ugUHgjL68ACxcu5t45EYmKpT4JnZ33odUWoafnERYtWoKMjBz4+nIAFxGJj6U+ASMjI6itrcLFixfg5+ePgoKtWLAg2vENiYimCUt9nDo6bkOrPYOBgX7Exi7H6tWZPHUfEc04LHUHzGYzqqt1uHLlEgIDVdi6dRfmzZsvdiwiop/EUh/DrVttKCsrhtFoREJCMlJTOYCLiGY2lvpPMJmMTwZwtSAoKBgbN25FaOhcsWMRETnEUv8BQRBw7dpVVFRoYbFYkJq6GklJaZDJpHuiayKSFpb6E4ODg9DpitHefhOhoXORm7sBQUHBYsciIpoQty91QRBw5crjAVyCYEd6ejbi4xM5gIuIXJJbl3pfXy9KS8/g3r27mDdvPnJy8qBSqcWORUQ0aW5Z6na7HY2N9aitrYJMJkdOTh6WLo3jIf5E5PLcrtQfPuyGVluE7u5OREVFIytrHfz9lWLHIiJyCrcpdZvNivr6Gly4cB7e3t7YsKEQ0dEx3DsnIklxi1J/8OAetNoz6O19hJiYpcjIyIGPj6/YsYiInE7SpT4yMoKamkpcvHgB/v5KFBZuQ2Tkc2LHIiKaMpIt9bt3b6O09PEArri4FVi1KgMKBQdwEZG0Sa7UzeZhVFXp0NzcBJVKjW3bdiM8PELsWERE00JSpX7jxnXodGdhMhmRmJiK1NRVkMs5gIuI3IckSt1oHEJ5uRZtba2YPTsEBQXbMGdOqNixiIimnUuXuiAIaG1tRkVFKUZGRpCWlo7ExBQO4CIit+WypT44OICysmLcvn0LoaFhTwZwzRY7FhGRqByWut1ux8GDB9HS0gKFQoHDhw8jMjJydPvJkyfx97//HXK5HL/85S+Rm5s7pYEFQUBTkx7V1eUQBAEZGTmIi0vgAC4iIoyj1IuLi2GxWHDixAno9XocPXoUx44dAwB0d3fjyy+/xD//+U+YzWbs27cP6enpUCgUUxI20GMY/+/bf6DzwT1EREQiJ2c9AgNVU3JfRESuyGGp19fXIzMzEwCQkJCApqam0W0XL15EYmIiFAoFFAoFNBoNrl69iuXLlzs9qM3wCFu9L6O3R4G1a/OxeHEsD/EnIvoPDkvdYDBAqfz3wCuZTAar1Qq5XA6DwYCAgIDRbf7+/jAYDGP+PpnMA2q134SDrl8RiaLBpXh1Rx6UygDHN5AImcxzUo+XK+Oa3QPXPDUclrpSqcTQ0NDoZbvdDrlc/pPbhoaGnir5n2KzCejrM044aETQLLz1XzvQ12ec1O1dlVrt51brBbhmd8E1T0xIyPh2Zh1+upiUlASdTgcA0Ov1iImJGd22fPly1NfXw2w2Y3BwEG1tbU9tJyKi6eVwTz0vLw+VlZXYu3cvBEHAkSNHcPz4cWg0Gqxbtw779+/Hvn37IAgCfvOb38Dbm/NViIjE4iEIgjCddzgyYpv0nx/8c809cM3ugWueGKe9/UJERK6DpU5J2Cx4AAAEIElEQVREJCEsdSIiCWGpExFJCEudiEhCpv3bL0RENHW4p05EJCEsdSIiCWGpExFJCEudiEhCWOpERBLCUicikpAZWep2ux2/+93vsGfPHuzfvx/t7e1PbT958iR27NiB3bt3Q6vVipTSuRyt+YsvvsCuXbuwa9cu/PnPfxYppfM4Wu+/fuaVV17B3/72NxESOp+jNZeVlWH37t3YvXs3Dh48CCl829jRmj///HPs2LEDO3fuxJkzZ0RKOTUaGxuxf//+H11fUlKCnTt3Ys+ePTh58qTz71iYgU6fPi389re/FQRBEBoaGoTXX399dFtXV5ewadMmwWw2CwMDA6P/dnVjrfn27dvC9u3bBavVKthsNmHPnj1Cc3OzWFGdYqz1/suf/vQn4YUXXhD++te/Tne8KTHWmgcHB4XCwkLh0aNHgiAIwieffDL6b1c21pr7+/uF7OxswWw2C319fUJOTo5YMZ3uk08+ETZt2iTs2rXrqestFouwfv16oa+vTzCbzcKOHTuErq4up973jNxTH+95UQMCAkbPi+rqxlrz3Llz8dlnn0Emk8HT0xNWq9Xl59aPtV4A+P777+Hh4YGsrCwx4k2Jsdbc0NCAmJgYfPDBB9i3bx+Cg4MRFBQkVlSnGWvNvr6+CA8Ph8lkgslkktQ5hzUaDT766KMfXd/W1gaNRgOVSgWFQoHk5GTU1dU59b4dniRDDM4+L6orGGvNXl5eCAoKgiAI+MMf/oDY2FhERUWJmPbZjbXe1tZWfPvtt/jwww/x8ccfi5jSucZac29vL2pqanDq1Cn4+fnhpZdeQkJCgqSfZwAICwtDYWEhbDYbXnvtNbFiOl1+fj7u3r37o+uno79mZKk7+7yormCsNQOA2WzGe++9B39/f7z//vtiRHSqsdZ76tQpdHZ24uWXX0ZHRwe8vLwwb948l99rH2vNarUa8fHxCAkJAQCkpKSgubnZ5Ut9rDXrdDp0dXXh7NmzAIADBw4gKSkJy5cvFyXrdJiO/pqRb7+443lRx1qzIAh44403sHjxYhw6dAgymUysmE4z1nrfeecdfPXVV/jyyy+xfft2/OIXv3D5QgfGXnNcXBxaW1vR09MDq9WKxsZGLFy4UKyoTjPWmlUqFXx8fKBQKODt7Y2AgAAMDAyIFXVaREdHo729HX19fbBYLKirq0NiYqJT72NG7qm743lRx1qz3W5HbW0tLBYLysvLAQBvvfWW018M08nRcyxFjtb89ttv45VXXgEAbNy4URI7K47WXFVVhd27d8PT0xNJSUlIT08XO/KU+Oabb2A0GrFnzx68++67OHDgAARBwM6dOxEaGurU++KURiIiCZmRb78QEdHksNSJiCSEpU5EJCEsdSIiCWGpExFJCEudiEhCWOpERBLCUicikpD/D6LQZK8MSYWIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr, tpr, _ = roc_curve(y_test, y_test_p[:,1])\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "sb.set_style(\"darkgrid\")\n",
    "plt.plot(fpr, tpr)\n",
    "plt.plot([0,1],[0,1], \"k-\", color=\"grey\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9999911590487137"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_test, y_test_p[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[226132,     88],\n",
       "       [     0,     30]], dtype=int64)"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True,  True,  True,  True,  True,  True,  True])"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "variance_thresh.get_support()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Item wrong length 7 instead of 54150.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-90-ce6b772e9f34>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mselected_variables1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvariance_thresh\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_support\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mselect_k\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_support\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   2916\u001b[0m         \u001b[1;31m# Do we have a (boolean) 1d indexer?\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2917\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_bool_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2918\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_bool_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2919\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2920\u001b[0m         \u001b[1;31m# We are left with two options: a single key, and a collection of keys,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m_getitem_bool_array\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   2961\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2962\u001b[0m             raise ValueError('Item wrong length %d instead of %d.' %\n\u001b[1;32m-> 2963\u001b[1;33m                              (len(key), len(self.index)))\n\u001b[0m\u001b[0;32m   2964\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2965\u001b[0m         \u001b[1;31m# check_bool_indexer will throw exception if Series key cannot\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Item wrong length 7 instead of 54150."
     ]
    }
   ],
   "source": [
    "selected_variables1 = independent_labels[variance_thresh.get_support()]\n",
    "select_k.get_support().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_variables2 = selected_variables1[select_k.get_support()]\n",
    "selected_variables2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_importances = random_forest.feature_importances_\n",
    "random_forest.feature_importances_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decreasing_order_of_features = np.argsort(-1*random_forest.feature_importances_,)\n",
    "[(selected_variables2[decreasing_order_of_features][i], features_importances[decreasing_order_of_features][i]) \n",
    " for i in range(features_importances.shape[0])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logistic = linear_model.LogisticRegression()\n",
    "\n",
    "pca = decomposition.PCA()\n",
    "pipe = Pipeline(steps=[('pca', pca), ('logistic', logistic)])\n",
    "\n",
    "digits = datasets.load_digits()\n",
    "X_digits = digits.data\n",
    "y_digits = digits.target\n",
    "\n",
    "pca.fit(X_digits)\n",
    "\n",
    "plt.figure(1, figsize=(4, 3))\n",
    "plt.clf()\n",
    "plt.axes([.2, .2, .7, .7])\n",
    "plt.plot(pca.explained_variance_, linewidth=2)\n",
    "plt.axis('tight')\n",
    "plt.xlabel('n_components')\n",
    "plt.ylabel('explained_variance_')\n",
    "\n",
    "n_components = [20, 40, 64]\n",
    "Cs = np.logspace(-4, 4, 3)\n",
    "\n",
    "#Parameters of pipelines can be set using ‘__’ separated parameter names:\n",
    "\n",
    "estimator = GridSearchCV(pipe,\n",
    "                         dict(pca__n_components=n_components,\n",
    "                              logistic__C=Cs))\n",
    "estimator.fit(X_digits, y_digits)\n",
    "\n",
    "plt.axvline(estimator.best_estimator_.named_steps['pca'].n_components,\n",
    "            linestyle=':', label='n_components chosen')\n",
    "plt.legend(prop=dict(size=12))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import keras as K\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "import numpy as np\n",
    "\n",
    "py_ver = sys.version\n",
    "k_ver = K.__version__\n",
    "tf_ver = tf.__version__\n",
    "\n",
    "print(\"Using Python version \" + str(py_ver))\n",
    "print(\"Using Keras version \" + str(k_ver))\n",
    "print(\"Using TensorFlow version \" + str(tf_ver))\n",
    "\n",
    "# 读取CSV数据集，并拆分为训练集和测试集\n",
    "# 该函数的传入参数为CSV_FILE_PATH: csv文件路径\n",
    "def load_data(CSV_FILE_PATH):\n",
    "    IRIS = pd.read_csv(CSV_FILE_PATH)\n",
    "    target_var = 'class'  # 目标变量\n",
    "    # 数据集的特征\n",
    "    features = list(IRIS.columns)\n",
    "    features.remove(target_var)\n",
    "    # 目标变量的类别\n",
    "    Class = IRIS[target_var].unique()\n",
    "    # 目标变量的类别字典\n",
    "    Class_dict = dict(zip(Class, range(len(Class))))\n",
    "    # 增加一列target, 将目标变量进行编码\n",
    "    IRIS['target'] = IRIS[target_var].apply(lambda x: Class_dict[x])\n",
    "    # 对目标变量进行0-1编码(One-hot Encoding)\n",
    "    lb = LabelBinarizer()\n",
    "    lb.fit(list(Class_dict.values()))\n",
    "    transformed_labels = lb.transform(IRIS['target'])\n",
    "    y_bin_labels = []  # 对多分类进行0-1编码的变量\n",
    "    for i in range(transformed_labels.shape[1]):\n",
    "        y_bin_labels.append('y' + str(i))\n",
    "        IRIS['y' + str(i)] = transformed_labels[:, i]\n",
    "    # 将数据集分为训练集和测试集\n",
    "    train_x, test_x, train_y, test_y = train_test_split(IRIS[features], IRIS[y_bin_labels], \\\n",
    "                                                        train_size=0.7, test_size=0.3, random_state=0)\n",
    "    return train_x, test_x, train_y, test_y, Class_dict\n",
    "\n",
    "# 2. 定义模型\n",
    "init = K.initializers.glorot_uniform(seed=1)\n",
    "simple_adam = K.optimizers.Adam()\n",
    "model = K.models.Sequential()\n",
    "model.add(K.layers.Dense(units=5, input_dim=4, kernel_initializer=init, activation='relu'))\n",
    "model.add(K.layers.Dense(units=6, kernel_initializer=init, activation='relu'))\n",
    "model.add(K.layers.Dense(units=3, kernel_initializer=init, activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer=simple_adam, metrics=['accuracy'])\n",
    "    \n",
    "# 3. 训练模型\n",
    "b_size = 1\n",
    "max_epochs = 100\n",
    "print(\"Starting training \")\n",
    "h = model.fit(train_x, train_y, batch_size=b_size, epochs=max_epochs, shuffle=True, verbose=1)\n",
    "print(\"Training finished \\n\")\n",
    "    \n",
    "# 4. 评估模型\n",
    "eval = model.evaluate(test_x, test_y, verbose=0)\n",
    "print(\"Evaluation on test data: loss = %0.6f accuracy = %0.2f%% \\n\" % (eval[0], eval[1] * 100) )\n",
    "\n",
    "# 5. 使用模型进行预测\n",
    "np.set_printoptions(precision=4)\n",
    "unknown = np.array([[6.1, 3.1, 5.1, 1.1]], dtype=np.float32)\n",
    "predicted = model.predict(unknown)\n",
    "print(\"Using model to predict species for features: \")\n",
    "print(unknown)\n",
    "print(\"\\nPredicted softmax vector is: \")\n",
    "print(predicted)\n",
    "species_dict = {v:k for k,v in Class_dict.items()}\n",
    "print(\"\\nPredicted species is: \")\n",
    "print(species_dict[np.argmax(predicted)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "metrics_df = pd.read_csv(\"\")\n",
    "metrics_df.head()   \n",
    "# table should display these columns: Input column (features selected), column for each metric, and Model column\n",
    "\n",
    "indices = []\n",
    "for features in range(len(metrics_df)):\n",
    "    if len(metrics_df.loc[features][0].split(\",\")) > 7:\n",
    "        indices.append(features)\n",
    "metrics_df.drop(indices, inplace = True)\n",
    "\n",
    "top_scores = metrics_df.sort_values(\"AUC\").groupby(\"Model\").last()\n",
    "top_scores.sort_values(\"AUC\", ascending = False, inplace = True)\n",
    "top_scores.head()\n",
    "\n",
    "top_scores.to_csv(\"Top_Scores.csv\")\n",
    "\n",
    "#create plot\n",
    "fig, (ax1, ax2) = plt.subplots(1, 2, figsize = (12,6))\n",
    "index = np.arrange(4)\n",
    "bar_width = 0.5\n",
    "opacity = 0.8\n",
    "\n",
    "ax1.bar(index, Top_Scores[\"\"].tolist(), bar_width, color = \"red\")\n",
    "ax1.set_title('')\n",
    "ax1.set_xlabel('Models', fontsize = 15)\n",
    "ax1.set_ylabel('Scores', fontsize = 15)\n",
    "ax1.set_xticklabels([\"\", \"RF\", \"\", \"\", \"\", \"\", \"\",     ])\n",
    "\n",
    "ax2.bar(index, Top_Scores[\"\"].tolist(), bar_width, color = \"blue\")\n",
    "ax2.set_title('')\n",
    "ax2.set_xlabel('Models', fontsize = 15)\n",
    "ax2.set_xticklabels([\"\", \"RF\", \"\", \"\", \"\", \"\", \"\",     ])\n",
    "\n",
    "fig.tight_layout()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}