Ensemble_alternative.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning Models and (Alternative) Ensemble Method\n",
    "---\n",
    "\n",
    "1. Split X-Features and y-labels\n",
    "2. Stratified k-fold Validation for range of models:\n",
    "    1. SVM\n",
    "    2. DT\n",
    "    3. AdaBoost\n",
    "    4. RF\n",
    "    5. ExtraTrees\n",
    "    6. GradientBoosting\n",
    "    7. MLP\n",
    "    8. KNN\n",
    "    9. LogReg\n",
    "    10. LDA\n",
    "    11. XGB\n",
    "3. Choosing best models and training them\n",
    "4. __Better__ Ensemble Method:\n",
    "5. Evaluate Ensemble on TEST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"-1\" # disable GPU\n",
    "#from tqdm import tqdm # progress bar\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "# processing / validation\n",
    "from imblearn.over_sampling import ADASYN # over-sampling\n",
    "from sklearn.model_selection import cross_val_score, StratifiedKFold\n",
    "\n",
    "# models\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier, ExtraTreesClassifier, GradientBoostingClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "# metrics\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "# constant seed for reproducibility\n",
    "#SEED = 111\n",
    "#os.environ['PYTHONHASHSEED'] = str(SEED)\n",
    "#np.random.seed(SEED)\n",
    "#tf.random.set_seed(SEED)\n",
    "\n",
    "# cpu worker\n",
    "WORKERS = 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1- X/y Split:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2985\n",
       "1    2910\n",
       "dtype: int64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/UFC_TRAIN.csv\")\n",
    "X = df.drop([\"date\",\"Winner\",\"B_fighter\",\"R_fighter\"], axis=1).values\n",
    "y = df[\"Winner\"].values\n",
    "\n",
    "# over-sampling\n",
    "over_sampler = ADASYN()\n",
    "X, y = over_sampler.fit_sample(X, y)\n",
    "\n",
    "pd.Series(y).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2- Stratified k-Fold-Validation with a Range of models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Mean_Accuracy  CrossValerrors         Algorithm\n",
      "1       0.808580        0.116597        ExtraTrees\n",
      "0       0.766886        0.159269                RF\n",
      "2       0.748574        0.171188  GradientBoosting\n",
      "4       0.620527        0.021079               LDA\n",
      "3       0.619505        0.023554            LogReg\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA78AAAGDCAYAAADjx6dlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5wlVZ338c8XRgUBySJIGEUyIgoYVmExYVgzq4ARE+qj7q45rOtiWEFddTGnR0EURMWAPrqKhIVFEckIggqKgIIEiaKk3/NHnZY7bYc7Pd19p2s+79erXlO3wqlT51b33G+fU3VTVUiSJEmS1GcrjboCkiRJkiTNNcOvJEmSJKn3DL+SJEmSpN4z/EqSJEmSes/wK0mSJEnqPcOvJEmSJKn3DL+SJM2zJKsm+XaS65J8dZ6PvXuSS4fcdv8kX5zrOi1vknwyyb8NvH5FkiuS3Jhk3fbvfWdYdiW53yTrjk/ykqUo691Jrkpy+RDb/ibJYyZZN/Q1sTSS7JvkfwdeT9pu47edwbG+l+QFM91f0orB8CtJy7H2YfiPSe426rrMpiQHJ7ktyYajrsuI/COwAbBuVT1z1JVZniV5cZLzk9zQAuh3k6zR1h2c5N3LWP7fhK6qenlVvautvwvwQWCPqlq9qq5u/160LMddVkk2BV4HbFtV9xplXYY1W+020R9lquoJVXXIspYtqd8Mv5K0nEqyGNgVKOApc3SMRXNR7jTHXA3YE7gOeO48H3vez3cSmwG/qKrbRl2R5VmSvwfeA+xTVWsA2wBHLMX+s/F+bwCsApw7C2XNpk2Bq6vqD6OuiCQtFIZfSVp+PR84GTgY+OtwviQPSXJ5kpUHlj09ydltfqUkb05yYZKrk3wlyTpt3eI27PLFSX4LHNuWf7WVeV2SE5JsN1D2um2I7vVJftqGWg4OZdw6ydFJrklyQZJnTXNeewLXAu8cPK9W1spJ3trqfkOS05Js0tZtN3CcK5K8tS1fovdv/BDONtzzTa19bkqyaKB9bkhyXpKnj6vHS5P8fGD9g5K8IcmR47b7cJKDJjrJJNu0nvtrk5yb5Clt+TuAtwN7tWGgL55g3/3be/LFVodzkmyZ5C1J/pDkkiR7DGy/UZKjWtv8KslLB9at2troj0nOA3YZd6yNkhyZ5Mokv07yT5OczyqtPle3c/ppkg2W5tzbuoOTfCzJ/2vn9pMkm09UTqvrj6vqDICquqaqDqmqG5LsBzwHeGNrx2+38od+v5NsA3wSeFgr49qBOr47yZbABa0u1yYZ+3n569DlJHdL8p9Jftuuy08mWXXgfN+Q5PdJfpfkRZOc50RtuGGSs5O8YYJ1jwGOBjZq9T64LX9Ka+9rW/tvM0nZU14T47b9RJL/HLfsW0le2+an/Fkat99gu63brtnrk5wCbD5u24PadX59ut8Du7bljwfeyp0/P2e15X8dMp7ud+Dbklzcfl6+kGTNtm7sd+AL2nt2VZJ/nazOknqmqpycnJyclsMJ+BXwf4CdgFuBDQbWXQg8duD1V4E3t/l/pgvNGwN3Az4FHN7WLabrSf4CsBqwalv+ImCNtv1/AWcOlP3lNt0d2Ba4BPjftm619vqFwCLggcBVdEMxJzuvY4D30fWo3QbsNLDuDcA5wFZAgAcA67a6/Z5umOcq7fVD2j4HA+8eKGN34NKB178BzgQ2GTjfZwIb0f0ReC/gJmDDgXWX0QWCAPej66ndsG23VttuEfCHwfoPHPMu7f17K3BX4FHADcBWbf3+wBenaKP9gT8Dj2vH+QLwa+BfW9kvBX49sP0JwMdb2+wIXAk8qq07EDgRWKe1wc/G2qed/2l0YfyuwH2Bi4DHja8n8DLg2+06WJnuurzHDM79YOBq4MHt3L4EfHmSdtgVuBl4B/Bw4G7j1i/x3s/w/d6Xdj1PVC53/swsGlhfwP3a/IeAo1r7rtHa6IC27vHAFcD2dD8rhw3uO8H5Hg+8BLgP8Atgvymukd1Z8jrfsp3XY9t78Mb2Ptx1oF0eM901McFxdqP7GU97vXZ7TzZa2rYd125fBr7S2mV7up+5wW2fS/ezv4ju5/5yYJXJfn7G2m7g99mv6K7n1YGvA4eOez8/A6xK9zvmL8A2c/073cnJafTTyCvg5OTk5PS3E/AIusC7Xnt9PvCagfXvBj7X5tdoHzg3a69/Djx6YNsNW1mLBj743XeKY6/VtlmTLuTcSgsuA8ceC797ASeO2/9TwL9PUvamwB3Aju3194GDBtZfADx1gv32Ac6YpMyDmT78vmia9j5z7LitTv88yXbfA17a5p8EnDfJdru2D+srDSw7HNi/zf/Nh/dx++8PHD3w+snAjcDKA+95tfdqE+B2YI2B7Q8ADm7zFwGPH1i3H3eG34cAvx137LcAnx9fT7pA8SNgh2nacrpzPxj47MC6JwLnT1HeE+gC5bWtDT440A5LvPczfL/3ZYbhl+6PIzcBmw+sexjtDxPA54ADB9ZtyfTh94PtHPaZ5hx2Z8nr/N+Arwy8XokuUO4+0C5j4XfSa2KC4wT4LbBbe/1S4NiZtO1Au439Xtl6YN17xr8P48r9I/CAyX5+WDL8HgP8n4F1W/G3vwM3Hlh/CrD3VO3t5OTUj8lhz5K0fHoB8IOquqq9PowlhwgfBjwj3YOwngGcXlUXt3WbAd9oQx+vpQvDt9P1tI65ZGwm3VDjA9vQxevpPiQDrAesT/eB8ZKJ9m3HesjYsdrxngNM9gCe5wE/r6oz2+svAc9O91Ah6ILchRPsN9nyYQ3WmSTPT3LmQJ23pzvf6Y51CHfep/xc4NBJttsIuKSq7hhYdjFw76Wo8xUD8zcDV1XV7QOvoevV2gi4pqpumORYG7Hk+V88ML8Z3dDZwffvrSx5rYw5lO4PA19uQ3jfN/C+DRrm3AefTvyndh4TqqrvVdWT6Xopn0oXqKZ7IvLSvN/LYn26nvDTBsr+77Ycpm77yTyHLrR+baD+u7Yhvjcmmeze440Gy2/tfwkTX3ND16uqiq6Xdp+26Nl0P7djdZtJ2070e2WJOiR5fbpbD65r5a45RLljlmiLNr+IJa/roa9BSf1h+JWk5Uy7X/BZwN+nuw/3cuA1wAOSPACgqs6j+0D3BLoPo4cNFHEJ8ISqWmtgWqWqLhvYpgbmn00XKh5D9wFz8VhV6IbP3kY3hHrMJuOO9T/jjrV6Vb1iktN7PnDfgfP6IN0H2icOlDfR/Z+X0A1hnMhNdAFkzETB+6/nm2QzuiGPr6J72vJadMM+M00dAL4J7JBke7qe3y9Nst3vgE2SDP4/uyldqJltvwPWSXsC8gTH+j1LvmebDsxfQtdLOfj+rVFVT2Scqrq1qt5RVdsCf0d3/s+fpD6zfu5VdUdVHUN3n/r2Y4sn23xsZoj3e7IyhnEV3R8ithtovzWraixITdX2k9m/lXtY2n39VXVi+7lavaq2m2S/39H9MQOAJGnHnqjdl7ZehwP/2NryIcCR7RjTte1kxn6vTFiHdn/vG+l+D67dyr2O4d+zJdqilX0bS/5BSdIKyPArScufp9H11G5Ld//mjnRPuT2RJcPGYXT39+5Gd8/vmE8C/9E+mJJk/SRPneJ4a9Dd83Y1XYh8z9iK1tP4dWD/JHdPsvW4OnwH2DLJ85LcpU27TPSgnSQPowuVDx44r+3beYyV+VngXUm2SGeHJOu242yY5F/SPWBojSQPafucCTwxyTpJ7gX8yxTnCt09hkX3AZwkL+TOMDVWh9cn2anV4X5jbVlVf6brkTsMOKWqfjvJMX5C15v0xtYmu9MNXf7yNHVbalV1Cd1w5APSPZRqB+DFwNhXwXwFeEuStZNsDLx6YPdTgBvSPSBq1TYKYPskf/MApCSPTHL/FsiupxtGesf47ZjFc0/y1CR7t7onyYOBv6e7px26MDPd9+1O935fAWyc5K5LW7/Wu/oZ4ENJ7tnKv3eSx7VNvgLsm2TbJHcH/n2IYm+lu492NeAL4/6IMJWvAP+Q5NGtR/51dD/XP5pk28muib9R3QPHrqL72fh+VV3bVk3XtpOVN/73yrYsObJlDbqweiWwKMnbgXsMrL8CWDxF2xwOvCbJfZKsTvc77Yjy6erSCs/wK0nLnxfQ3XP526q6fGwCPgo8J3d+fcvhdEHg2IHh0QAH0T2A5wdJbqALCg9hcl+g60W+DDiPO4PFmFfR9QhfTjf09XC6D9W0obZ7AHvT9bZcDryX7sFZE53Xt6rqnHHndRDwpHRPpP4g3QfzH9AFrP9L99CiG+ge5PPkdoxfAo9s5R4KnEU3XPsHTPNVOK3X/APAj+k+RN8fOGlg/VeB/6ALuDfQ9fauM1DEIW2fyYY8U1W3tLo+gS40fBx4flWdP1XdlsE+dD32vwO+QXfP9Q/bunfQvb+/pmufv9a7hZAn0f0h4tfcGXDWnOAY96IL/tfTDaX/HyZog1k+9z/S3WP6y3bcLwLvr6qxHvf/C2zbhtx+c6ICpnu/6XqSzwUuT3LV35YwrTfRPVzp5HS3DfyQ7h5Tqup7dA+QO7Ztc+wwBbY2fAbdMN3PDROAq+oCuqH4H6Fr9ycDT25ljTfpNTGFw+hGh/x1lMkQbTuVV9ENNb6c7h7rzw+s+z7d8PFftHr+mSWHSI/9se/qJKdPUPbn6M7pBLpz/DPTBHxJK4axJ/dJkjSUJO8F7lVVL5h24x5KsindA8juVVXXj7o+kiRpOPb8SpKmlO57fHcYGHb6YrrexRVO64F7Ld1X8xh8JUlaQBZNv4kkaQW3Bt1Q543ohjZ+APjWSGs0AklWozv/i+m+v1WSJC0gDnuWJEmSJPWew54lSZIkSb1n+JUkSZIk9Z73/PbIeuutV4sXLx51NSRJkiRpJE477bSrqmr9idYZfntk8eLFnHrqqaOuhiRJkiSNRJKLJ1vnsGdJkiRJUu8ZfiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7PvCqR6686UY+8dMTRl0NSZIkaUKv2GW3UVdBKzB7fiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8ZfiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8ZfiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8ZfiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8ZfiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8ZfiVJkiRJvWf4lSRJkiT13qJRV0CQ5HbgHLr349fA86rq2iSLgZ8DFwxs/uCqumXeKylJkiRJC5g9v8uHm6tqx6raHrgGeOXAugvburHJ4CtJkiRJS8nwu/z5MXDvUVdCkiRJkvrE8LscSbIy8GjgqIHFmyc5s00fm2Cf/ZKcmuTUG6+9dt7qKkmSJEkLiff8Lh9WTXImXY/vz4GjB9ZdWFU7TrZjVX0a+DTAZttsXXNaS0mSJElaoOz5XT7c3ALuZkBY8p5fSZIkSdIyMvwuR6rqT8A/Aa9LYq+8JEmSJM0Sw+9ypqrOAM4G9hl1XSRJkiSpL+xdXA5U1erjXj954OX281wdSZIkSeode34lSZIkSb1n+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb1n+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb1n+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb1n+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb1n+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvLRp1BTR71l9tdV6xy26jroYkSZIkLXfs+ZUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb1n+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb1n+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb23aNQV0Oy5444buPnmY0ZdDUmSJEmzaNVVHz3qKvSCPb+SJEmSpN4z/EqSJEmSes/wK0mSJEnqPcOvJEmSJKn3DL+SJEmSpN4z/EqSJEmSes/wK0mSJEnqPcOvJEmSJKn3DL+SJEmSpN4z/EqSJEmSes/wK0mSJEnqPcOvJEmSJKn3DL+SJEmSpN4z/EqSJEmSes/wK0mSJEnqPcOvJEmSJKn3DL+SJEmSpN4z/EqSJEmSes/wK0mSJEnqPcOvJEmSJKn3DL+SJEmSpN4z/EqSJEmSeq/34TfJ7UnOHJjePM32b53BMb7Ryv5VkusGjvV3M6+5JEmSJGm2LBp1BebBzVW141Js/1bgPeMXJgmQqrpj/LqqenrbZnfg9VX1pIkKTrKoqm5birpIkiRJkmZB73t+J5JkzSQXJNmqvT48yUuTHAis2nptv5RkcdvuC8DPgE2SfCLJqUnOTfKOIY51aZIDk5wBPD3JFkm+n+S0JCck2bJtt0GSr7eyT0ny0Lb8UUnOanU6Pclqc9cykiRJktRPK0LP76pJzhx4fUBVHZHkVcDBSQ4C1q6qzwAkedVYT3GSxcAWwAuq6uS27F+r6pokKwPHJNmhqs6epg5/qKoHtv2PA15SVRcmeTjwUWAP4MPA+6rq5Hbc7wDbA28A9quqnyRZHfjzYMFJ9gP2A9hkk3vOpH0kSZIkqfdWhPA74bDnqjo6yTOBjwEPmGL/i8eCb/OsFjgXARsC2wLThd8jAJKsBTwUOLIbRQ3c+R48BthqYPnaSVYFTgIOSvIl4MiqunHceXwa+DTAgx60VU1TD0mSJElaIa0I4XdCSVYCtgH+BKwNXDrJpjcN7HMf4PXALlX1xyQHA6sMcbixMgJcNck9yAEeXFW3jFv+7iRHAf8AnJzk0VX1yyGOKUmSJElqVsh7fpvXAD8Hng18Psld2vJbB+bHuwddkL0uyQbAE5bmgFX1R+D3ScYekLVSkrFe5x8CrxzbNsnY0OvNq+rsqjoAOB3YammOKUmSJElaMcLv2AOsxqYD24OuXgK8rqpOBE4A3ta2/zRwdhtmvISqOgs4AzgfOIxuSPLS2ht4eZKzgHOBsSdDvxJ4eJKzk5wHvLQtf32SnyU5G7gR+MEMjilJkiRJK7RUeZtoXzzoQVvVSSd9fNTVkCRJkjSLVl310aOuwoKR5LSq2nmidStCz68kSZIkaQVn+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb1n+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb1n+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb1n+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb1n+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb23aNQV0OxZaaU1WHXVR4+6GpIkSZK03LHnV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb1n+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb1n+JUkSZIk9Z7hV5IkSZLUe4ZfSZIkSVLvGX4lSZIkSb1n+JUkSZIk9d6iUVdAs+fGG/7MCcf9fNTVkCRJ0nJmt0duM+oqSCNnz68kSZIkqfcMv5IkSZKk3jP8SpIkSZJ6b6h7fpOsDWwyuH1VnT5XlZIkSZIkaTZNG36TvAvYF7gQqLa4gEfNXbUkSZIkSZo9w/T8PgvYvKpumevKSJIkSZI0F4a55/dnwFpzXRFJkiRJkubKMD2/BwBnJPkZ8JexhVX1lDmrlSRJkiRJs2iY8HsI8F7gHOCOua2OJEmSJEmzb5jw+6eq+vCc10SSJEmSpDkyTPg9MckBwFEsOezZrzqSJEmSJC0Iw4TfB7Z/HzqwzK86kiRJkiQtGNOG36p65HxURJIkSZKkuTJt+E1yN2BPYPHg9lX1zrmrliRJkiRJs2eYYc/fAq4DTmPgnl9JkiRJkhaKYcLvxlX1+DmviSRJkiRJc2SlIbb5UZL7z3lNJEmSJEmaI5P2/CY5h+6pzouAFya5iG7Yc4Cqqh3mp4qSJEmSJC2bqYY9P2neaiFJkiRJ0hyaNPxW1cUASQ6tqucNrktyKPC8CXeUJEmSJGk5M8w9v9sNvkiyMrDT3FRHkiRJkqTZN2n4TfKWJDcAOyS5vk03AH+g+/qjaSXZIMlhSS5KclqSHyd5+kwrm2T/JK9v8+9M8pgZlrNjkicOvN43yZVJzkxybpKvJbn7TOs5xPGekuTNs1W+JEmSJGlqk4bfqjqgqtYA3l9V92jTGlW1blW9ZbqCkwT4JnBCVd23qnYC9gY2HrfdMF+3NFH93l5VP5zJvsCOwBPHLTuiqnasqu2AW4C9Zlj2tMerqqOq6sBZLF+SJEmSNIWpen63brNfTfKg8dMQZT8KuKWqPjm2oKourqqPtJ7Wo5IcCxyTZPUkxyQ5Pck5SZ46UI9/TfKLJP8LbDWw/OAk/9jmd0ryP613+ftJNmzLj0/y3iSntDJ2TXJX4J3AXq2nd4mQ28L4asAf2+vFSY5Ncnar46bTLH9mkp8lOSvJCRMdr53/RwfO48NJftR6yMfOaaUkH09yfpKjk3x3bJ0kSZIkaelM1ev6WmA/4AMTrCu6cDuV7YDTp1j/IGCHqrqmBc6nV9X1SdYDTk5yVNtmb7qe00WtvNMGC0lyF+AjwFOr6soWZv8DeNHYOVbVg9uw43+vqsckeTuwc1W9qpWxL104fQSwIfAL4Ntt/48Ah1TVIUleBHwYeNoUy98OPK6qLkuyVlXdMsnxBm0IPALYGjgK+BrwDGAxsC1wT+DnwOemaE9JkiRJ0iSmetrzfklWAt5WVSct64GSfIwu4N0CfAw4uqquGVsNvCfJbsAdwL2BDYBdgW9U1Z9aGUdNUPRWwPbA0d1Ia1YGfj+w/uvt39PowuRkjqiqV7Xh2h8D3gAcCDyMLogCHAq8r81Ptvwk4OAkXxk49nS+WVV3AOcl2aAtewTw1bb88iTHTbRjkv3o/kjBBhtsOOThJEmSJGnFMuXTnlvw+ugMyz6Xrud2rKxXAo8G1m+LbhrY9jlt+U5VtSNwBbDKkMcJcG67X3fHqrp/Ve0xsP4v7d/bmbqne6yeRdfru9uQxx+//8uBtwGbAKclWXeI3f4yMJ+lPN6nq2rnqtp5rTXXWZpdJUmSJGmFMcxXHR2TZM/WI7o0jgVWSfKKgWWTPUF5TeAPVXVrkkcCm7XlJwBPS7JqkjWAJ0+w7wXA+kkeBt0w6CTbTbDdoBuANaZY/wjgwjb/I7qh19CF9BOnWp5k86r6SVW9HbiSLgRPd7yJnATs2e793QDYfSn3lyRJkiQ1wzxp+WV09//enuRmup7Jqqp7TLVTVVWSpwEfSvJGuiB4E/AmYNVxm38J+HaSc4BTgfNbGacnOQI4i+4rln46wXFuaQ+C+nCSNds5/Rddz/NkjgPenORM4IC2bOye35WAS4F92/JXA59P8oZ2Di+cZvn7k2xB107HtLr/doLjTedIup7y84BL6O53vm7IfSVJkiRJA9KN8tXyKMnqVXVjGzp9CvDwqrp8su233mr7+vQnvzp/FZQkSdKCsNsjtxl1FaR5keS0qtp5onVDfcdukqdw5z2wx1fVd2arcprSd5KsBdwVeNdUwVeSJEmSNLlpw2+SA4Fd6IYmA/xzkodX1VvmtGaiqnYfdR0kSZIkqQ+G6fl9IrBje/IzSQ4BzgAMv5IkSZKkBWGYpz0DrDUwv+ZcVESSJEmSpLkyTM/vAcAZSY6je4LxbsCb57RWkiRJkiTNomnDb1UdnuR4uvt+Ad7kg5ckSZIkSQvJMA+8elCbvbT9u1GS1YCLq+q2OauZJEmSJEmzZJhhzx8HHgScTTfseXvgXGDNJK+oqh/MYf0kSZIkSVpmwzzw6nfAA6tq56raCXggcBHwWOB9c1k5SZIkSZJmwzDhd8uqOnfsRVWdB2xdVRfNXbUkSZIkSZo9wwx7PjfJJ4Avt9d7AecluRtw65zVTJIkSZKkWTJMz+++wK+Af2nTRW3ZrcAj56pikiRJkiTNlmG+6uhm4ANtGu/GWa+RJEmSJEmzbNLwm+QcoCZZXVX1gLmpkiRJkiRJs2uqnt8nTbAswCbAW+amOpIkSZIkzb5Jw29VXTw2n+SBwLOBZwK/Bo6c+6pJkiRJkjQ7phr2vCWwT5uuAo4AUlU+5EqSJEmStKBMNez5fOBE4ElV9SuAJK+Zl1pJkiRJkjSLpvqqo2cAvweOS/KZJI+mu+dXkiRJkqQFZdLwW1XfrKq9ga2B4+i+4/eeST6RZI/5qqAkSZIkSctqqp5fAKrqpqo6rKqeDGwMnAG8ac5rJkmSJEnSLEnVZF/lq4Vm5513rlNPPXXU1ZAkSZKkkUhyWlXtPNG6aXt+JUmSJEla6Ay/kiRJkqTeM/xKkiRJknrP8CtJkiRJ6j3DryRJkiSp9wy/kiRJkqTeM/xKkiRJknrP8CtJkiRJ6j3DryRJkiSp9wy/kiRJkqTeM/xKkiRJknrP8CtJkiRJ6r1Fo66AZs9tV/yeKz/07lFXQ5LUU+u/5m2jroIkSTNmz68kSZIkqfcMv5IkSZKk3jP8SpIkSZJ6z/ArSZIkSeo9w68kSZIkqfcMv5IkSZKk3jP8SpIkSZJ6z/ArSZIkSeo9w68kSZIkqfcMv5IkSZKk3jP8SpIkSZJ6z/ArSZIkSeo9w68kSZIkqfcMv5IkSZKk3jP8SpIkSZJ6z/ArSZIkSeo9w68kSZIkqfcMv5IkSZKk3jP8SpIkSZJ6z/ArSZIkSeo9w68kSZIkqfcMv5IkSZKk3jP8TiPJjbNQxuIkNyc5M8l5Sb6Q5C6zUT9JkiRJ0vQMv/PnwqraEbg/sDHwrBHXR5IkSZJWGIbfGWg9uccmOTvJMUk2bcs3T3JyknOSvHuiXuOquh04Bbh322flJO9P8tNW3sva8pWSfDzJ+UmOTvLdJP84n+cpSZIkSX1h+J2ZjwCHVNUOwJeAD7flBwEHVdX9gUsn2jHJKsBDgP9ui14MXFdVuwC7AC9Nch/gGcBiYFvgecDDJilvvySnJjn16ptumo1zkyRJkqTeMfzOzMOAw9r8ocAjBpZ/tc0fNm6fzZOcCbPloZYAAA5cSURBVFwB/L6qzm7L9wCe39b9BFgX2KKV+dWquqOqLgeOm6giVfXpqtq5qnZed7XVZuHUJEmSJKl/DL/zZ+ye382BnZI8pS0P8Oqq2rFN96mqH4yumpIkSZLUP4bfmfkRsHebfw5wYps/Gdizze89fieAqroKeDPwlrbo+8Arxp7+nGTLJKsBJwF7tnt/NwB2n+2TkCRJkqQVheF3endPcunA9Frg1cALk5xNdz/uP7dt/wV4bVt+P+C6Scr8Zit3V+CzwHnA6Ul+BnwKWAQcSXff8HnAF4HTpyhPkiRJkjSFRaOuwPKuqib7A8GjJlh2GfDQqqokewNbtTJ+A2w/UGYBDxjY761tWkKS11fVjUnWpXtC9DkzOglJkiRJWsEZfmfXTsBHkwS4FnjRMpb3nSRrAXcF3tUefCVJkiRJWkqG31lUVSeyZI/uspa3+2yVJUmSJEkrMu/5lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8ZfiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8ZfiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8ZfiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8ZfiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7i0ZdAc2eRRtsyPqveduoqyFJkiRJyx17fiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8ZfiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8ZfiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8tGnUFNHsuv/Ym3v/NU0ZdDUlSz73haQ8edRUkSVpq9vxKkiRJknrP8CtJkiRJ6j3DryRJkiSp9wy/kiRJkqTeM/xKkiRJknrP8CtJkiRJ6j3DryRJkiSp9wy/kiRJkqTeM/xKkiRJknrP8CtJkiRJ6j3DryRJkiSp9wy/kiRJkqTeM/xKkiRJknrP8CtJkiRJ6j3DryRJkiSp9wy/kiRJkqTeM/xKkiRJknrP8CtJkiRJ6j3DryRJkiSp9wy/kiRJkqTeM/xKkiRJknrP8CtJkiRJ6j3D7zxIcuMEy/ZPclmSM5P8MsnXk2w7bpv1ktya5OXzV1tJkiRJ6h/D72h9qKp2rKotgCOAY5OsP7D+mcDJwD4jqZ0kSZIk9YThdzlRVUcAPwCePbB4H+B1wL2TbDySikmSJElSDxh+ly+nA1sDJNkE2LCqTgG+Auw10Q5J9ktyapJTb7r+2vmrqSRJkiQtIIbf5UsG5veiC70AX2aSoc9V9emq2rmqdl7tHmvNdf0kSZIkaUFaNOoKaAkPBE5t8/sA90rynPZ6oyRbVNUvR1M1SZIkSVq47PldTiTZE9gDODzJlsDqVXXvqlpcVYuBA/DBV5IkSZI0I4bf+XH3JJcOTK9ty18z9lVHwHOBR1XVlXQh9xvjyjgSw68kSZIkzYjDnudBVU32R4b9J9n+HRMsOxvYZharJUmSJEkrDHt+JUmSJEm9Z/iVJEmSJPWe4VeSJEmS1HuGX0mSJElS7xl+JUmSJEm9Z/iVJEmSJPWe4VeSJEmS1HuGX0mSJElS7xl+JUmSJEm9Z/iVJEmSJPWe4VeSJEmS1HuGX0mSJElS7xl+JUmSJEm9Z/iVJEmSJPWe4VeSJEmS1HuGX0mSJElS7xl+JUmSJEm9Z/iVJEmSJPWe4VeSJEmS1HuGX0mSJElS7y0adQU0e+611mq84WkPHnU1JEmSJGm5Y8+vJEmSJKn3DL+SJEmSpN4z/EqSJEmSes/wK0mSJEnqPcOvJEmSJKn3DL+SJEmSpN5LVY26DpolSW4ALhh1PVZA6wFXjboSKxjbfP7Z5vPPNp9/tvlo2O7zzzaff7b5/NmsqtafaIXf89svF1TVzqOuxIomyam2+/yyzeefbT7/bPP5Z5uPhu0+/2zz+WebLx8c9ixJkiRJ6j3DryRJkiSp9wy//fLpUVdgBWW7zz/bfP7Z5vPPNp9/tvlo2O7zzzaff7b5csAHXkmSJEmSes+eX0mSJElS7xl+F4gkj09yQZJfJXnzFNvtmaSS7Dyw7C1tvwuSPG5+arzwzbTNkyxOcnOSM9v0yfmr9cI2XZsn2TfJlQNt+5KBdS9I8ss2vWB+a76wLWO73z6w/Kj5rfnCNczvlyTPSnJeknOTHDaw3Gt9Bpaxzb3OZ2CI3y0fGmjXXyS5dmCd1/kMLGObe53P0BDtvmmS45KckeTsJE8cWOfn9PlUVU7L+QSsDFwI3Be4K3AWsO0E260BnACcDOzclm3btr8bcJ9WzsqjPqflfVrGNl8M/GzU57DQpmHaHNgX+OgE+64DXNT+XbvNrz3qc1oI07K0e1t346jPYaFNQ7b5FsAZY9cxcM/2r9f6PLd5m/c6n4M2H7f9q4HPtXmv83lu8/ba63yO2p3uft9XtPltgd8MzPs5fR4ne34XhgcDv6qqi6rqFuDLwFMn2O5dwHuBPw8seyrw5ar6S1X9GvhVK09TW5Y218wM2+YTeRxwdFVdU1V/BI4GHj9H9eybZWl3zcwwbf5S4GPteqaq/tCWe63PzLK0uWZmaX+37AMc3ua9zmdmWdpcMzdMuxdwjza/JvC7Nu/n9Hlm+F0Y7g1cMvD60rbsr5I8CNikqv7f0u6rCS1LmwPcpw1t+Z8ku85hPftk2Gt1zzZk6GtJNlnKffW3lqXdAVZJcmqSk5M8bU5r2h/DtPmWwJZJTmpt+/il2Fd/a1naHLzOZ2LoazXJZnS9Xscu7b5awrK0OXidz9Qw7b4/8NwklwLfpet1H3ZfzaJFo66All2SlYAP0g1N1DyYps1/D2xaVVcn2Qn4ZpLtqur6+axjT30bOLyq/pLkZcAhwKNGXKcVwVTtvllVXZbkvsCxSc6pqgtHVtP+WEQ3DHd3YGPghCT3H2mN+m/CNq+qa/E6n2t7A1+rqttHXZEVyERt7nU+d/YBDq6qDyR5GHBoku1HXakVkT2/C8NlwGBPy8Zt2Zg1gO2B45P8BngocFS6BzBNt68mNuM2b0NXrgaoqtPo7t/Ycl5qvbBNe61W1dVV9Zf28rPATsPuq0ktS7tTVZe1fy8CjgceOJeV7YlhrtdLgaOq6tY2FO4XdMHMa31mlqXNvc5nZmmu1b1Zcvit1/nMLEube53P3DDt/mLgKwBV9WNgFWC9IffVbBr1TcdO0090f42+iG54ytiN9NtNsf3x3Pnwpe1Y8kb6i/BG+rlu8/XH2pju4QeXAeuM+pyW92mYNgc2HJh/OnBym18H+DXdg1HWbvO2+dy3+9rA3dr8esAvmeLhKk5L1eaPBw4ZaNtLgHW91kfS5l7nc9Tmbbutgd8AGVjmdT7/be51PoftDnwP2LfNb0N3z2/wc/q8Tw57XgCq6rYkrwK+T/dEuc9V1blJ3gmcWlWTPo6+bfcV4DzgNuCV5bCiaS1LmwO7Ae9McitwB/Dyqrpm7mu9sA3Z5v+U5Cl01/I1tGHnVXVNkncBP23FvdM2H86ytDvdf+CfSnIH3UiiA6vqvHk/iQVmyDb/PrBHkvOA24E3VBtR4rW+9JalzZP8HV7nS20p/h/dm+6BPzWwr7/TZ2BZ2hx/n8/YkO3+OuAzSV5D9/CrfVv7+zl9nmXJ616SJEmSpP7xnl9JkiRJUu8ZfiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8ZfiVJWqCSVJIvDrxelOTKJN+Zh2OPHevAuT6WJEmzwfArSdLCdROwfZJV2+vHApfN07EfC/wCeGaSzNVBkiyaq7IlSSsWw68kSQvbd4F/aPP7AIePrUiyWpLPJTklyRlJntqWL05yYpLT2/R3bfnuSY5P8rUk5yf50hTBdh/gIOC3wMMGjrlLkh8lOasdd40kKyf5zyQ/S3J2kle3bX+TZL02v3OS49v8/kkOTXIScOhk9W3bvinJOe14BybZPMnpA+u3GHwtSVpx+ddUSZIWti8Db29DnXcAPgfs2tb9K3BsVb0oyVrAKUl+CPwBeGxV/TnJFnSBeee2zwOB7YDfAScBDwf+d/CASVYBHgO8DFiLLgj/KMldgSOAvarqp0nuAdwM7AcsBnasqtuSrDPEeW0LPKKqbk5y94nqm+QJwFOBh1TVn5KsU1XXJLkuyY5VdSbwQuDzwzenJKmv7PmVJGkBq6qz6YLlPnS9wIP2AN6c5EzgeGAVYFPgLsBnkpwDfJUuaI45paourao7gDNb2eM9CTiuqm4GjgSelmRlYCvg91X101a366vqNrqg/Kk2T1VdM8SpHdXKZ4r6Pgb4fFX9aVy5nwVe2Oq0F3DYEMeTJPWcPb+SJC18RwH/CewOrDuwPMCeVXXB4MZJ9geuAB5A94fwPw+s/svA/O1M/FlhH+ARSX7TXq8LPAq4fCnrfRt3/iF+lXHrbhqYf80U9Z3IkcC/A8cCp1XV1UtZL0lSD9nzK0nSwvc54B1Vdc645d8HXj12326SB7bla9L10N4BPA9YedgDtaHMuwKbVtXiqloMvJIuEF8AbJhkl7btGu2BVUcDLxt7eNXAsOffADu1+T2nOOxk9T2arof37oPlVtWf27l/Aoc8S5Iaw68kSQtcG6b84QlWvYtuyPDZSc5trwE+DrwgyVnA1izZyzqdp9PdRzzYQ/wt4Ml0Pc17AR9pZR9N16P7WboHY53dlj+77fcO4KAkp9L1Mk9mwvpW1X/T9Xqf2oZ2v35gny8BdwA/WIpzkyT1WKpq1HWQJEmaVUleD6xZVf826rpIkpYP3vMrSZJ6Jck3gM3p7kOWJAmw51eSJEmStALwnl9JkiRJUu8ZfiVJkiRJvWf4lSRJkiT1nuFXkiRJktR7hl9JkiRJUu8ZfiVJkiRJvff/ARB8SetV4TnLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "classifiers = [\n",
    "    \n",
    "    #SVC(random_state=SEED),\n",
    "    #DecisionTreeClassifier(random_state=SEED),\n",
    "    #AdaBoostClassifier(DecisionTreeClassifier(random_state=SEED),random_state=SEED,learning_rate=0.1),\n",
    "    #RandomForestClassifier(random_state=SEED),\n",
    "    #ExtraTreesClassifier(random_state=SEED),\n",
    "    #GradientBoostingClassifier(random_state=SEED),\n",
    "    #MLPClassifier(random_state=SEED),\n",
    "    #KNeighborsClassifier(),\n",
    "    #LogisticRegression(random_state = SEED),\n",
    "    #LinearDiscriminantAnalysis()\n",
    "    \n",
    "\n",
    "    RandomForestClassifier(),\n",
    "    ExtraTreesClassifier(),\n",
    "    GradientBoostingClassifier(),\n",
    "    LogisticRegression(),\n",
    "    LinearDiscriminantAnalysis()\n",
    "]\n",
    "#classifiers.append(XGBClassifier(n_estimators=200)) # we can append models \n",
    "\n",
    "kfold = StratifiedKFold( n_splits=10 ) # k = 10 stratified folds\n",
    "cv_results = [] # for EACH model holds k, i.e:10, cv-accuracy results (shape: no. of models x K)\n",
    "for classifier in classifiers:\n",
    "    cv_results.append(cross_val_score(classifier, X=X, y = y, scoring = \"accuracy\", cv = kfold, n_jobs=WORKERS))\n",
    "\n",
    "# mean accuracy and std of each model (shape: no. of models)\n",
    "cv_means = []\n",
    "cv_std = []\n",
    "for cv_result in cv_results:\n",
    "    cv_means.append(cv_result.mean())\n",
    "    cv_std.append(cv_result.std())\n",
    "\n",
    "cv_res = pd.DataFrame({\"Mean_Accuracy\":cv_means,\n",
    "                       \"CrossValerrors\": cv_std,\n",
    "                       \"Algorithm\":\n",
    "                       [\n",
    "                           #\"SVM\",\n",
    "                           #\"DT\",\n",
    "                           #\"AdaBoost\",\n",
    "                           \"RF\",\n",
    "                           \"ExtraTrees\",\n",
    "                           \"GradientBoosting\",\n",
    "                           #\"MLP\",\n",
    "                           #\"KNN\",\n",
    "                           \"LogReg\",\n",
    "                           \"LDA\",\n",
    "                           #\"XGB\"\n",
    "                       ]\n",
    "                      }\n",
    "                     )\n",
    "print(cv_res.sort_values(\"Mean_Accuracy\", ascending=False))\n",
    "plt.figure(figsize=(15,6))\n",
    "g = sns.barplot(x=\"Mean_Accuracy\",y=\"Algorithm\",data = cv_res, palette=\"Set3\",orient = \"h\")\n",
    "g.set(xlim=(0.4))\n",
    "g.set_xlabel(\"Mean Accuracy\")\n",
    "g = g.set_title(\"Average Accuracy of models on Stratified k-fold validation\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3- Model Selection:\n",
    "__LDA, GradientBoosting and LogReg are chosen to be in Ensemble model__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "ldaClf = LinearDiscriminantAnalysis()\n",
    "ldaClf = ldaClf.fit(X,y)\n",
    "\n",
    "#gboostClf = GradientBoostingClassifier(random_state=SEED)\n",
    "gboostClf = GradientBoostingClassifier()\n",
    "gboostClf = gboostClf.fit(X,y)\n",
    "\n",
    "#lrClf = LogisticRegression(random_state = SEED,n_jobs=WORKERS)\n",
    "lrClf = LogisticRegression(n_jobs=WORKERS)\n",
    "lrClf = lrClf.fit(X,y)\n",
    "\n",
    "#exTreeClf = ExtraTreesClassifier(random_state=SEED)\n",
    "exTreeClf = ExtraTreesClassifier()\n",
    "exTreeClf = exTreeClf.fit(X,y)\n",
    "\n",
    "#rfClf = RandomForestClassifier(random_state=SEED)\n",
    "rfClf = RandomForestClassifier()\n",
    "rfClf = rfClf.fit(X,y)\n",
    "\n",
    "\n",
    "# the following is redundant as k-fold-val has been performed previously\n",
    "#ldaScore = cross_val_score(ldaClf, X=X, y = y, scoring = \"accuracy\", cv = kfold, n_jobs=WORKERS).mean()\n",
    "#print(f\"LDA k-fold-val-avg-Accuracy: {ldaScore}\")\n",
    "#gboostScore = cross_val_score(gboostClf, X=X, y = y, scoring = \"accuracy\", cv = kfold, n_jobs=WORKERS).mean()\n",
    "#print(f\"GBOOST k-fold-val-avg-Accuracy: {gboostScore}\")\n",
    "#lraScore = cross_val_score(lrClf, X=X, y = y, scoring = \"accuracy\", cv = kfold, n_jobs=WORKERS).mean()\n",
    "#print(f\"LogReg k-fold-val-avg-Accuracy: {lraScore}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4- Ensemble Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 Models in Ensemble: ['ldaClf', 'gboostClf', 'lrClf', 'exTreeClf', 'rfClf']\n"
     ]
    }
   ],
   "source": [
    "ensClf = VotingClassifier(estimators=\n",
    "                          [\n",
    "                              ('ldaClf', ldaClf),\n",
    "                              ('gboostClf', gboostClf),\n",
    "                              ('lrClf', lrClf),\n",
    "                              ('exTreeClf', exTreeClf),\n",
    "                              ('rfClf', rfClf),\n",
    "                              #('xgbClf', xgbClf)\n",
    "                          ],\n",
    "                          voting='soft', n_jobs=WORKERS)\n",
    "ensClf = ensClf.fit(X, y)\n",
    "print(f\"{len([e[0] for e in ensClf.estimators])} Models in Ensemble: {[e[0] for e in ensClf.estimators]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5- Performance Evaluation on TEST (unseen data):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ensemble TEST Performance on BALANCED:\n",
      "------------------\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "     class 0       0.75      0.84      0.79       157\n",
      "     class 1       0.55      0.41      0.47        75\n",
      "\n",
      "    accuracy                           0.70       232\n",
      "   macro avg       0.65      0.63      0.63       232\n",
      "weighted avg       0.69      0.70      0.69       232\n",
      "\n",
      "AUC:  0.6270488322717622\n"
     ]
    }
   ],
   "source": [
    "TEST = pd.read_csv(\"data/UFC_TEST.csv\")\n",
    "X_TEST = TEST.drop([\"date\",\"B_fighter\",\"R_fighter\",\"Winner\"],axis=1).values\n",
    "y_TEST = TEST[\"Winner\"].values\n",
    "target_names = ['class 0', 'class 1']\n",
    "ensPreds_TEST = ensClf.predict(X_TEST)\n",
    "print(\"Ensemble TEST Performance on BALANCED:\\n------------------\\n\",classification_report(y_TEST, ensPreds_TEST , target_names=target_names))\n",
    "print(\"AUC: \",roc_auc_score(y_TEST, ensPreds_TEST))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save models to disk\n",
    "#import pickle\n",
    "#pickle.dump(ensClf, open('resources/ens_method.sav', 'wb'))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}