HyperclassifierSearch examples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HyperclassifierSearch Examples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example 1: train multiple classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import of the package (after e.g. 'pip install HyperclassifierSearch')\n",
    "from HyperclassifierSearch import HyperclassifierSearch\n",
    "\n",
    "# usage dependent imports\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# example dataset\n",
    "from sklearn import datasets\n",
    "breast_cancer = datasets.load_breast_cancer()\n",
    "X = breast_cancer.data\n",
    "y = breast_cancer.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search for LogisticRegression ...\n",
      "Search for RandomForestClassifier ...\n",
      "Search is done.\n"
     ]
    }
   ],
   "source": [
    "# define classifiers and parameters for hyperfitting\n",
    "models = {\n",
    "    'LogisticRegression': LogisticRegression(solver='lbfgs', max_iter=10000),\n",
    "    'RandomForestClassifier': RandomForestClassifier()\n",
    "}\n",
    "params = { \n",
    "    'LogisticRegression': { 'C': [0.1, 1, 2] },\n",
    "    'RandomForestClassifier': { 'n_estimators': [16, 32] }\n",
    "}\n",
    "\n",
    "# run search\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "search = HyperclassifierSearch(models, params)\n",
    "best_model = search.train_model(X_train, y_train, cv=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Estimator</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'C': 0.1}</td>\n",
       "      <td>0.948598</td>\n",
       "      <td>0.957547</td>\n",
       "      <td>0.953052</td>\n",
       "      <td>0.004474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'C': 2}</td>\n",
       "      <td>0.939252</td>\n",
       "      <td>0.966981</td>\n",
       "      <td>0.953052</td>\n",
       "      <td>0.013864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'C': 1}</td>\n",
       "      <td>0.934579</td>\n",
       "      <td>0.966981</td>\n",
       "      <td>0.950704</td>\n",
       "      <td>0.016201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'n_estimators': 16}</td>\n",
       "      <td>0.953271</td>\n",
       "      <td>0.938679</td>\n",
       "      <td>0.946009</td>\n",
       "      <td>0.007296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'n_estimators': 32}</td>\n",
       "      <td>0.943925</td>\n",
       "      <td>0.929245</td>\n",
       "      <td>0.936620</td>\n",
       "      <td>0.007340</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Estimator                params  split0_test_score  \\\n",
       "0      LogisticRegression            {'C': 0.1}           0.948598   \n",
       "1      LogisticRegression              {'C': 2}           0.939252   \n",
       "2      LogisticRegression              {'C': 1}           0.934579   \n",
       "3  RandomForestClassifier  {'n_estimators': 16}           0.953271   \n",
       "4  RandomForestClassifier  {'n_estimators': 32}           0.943925   \n",
       "\n",
       "   split1_test_score  mean_test_score  std_test_score  \n",
       "0           0.957547         0.953052        0.004474  \n",
       "1           0.966981         0.953052        0.013864  \n",
       "2           0.966981         0.950704        0.016201  \n",
       "3           0.938679         0.946009        0.007296  \n",
       "4           0.929245         0.936620        0.007340  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search.evaluate_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example 2: add multiple pipelines to example 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search for LogisticRegression ...\n",
      "Search for RandomForestClassifier ...\n",
      "Search is done.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda2/envs/py3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Estimator</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>split5_test_score</th>\n",
       "      <th>split6_test_score</th>\n",
       "      <th>split7_test_score</th>\n",
       "      <th>split8_test_score</th>\n",
       "      <th>split9_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'clf__C': 0.1}</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.931818</td>\n",
       "      <td>0.977273</td>\n",
       "      <td>0.952381</td>\n",
       "      <td>0.97619</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.952381</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.974178</td>\n",
       "      <td>0.021944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'clf__C': 1}</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.954545</td>\n",
       "      <td>0.954545</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.97619</td>\n",
       "      <td>0.952381</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.952381</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.974178</td>\n",
       "      <td>0.019361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'clf__C': 2}</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.954545</td>\n",
       "      <td>0.931818</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.97619</td>\n",
       "      <td>0.928571</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.952381</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.969484</td>\n",
       "      <td>0.025623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'clf__n_estimators': 16}</td>\n",
       "      <td>0.954545</td>\n",
       "      <td>0.954545</td>\n",
       "      <td>0.954545</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.97619</td>\n",
       "      <td>0.952381</td>\n",
       "      <td>0.97619</td>\n",
       "      <td>0.952381</td>\n",
       "      <td>0.97619</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.967136</td>\n",
       "      <td>0.015194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'clf__n_estimators': 32}</td>\n",
       "      <td>0.931818</td>\n",
       "      <td>0.931818</td>\n",
       "      <td>0.954545</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.97619</td>\n",
       "      <td>0.952381</td>\n",
       "      <td>0.97619</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.97619</td>\n",
       "      <td>0.952381</td>\n",
       "      <td>0.960094</td>\n",
       "      <td>0.017544</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Estimator                     params  split0_test_score  \\\n",
       "0      LogisticRegression            {'clf__C': 0.1}           1.000000   \n",
       "1      LogisticRegression              {'clf__C': 1}           1.000000   \n",
       "2      LogisticRegression              {'clf__C': 2}           1.000000   \n",
       "3  RandomForestClassifier  {'clf__n_estimators': 16}           0.954545   \n",
       "4  RandomForestClassifier  {'clf__n_estimators': 32}           0.931818   \n",
       "\n",
       "   split1_test_score  split2_test_score  split3_test_score  split4_test_score  \\\n",
       "0           0.931818           0.977273           0.952381            0.97619   \n",
       "1           0.954545           0.954545           0.976190            0.97619   \n",
       "2           0.954545           0.931818           0.976190            0.97619   \n",
       "3           0.954545           0.954545           0.976190            0.97619   \n",
       "4           0.931818           0.954545           0.976190            0.97619   \n",
       "\n",
       "   split5_test_score  split6_test_score  split7_test_score  split8_test_score  \\\n",
       "0           0.976190            1.00000           0.952381            1.00000   \n",
       "1           0.952381            1.00000           0.952381            1.00000   \n",
       "2           0.928571            1.00000           0.952381            1.00000   \n",
       "3           0.952381            0.97619           0.952381            0.97619   \n",
       "4           0.952381            0.97619           0.976190            0.97619   \n",
       "\n",
       "   split9_test_score  mean_test_score  std_test_score  \n",
       "0           0.976190         0.974178        0.021944  \n",
       "1           0.976190         0.974178        0.019361  \n",
       "2           0.976190         0.969484        0.025623  \n",
       "3           1.000000         0.967136        0.015194  \n",
       "4           0.952381         0.960094        0.017544  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# additional imports\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# define the model including pipelines\n",
    "models = {\n",
    "    'LogisticRegression': Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf', LogisticRegression(solver='lbfgs', max_iter=200))\n",
    "    ]),\n",
    "    'RandomForestClassifier': Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf', RandomForestClassifier())\n",
    "    ])\n",
    "}\n",
    "params = { \n",
    "    'LogisticRegression': { 'clf__C': [0.1, 1, 2] },\n",
    "    'RandomForestClassifier': { 'clf__n_estimators': [16, 32] }\n",
    "}\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "search = HyperclassifierSearch(models, params)\n",
    "best_model = search.train_model(X_train, y_train, cv=10)\n",
    "search.evaluate_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "highest score: 0.9742 \n",
      "\n",
      "... with this estimator: \n",
      "\n",
      " Pipeline(memory=None,\n",
      "         steps=[('scale',\n",
      "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
      "                ('clf',\n",
      "                 LogisticRegression(C=0.1, class_weight=None, dual=False,\n",
      "                                    fit_intercept=True, intercept_scaling=1,\n",
      "                                    l1_ratio=None, max_iter=200,\n",
      "                                    multi_class='warn', n_jobs=None,\n",
      "                                    penalty='l2', random_state=None,\n",
      "                                    solver='lbfgs', tol=0.0001, verbose=0,\n",
      "                                    warm_start=False))],\n",
      "         verbose=False)\n"
     ]
    }
   ],
   "source": [
    "# the usual parameters from GridSearchCV can be obtained:\n",
    "\n",
    "print('highest score:', round(best_model.best_score_, 4), \n",
    "      '\\n\\n... with this estimator: \\n\\n', best_model.best_estimator_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example 3: using RandomizedSearchCV and more exhaustive search compared to example 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# additional imports:\n",
    "import numpy as np\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import classification_report, f1_score, make_scorer, accuracy_score, balanced_accuracy_score, fbeta_score\n",
    "\n",
    "# model and parameter definition in a function\n",
    "def build_model():\n",
    "    models = {\n",
    "        \n",
    "        'LogisticRegression': Pipeline([\n",
    "            ('scale', StandardScaler()),\n",
    "            ('clf', LogisticRegression(solver='lbfgs'))\n",
    "        ]),\n",
    "        'RandomForestClassifier': Pipeline([\n",
    "            ('scale', StandardScaler()),\n",
    "            ('clf', RandomForestClassifier())\n",
    "        ]),\n",
    "        'AdaBoost': Pipeline([\n",
    "            ('tfidf', StandardScaler()),\n",
    "            ('clf', AdaBoostClassifier())  \n",
    "        ])\n",
    "    }\n",
    "    params = { \n",
    "        'LogisticRegression': { 'clf__C': np.linspace(0.1, 1.0, num=10) },\n",
    "        'RandomForestClassifier': { 'clf__n_estimators': np.arange(16,32+1) },\n",
    "        'AdaBoost': { 'clf__n_estimators': np.arange(16,32+1) }\n",
    "    }\n",
    "    scorer = make_scorer(fbeta_score, beta=2, average='weighted')\n",
    "    return models, params, scorer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search for LogisticRegression ...\n",
      "Search for RandomForestClassifier ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda2/envs/py3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search for AdaBoost ...\n",
      "Search is done.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Estimator</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'clf__C': 0.2}</td>\n",
       "      <td>0.976535</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>0.988353</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988040</td>\n",
       "      <td>0.981120</td>\n",
       "      <td>0.005775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'clf__C': 0.7000000000000001}</td>\n",
       "      <td>0.976535</td>\n",
       "      <td>0.988320</td>\n",
       "      <td>0.976667</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988076</td>\n",
       "      <td>0.981105</td>\n",
       "      <td>0.005787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'clf__C': 1.0}</td>\n",
       "      <td>0.976535</td>\n",
       "      <td>0.988320</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988076</td>\n",
       "      <td>0.978775</td>\n",
       "      <td>0.008695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'clf__C': 0.8}</td>\n",
       "      <td>0.976535</td>\n",
       "      <td>0.988320</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988076</td>\n",
       "      <td>0.978775</td>\n",
       "      <td>0.008695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'clf__C': 0.9}</td>\n",
       "      <td>0.976535</td>\n",
       "      <td>0.988320</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988076</td>\n",
       "      <td>0.978775</td>\n",
       "      <td>0.008695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'clf__C': 0.30000000000000004}</td>\n",
       "      <td>0.976535</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>0.976667</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988076</td>\n",
       "      <td>0.978768</td>\n",
       "      <td>0.004621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'clf__C': 0.4}</td>\n",
       "      <td>0.976535</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>0.976667</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988076</td>\n",
       "      <td>0.978768</td>\n",
       "      <td>0.004621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'clf__C': 0.5}</td>\n",
       "      <td>0.976535</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>0.976667</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988076</td>\n",
       "      <td>0.978768</td>\n",
       "      <td>0.004621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'clf__C': 0.6}</td>\n",
       "      <td>0.976535</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>0.976667</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988076</td>\n",
       "      <td>0.978768</td>\n",
       "      <td>0.004621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>{'clf__C': 0.1}</td>\n",
       "      <td>0.964641</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>0.988353</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988040</td>\n",
       "      <td>0.978719</td>\n",
       "      <td>0.008841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AdaBoost</td>\n",
       "      <td>{'clf__n_estimators': 24}</td>\n",
       "      <td>0.964641</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.965031</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.974024</td>\n",
       "      <td>0.013560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AdaBoost</td>\n",
       "      <td>{'clf__n_estimators': 19}</td>\n",
       "      <td>0.953211</td>\n",
       "      <td>0.953488</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.971775</td>\n",
       "      <td>0.017407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>AdaBoost</td>\n",
       "      <td>{'clf__n_estimators': 27}</td>\n",
       "      <td>0.953211</td>\n",
       "      <td>0.953474</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.971728</td>\n",
       "      <td>0.017399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>AdaBoost</td>\n",
       "      <td>{'clf__n_estimators': 25}</td>\n",
       "      <td>0.953211</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988040</td>\n",
       "      <td>0.971723</td>\n",
       "      <td>0.011794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>AdaBoost</td>\n",
       "      <td>{'clf__n_estimators': 23}</td>\n",
       "      <td>0.953211</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.965031</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.971716</td>\n",
       "      <td>0.015755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>AdaBoost</td>\n",
       "      <td>{'clf__n_estimators': 22}</td>\n",
       "      <td>0.953211</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.965031</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988040</td>\n",
       "      <td>0.969358</td>\n",
       "      <td>0.011724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'clf__n_estimators': 19}</td>\n",
       "      <td>0.964641</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.953474</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988076</td>\n",
       "      <td>0.969340</td>\n",
       "      <td>0.011693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>AdaBoost</td>\n",
       "      <td>{'clf__n_estimators': 28}</td>\n",
       "      <td>0.941281</td>\n",
       "      <td>0.953474</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.969320</td>\n",
       "      <td>0.020368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>AdaBoost</td>\n",
       "      <td>{'clf__n_estimators': 26}</td>\n",
       "      <td>0.941281</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988040</td>\n",
       "      <td>0.969314</td>\n",
       "      <td>0.015849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'clf__n_estimators': 31}</td>\n",
       "      <td>0.964641</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.953474</td>\n",
       "      <td>0.963781</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.969288</td>\n",
       "      <td>0.015821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'clf__n_estimators': 20}</td>\n",
       "      <td>0.976535</td>\n",
       "      <td>0.953474</td>\n",
       "      <td>0.941778</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.988076</td>\n",
       "      <td>0.967027</td>\n",
       "      <td>0.016946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>AdaBoost</td>\n",
       "      <td>{'clf__n_estimators': 29}</td>\n",
       "      <td>0.941281</td>\n",
       "      <td>0.953474</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.966975</td>\n",
       "      <td>0.020044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'clf__n_estimators': 28}</td>\n",
       "      <td>0.941742</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.965031</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.976111</td>\n",
       "      <td>0.964691</td>\n",
       "      <td>0.012537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'clf__n_estimators': 27}</td>\n",
       "      <td>0.941742</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>0.953474</td>\n",
       "      <td>0.963781</td>\n",
       "      <td>0.988076</td>\n",
       "      <td>0.964659</td>\n",
       "      <td>0.016402</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'clf__n_estimators': 21}</td>\n",
       "      <td>0.929239</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.976667</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.976111</td>\n",
       "      <td>0.964516</td>\n",
       "      <td>0.018262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>AdaBoost</td>\n",
       "      <td>{'clf__n_estimators': 16}</td>\n",
       "      <td>0.941281</td>\n",
       "      <td>0.953474</td>\n",
       "      <td>0.965031</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.962304</td>\n",
       "      <td>0.013510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'clf__n_estimators': 17}</td>\n",
       "      <td>0.964641</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.930038</td>\n",
       "      <td>0.963781</td>\n",
       "      <td>0.988076</td>\n",
       "      <td>0.962205</td>\n",
       "      <td>0.018554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'clf__n_estimators': 30}</td>\n",
       "      <td>0.953211</td>\n",
       "      <td>0.965129</td>\n",
       "      <td>0.930281</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.976111</td>\n",
       "      <td>0.959991</td>\n",
       "      <td>0.017157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'clf__n_estimators': 26}</td>\n",
       "      <td>0.941742</td>\n",
       "      <td>0.953474</td>\n",
       "      <td>0.918682</td>\n",
       "      <td>0.975968</td>\n",
       "      <td>0.976111</td>\n",
       "      <td>0.952981</td>\n",
       "      <td>0.021742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>{'clf__n_estimators': 16}</td>\n",
       "      <td>0.941742</td>\n",
       "      <td>0.953488</td>\n",
       "      <td>0.930281</td>\n",
       "      <td>0.963781</td>\n",
       "      <td>0.976111</td>\n",
       "      <td>0.952922</td>\n",
       "      <td>0.016064</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Estimator                           params  \\\n",
       "0       LogisticRegression                  {'clf__C': 0.2}   \n",
       "1       LogisticRegression   {'clf__C': 0.7000000000000001}   \n",
       "2       LogisticRegression                  {'clf__C': 1.0}   \n",
       "3       LogisticRegression                  {'clf__C': 0.8}   \n",
       "4       LogisticRegression                  {'clf__C': 0.9}   \n",
       "5       LogisticRegression  {'clf__C': 0.30000000000000004}   \n",
       "6       LogisticRegression                  {'clf__C': 0.4}   \n",
       "7       LogisticRegression                  {'clf__C': 0.5}   \n",
       "8       LogisticRegression                  {'clf__C': 0.6}   \n",
       "9       LogisticRegression                  {'clf__C': 0.1}   \n",
       "10                AdaBoost        {'clf__n_estimators': 24}   \n",
       "11                AdaBoost        {'clf__n_estimators': 19}   \n",
       "12                AdaBoost        {'clf__n_estimators': 27}   \n",
       "13                AdaBoost        {'clf__n_estimators': 25}   \n",
       "14                AdaBoost        {'clf__n_estimators': 23}   \n",
       "15                AdaBoost        {'clf__n_estimators': 22}   \n",
       "16  RandomForestClassifier        {'clf__n_estimators': 19}   \n",
       "17                AdaBoost        {'clf__n_estimators': 28}   \n",
       "18                AdaBoost        {'clf__n_estimators': 26}   \n",
       "19  RandomForestClassifier        {'clf__n_estimators': 31}   \n",
       "20  RandomForestClassifier        {'clf__n_estimators': 20}   \n",
       "21                AdaBoost        {'clf__n_estimators': 29}   \n",
       "22  RandomForestClassifier        {'clf__n_estimators': 28}   \n",
       "23  RandomForestClassifier        {'clf__n_estimators': 27}   \n",
       "24  RandomForestClassifier        {'clf__n_estimators': 21}   \n",
       "25                AdaBoost        {'clf__n_estimators': 16}   \n",
       "26  RandomForestClassifier        {'clf__n_estimators': 17}   \n",
       "27  RandomForestClassifier        {'clf__n_estimators': 30}   \n",
       "28  RandomForestClassifier        {'clf__n_estimators': 26}   \n",
       "29  RandomForestClassifier        {'clf__n_estimators': 16}   \n",
       "\n",
       "    split0_test_score  split1_test_score  split2_test_score  \\\n",
       "0            0.976535           0.976744           0.988353   \n",
       "1            0.976535           0.988320           0.976667   \n",
       "2            0.976535           0.988320           0.965129   \n",
       "3            0.976535           0.988320           0.965129   \n",
       "4            0.976535           0.988320           0.965129   \n",
       "5            0.976535           0.976744           0.976667   \n",
       "6            0.976535           0.976744           0.976667   \n",
       "7            0.976535           0.976744           0.976667   \n",
       "8            0.976535           0.976744           0.976667   \n",
       "9            0.964641           0.976744           0.988353   \n",
       "10           0.964641           0.965129           0.965031   \n",
       "11           0.953211           0.953488           0.976744   \n",
       "12           0.953211           0.953474           0.976744   \n",
       "13           0.953211           0.965129           0.976744   \n",
       "14           0.953211           0.965129           0.965031   \n",
       "15           0.953211           0.965129           0.965031   \n",
       "16           0.964641           0.965129           0.953474   \n",
       "17           0.941281           0.953474           0.976744   \n",
       "18           0.941281           0.965129           0.976744   \n",
       "19           0.964641           0.965129           0.953474   \n",
       "20           0.976535           0.953474           0.941778   \n",
       "21           0.941281           0.953474           0.965129   \n",
       "22           0.941742           0.965129           0.965031   \n",
       "23           0.941742           0.976744           0.953474   \n",
       "24           0.929239           0.965129           0.976667   \n",
       "25           0.941281           0.953474           0.965031   \n",
       "26           0.964641           0.965129           0.930038   \n",
       "27           0.953211           0.965129           0.930281   \n",
       "28           0.941742           0.953474           0.918682   \n",
       "29           0.941742           0.953488           0.930281   \n",
       "\n",
       "    split3_test_score  split4_test_score  mean_test_score  std_test_score  \n",
       "0            0.975968           0.988040         0.981120        0.005775  \n",
       "1            0.975968           0.988076         0.981105        0.005787  \n",
       "2            0.975968           0.988076         0.978775        0.008695  \n",
       "3            0.975968           0.988076         0.978775        0.008695  \n",
       "4            0.975968           0.988076         0.978775        0.008695  \n",
       "5            0.975968           0.988076         0.978768        0.004621  \n",
       "6            0.975968           0.988076         0.978768        0.004621  \n",
       "7            0.975968           0.988076         0.978768        0.004621  \n",
       "8            0.975968           0.988076         0.978768        0.004621  \n",
       "9            0.975968           0.988040         0.978719        0.008841  \n",
       "10           0.975968           1.000000         0.974024        0.013560  \n",
       "11           0.976190           1.000000         0.971775        0.017407  \n",
       "12           0.975968           1.000000         0.971728        0.017399  \n",
       "13           0.975968           0.988040         0.971723        0.011794  \n",
       "14           0.975968           1.000000         0.971716        0.015755  \n",
       "15           0.975968           0.988040         0.969358        0.011724  \n",
       "16           0.975968           0.988076         0.969340        0.011693  \n",
       "17           0.975968           1.000000         0.969320        0.020368  \n",
       "18           0.975968           0.988040         0.969314        0.015849  \n",
       "19           0.963781           1.000000         0.969288        0.015821  \n",
       "20           0.975968           0.988076         0.967027        0.016946  \n",
       "21           0.975968           1.000000         0.966975        0.020044  \n",
       "22           0.975968           0.976111         0.964691        0.012537  \n",
       "23           0.963781           0.988076         0.964659        0.016402  \n",
       "24           0.975968           0.976111         0.964516        0.018262  \n",
       "25           0.976190           0.976190         0.962304        0.013510  \n",
       "26           0.963781           0.988076         0.962205        0.018554  \n",
       "27           0.975968           0.976111         0.959991        0.017157  \n",
       "28           0.975968           0.976111         0.952981        0.021742  \n",
       "29           0.963781           0.976111         0.952922        0.016064  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# with the model defined in build_model() above the search reduces to:\n",
    "models, params, scorer = build_model()\n",
    "search = HyperclassifierSearch(models, params)\n",
    "skf = StratifiedKFold(n_splits=5)\n",
    "best_model = search.train_model(X_train, y_train, search='random', scoring=scorer, cv=skf)\n",
    "search.evaluate_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use best model for prediction: best_model.predict(X_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}