03_classification_exercises.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import fetch_openml\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import cross_val_predict, GridSearchCV\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "%matplotlib inline\n",
    "plt.style.use('jt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use RELOAD=True mode to import pickles and avoid retraining models \n",
    "from joblib import dump, load\n",
    "model_fpath = 'saved_models/3_'\n",
    "RELOAD = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise 1)\n",
    "Build a classifier for the MNIST dataset tha achieves over 97% accuracy on the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if RELOAD:\n",
    "    mnist = load(model_fpath+'mnist.joblib')\n",
    "else:\n",
    "    mnist = fetch_openml('mnist_784', version=1)\n",
    "    dump(mnist, model_fpath+'mnist.joblib')\n",
    "\n",
    "X, y = mnist['data'], mnist['target'].astype(np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recall data is already shuffled into training and test set\n",
    "X_train, y_train, X_test, y_test = X[:60000], y[:60000], X[60000:], y[60000:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll start with a small set and to write the code and then run on the full training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_small, y_train_small = X_train[:5000], y_train[:5000]\n",
    "X_test_small, y_test_small = X_test[:5000], y_test[:5000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    8.5s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   24.7s finished\n"
     ]
    }
   ],
   "source": [
    "knn_clf = KNeighborsClassifier()\n",
    "knn_preds = cross_val_predict(knn_clf, X_train_small, y_train_small, cv=3, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_model_scores(y_actual, y_preds, average=None):\n",
    "    precision = precision_score(y_actual, y_preds, average=average)\n",
    "    recall = recall_score(y_actual, y_preds, average=average)\n",
    "    f1 = f1_score(y_actual, y_preds, average=average)\n",
    "    accuracy = accuracy_score(y_actual, y_preds)\n",
    "\n",
    "    print('Model results')\n",
    "    print(f'Accuracy: {accuracy}')\n",
    "    print(f'Precision: {precision}')\n",
    "    print(f'Recall: {recall}')\n",
    "    print(f'F1: {f1}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model results\n",
      "Accuracy: 0.9216\n",
      "Precision: 0.9250926365448813\n",
      "Recall: 0.9200168298628594\n",
      "F1: 0.921287739432865\n"
     ]
    }
   ],
   "source": [
    "print_model_scores(y_train_small, knn_preds, 'macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "             estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30,\n",
       "                                            metric='minkowski',\n",
       "                                            metric_params=None, n_jobs=None,\n",
       "                                            n_neighbors=5, p=2,\n",
       "                                            weights='uniform'),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid=[{'n_neighbors': array([1, 2, 3, 4, 5, 6, 7, 8, 9]),\n",
       "                          'weights': ['uniform', 'distance']}],\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_grid = [\n",
    "    {'weights': ['uniform', 'distance'], 'n_neighbors': np.r_[1:10]}\n",
    "]\n",
    "grid_search = GridSearchCV(knn_clf, param_grid, cv=3)\n",
    "grid_search.fit(X_train_small, y_train_small)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_neighbors': 4, 'weights': 'distance'}"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_clf = grid_search.best_estimator_\n",
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model results\n",
      "Precision: 0.9316239364223113\n",
      "Recall: 0.9279879556944662\n",
      "F1: 0.9289948152946156\n"
     ]
    }
   ],
   "source": [
    "grid_preds = cross_val_predict(grid_clf, X_train_small, y_train_small, cv=3)\n",
    "print_model_scores(y_train_small, grid_preds, 'macro')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Grid search on this model trained with a small sample slightly improved results and yields n_neighbours = 4. With this, I can reduce the search space for n_neighbors to say range(3, 6) for the full dataset to save some time during training as the full dataset is very large"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initiating GridSearch\n",
      "GridSearch complete in 568.43 minutes\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'GridSearchCV' object has no attribute 'best_estimator'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-55-694fdb3bc841>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'GridSearch complete in {(time.time()-start_time)/60.0:.2f} minutes'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mgrid_clf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgrid_search\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_estimator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Found best params {grid_search.best_params}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'GridSearchCV' object has no attribute 'best_estimator'"
     ]
    }
   ],
   "source": [
    "# Grid Search for full dataset\n",
    "if RELOAD:\n",
    "    grid_search = load(model_fpath + 'ex1_knn.joblib')\n",
    "else:\n",
    "    start_time = time.time()\n",
    "    param_grid = [{'weights':['uniform', 'distance'], 'n_neighbors':np.r_[3:7]}]\n",
    "    print(\"Initiating GridSearch\")\n",
    "    grid_search = GridSearchCV(knn_clf, param_grid, cv=5)\n",
    "    grid_search.fit(X_train, y_train)\n",
    "    print(f'GridSearch complete in {(time.time()-start_time)/60.0:.2f} minutes')\n",
    "    dump(grid_search, model_fpath+'ex1_knn.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best params {'n_neighbors': 4, 'weights': 'distance'}\n",
      "Initiating CrossValPredict on training set\n",
      "Validation completed in 800.63 minutes\n",
      "Model results\n",
      "Precision: [0.97985012 0.96184739 0.98721934 0.96971193 0.97932233 0.96290829\n",
      " 0.97712855 0.96174433 0.97943123 0.94711618]\n",
      "Recall: [0.9934155  0.99466034 0.95938234 0.96085467 0.9647381  0.96255303\n",
      " 0.98901656 0.97509976 0.93590839 0.96335519]\n",
      "F1: [0.98658618 0.97797871 0.9731018  0.96526299 0.97197551 0.96273063\n",
      " 0.98303661 0.968376   0.95717532 0.95516667]\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "grid_clf = grid_search.best_estimator_\n",
    "print(f'Found best params {grid_search.best_params_}')\n",
    "\n",
    "print(f'Initiating CrossValPredict on training set')\n",
    "grid_preds = cross_val_predict(grid_clf, X_train, y_train, cv=3)\n",
    "print(f'Validation completed in {(time.time()-start_time)/60.0:.2f} minutes')\n",
    "print_model_scores(y_train, grid_preds, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_set_preds = cross_val_predict(grid_clf, X_test, y_test, cv=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model results\n",
      "Precision: 0.9421027040033261\n",
      "Recall: 0.9402735042228934\n",
      "F1: 0.9407514123508827\n"
     ]
    }
   ],
   "source": [
    "print_model_scores(y_test, test_set_preds, average='macro')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model is slightly overfitting the train set. Perhaps scaling can help. Since we found the best parameters, there's no need to continue with the gridsearch, but just in case I will perform another gridsearch on the smaller to check if the scaling affects the model parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.10957143, 0.12503175, 0.11095238, 0.11334921, 0.10831746,\n",
       "        0.10020635, 0.10914286, 0.1157619 , 0.10833333, 0.11044444]),\n",
       " array([0. , 0.9, 1.8, 2.7, 3.6, 4.5, 5.4, 6.3, 7.2, 8.1, 9. ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAecAAAFKCAYAAAAnj5dkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAXAklEQVR4nO3dX2jb9f7H8VeaVKtbVpI2/bNZ0ZueCL1woBdnF2mhK6LsFBI4axA3d+FV5QgFj5Yi6MB6sl1s4cipx8HOeuGwG7NHEc4uXO08K5yWIb1oy2QwBCvdlj/txuiqa7f8Lg6/SPA06Wb6zTvZ83Hlt998v/3k7eCZ7zdZ5spkMhkBAAAzqkq9AAAAkIs4AwBgDHEGAMAY4gwAgDHEGQAAY4gzAADGeEq9gP+XTN4q6vl8vse1tHS7qOfErzFnZzBn5zBrZzBnKRDwrruvYq+cPR53qZfwUGDOzmDOzmHWzmDO+VVsnAEAKFfEGQAAY4gzAADGEGcAAIwhzgAAGEOcAQAwhjgDAGAMcQYAwBjiDACAMcQZAABjiDMAAMYQZwAAjDHzr1I9jA4OXyz1EvJ698DzpV4CADyUuHIGAMAY4gwAgDHEGQAAY4gzAADGEGcAAIwhzgAAGEOcAQAwZkNxjsfjikajikQimpmZydn3888/6+2331YkEsn5+ZEjR9TT06NIJKKzZ88Wb8UAAFS4gnGenJzU7OysRkZGFIvFFIvFcvYfPnxYzzzzTM7PLl68qEuXLunUqVP6xz/+ob/85S/FXTUAABWsYJynpqbU2dkpSWptbVUikdDKykp2f19fn3bv3p1zzM6dOxWPxyVJXq9Xq6urunfvXjHXDQBAxSoY52QyKb/fn932+/1KpVLZ7a1bt/7qGI/Hoy1btkiSzpw5o/b2dlVV8fY2AAAbUfC7taurq3O2M5mMXC7Xhk5+7tw5nT59WidOnCj4WJ/vcXk87g2dd6MCAW9Rz1ds1R7bL1g2Oj/rc64UzNk5zNoZzHl9BeMcCASUTqez24uLi6qvry944gsXLmhoaEjHjx/Xtm3bCj5+ael2wcfcj0DAq2TyVlHPWWyra7Zv9W9kfuUw50rAnJ3DrJ3BnPO/OCl46RYKhTQ2NiZJmpubU0tLi2pqavIec+vWLcViMR07dkw+n+8+lwsAwMOt4JVzW1ubgsGgwuGw3G63BgcHNTo6Kq/Xq66uLr3xxhu6du2avv/+e+3bt0979+7V7du3dfPmTfX19WXPc+jQIW3fvn1TnwwAAJVgQ/+e85///Oec7d/97nfZ//7rX//6P4/p6en5DcsCAODhZfsTSQAAPISIMwAAxhBnAACMIc4AABizoQ+EAQDWd3D4YqmXUNC7B54v9RJwH7hyBgDAGOIMAIAxxBkAAGOIMwAAxhBnAACM4dPaKGvWPyXLJ2QBPAiunAEAMIY4AwBgDHEGAMAY4gwAgDF8IAzr2siHrao9VVpdu+fAagDg4UGcAQAlZ/1vXkjO/u0LbmsDAGBMxV459x09z+1WAEBZqtg4A6gcTt7y5HMUsIDb2gAAGMOVM7CJinXFt5lXc3zF6MPB2geuuEORH1fOAAAYQ5wBADCGOAMAYAxxBgDAGOIMAIAxxBkAAGOIMwAAxhBnAACMIc4AABhDnAEAMIY4AwBgDHEGAMAY4gwAgDHEGQAAYzYU53g8rmg0qkgkopmZmZx9P//8s95++21FIpENHwMAANZXMM6Tk5OanZ3VyMiIYrGYYrFYzv7Dhw/rmWeeua9jAADA+grGeWpqSp2dnZKk1tZWJRIJraysZPf39fVp9+7d93UMAABYX8E4J5NJ+f3+7Lbf71cqlcpub9269b6PAQAA6/MUekB1dXXOdiaTkcvlKvoxPt/j8njchZZzX6o9fN7NCczZGZs15w8++XZTzltMTv8Z48+0M8ptzoGA17HfVTDOgUBA6XQ6u724uKj6+vqiH7O0dLvQUu7b6tq9op8Tuao9VczZAczZOczaGeU452TyVlHPly/2BV+2hEIhjY2NSZLm5ubU0tKimpqaoh8DAAD+q+CVc1tbm4LBoMLhsNxutwYHBzU6Oiqv16uuri698cYbunbtmr7//nvt27dPe/fu1R/+8IdfHQMAADbGlclkMqVehFT82wUffPJt2d0yKUfleGuqHDFn5zBrZ5TjnN898HxRz/ebbmsDAABnEWcAAIwhzgAAGEOcAQAwhjgDAGAMcQYAwBjiDACAMcQZAABjiDMAAMYQZwAAjCHOAAAYQ5wBADCGOAMAYAxxBgDAGOIMAIAxxBkAAGOIMwAAxhBnAACMIc4AABhDnAEAMIY4AwBgDHEGAMAY4gwAgDHEGQAAY4gzAADGEGcAAIwhzgAAGEOcAQAwhjgDAGAMcQYAwBjiDACAMcQZAABjiDMAAMYQZwAAjCHOAAAYQ5wBADCGOAMAYMyG4hyPxxWNRhWJRDQzM5Ozb3p6WtFoVOFwWENDQ5Kk5eVlvf7669q3b5/27t2rb775pvgrBwCgQnkKPWByclKzs7MaGRnR5cuXdfDgQZ08eTK7v7+/X8PDw2psbFRPT4/27Nmjf//733r66af15ptv6vr169q/f7/a29s39YkAAFApCl45T01NqbOzU5LU2tqqRCKhlZUVSdL8/Lxqa2vV3NysqqoqdXR0aGJiQj6fT+l0WpJ048YN+f3+TXwKAABUloJxTiaTOXH1+/1KpVKSpEQikbOvrq5OqVRKL730kq5evaoXXnhBr776qt56661NWDoAAJWp4G3t6urqnO1MJiOXy5V33xdffKEdO3ZoeHhY3333nQYGBjQ6Opr39/h8j8vjcd/v+vOv3cPn3ZzAnJ3BnJ3DrJ1RbnMOBLyO/a6CcQ4EAtlb1JK0uLio+vp6SVJDQ0POvlQqpYaGBk1PTysUCkmSgsGgEomE1tbW5PGs/+uWlm4/8JNYz+ravaKfE7mqPVXM2QHM2TnM2hnlOOdk8lZRz5cv9gVftoRCIY2NjUmS5ubm1NLSopqaGklSU1OT1tbWtLCwoLt372p8fFyhUEhPPvmkZmdnJUnXr1/Xli1b8oYZAAD8omAx29raFAwGFQ6H5Xa7NTg4qNHRUXm9XnV1dWlgYEC9vb1yuVzq7u5Wc3OzotGo+vv79corr2h1dVXvvfeeA08FAIDK4MpkMplSL0Iq/u2CDz75tuxumZSjcrw1VY6Ys3OYtTPKcc7vHni+qOf7Tbe1AQCAs4gzAADGEGcAAIwhzgAAGEOcAQAwhjgDAGAMcQYAwBjiDACAMcQZAABjiDMAAMYQZwAAjCHOAAAYQ5wBADCGOAMAYAxxBgDAGOIMAIAxxBkAAGOIMwAAxhBnAACMIc4AABhDnAEAMIY4AwBgDHEGAMAY4gwAgDHEGQAAY4gzAADGEGcAAIwhzgAAGEOcAQAwhjgDAGAMcQYAwBjiDACAMcQZAABjiDMAAMYQZwAAjCHOAAAYQ5wBADBmQ3GOx+OKRqOKRCKamZnJ2Tc9Pa1oNKpwOKyhoaHsz7/88ktFIhGFw2GdP3++qIsGAKCSFYzz5OSkZmdnNTIyolgsplgslrO/v79fR48e1Weffabx8XH98MMPWl5e1okTJ/Tpp5/q73//u86dO7dpTwAAgErjKfSAqakpdXZ2SpJaW1uVSCS0srKixx57TPPz86qtrVVzc7MkqaOjQxMTE6qrq1N7e7seffRRNTY26v3339/cZwEAQAUpGOdkMqlgMJjd9vv9SqVSamlpUSKRkN/vz+6rq6tTIpHQnTt3tLi4qNdee023b9/Wn/70J/3+97/P+3t8vsfl8bh/w1P5tWoPb6k7gTk7gzk7h1k7o9zmHAh4HftdBeNcXV2ds53JZORyufLuu3PnjtLptD7++GPNz8/rwIED+vrrr1VVtf7/iKWl2w+y/rxW1+4V/ZzIVe2pYs4OYM7OYdbOKMc5J5O3inq+fLEv+LIlEAgonU5ntxcXF1VfXy9JamhoyNmXSqXU0NCgQCCgZ599Vm63W0899ZS2bt2qpaWl3/IcAAB4aBSMcygU0tjYmCRpbm5OLS0tqqmpkSQ1NTVpbW1NCwsLunv3rsbHxxUKhbRr1y5NTk4qk8konU5reXlZPp9vc58JAAAVouBt7ba2NgWDQYXDYbndbg0ODmp0dFRer1ddXV0aGBhQb2+vXC6Xuru7sx8O6+rq0v79+7W8vKx33nkn7y1tAADwC1cmk8mUehFS8e/lf/DJt2X3fkY5Ksf3jcoRc3YOs3ZGOc753QPPF/V8v+k9ZwAA4CziDACAMcQZAABjiDMAAMYQZwAAjCHOAAAYQ5wBADCGOAMAYAxxBgDAGOIMAIAxxBkAAGOIMwAAxhBnAACMIc4AABhDnAEAMIY4AwBgDHEGAMAY4gwAgDHEGQAAY4gzAADGEGcAAIwhzgAAGEOcAQAwhjgDAGAMcQYAwBjiDACAMcQZAABjiDMAAMYQZwAAjCHOAAAYQ5wBADCGOAMAYAxxBgDAGOIMAIAxxBkAAGOIMwAAxmwozvF4XNFoVJFIRDMzMzn7pqenFY1GFQ6HNTQ0lLPvp59+0u7duzU6Olq8FQMAUOEKxnlyclKzs7MaGRlRLBZTLBbL2d/f36+jR4/qs88+0/j4uH744Yfsvo8++ki1tbXFXzUAABWsYJynpqbU2dkpSWptbVUikdDKyookaX5+XrW1tWpublZVVZU6Ojo0MTEhSbpy5YquXLmijo6OzVs9AAAVqGCck8mk/H5/dtvv9yuVSkmSEolEzr66urrsvsOHD6u/v7/Y6wUAoOJ5Cj2guro6ZzuTycjlcuXd9/nnn+u5557TE088seGF+HyPy+Nxb/jxG1Ht4fNuTmDOzmDOzmHWzii3OQcCXsd+V8E4BwIBpdPp7Pbi4qLq6+slSQ0NDTn7UqmUGhoadP78ef3444/66quvdO3aNT3yyCNqamrSrl271v09S0u3f8vz+J9W1+4V/ZzIVe2pYs4OYM7OYdbOKMc5J5O3inq+fLEvGOdQKKSjR4/q5Zdf1tzcnFpaWlRTUyNJampq0tramhYWFtTY2Kjx8XH97W9/U09PT/b4Dz/8UDt27MgbZgAA8IuCcW5ra1MwGFQ4HJbb7dbg4KBGR0fl9XrV1dWlgYEB9fb2yuVyqbu7W83NzU6sGwCAiuXKZDKZUi9CKv7tgg8++bbsbpmUo3K8NVWOmLNzmLUzynHO7x54vqjny3dbu7zejQcA4CFAnAEAMIY4AwBgDHEGAMAY4gwAgDHEGQAAY4gzAADGEGcAAIwhzgAAGEOcAQAwhjgDAGAMcQYAwBjiDACAMcQZAABjiDMAAMYQZwAAjCHOAAAYQ5wBADCGOAMAYAxxBgDAGOIMAIAxxBkAAGOIMwAAxhBnAACMIc4AABhDnAEAMIY4AwBgDHEGAMAY4gwAgDHEGQAAY4gzAADGEGcAAIwhzgAAGEOcAQAwhjgDAGAMcQYAwBjiDACAMRuKczweVzQaVSQS0czMTM6+6elpRaNRhcNhDQ0NZX9+5MgR9fT0KBKJ6OzZs8VdNQAAFcxT6AGTk5OanZ3VyMiILl++rIMHD+rkyZPZ/f39/RoeHlZjY6N6enq0Z88eXb9+XZcuXdKpU6d048YNdXd368UXX9zUJwIAQKUoeOU8NTWlzs5OSVJra6sSiYRWVlYkSfPz86qtrVVzc7OqqqrU0dGhiYkJ7dy5U/F4XJLk9Xq1urqqe/fubeLTAACgchS8ck4mkwoGg9ltv9+vVCqllpYWJRIJ+f3+7L66ujolEgl5PB55PP899ZkzZ9Te3q6qqvyvA3y+x+XxuB/0efxP1R7eUncCc3YGc3YOs3ZGuc05EPA69rsKxrm6ujpnO5PJyOVyFdwnSefOndPp06d14sSJggtZWrq9oQXfj9U1rtY3W7Wnijk7gDk7h1k7oxznnEzeKur58sW+YJwDgYDS6XR2e3FxUfX19ZKkhoaGnH2pVEoNDQ2SpAsXLmhoaEjHjx/Xtm3bHnjxAAA8bAreUwiFQhobG5Mkzc3NqaWlRTU1NZKkpqYmra2taWFhQXfv3tX4+LhCoZBu3bqlWCymY8eOyefzbe4zAACgwhS8cm5ra1MwGFQ4HJbb7dbg4KBGR0fl9XrV1dWlgYEB9fb2yuVyqbu7W83NzTp16pRu3rypvr6+7HkOHTqk7du3b+qTAQCgErgymUym1IuQin8v/4NPvi279zPKUTm+b1SOmLNzmLUzynHO7x54vqjny/eec3l9VA4AgIcAcQYAwBjiDACAMcQZAABjiDMAAMYQZwAAjCHOAAAYQ5wBADCGOAMAYAxxBgDAGOIMAIAxxBkAAGOIMwAAxhBnAACMIc4AABhDnAEAMIY4AwBgDHEGAMAY4gwAgDHEGQAAY4gzAADGEGcAAIwhzgAAGEOcAQAwhjgDAGAMcQYAwBjiDACAMcQZAABjiDMAAMYQZwAAjCHOAAAYQ5wBADCGOAMAYAxxBgDAGOIMAIAxxBkAAGM2FOd4PK5oNKpIJKKZmZmcfdPT04pGowqHwxoaGtrQMQAAYH2eQg+YnJzU7OysRkZGdPnyZR08eFAnT57M7u/v79fw8LAaGxvV09OjPXv2aGFhIe8xAABgfQXjPDU1pc7OTklSa2urEomEVlZW9Nhjj2l+fl61tbVqbm6WJHV0dGhiYkLJZHLdYwAAQH4Fb2snk0n5/f7stt/vVyqVkiQlEomcfXV1dUqlUnmPAQAA+RW8cq6urs7ZzmQycrlcefflO2Y9gYB3QwveqKN9HUU9HwAATikY50AgoHQ6nd1eXFxUfX29JKmhoSFnXyqVUkNDg9xu97rHAACA/Are1g6FQhobG5Mkzc3NqaWlRTU1NZKkpqYmra2taWFhQXfv3tX4+LhCoVDeYwAAQH4Fr5zb2toUDAYVDofldrs1ODio0dFReb1edXV1aWBgQL29vXK5XOru7lZzc7Oam5t/dQwAANgYVyaTyZR6EQAA4Bd8QxgAAMYQZwAAjKnIOPPVoc44cuSIenp6FIlEdPbs2VIvp6L99NNP2r17t0ZHR0u9lIr15ZdfKhKJKBwO6/z586VeTkVaXl7W66+/rn379mnv3r365ptvSr0kswp+IKzcFPq6URTHxYsXdenSJZ06dUo3btxQd3e3XnzxxVIvq2J99NFHqq2tLfUyKtby8rJOnDihTz/9VDdu3NCHH36ojo6OUi+r4vzzn//U008/rTfffFPXr1/X/v371d7eXuplmVRxV87rfd0oimvnzp2Kx+OSJK/Xq9XVVd27d6/Eq6pMV65c0ZUrV4jFJpqYmFB7e7seffRRNTY26v333y/1kiqSz+fLfgfGjRs3cr5JErkqLs58dagzPB6PtmzZIkk6c+aM2tvbVVVVcX+cTDh8+LD6+/tLvYyKdvXqVS0uLuq1117Tyy+/rP/85z+lXlJFeumll3T16lW98MILevXVV/XWW2+VeklmVdxt7Qf56lA8uHPnzun06dM6ceJEqZdSkT7//HM999xzeuKJJ0q9lIp2584dpdNpffzxx5qfn9eBAwf09ddf84KzyL744gvt2LFDw8PD+u677zQwMMDnKNZRcXHO93WjKK4LFy5oaGhIx48f17Zt20q9nIp0/vx5/fjjj/rqq6907do1PfLII2pqatKuXbtKvbSKEggE9Oyzz8rtduupp57S1q1btbS0pLq6ulIvraJMT08rFApJkoLBoBKJhNbW1uTxVFyKfrOKe1nIV4c649atW4rFYjp27Jh8Pl+pl1Ox4vG4zpw5o9OnT+uPf/yjent7CfMm2LVrlyYnJ5XJZJROp7W8vMyf603w5JNPanZ2VpJ0/fp1bdmyhTCvo+Km8r++bhTF969//Us3b95UX19f9meHDh3S9u3bS7gq4ME0Njaqq6tL+/fv1/Lyst555x1uaW+CaDSq/v5+vfLKK1pdXdV7771X6iWZxdd3AgBgDC8NAQAwhjgDAGAMcQYAwBjiDACAMcQZAABjiDMAAMYQZwAAjCHOAAAY83+kI312k4jFDwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(y, density=True, alpha=0.8)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks like the numbers are somewhat uniformly distributed. What if we apply normalisation instead of Standardisation?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing small scale grid search\n",
      "Done in 7.10 minutes\n",
      "Found best parameters: {'n_neighbors': 4, 'weights': 'distance'}\n",
      "Performing predictions on training set...\n",
      "Done in 0.38 minutes\n",
      "Small scale training set predictions\n",
      "Model results\n",
      "Precision: 0.9318254917568485\n",
      "Recall: 0.9281655755878944\n",
      "F1: 0.9291717352860764\n",
      "Performing predictions on test set...\n",
      "Done in 0.39 minutes\n",
      "Small scale test set predictions\n",
      "Model results\n",
      "Precision: 0.9208179213265941\n",
      "Recall: 0.91791024337353\n",
      "F1: 0.9183627833232635\n"
     ]
    }
   ],
   "source": [
    "small_scale_knn = KNeighborsClassifier()\n",
    "scaler = MinMaxScaler()\n",
    "X_train_small_scaled = scaler.fit_transform(X_train_small)\n",
    "X_test_small_scaled = scaler.fit_transform(X_test_small)\n",
    "\n",
    "start_time = time.time()\n",
    "print(f'Initializing small scale grid search')\n",
    "scaled_grid_params = [{\n",
    "    'weights':['uniform', 'distance'], 'n_neighbors':np.r_[1:10]\n",
    "}]\n",
    "scaled_gs = GridSearchCV(small_scale_knn, scaled_grid_params, cv=3)\n",
    "scaled_gs.fit(X_train_small_scaled, y_train_small)\n",
    "print(f'Done in {(time.time() - start_time)/60.0 :.2f} minutes')\n",
    "print(f'Found best parameters: {scaled_gs.best_params_}')\n",
    "best_small_scaled = scaled_gs.best_estimator_\n",
    "\n",
    "start_time = time.time()\n",
    "print(f'Performing predictions on training set...')\n",
    "small_scaled_train_pred = cross_val_predict(best_small_scaled, \n",
    "                                            X_train_small_scaled, y_train_small, cv=3)\n",
    "print(f'Done in {(time.time() - start_time)/60.0 :.2f} minutes')\n",
    "print(f'Small scale training set predictions')\n",
    "print_model_scores(y_train_small, small_scaled_train_pred, average='macro')\n",
    "\n",
    "start_time = time.time()\n",
    "print(f'Performing predictions on test set...')\n",
    "small_scaled_test_pred = cross_val_predict(best_small_scaled, \n",
    "                                           X_test_small_scaled, y_test_small, cv=3)\n",
    "print(f'Done in {(time.time() - start_time)/60.0 :.2f} minutes')\n",
    "print(f'Small scale test set predictions')\n",
    "print_model_scores(y_test_small, small_scaled_test_pred, average='macro')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks like MinMaxScaling helps! Let's use this and the best parameters to create our predictor."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = MinMaxScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_test_scaled = scaler.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'scaled_gs' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-dc8cf6483182>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mweights\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscaled_gs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_params_\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'weights'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mn_neighbors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscaled_gs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_params_\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'n_neighbors'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mknn_clf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mKNeighborsClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_neighbors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_neighbors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mweights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'scaled_gs' is not defined"
     ]
    }
   ],
   "source": [
    "weights = scaled_gs.best_params_['weights'] # distance\n",
    "n_neighbors = scaled_gs.best_params_['n_neighbors'] # 4\n",
    "\n",
    "knn_clf = KNeighborsClassifier(n_neighbors=n_neighbors, weights=weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting to training set...\n",
      "Done in 0.25 minutes\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "print(f'Fitting to training set...')\n",
    "knn_clf.fit(X_train_scaled, y_train)\n",
    "print(f'Done in {(time.time() - start_time)/60.0 :.2f} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting test set...\n",
      "Done in 13.21 minutes\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "print(f'Predicting test set...')\n",
    "y_preds = knn_clf.predict(X_test_scaled)\n",
    "print(f'Done in {(time.time() - start_time)/60.0 :.2f} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model results\n",
      "Accuracy: 0.9714\n",
      "Precision: 0.9715597201945959\n",
      "Recall: 0.9710823052664403\n",
      "F1: 0.971224084176584\n"
     ]
    }
   ],
   "source": [
    "print_model_scores(y_test, y_preds, average='macro')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Done!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise 2)\n",
    "Create a function that can shift an MNIST image in any direction. Then for each image in the training set, create four shifted copies and add them to the training set. Re-train the model and measure its accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 2, 3, 4, 5],\n",
       "       [1, 2, 3, 4, 5],\n",
       "       [1, 2, 3, 4, 5],\n",
       "       [1, 2, 3, 4, 5],\n",
       "       [1, 2, 3, 4, 5]])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = np.ones((5,5), dtype='uint8') * np.r_[1:6]\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1, 2, 3, 4],\n",
       "       [0, 1, 2, 3, 4],\n",
       "       [0, 1, 2, 3, 4],\n",
       "       [0, 1, 2, 3, 4],\n",
       "       [0, 1, 2, 3, 4]])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shift(test, [0,1]) # shifts col to right"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0],\n",
       "       [1, 2, 3, 4, 5],\n",
       "       [1, 2, 3, 4, 5],\n",
       "       [1, 2, 3, 4, 5],\n",
       "       [1, 2, 3, 4, 5]])"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shift(test, [1,0]) # shifts row down"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.ndimage import shift\n",
    "\n",
    "def create_shifted_samples(arr, shift_size=1):\n",
    "    original_shape = arr.shape\n",
    "    img_shape = (28,28)\n",
    "    right_shift = np.array([0,0,1]) * shift_size\n",
    "    down_shift = np.array([0,1,0]) * shift_size\n",
    "    left_shift, up_shift = -1*right_shift, -1*down_shift\n",
    "    \n",
    "    shaped_arr = np.apply_along_axis(np.reshape, 1, arr, img_shape)\n",
    "    right_shifted = shift(shaped_arr, right_shift).reshape(original_shape)\n",
    "    down_shifted = shift(shaped_arr, down_shift).reshape(original_shape)\n",
    "    left_shifted = shift(shaped_arr, left_shift).reshape(original_shape)\n",
    "    up_shifted = shift(shaped_arr, up_shift).reshape(original_shape)\n",
    "    \n",
    "    return np.r_[right_shifted, down_shifted, left_shifted, up_shifted]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "shifted_train_set = create_shifted_samples(X_train_scaled)\n",
    "X_train_scaled_shifted = np.r_[X_train_scaled, shifted_train_set]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_test_scaled_shifted = create_shifted_samples(X_test_scaled)\n",
    "# X_test_scaled_shifted = np.r_[X_test_scaled, X_test_scaled_shifted]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_shifted = np.tile(y_train, 5)\n",
    "# y_test_shifted = np.tile(y_test, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check that there are the same number of observations in X and y\n",
    "# X_train_scaled_shifted.shape[0] == y_train_shifted.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting to training set...\n",
      "Done in 0.84 minutes\n"
     ]
    }
   ],
   "source": [
    "n_neighbors = 4\n",
    "weights = 'distance'\n",
    "\n",
    "shifted_knn_clf = KNeighborsClassifier(n_neighbors=n_neighbors, weights=weights)\n",
    "\n",
    "start_time = time.time()\n",
    "print(f'Fitting to training set...')\n",
    "shifted_knn_clf.fit(X_train_scaled_shifted, y_train_shifted)\n",
    "print(f'Done in {(time.time() - start_time)/60.0 :.2f} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting test set...\n",
      "Done in 69.92 minutes\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'print_model_scores' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-5e614fb02182>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Done in {(time.time() - start_time)/60.0 :.2f} minutes'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mprint_model_scores\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_preds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maverage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'macro'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'print_model_scores' is not defined"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "print(f'Predicting test set...')\n",
    "y_preds = shifted_knn_clf.predict(X_test_scaled)\n",
    "print(f'Done in {(time.time() - start_time)/60.0 :.2f} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model results\n",
      "Accuracy: 0.9764\n",
      "Precision: 0.9767313811568235\n",
      "Recall: 0.9761507279337562\n",
      "F1: 0.97636443628852\n"
     ]
    }
   ],
   "source": [
    "print_model_scores(y_test, y_preds, average='macro')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Slight gains"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3) Tackle the titanic dataset\n",
    "I'll keep this attempt simple and short so I don't spend too much time on it for now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "titanic_fpath = 'datasets/titanic/train.csv'\n",
    "test_set_fpath = 'datasets/titanic/test.csv'\n",
    "test_df = pd.read_csv(test_set_fpath)\n",
    "df = pd.read_csv(titanic_fpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApMAAAHICAYAAAASt2FUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAWDElEQVR4nO3df4ykB13H8c92T9twOSm7V8WrVjDVJiqVCkIa2lBqjRgoigEpFQppaiApwdiiKX9YiRAlEBLAimgVTbW/6A9y9g+Qoj2EFLgCVfAPpcaQ9hIOrrfYHg2/2j7+8czm5pbnmZ393tzs7M7rlWxunnlmdp779rmn731mZmehaZoAAEDFSZu9AQAAbF1iEgCAMjEJAECZmAQAoExMAgBQJiYBACjbsZkPfujQkan+XqKlpZ1ZWXlsmg+5ZZhNN3PpZzbdzKWf2XQzl35m023acznttF0Lo9bPzZnJhYVkcfGkLIwcx3wym27m0s9suplLP7PpZi79zKbbLM5lbmISAIDJE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAynZs9gYAAEzS5e/8183ehBPqrvf8xmZvwjGcmQQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAEDZjg3c9qoklyR5Isn/Jrk8yXeH1jdJPjm0/JdJbj3eDQQAYHaNG5PnJnl9kuck+X6SO5JcluT6Nbe7YFIbBgDA7Bv3ae79Sc5LG5JJcijJU0/IFgEAsGWMe2byiSSPDi6fmeQlSV7QcbsbkjwzyYNpnxb/+nrfeGFhzC04TquPM63H20rMppu59DObbubSz2y6mUs/sxltluaykddMJsnZaZ/iviJtMA67MsmdSQ4muTbJdUleOeqbLS3tzOLidN8DtLy8a6qPt5WYTTdz6Wc23cyln9l0M5d+ZtNtluaykZg8J+0bai5Ncl/H+g8MXb497WsqR1pZeWyqZyaXl3fl8OEjaZrpPOZWYTbdzKWf2XQzl35m081c+pnNaNOcy+7do8N13JjcmeS2JL+V5Esd689K8hdJXprkO0kuSnL/ON942jtI00z/MbcKs+lmLv3Mppu59DObbubSz2y6zdJcxo3JS5MsJXn/0HV3JzktyU1p36CzL8m9SY4Mvt4wsa0EAGAmjRuT1+cHfw3QWu8YfAEAMCd8Ag4AAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGUbicmrkuxP8pkkNyY5ec3685Pcm+RzSe5KcuokNhAAgNk1bkyem+T1SV4wuHxKksuG1u9IcnOS1yV5ftrovGZiWwkAwEzaMebt9ic5L8n3B8uHkjx1aP3zkhxI8sBg+bYkezNGUC4sjLkFx2n1cab1eFuJ2XQzl35m081c+plNN3PpZzajzdJcxo3JJ5I8Orh8ZpKXpD1LuWpPkq8NLR9Mcvp633RpaWcWF6f7ss3l5V1TfbytxGy6mUs/s+lmLv3Mppu59DObbrM0l3FjctXZSe5IckWSB4eub5IMN/JJSZ5c75utrDw21TOTy8u7cvjwkTTNdB5zqzCbbubSz2y6mUs/s+lmLv3MZrRpzmX37tHhupGYPCfJrUkuTXLfmnUH0p6dXLUnyUPjfNNp7yBNM/3H3CrMppu59DObbubSz2y6mUs/s+k2S3MZ9znmnWlfB/mK/GBIJsnnk5yW5KzB8msGtwcAYBsb98zkpUmWkrx/6Lq70wbkTWnfoHN5kluSPJ72rORlAQBgWxs3Jq8ffI1yT9qnwgEAmBM+AQcAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlG4nJU5PcmuRgz/omyb6hr1cdx3YBALAF7NjAbW9OsjfJC0fc5oLj2hoAALaUjcTkJUmeluRtk9yAhYVJfrf1H2daj7eVmE03c+lnNt3MpZ/ZdDOXfmYz2izNZSMx+UjamBzlhiTPTPJgkquSfH3UjZeWdmZxcbov21xe3jXVx9tKzKabufQzm27m0s9suplLP7PpNktz2UhMrufKJHemfU3ltUmuS/LKUXdYWXlsqmcml5d35fDhI2ma6TzmVmE23cyln9l0M5d+ZtPNXPqZzWjTnMvu3aPDdZIx+YGhy7cnuWycO017B2ma6T/mVmE23cyln9l0M5d+ZtPNXPqZTbdZmsuknmM+K8knkpwyWL4oyf0T+t4AAMyocc9MLqV9CvuUweV9Sb6c5IkkNyXZP7ju3iRHBl9vmOymAgAwa8aNyZWs/2t/3jH4AgBgTvgEHAAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBA2Y7N3oBpuvjqvZu9CSfch665cLM3AQCYI85MAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQtmOzNwBgO7n46r2bvQkn3IeuuXCzNwGYIc5MAgBQtpGYPDXJrUkO9qw/P8m9ST6X5K7B7QEA2MY2EpM3J7mnZ92OwfrXJXl+kv1Jrjm+TQMAYNZt5DWTlyR5WpK3dax7XpIDSR4YLN+WZG/GCMqFhQ1swXGY1uNstsrfc/U+8zKjcZlLP7PpNi/zcJyZHHPpZzajzdJcNhKTj6SNyS57knxtaPlgktPX+4ZLSzuzuOhlm5O0e/eu8n2Xl+v33c7MpZ/ZzCfHmckzl35m022W5jKpd3M3SYYb+aQkT653p5WVx5yZnLCHHz6y4fssLLQ75eHDR9I0J2Cjtihz6Wc23Rxn+tlnuplLP7MZbZpzWe8HyEnF5IG0ZydX7Uny0Dh3tINM1vHMs2n89+hiLv3MZj45zkyeufQzm26zNJdJPcf8+SSnJTlrsPyatK+bBABgGxv3zORSkjuTnDK4vC/Jl5M8keSmtO/evjzJLUkeT3tW8rIJbysAADNm3JhcSXLBOre5J8k5x7U1AABsKd5KDQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgLJxY/JNST6T5ItJ3rNm3TOSfCfJvqGvF05i4wAAmG07xrjNs5JcmeSX0kbjx5O8OMnHhm5zMMkFk944AABm2zgxeXGSvUm+PVi+I8nLc2xMli0sTOK7zM7jbLbK33P1PvMyo3GZSz+z6TYv83CcmRxz6Wc2o83SXMaJyT1JHhhaPpjk9DW3+ZEkHx5c/6Ukf5DkW+t946WlnVlc9LLNSdq9e1f5vsvL9ftuZ+bSz2zmk+PM5JlLP7PpNktzGScmmyTD/XtSkieHlg8neWuSm9MG5F8n+eO0QTnSyspjzkxO2MMPH9nwfRYW2p3y8OEjaZoTsFFblLn0M5tujjP97DPdzKWf2Yw2zbms9wPkODF5IO3ZyVV7kjw0tHwkyV8NLd+Z5Koxt88OMmHHM8+m8d+ji7n0M5v55DgzeebSz2y6zdJcxnmO+SNJXpbkKUkWk7w6yW1D61+U5Jah73VRkvsnuI0AAMyocWLyK0k+mOSzSb6Q5J60v/7nliRnJPm3JN9Isj/Jp5M8PcnbT8C2AgAwY8Z5mjtJ3jv4GnbJ0OU3T2ZzAADYSryVGgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGXjxuSbknwmyReTvKdj/SsG6+9L8g9JfngiWwcAwEwbJyafleTKJBcmeU6Ss5O8eGj9cpL3JXlZkl9O8r0kl092MwEAmEU7xrjNxUn2Jvn2YPmOJC9P8rHB8q8l+VSSQ4Pl25L8fpIPjrMBCwvjburxmdbjbLbK33P1PvMyo3GZSz+z6TYv83CcmRxz6Wc2o83SXBaaplnvNtcleSDt2cck+c0kVyR56WD5LUlOTxuQSfLsJP+Y5BcmuqUAAMyccZ7mbpIM9+9JSZ7cwHoAALapcWLyQJI9Q8t7kjy0gfUAAGxT48TkR9K+ueYpSRaTvDrt6yJXfSzJuUl+dLD8mjXrAQDYpsaJya+kfTPNZ5N8Ick9SfYluSXJGUkeSfu6yU8M1h9J+5pJAAC2uXHegAMAAJ18Ag4AAGViEgCAMjEJAEDZdorJU5PcmuRgz/rzk9yb5HNJ7hrcPkl+Lu0biu4d/PlTJ3IjN8l6s/nVtJ+t/sm0b7D66cH1+5L8++DPfUn+7ARu42ZYby5Njv7d9yV51eD6ed9nzsixc/ls2jfqJclXB8ur637vBG7jZrgqyf60/15uTHLymvXzepxZby7zeoxJ1p/NvB5nRs1lno8xC0nelXYu96b9BMKnrrnN7B1nmqbZLl8fbZrmjU3THOxYt6NpmgNN0/zMYPmPmqZ55+DyF5qmedHg8mubprllBv4u05zNyU3THGqa5ozB8pubprlxcHlf0zQXzMD2b8Zc0rS6rp/3fWbt1580TXPl4PJXm6Z5xgxs/4n4Ordpmi81TfNDg+U7mqb53aH183qcWW8u83yMWW82aVpd953nfWbt17wcY9I0zflN09w0tHxD0zRvGVqeyePMdjozeUmOfl74Ws9L+8vVHxgs35b288X3pP0oyHsG19+e9ndqztAnXk7EqNl8N8nPJnlwsPyN/OBPQdvVqLn0sc8c6xlJXpL214dtd/uTnJfk+4PlQzn238q8HmfWm8s8H2PWm02fed9nhj0j83OMSZJPJbl0cPnkJE/P0X87yYweZ7ZTTD4yYt2eJF8bWj6Yduhrr/92ku8lWZ741m2uUbNJkm8O/jwlydVJ/nZo3dVpd86PJ3nu5DdtU603lyS5Ie0/7huT/FjsM2u9Ncm7kzwxdN270z6duTfJmRPers30RJJHB5fPTPs/uA8PrZ/X48x6c0nm9xgzzmyS+TvOjDuXZL6OMcPelfbTBP8rW+A4s51icpS+zw9fe/3wunlzapK7k/xT2k89SpL3Jbk2yYuS/Olg3Y5N2brNcWWSP0z7+pT/TnJd7DPDlpJcnOTOoeuuTXvwf2GSO5LcvAnbdaKdneSjSa7IsWcM5v040zeXVfN8jBk1m3k+zqy3z8zrMSZp94mfTPvDxdVD18/kcWZeYrLv88MPJPnxoet3pZ3JyvQ2bSacmuRfkvx9krcPXf+RJPcPLu9Le5B/+jQ3bJN9IEffgHJ7knNinxn262nPJn1v6LobkvzP4PLtSZ6d7fPUXNLuA7enfRrqn9esm+fjzKi5JPN9jFlvNvN6nFlvLsl8HmN+Pu1skvYlIrcmuXBo/UweZ+YlJj+f5LQkZw2WVz8//Otpd8pfGVz/O2l3znnzd0n+Ksc+9bSY5NM5+hTCLyZ5PMeeRt/Ozkr7EaGnDJYvSvs/PfvMUecn+Y+h5VPTvrtw9WmVi9K+U3e7fMzWzrTHjVckua9j/bweZ9abSzK/x5j1ZjOvx5lx9plk/o4xSfv64j/P0TP05yX5z6H1M3mc2S5PJyylPQ1+yuDyviRfTvsai5vSvtj38rSfJ/542oq/bHDfNyb5m7Q/2Tya5LVT3O5pWG8230z7It2n5eiLfg8leWXa12zckuRbafeV386xr1vZysbZZ/al/RULRwZfbxjcd973mf2D2/1E2rNNq/4vyfVpn8o8Mrj966ayxdNxadp5vH/ourvTHtjn+Tiz3lzm9RiTjLfP7Mv8HWfGmUsyf8eYpD1b/9y0vxrou2l/uLoiyXszw8cZn80NAEDZvDzNDQDACSAmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKDs/wE5FEzvJJnzNAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 800x560 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "dark"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['Pclass'].hist(density=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApMAAAHICAYAAAASt2FUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAWCElEQVR4nO3dbaxkB13H8d/trrZhs1Lu3SputRZTbaJSqSCkoQ2l1oiBoiRFSoVCmhpISjC2aMoLayJECYQEsCJaRVPtE30gtS+oFO0ipMAWqIIvhBpD2k1Y2O5iuzQ8tT2+OHOzs5dzZub+d3bu3DufT3Kzc+bM3Dn779nT7z0zc2epaZoAAEDFCRu9AQAAbF5iEgCAMjEJAECZmAQAoExMAgBQJiYBACjbvpEPfuDA4Zn+XqLl5R05dOiJWT7kpmE23cyln9l0M5d+ZtPNXPqZTbdZz+WUU3YujVq/MGcml5aSbdtOyNLIcSwms+lmLv3Mppu59DObbubSz2y6zeNcFiYmAQCYPjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoGz7Rm8AAMA0Xf6uf9voTTiu7n7vb230JhzFmUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFAmJgEAKBOTAACUiUkAAMrEJAAAZWISAIAyMQkAQNn2ddz2qiSXJHkqyf8muTzJ94bWN0k+ObT8V0luPdYNBABgfk0ak+ckeWOS5yf5QZI7klyW5Po1tzt/WhsGAMD8mzQm9yY5N21IJsmBJM+cxgYsLU3ju0z+OLN6vM3EbLqZSz+z6WYu/cymm7n0M5vR5mkuk8bkU0keH1w+I8nLk7y443Y3JHlOkofTPi3+jVHfdHl5R7Ztm+3LNldWds708TYTs+lmLv3Mppu59DObbubSz2y6zdNc1vOaySQ5K+1T3FekDcZhVya5M8n+JNcmuS7Jq0d9s0OHnpjpmcmVlZ05ePBwmmY2j7lZmE03c+lnNt3MpZ/ZdDOXfmYz2iznsmvX6HBdT0yenfYNNZcmeaBj/QeHLt+e9jWVY816B2ma2T/mZmE23cyln9l0M5d+ZtPNXPqZTbd5msukzzHvSHJbkovTHZJnJvlEkpMGyxcmefCYtw4AgLk26ZnJS5MsJ/nA0HX3JjklyU1p36CzJ8n9SQ4Pvt40ta0EAGAuTRqT1+eHfw3QWu8cfAEAsCB8Ag4AAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoGw9MXlVkr1JPpPkxiQnrll/XpL7k3wuyd1JTp7GBgIAML8mjclzkrwxyYsHl09KctnQ+u1Jbk7yhiQvShud10xtKwEAmEvbJ7zd3iTnJvnBYPlAkmcOrX9hkn1JHhos35bkrkwQlEtLE27BMVp9nFk93mZiNt3MpZ/ZdDOXfmbTzVz6mc1o8zSXSWPyqSSPDy6fkeTlac9Srtqd5OtDy/uTnDrumy4v78i2bbN92ebKys6ZPt5mYjbdzKWf2XQzl35m081c+plNt3may6QxueqsJHckuSLJw0PXN0mGG/mEJE+P+2aHDj0x0zOTKys7c/Dg4TTNbB5zszCbbubSz2y6mUs/s+lmLv3MZrRZzmXXrtHhup6YPDvJrUkuTfLAmnX70p6dXLU7ySOTfNNZ7yBNM/vH3CzMppu59DObbubSz2y6mUs/s+k2T3OZ9DnmHWlfB3lxfjgkk+TzSU5JcuZg+XWD2wMAsIVNemby0iTLST4wdN29aQPyprRv0Lk8yS1Jnkx7VvKyAACwpU0ak9cPvka5L+1T4QAALAifgAMAQJmYBACgTEwCAFAmJgEAKBOTAACUiUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFAmJgEAKBOTAACUiUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFAmJgEAKBOTAACUiUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFAmJgEAKBOTAACUiUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFAmJgEAKBOTAACUiUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFAmJgEAKBOTAACUiUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFAmJgEAKBOTAACUiUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFAmJgEAKBOTAACUiUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFAmJgEAKFtPTJ6c5NYk+3vWN0n2DH295hi2CwCATWD7Om57c5K7krxkxG3OP6atAQBgU1nPmclLktxzvDYEAIDNZz1nJh9L8qwxt7khyXOSPJzkqiTfGPdNl5bWsQXHYPVxZvV4m4nZdDOXfmbTzVz6mU03c+lnNqPN01zWE5PjXJnkzrSvqbw2yXVJXj3qDsvLO7Jt22zfA7SysnOmj7eZmE03c+lnNt3MpZ/ZdDOXfmbTbZ7mMs2Y/ODQ5duTXDbuDocOPTHTM5MrKztz8ODhNM1sHnOzMJtu5tLPbLqZSz+z6WYu/cxmtFnOZdeu0eE6rZg8M8lfJnlFku8muTDJg5PccdY7SNPM/jE3C7PpZi79zKabufQzm27m0s9sus3TXCaNyeW0T2GfNLi8J8mXkzyV5KYkewfX3Z/k8ODrTdPdVAAA5s2kMXko43/tzzsHXwAALAifgAMAQJmYBACgTEwCAFAmJgEAKBOTAACUiUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFAmJgEAKBOTAACUiUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFAmJgEAKBOTAACUiUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFAmJgEAKBOTAACUbd/oDZili66+a6M34bj78DUXbPQmAAALxJlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlIlJAADKxCQAAGViEgCAMjEJAECZmAQAoExMAgBQJiYBACgTkwAAlG3f6A0A2Eouuvqujd6E4+7D11yw0ZsAzBFnJgEAKBOTAACUiUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFAmJgEAKBOTAACUiUkAAMrEJAAAZWISAICy9cTkyUluTbK/Z/15Se5P8rkkdw9uDwDAFraemLw5yX0967YP1r8hyYuS7E1yzbFtGgAA8249MXlJknt61r0wyb4kDw2Wb0vyqmPYLgAANoHt67jtY0me1bNud5KvDy3vT3LqJN90aWkdW3AMZvU4G63y91y9z6LMaFLm0s9sui3KPBxnpsdc+pnNaPM0l/XE5ChNkuG/1glJnh53p+XlHdm2zXuApmnXrp3l+66s1O+7lZlLP7NZTI4z02cu/cym2zzNZVoxuS/t2clVu5M8Mu5Ohw494czklD366OF132dpqd0pDx48nKY5Dhu1SZlLP7Pp5jjTzz7TzVz6mc1os5zLuB8gpxWTn09ySpIzk3wlyevSvm5yLDvIdB3LPJvGf48u5tLPbBaT48z0mUs/s+k2T3OZNCaXk9yZ5KTB5T1JvpzkqSQ3pX339uVJbknyZNqzkpdNeVsBAJgzk8bkoSTnj7nNfUnOPqatAQBgU/HuFwAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBs0ph8S5LPJPlikveuWXd6ku8m2TP09ZJpbBwAAPNt+wS3eW6SK5P8Stpo/HiSlyW5Z+g2+5OcP+2NAwBgvk0SkxcluSvJdwbLdyR5VY6OybKlpWl8l/l5nI1W+Xuu3mdRZjQpc+lnNt0WZR6OM9NjLv3MZrR5msskMbk7yUNDy/uTnLrmNj+W5COD67+U5A+TfHvcN15e3pFt27xsc5p27dpZvu/KSv2+W5m59DObxeQ4M33m0s9sus3TXCaJySbJcP+ekOTpoeWDSd6e5Oa0Afk3Sf4kbVCOdOjQE85MTtmjjx5e932Wltqd8uDBw2ma47BRm5S59DObbo4z/ewz3cyln9mMNsu5jPsBcpKY3Jf27OSq3UkeGVo+nOSvh5bvTHLVhNtnB5myY5ln0/jv0cVc+pnNYnKcmT5z6Wc23eZpLpM8x/zRJK9M8owk25K8NsltQ+tfmuSWoe91YZIHp7iNAADMqUli8qtJPpTks0m+kOS+tL/+55YkpyX59yTfTLI3yaeTPDvJO47DtgIAMGcmeZo7Sd43+Bp2ydDlt05ncwAA2Ey8lRoAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUCYmAQAoE5MAAJSJSQAAysQkAABlYhIAgDIxCQBAmZgEAKBMTAIAUDZpTL4lyWeSfDHJezvWXzxY/0CSf0zyo1PZOgAA5tokMfncJFcmuSDJ85OcleRlQ+tXkrw/ySuT/GqS7ye5fLqbCQDAPNo+wW0uSnJXku8Mlu9I8qok9wyWfyPJp5IcGCzfluQPknxokg1YWpp0U4/NrB5no1X+nqv3WZQZTcpc+plNt0WZh+PM9JhLP7MZbZ7mstQ0zbjbXJfkobRnH5Pkt5NckeQVg+W3JTk1bUAmyfOS/FOSX5rqlgIAMHcmeZq7STLcvyckeXod6wEA2KImicl9SXYPLe9O8sg61gMAsEVNEpMfTfvmmmck2ZbktWlfF7nqniTnJPnxwfLr1qwHAGCLmiQmv5r2zTSfTfKFJPcl2ZPkliSnJXks7esmPzFYfzjtayYBANjiJnkDDgAAdPIJOAAAlIlJAADKxCQAAGVbKSZPTnJrkv09689Lcn+SzyW5e3D7JPmFtG8oun/w588cz43cIONm8+tpP1v9k2nfYPWzg+v3JPmPwZ97kvz5cdzGjTBuLk2O/N33JHnN4PpF32dOy9Fz+WzaN+olydcGy6vrfv84buNGuCrJ3rT/Xm5McuKa9Yt6nBk3l0U9xiTjZ7Oox5lRc1nkY8xSknenncv9aT+B8JlrbjN/x5mmabbK18eapnlz0zT7O9Ztb5pmX9M0PzdY/uOmad41uPyFpmleOrj8+qZpbpmDv8ssZ3Ni0zQHmqY5bbD81qZpbhxc3tM0zflzsP0bMZc0ra7rF32fWfv1p03TXDm4/LWmaU6fg+0/Hl/nNE3zpaZpfmSwfEfTNL83tH5RjzPj5rLIx5hxs0nT6rrvIu8za78W5RiTpmnOa5rmpqHlG5qmedvQ8lweZ7bSmclLcuTzwtd6Ydpfrv7QYPm2tJ8vvjvtR0HeN7j+9rS/U3OOPvFyKkbN5ntJfj7Jw4Plb+aHfwraqkbNpY995minJ3l52l8fttXtTXJukh8Mlg/k6H8ri3qcGTeXRT7GjJtNn0XfZ4adnsU5xiTJp5JcOrh8YpJn58i/nWROjzNbKSYfG7Fud5KvDy3vTzv0tdd/J8n3k6xMfes21qjZJMm3Bn+elOTqJH83tO7qtDvnx5O8YPqbtqHGzSVJbkj7j/vGJD8R+8xab0/yniRPDV33nrRPZ96V5Iwpb9dGeirJ44PLZ6T9H9xHhtYv6nFm3FySxT3GTDKbZPGOM5POJVmsY8ywd6f9NMH/ziY4zmylmByl7/PD114/vG7RnJzk3iT/nPZTj5Lk/UmuTfLSJH82WLd9Q7ZuY1yZ5I/Svj7lK0mui31m2HKSi5LcOXTdtWkP/i9JckeSmzdgu463s5J8LMkVOfqMwaIfZ/rmsmqRjzGjZrPIx5lx+8yiHmOSdp/46bQ/XFw9dP1cHmcWJSb7Pj98X5KfHLp+Z9qZHJrdps2Fk5P8a5J/SPKOoes/muTBweU9aQ/yz57lhm2wD+bIG1BuT3J27DPDfjPt2aTvD113Q5L/GVy+PcnzsnWemkvafeD2tE9D/cuadYt8nBk1l2SxjzHjZrOox5lxc0kW8xjzi2lnk7QvEbk1yQVD6+fyOLMoMfn5JKckOXOwvPr54d9Iu1P+2uD63027cy6av0/y1zn6qadtST6dI08h/HKSJ3P0afSt7My0HxF60mD5wrT/07PPHHFekv8cWj457bsLV59WuTDtO3W3ysds7Uh73Lg4yQMd6xf1ODNuLsniHmPGzWZRjzOT7DPJ4h1jkvb1xX+RI2foz03yX0Pr5/I4s1WeTlhOexr8pMHlPUm+nPY1FjelfbHv5Wk/T/zJtBV/2eC+b07yt2l/snk8yetnuN2zMG4230r7It1n5ciLfg8keXXa12zckuTbafeV38nRr1vZzCbZZ/ak/RULhwdfbxrcd9H3mb2D2/1U2rNNq/4vyfVpn8o8PLj9G2ayxbNxadp5fGDounvTHtgX+Tgzbi6LeoxJJttn9mTxjjOTzCVZvGNM0p6tf0HaXw30vbQ/XF2R5H2Z4+OMz+YGAKBsUZ7mBgDgOBCTAACUiUkAAMrEJAAAZWISAIAyMQkAQJmYBACgTEwCAFD2/w7kTTnbvSZPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 800x560 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "dark"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_df['Pclass'].hist(density=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAooAAAHICAYAAADJDMt2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAQh0lEQVR4nO3dfaxkd13H8c9wV1tdl8K9twaBIIRGEh8gCiSCFUslkUSQhxjEgligpihqFBolMURCYgRME0FMJCoQIoGWYCngUxQogVCoCvKQkJQYDQ9R6d2LcF2J0jL+cWbj0nx277nLPXemO69Xstk5956Z853ZP847v5mzM5vP5wEAgLu717IHAABgNQlFAAAqoQgAQCUUAQCohCIAAJVQBACgOjbVA99xx96R/r87m5vHs7t76igPCQBwqI66Zy699MTsXL+/IFYUZ7NkY+NemZ3zqQIArK5V7JkLIhQBADh8QhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAdW/YAh+XJL7552SNM7vUvuXLZIwAAa8SKIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAA1UFC8ZFJPpLkQ0n+Ism9J5kIAICVcJBQfEOSa5I8NkMwXjHFQAAArIZjI/d7RJKdJJ9cbL98zJ1ms/MZ6eCO6jjLti7PEwDW0enz/Cqd78eG4oOTfDHJ65M8LMmnk7woyVfOdofNzePZ2PARyMO0vX1i2SMAABPb2lqd8/3YULw4yQ8neXSGlcXXJXlZhlisdndPWVE8ZDs7e8seAQCYyGw2ROLJk3uZz4/mmPstQo0Nxc8n+USSOxbbNyW5br87HdWTXBdeTwC48M3nq3POH/ve8G0Z3n6+dLF9eYZwBADgAjV2RfFrSV6Q5MbFfb6U5OqJZgIAYAWMDcVk+P8THz/VIAAArBaXJQMAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAANXYULwiyX8mueWMPw+bYiAAAFbDsQPs+08ZghEAgDVwkFA8sNlsykc/+uMs27o8TwBYR6fP86t0vj9IKD4oyc1JLk3y/iQvTXLn2Xbe3DyejQ0fgTxM29snlj0CADCxra3VOd+PDcXPJHl5krcstt+R5Nokf3i2O+zunrKieMh2dvaWPQIAMJHZbIjEkyf3Mp8fzTH3W4QaG4pfSPLGM7bfleSH9rvTUT3JdeH1BIAL33y+Ouf8se8NPzvJ7y9uz5JcmeRjk0wEAMBKGBuKNyX5ziQfSfKhJP+R5HVTDQUAwPKNfev5VJKrphwEAIDV4rJkAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgOmgovjbJLRPMAQDAijlIKD4hySOmGgQAgNUyNhQvSfKKJL8+4SwAAKyQYyP3+4MkL0+yc5AHn80OPM95OarjLNu6PE8AWEenz/OrdL4fE4pPT/K1JO9M8uCxD7y5eTwbG66VOUzb2yeWPQIAMLGtrdU5348JxZ9JclmSDye5KMlDk7w5ybPOdafd3VNWFA/Zzs7eskcAACYymw2RePLkXubzoznmfotQY0PxtAcneWP2icTTjupJrguvJwBc+Obz1Tnne28YAIDqoKH4r0muOPwxAABYNVYUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQHVs2QMAAIzxvFe8d9kjTO5d1z9l2SN8AyuKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAAKpjI/aZJXllkh9NMk9yR5LnJPnyhHMBALBkY1YUL0/ywCSPSfLYDIH4C1MOBQDA8o0JxQ8kuWpx+6Ik90vy2ckmAgBgJYx56/m0VyW5Oslbk9w45g6z2XlMdB6O6jjLti7PEwDW2Sqd7w8Sir+R5KVJ3pTkxUmuP9fOm5vHs7HhWpnDtL19YtkjAAAT29panfP9mFD8viTfmuRjSf4nyQ1Jnp99QnF395QVxUO2s7O37BEAgImdPLmX+fxojrXfItSYUPyeDCuIVyS5M8PFLZ8ac/CjepLrwusJABe++Xx1zvljQvGmJI9KcmuGFcV/S3LNlEMBALB8Yz+j+FuLPwAArAlXmwAAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAANVBQvFFSW5LcmuSNye5aJKJAABYCWND8TFJrk7yI4vbFyd5zkQzAQCwAsaG4m1JLk/ytcX2HUkumWQiAABWwrGR+92V5CuL25cl+ckMq4vnNJud51QHdFTHWbZ1eZ4AsM5W6Xw/NhRPe3iStye5Jslnz7Xj5ubxbGy4VuYwbW+fWPYIAMDEtrZW53x/kFD8wSQ3JLkqyd/vt/Pu7ikriodsZ2dv2SMAABM7eXIv8/nRHGu/RaixoXg8yduSPD3JJ8Ye/Kie5LrwegLAhW8+X51z/thQvCrJZpLXnPGzv03yO4c+EQAAK2FsKP7x4g8AAGvC1SYAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqA4SivdJckOSf59oFgAAVshBQvEtSd431SAAAKyWYwfY95lJ7pvkZWPvMJsddJzzc1THWbZ1eZ4AsM5W6Xx/kFD8coZQHGVz83g2NnwE8jBtb59Y9ggAwMS2tlbnfH+QUDyQ3d1TVhQP2c7O3rJHAAAmdvLkXubzoznWfotQk4VikiN7kuvC6wkAF775fHXO+d4bBgCgGruiuJnkz5NcvLh9S5JPJvmVacYCAGDZxobibpIrJpwDAIAV461nAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgGhuKv5zk1iQfTXL9dOMAALAqxoTiDyR5YZIrkzwyycOTPHHKoQAAWL5jI/Z5cpKbk3x1sf32JE9L8tf73XE2O//BDuKojrNs6/I8AWCdrdL5fjafz/fb57VJPpPk1Yvtpya5JsmTJpwLAIAlG/PW8zzJmW17ryRfn2YcAABWxZhQ/HyS+5+xff8kn5tmHAAAVsWYULwpyU8l+fYkG0l+NsnbphwKAIDlG3Mxy+1J/ijJhzO85fzuJLdMOBMAACtgzMUsAACsId/MAgBAJRQBAKiEIgAA1YUcirfEVw0CAEfrhiQfT/LdEzz21UneOsHjntWYq54BABjnp5NckuS/lj3IYVj1FcUfT/KeJH+a5FNJ/izJC5O8L8mnkzwkyfOT3Lb42fuTPKg8zq8m+UCGVcYbM/wDAgAcpjdkaKu/TPKbGdrjvUn+KskDF/t8LslLFj+/PcNXIr8zyT8nuXaxz0MzNM37kvxjkmeVYz0iyd8tHueDSR536M8mqx+KdyV5dIYX+5EZvmf6f5M8PsOL8owkx5M8bfGz92SIwjM9avH7H0tyRYbgvG760QGANfPcxd/PTPLsJE9IcmWStyT53cXv7kqyu/j5+zM0zlOTPCfJry32eUCSV2Vom6cleXU51puSvGDxOD+32D5094S3nm9PsrO4vZPk1sXtLyS5T5I7MvwDzJN8V5IP3e3+lye5LENxJ8m3ZYhFAIApPC7J/ZL8zWL7W/KNb0WfbpUvJPlyhi80Od01ydA712WIyCTZutvjX5Lke5P8yRk/uyvJfZN86Zsf///dE0LxznNsPyTJLyb5/gzfSX3d4vaZvp5hCfjaAABM77+TfDTJT5zl93ee5fZs8fdrk7wjyWuSbGdYFDvTfHG/xy9uT2bV33rez32TfDFDJH5HhqXbi+62zwczXP1878X2z+fs/3AAAN+sf8jwkbkHLLafmOGt5bG2Mnw2MYv7zfONffOVJJ9J8pTF9gOS/N75Dnsu94QVxXP5eJK9DBezfDbJb2e44OUZZ+zz0STXZ/hA6FczfC7guQEAmMZukl9KcnOG1cW7kjzvAPd/ZYYLef8lyeszXLTy5iTvPmOfZ2dYeXxRkouTvPSbnrrwXc8AAFT39LeeAQCYiFAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCo/g+PYI6CMM+dowAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 800x560 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "dark"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['Sex'].hist(density=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAooAAAHICAYAAADJDMt2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAQiElEQVR4nO3dfYxsd13H8c+wV1u9Xgq7W4NAEEIjiQ80CiSCFUslkUSRhxhEQCxQA4oaAo2SGCIhMQLaP3gwkahAiAQKwVLApyi0BEKhKspDQgIxGh6i0r2LsF6JQhn/OHPDpfl09+ztnp3pndcr2dw5u2fmfM/sH+ed38zcnc3n8wAAwB3dY9kDAACwmoQiAACVUAQAoBKKAABUQhEAgEooAgBQnZjqgW+7be9Y/9+dzc2T2d09c5yHBAA4UsfdM5deemq2388viBXF2SzZ2LhHZvueKgDA6lrFnrkgQhEAgKMnFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAANWJZQ9wVB7/ohuXPcLkXv/iq5Y9AgCwRqwoAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAqsOE4sOSfCTJh5L8RZJ7TjIRAAAr4TCh+IYk1yR5VIZgvHKKgQAAWA0nRu53eZKdJJ9YbL9smnEAAFgVY0PxgUm+mOT1SR6S5FNJXpjkK/vdaTa7K6ONd1zHWbZ1OU8AWEdnr/OrdL0fG4oXJ/nRJI/IsLL4uiQvzRCL1ebmyWxs+KzMUdrePrXsEQCAiW1trc71fmwofj7Jx5Pctti+Icm1+91hd/eMFcUjtrOzt+wRAICJzGZDJJ4+vZf5/HiOedAi1NhQvDXDy8+XZojFKzKE476O6yTXhecTAC588/nqXPPHhuLXkjwvydsW9/lSkqsnmgkAgBUwNhST4f9PfMxUgwAAsFp82gQAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAA1dhQvDLJfyW5+Zyvh0wxEAAAq+HEIfb95wzBCADAGvDSMwAA1WFWFB+Q5MYklyZ5f5KXJPn6fneYzc5/sMM4ruMs27qcJwCso7PX+VW63o8Nxc8keVmStyy235nkuUn+8M7usLl5MhsbFiyP0vb2qWWPAABMbGtrda73Y0PxC0neeM72u5P8yH532N09Y0XxiO3s7C17BABgIrPZEImnT+9lPj+eYx60CDU2FJ+R5OFJXpBkluSqJDcddKfjOsl14fkEgAvffL461/yxrw3fkOS7k3wkyYeS/GeS1001FAAAyzd2RfFMkqdNOQgAAKvFp00AAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAACqw4bia5PcPMEcAACsmMOE4mOTXD7VIAAArJYTI/e7JMnLkzwvyR+MffDZ7HxGOrzjOs6yrct5AsA6OnudX6Xr/dhQfE2SlyXZGfvAm5sns7HhLZBHaXv71LJHAAAmtrW1Otf7MaH45CRfS/KuJA8c+8C7u2esKB6xnZ29ZY8AAExkNhsi8fTpvcznx3PMgxahxoTizye5LMmHk1yU5MFJ3pzk6Qfd8bhOcl14PgHgwjefr841f2wonvXAJG/MiEgEAODuzZsIAQCoDhuK/5bkyqMfAwCAVWNFEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIDqxLIHAAAY49kvf9+yR5jcu697wrJH+BZWFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVCdG7DNL8ookP55knuS2JM9M8uUJ5wIAYMnGrChekeT+SR6Z5FEZAvGXpxwKAIDlG7Oi+IHFV5JclOQ+Sd4z5sFns/Oc6pCO6zjLti7nCQDrbJWu92NC8axXJrk6yVuTvO2gnTc3T2Zjw1sgj9L29qlljwAATGxra3Wu94cJxd9M8pIkb0ryoiTX7bfz7u4ZK4pHbGdnb9kjAAATO316L/P58RzroEWoMaH4A0m+Pck/JfnfJNcneU4OCMUkx3aS68LzCQAXvvl8da75Y14b/r4kr8k3o/KKJJ+cbCIAAFbCmBXFG5I8PMktGVYU/z3JNVMOBQDA8o19j+JvL74AAFgTPpYMAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgOowofjCJLcmuSXJm5NcNMlEAACshLGh+MgkVyf5scXti5M8c6KZAABYAWND8dYkVyT52mL7tiSXTDIRAAAr4cTI/W5P8pXF7cuS/HSG1cV9zWbnOdUhHddxlm1dzhMA1tkqXe/HhuJZD03yjiTXJPnsfjtubp7MxobPyhyl7e1Tyx4BAJjY1tbqXO8PE4o/nOT6JE9L8vcH7by7e8aK4hHb2dlb9ggAwMROn97LfH48xzpoEWpsKJ5M8vYkT07y8bEHP66TXBeeTwC48M3nq3PNHxuKT0uymeTV53zvb5P87pFPBADAShgbin+8+AIAYE34tAkAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEB1mFC8V5Lrk/zHRLMAALBCDhOKb0ly01SDAACwWk4cYt+nJrl3kpeOvcNsdthxzs9xHWfZ1uU8AWCdrdL1/jCh+OUMoTjK5ubJbGx4C+RR2t4+tewRAICJbW2tzvX+MKF4KLu7Z6woHrGdnb1ljwAATOz06b3M58dzrIMWoSYLxSTHdpLrwvMJABe++Xx1rvleGwYAoBq7oriZ5M+TXLy4fXOSTyT59WnGAgBg2caG4m6SKyecAwCAFeOlZwAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUQhEAgEooAgBQCUUAACqhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUAlFAAAqoQgAQCUUAQCohCIAAJVQBACgEooAAFRCEQCASigCAFAJRQAAKqEIAEAlFAEAqIQiAACVUAQAoBKKAABUY0Px15LckuSjSa6bbhwAAFbFmFD8oSTPT3JVkocleWiSx005FAAAy3dixD6PT3Jjkq8utt+R5ElJ/vqgO85m5z/YYRzXcZZtXc4TANbZKl3vZ/P5/KB9XpvkM0letdh+YpJrkvzMhHMBALBkY156nic5t23vkeQb04wDAMCqGBOKn09y33O275vkc9OMAwDAqhgTijck+dkk35lkI8kvJHn7lEMBALB8Yz7M8ukkf5Tkwxlecn5PkpsnnAkAgBUw5sMsAACsIX+ZBQCASigCAFAJRQAAqgs5FG+OPzUIAByv65N8LMn3TvDYVyd56wSPe6fGfOoZAIBxfi7JJUn+e9mDHIVVX1H8ySTvTfKnST6Z5M+SPD/JTUk+leRBSZ6T5NbF996f5AHlcX4jyQcyrDK+LcMvEADgKL0hQ1v9ZZLfytAe70vyV0nuv9jnc0levPj+pzP8SeR3JfmXJM9d7PPgDE1zU5J/TPL0cqzLk/zd4nE+mOTRR342Wf1QvD3JIzI82Q/L8Hem/y/JYzI8KU9JcjLJkxbfe2+GKDzXwxc//4kkV2YIzmunHx0AWDPPWvz71CTPSPLYJFcleUuS31v87PYku4vvvz9D4zwxyTOTvGCxz/2SvDJD2zwpyavKsd6U5HmLx/nFxfaRuzu89PzpJDuL2ztJblnc/kKSeyW5LcMvYJ7ke5J86A73vyLJZRmKO0m+I0MsAgBM4dFJ7pPkbxbb35ZvfSn6bKt8IcmXM/xBk7Ndkwy9c22GiEySrTs8/iVJvj/Jn5zzvduT3DvJl+76+N90dwjFr++z/aAkv5LkBzP8TeprF7fP9Y0MS8DPDQDA9P4nyUeT/NSd/Pzrd3J7tvj3tUnemeTVSbYzLIqda76432MWtyez6i89H+TeSb6YIRK/K8PS7UV32OeDGT79fM/F9i/lzn9xAAB31T9keMvc/Rbbj8vw0vJYWxnem5jF/eb51r75SpLPJHnCYvt+SX7/fIfdz91hRXE/H0uyl+HDLJ9N8jsZPvDylHP2+WiS6zK8IfSrGd4X8KwAAExjN8mvJrkxw+ri7UmefYj7vyLDB3n/NcnrM3xo5c1J3nPOPs/IsPL4wiQXJ3nJXZ668LeeAQCo7u4vPQMAMBGhCABAJRQBAKiEIgAAlVAEAKASigAAVEIRAIBKKAIAUP0/T5qPG1pAjxgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 800x560 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "dark"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_df['Sex'].hist(density=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the quick plots above, it looks like the train and test set have already been stratified, let's make the assumption that it has"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 891 entries, 0 to 890\n",
      "Data columns (total 12 columns):\n",
      "PassengerId    891 non-null int64\n",
      "Survived       891 non-null int64\n",
      "Pclass         891 non-null int64\n",
      "Name           891 non-null object\n",
      "Sex            891 non-null object\n",
      "Age            714 non-null float64\n",
      "SibSp          891 non-null int64\n",
      "Parch          891 non-null int64\n",
      "Ticket         891 non-null object\n",
      "Fare           891 non-null float64\n",
      "Cabin          204 non-null object\n",
      "Embarked       889 non-null object\n",
      "dtypes: float64(2), int64(5), object(5)\n",
      "memory usage: 83.7+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    714.000000\n",
       "mean      29.699118\n",
       "std       14.526497\n",
       "min        0.420000\n",
       "25%       20.125000\n",
       "50%       28.000000\n",
       "75%       38.000000\n",
       "max       80.000000\n",
       "Name: Age, dtype: float64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Age'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some Age values are missing, let's pick the columns we need and apply transformations. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_cols = ['Age', 'SibSp', 'Parch']\n",
    "cat_cols = ['Pclass', 'Sex']\n",
    "\n",
    "num_attribs = df[num_cols]\n",
    "cat_attribs = df[cat_cols]\n",
    "\n",
    "X_train_df = df[num_cols + cat_cols]\n",
    "y_train_df = df['Survived'].copy()\n",
    "\n",
    "X_test_df = test_df[num_cols + cat_cols]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preprocessing stage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(891, 8)"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler, OneHotEncoder\n",
    "\n",
    "num_pipeline = Pipeline([\n",
    "    ('imputer', SimpleImputer(strategy='mean')),\n",
    "    ('scaler', MinMaxScaler())\n",
    "])\n",
    "\n",
    "prep_pipeline = ColumnTransformer([\n",
    "    ('numerical', num_pipeline, num_cols),\n",
    "    ('categorical', OneHotEncoder(), cat_cols)\n",
    "])\n",
    "\n",
    "X_train = prep_pipeline.fit_transform(X_train_df)\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's test some classifiers, see how they perform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "predict_pipeline = Pipeline([\n",
    "    ('prep', prep_pipeline),\n",
    "    ('predict', SGDClassifier())\n",
    "])\n",
    "\n",
    "n_jobs = -1\n",
    "random_state = 42\n",
    "\n",
    "param_grid = [\n",
    "    {\n",
    "        'prep__numerical__imputer__strategy': ['mean', 'median', 'most_frequent'],\n",
    "        'prep__numerical__scaler': [MinMaxScaler(), StandardScaler()],\n",
    "        'predict': [SGDClassifier(n_jobs=n_jobs, random_state=random_state)],\n",
    "        'predict__loss': ['hinge', 'log', 'perceptron'],\n",
    "        'predict__penalty': ['l2', 'l1', 'elasticnet'],\n",
    "        'predict__max_iter' : np.r_[1000 : 10000 : 1000]\n",
    "    },\n",
    "    {\n",
    "        'prep__numerical__imputer__strategy': ['mean', 'median', 'most_frequent'],\n",
    "        'prep__numerical__scaler': [MinMaxScaler(), StandardScaler()],\n",
    "        'predict': [RandomForestClassifier(n_jobs=n_jobs, random_state=random_state)],\n",
    "        'predict__n_estimators': np.r_[100:500:100],\n",
    "        'predict__criterion': ['gini', 'entropy'],\n",
    "        'predict__max_features': ['auto','log2']\n",
    "    },\n",
    "    {\n",
    "        'prep__numerical__imputer__strategy': ['mean', 'median', 'most_frequent'],\n",
    "        'prep__numerical__scaler': [MinMaxScaler(), StandardScaler()],\n",
    "        'predict': [SVC(random_state=random_state)],\n",
    "        'predict__C': np.r_[1:10],\n",
    "        'predict__kernel': ['linear', 'rbf', 'sigmoid'],\n",
    "        'predict__gamma': ['scale', 'auto']\n",
    "    },\n",
    "    {\n",
    "        'prep__numerical__imputer__strategy': ['mean', 'median', 'most_frequent'],\n",
    "        'prep__numerical__scaler': [MinMaxScaler(), StandardScaler()],\n",
    "        'predict': [KNeighborsClassifier(n_jobs=n_jobs)],\n",
    "        'predict__n_neighbors': np.r_[2:10],\n",
    "        'predict__weights': ['uniform', 'distance'],\n",
    "        'predict__algorithm': ['auto','ball_tree', 'kd_tree']\n",
    "    }\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1194 candidates, totalling 5970 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 5970 out of 5970 | elapsed:  5.0min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'predict': SVC(C=2, break_ties=False, cache_size=200, class_weight=None, coef0=0.0,\n",
       "     decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
       "     max_iter=-1, probability=False, random_state=42, shrinking=True, tol=0.001,\n",
       "     verbose=False),\n",
       " 'predict__C': 2,\n",
       " 'predict__gamma': 'auto',\n",
       " 'predict__kernel': 'rbf',\n",
       " 'prep__numerical__imputer__strategy': 'mean',\n",
       " 'prep__numerical__scaler': StandardScaler(copy=True, with_mean=True, with_std=True)}"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "grid_search = GridSearchCV(predict_pipeline, param_grid, cv=5, scoring='accuracy', verbose=1)\n",
    "grid_search.fit(X_train_df, y_train_df)\n",
    "\n",
    "best_estimator = grid_search.best_estimator_\n",
    "best_param = grid_search.best_params_\n",
    "best_param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8294017952419811"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Somewhat decent score for a quick attempt. There's more that can be done to improve the model but i'll leave this as future work"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* What is SibSp and Parch? how do the affect Survival?\n",
    "* Are there correlations between class/location of embark and survival?\n",
    "* Feature Engineering\n",
    "    * Some names have titles, can we extract them and use them?\n",
    "* Test other models\n",
    "* Test RandomSearchCV"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}