churn_predict.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"churn_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 7043 entries, 0 to 7042\n",
      "Data columns (total 21 columns):\n",
      " #   Column            Non-Null Count  Dtype  \n",
      "---  ------            --------------  -----  \n",
      " 0   customerID        7043 non-null   object \n",
      " 1   gender            7043 non-null   object \n",
      " 2   SeniorCitizen     7043 non-null   int64  \n",
      " 3   Partner           7043 non-null   object \n",
      " 4   Dependents        7043 non-null   object \n",
      " 5   tenure            7043 non-null   int64  \n",
      " 6   PhoneService      7043 non-null   object \n",
      " 7   MultipleLines     7043 non-null   object \n",
      " 8   InternetService   7043 non-null   object \n",
      " 9   OnlineSecurity    7043 non-null   object \n",
      " 10  OnlineBackup      7043 non-null   object \n",
      " 11  DeviceProtection  7043 non-null   object \n",
      " 12  TechSupport       7043 non-null   object \n",
      " 13  StreamingTV       7043 non-null   object \n",
      " 14  StreamingMovies   7043 non-null   object \n",
      " 15  Contract          7043 non-null   object \n",
      " 16  PaperlessBilling  7043 non-null   object \n",
      " 17  PaymentMethod     7043 non-null   object \n",
      " 18  MonthlyCharges    7043 non-null   float64\n",
      " 19  TotalCharges      7043 non-null   object \n",
      " 20  Churn             7043 non-null   object \n",
      "dtypes: float64(1), int64(2), object(18)\n",
      "memory usage: 1.1+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerID</th>\n",
       "      <th>gender</th>\n",
       "      <th>SeniorCitizen</th>\n",
       "      <th>Partner</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>tenure</th>\n",
       "      <th>PhoneService</th>\n",
       "      <th>MultipleLines</th>\n",
       "      <th>InternetService</th>\n",
       "      <th>OnlineSecurity</th>\n",
       "      <th>...</th>\n",
       "      <th>DeviceProtection</th>\n",
       "      <th>TechSupport</th>\n",
       "      <th>StreamingTV</th>\n",
       "      <th>StreamingMovies</th>\n",
       "      <th>Contract</th>\n",
       "      <th>PaperlessBilling</th>\n",
       "      <th>PaymentMethod</th>\n",
       "      <th>MonthlyCharges</th>\n",
       "      <th>TotalCharges</th>\n",
       "      <th>Churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7590-VHVEG</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>No phone service</td>\n",
       "      <td>DSL</td>\n",
       "      <td>No</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>29.85</td>\n",
       "      <td>29.85</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5575-GNVDE</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>34</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>One year</td>\n",
       "      <td>No</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>56.95</td>\n",
       "      <td>1889.5</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3668-QPYBK</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>53.85</td>\n",
       "      <td>108.15</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7795-CFOCW</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>45</td>\n",
       "      <td>No</td>\n",
       "      <td>No phone service</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>One year</td>\n",
       "      <td>No</td>\n",
       "      <td>Bank transfer (automatic)</td>\n",
       "      <td>42.30</td>\n",
       "      <td>1840.75</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9237-HQITU</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>No</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>70.70</td>\n",
       "      <td>151.65</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   customerID  gender  SeniorCitizen Partner Dependents  tenure PhoneService  \\\n",
       "0  7590-VHVEG  Female              0     Yes         No       1           No   \n",
       "1  5575-GNVDE    Male              0      No         No      34          Yes   \n",
       "2  3668-QPYBK    Male              0      No         No       2          Yes   \n",
       "3  7795-CFOCW    Male              0      No         No      45           No   \n",
       "4  9237-HQITU  Female              0      No         No       2          Yes   \n",
       "\n",
       "      MultipleLines InternetService OnlineSecurity  ... DeviceProtection  \\\n",
       "0  No phone service             DSL             No  ...               No   \n",
       "1                No             DSL            Yes  ...              Yes   \n",
       "2                No             DSL            Yes  ...               No   \n",
       "3  No phone service             DSL            Yes  ...              Yes   \n",
       "4                No     Fiber optic             No  ...               No   \n",
       "\n",
       "  TechSupport StreamingTV StreamingMovies        Contract PaperlessBilling  \\\n",
       "0          No          No              No  Month-to-month              Yes   \n",
       "1          No          No              No        One year               No   \n",
       "2          No          No              No  Month-to-month              Yes   \n",
       "3         Yes          No              No        One year               No   \n",
       "4          No          No              No  Month-to-month              Yes   \n",
       "\n",
       "               PaymentMethod MonthlyCharges  TotalCharges Churn  \n",
       "0           Electronic check          29.85         29.85    No  \n",
       "1               Mailed check          56.95        1889.5    No  \n",
       "2               Mailed check          53.85        108.15   Yes  \n",
       "3  Bank transfer (automatic)          42.30       1840.75    No  \n",
       "4           Electronic check          70.70        151.65   Yes  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X = data.loc[:, 'gender': 'TotalCharges']\n",
    "# Y = data['Churn']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_data = data[data.Churn == 'No'].reset_index()\n",
    "new_data = new_data.iloc[0:1500, ]\n",
    "new_data = pd.concat([new_data, data[data.Churn == 'Yes']], ignore_index=True)\n",
    "\n",
    "X = new_data.loc[:, 'gender': 'TotalCharges']\n",
    "Y = new_data['Churn']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3369 entries, 0 to 3368\n",
      "Data columns (total 22 columns):\n",
      " #   Column            Non-Null Count  Dtype  \n",
      "---  ------            --------------  -----  \n",
      " 0   index             1500 non-null   float64\n",
      " 1   customerID        3369 non-null   object \n",
      " 2   gender            3369 non-null   object \n",
      " 3   SeniorCitizen     3369 non-null   int64  \n",
      " 4   Partner           3369 non-null   object \n",
      " 5   Dependents        3369 non-null   object \n",
      " 6   tenure            3369 non-null   int64  \n",
      " 7   PhoneService      3369 non-null   object \n",
      " 8   MultipleLines     3369 non-null   object \n",
      " 9   InternetService   3369 non-null   object \n",
      " 10  OnlineSecurity    3369 non-null   object \n",
      " 11  OnlineBackup      3369 non-null   object \n",
      " 12  DeviceProtection  3369 non-null   object \n",
      " 13  TechSupport       3369 non-null   object \n",
      " 14  StreamingTV       3369 non-null   object \n",
      " 15  StreamingMovies   3369 non-null   object \n",
      " 16  Contract          3369 non-null   object \n",
      " 17  PaperlessBilling  3369 non-null   object \n",
      " 18  PaymentMethod     3369 non-null   object \n",
      " 19  MonthlyCharges    3369 non-null   float64\n",
      " 20  TotalCharges      3369 non-null   object \n",
      " 21  Churn             3369 non-null   object \n",
      "dtypes: float64(2), int64(2), object(18)\n",
      "memory usage: 579.2+ KB\n"
     ]
    }
   ],
   "source": [
    "new_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = Y.map({\"Yes\": 1, \"No\": 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1869"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y[Y == 1].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1500"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y[Y == 0].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.gender = X.gender.map({\"Male\": 1, \"Female\": 0})\n",
    "X.Partner = X.Partner.map({\"Yes\": 1, \"No\": 0})\n",
    "X.Dependents = X.Dependents.map({\"Yes\": 1, \"No\": 0})\n",
    "X.PaperlessBilling = X.PaperlessBilling.map({\"Yes\": 1, \"No\": 0})\n",
    "X.TotalCharges = pd.to_numeric(X.TotalCharges, errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "gender              0\n",
       "SeniorCitizen       0\n",
       "Partner             0\n",
       "Dependents          0\n",
       "tenure              0\n",
       "PhoneService        0\n",
       "MultipleLines       0\n",
       "InternetService     0\n",
       "OnlineSecurity      0\n",
       "OnlineBackup        0\n",
       "DeviceProtection    0\n",
       "TechSupport         0\n",
       "StreamingTV         0\n",
       "StreamingMovies     0\n",
       "Contract            0\n",
       "PaperlessBilling    0\n",
       "PaymentMethod       0\n",
       "MonthlyCharges      0\n",
       "TotalCharges        5\n",
       "dtype: int64"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = X.fillna({'TotalCharges': X.TotalCharges.mean()})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = pd.get_dummies(X, columns=['PhoneService', 'MultipleLines', 'InternetService', 'OnlineSecurity', 'OnlineBackup', \\\n",
    "                               'DeviceProtection', 'TechSupport', 'StreamingTV', 'StreamingMovies', 'PaymentMethod', \\\n",
    "                               'Contract'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3369 entries, 0 to 3368\n",
      "Data columns (total 41 columns):\n",
      " #   Column                                   Non-Null Count  Dtype  \n",
      "---  ------                                   --------------  -----  \n",
      " 0   gender                                   3369 non-null   int64  \n",
      " 1   SeniorCitizen                            3369 non-null   int64  \n",
      " 2   Partner                                  3369 non-null   int64  \n",
      " 3   Dependents                               3369 non-null   int64  \n",
      " 4   tenure                                   3369 non-null   int64  \n",
      " 5   PaperlessBilling                         3369 non-null   int64  \n",
      " 6   MonthlyCharges                           3369 non-null   float64\n",
      " 7   TotalCharges                             3369 non-null   float64\n",
      " 8   PhoneService_No                          3369 non-null   uint8  \n",
      " 9   PhoneService_Yes                         3369 non-null   uint8  \n",
      " 10  MultipleLines_No                         3369 non-null   uint8  \n",
      " 11  MultipleLines_No phone service           3369 non-null   uint8  \n",
      " 12  MultipleLines_Yes                        3369 non-null   uint8  \n",
      " 13  InternetService_DSL                      3369 non-null   uint8  \n",
      " 14  InternetService_Fiber optic              3369 non-null   uint8  \n",
      " 15  InternetService_No                       3369 non-null   uint8  \n",
      " 16  OnlineSecurity_No                        3369 non-null   uint8  \n",
      " 17  OnlineSecurity_No internet service       3369 non-null   uint8  \n",
      " 18  OnlineSecurity_Yes                       3369 non-null   uint8  \n",
      " 19  OnlineBackup_No                          3369 non-null   uint8  \n",
      " 20  OnlineBackup_No internet service         3369 non-null   uint8  \n",
      " 21  OnlineBackup_Yes                         3369 non-null   uint8  \n",
      " 22  DeviceProtection_No                      3369 non-null   uint8  \n",
      " 23  DeviceProtection_No internet service     3369 non-null   uint8  \n",
      " 24  DeviceProtection_Yes                     3369 non-null   uint8  \n",
      " 25  TechSupport_No                           3369 non-null   uint8  \n",
      " 26  TechSupport_No internet service          3369 non-null   uint8  \n",
      " 27  TechSupport_Yes                          3369 non-null   uint8  \n",
      " 28  StreamingTV_No                           3369 non-null   uint8  \n",
      " 29  StreamingTV_No internet service          3369 non-null   uint8  \n",
      " 30  StreamingTV_Yes                          3369 non-null   uint8  \n",
      " 31  StreamingMovies_No                       3369 non-null   uint8  \n",
      " 32  StreamingMovies_No internet service      3369 non-null   uint8  \n",
      " 33  StreamingMovies_Yes                      3369 non-null   uint8  \n",
      " 34  PaymentMethod_Bank transfer (automatic)  3369 non-null   uint8  \n",
      " 35  PaymentMethod_Credit card (automatic)    3369 non-null   uint8  \n",
      " 36  PaymentMethod_Electronic check           3369 non-null   uint8  \n",
      " 37  PaymentMethod_Mailed check               3369 non-null   uint8  \n",
      " 38  Contract_Month-to-month                  3369 non-null   uint8  \n",
      " 39  Contract_One year                        3369 non-null   uint8  \n",
      " 40  Contract_Two year                        3369 non-null   uint8  \n",
      "dtypes: float64(2), int64(6), uint8(33)\n",
      "memory usage: 319.3 KB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size = 0.25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "scaler.fit(X_train)\n",
    "X_train = scaler.transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = LogisticRegression(max_iter = 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(max_iter=1000)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7758007117437722"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(clf.predict(X_test), y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score, precision_score, recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApkAAAGeCAYAAADfQhp3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnQ0lEQVR4nO3de9hlZV3/8ffHQfIAI41gjQMjk6E2/jioIyOVqZkKqE3kAcHyp1ZAidVlB436aWWSpZVaGhES2c8RskTQULSDYioPBx05Gf4m0GEAL04qiKYi398fe23cPOznefbMrH183q/rmmuetda91/4O65q5Ptz3uu87VYUkSZLUpvuNuwBJkiTNHkOmJEmSWmfIlCRJUusMmZIkSWqdIVOSJEmtM2RKkiSpdYZMSZpBSd6T5GcGaPfFJD/V/Jwkf5fkK0kuSnJQkk8NvVhJM8mQKWnqNUHpm0m+nuTLSc5Isse8Nj+a5N+T3JHka0k+kGT9vDYrk7wlybbmXlub471H+yfaNUkOAg4GztnBj/448Axg36o6tKouA76a5Llt1yhp9hkyJc2K51bVHsAhwOOA3+leSHIY8BE6oevhwDrgc8Ank/xQ02Z34N+AxwKHAyuBHwVuBQ4dZuFJdmv5lscD764d323jEcAXq+rOnnPvbu4nSTvEkClpplTVl4Hz6YTNrj8F3lVVb62qO6rqtqr6PeBC4PebNi8B1gJHVdVVVXV3Vd1UVa+vqvP6fVeSFUlOSvLfTQ/ppUn2S7J/kuoNj0k+luQXm59fmuSTSf4iyW3A65N8Ncn/6mm/T9M7+7Dm+DlJtjTtPtX0Vi7kCODj82r9pSSfb+q8Ksnj513/BeA04LCmF/cPmksfA56e5PsW+T5Jug9DpqSZkmRfOiFra3P8IDo9ku/t0/wf6QwPA/wU8OGq+voOfN2rgGOAI+n0fL4c+MaAn90IXAM8DPhD4H3NvbpeCHy8qm5qAuHpdHoUHwr8DXBuv+CX5MF0emqv7jn3Ajph+iVNnT9Np4f2HlX1TuAE4NNVtUdVva45fz3wHeDRA/65JAkwZEqaHe9PcgdwHXAT8Lrm/Co6/9bd2OczNwLd9y0fukCbxfwi8HtVdXV1fK6qbl3yUx03VNVfVtVdVfVNYDP3DpnHNucAfgn4m6qaq6rvVtXfA98CntTnvns1v98xr84/raqLmzq3VtWXBqyze6+9lmokSb0MmZJmxc9U1Z7AU4HH8L3w+BXgbmB1n8+sBm5pfr51gTYAJHlxM4z89SQfak7vB/z3TtZ73bzjfwcemGRjkkfQGe4/u7n2COA3mqHyryb5avPdD+9z3682v+/Zc25X6uze66tLNZKkXoZMSTOlqj4OnAG8uTm+E/g08II+zV9IZ7IPwL8Cz2qGm/vd993NMPIeVXVEc/o64JF9mncnzjyo59wPzr/lvPvfTWf4/hg6vZgfrKpub+R1wBuqaq+eXw+qqvf0qfNOOoHyUT2nF6pzSUkeDuxOz/C7JA3CkClpFr0FeEaSQ5rj1wD/O8mvJtkzyfcn+SPgMKA7weUf6ISxf07ymCT3S/LQZmLPkQt8z2l0Ju0c0KwxeVCSh1bVzcD1wM81k4NezmAhbzNwNPBivjdUDvC3wAlNL2eSPDjJs5Ps2fcucB7wlHl1/maSJzSf/+Gmt3QQTwX+vaq+NWB7SQIMmZJmUBPy3gX8n+b4P4FnAT9L573LL9FZ5ujHq+r/NW2+RWfyz38BHwVuBy6iM+w+t8BX/Tmd3sePNO3fCTywufZLwG/RGYZ/LLDkouZVNUenF/ThwId6zl/S3O+v6Az/bwVeusitTgVenCTN598LvIFOcL0DeD+dd1UH8WLglAHbStI9suPLqEmSJl2SzcA/VtX7d+EeBwKnVtVhrRUmadkwZEqSJKl1DpdLkiSpdYZMSZIktc6QKUmSpNbttnSTybL33nvX/vvvP+4yJEmSlr1LL730lqrap9+1qQuZ+++/P5dccsm4y5AkSVr2kiy4Ra3D5ZIkSWqdIVOSJEmtM2RKkiSpdYZMSZIktc6QKUmSpNYZMiVJktQ6Q6YkSZJaZ8iUJElS6wyZkiRJap0hU5IkSa0zZEqSJKl1QwuZSU5PclOSKxa4niRvS7I1yWVJHj+sWiRJkjRaw+zJPAM4fJHrRwAHNL+OA/56iLVIkiRphIYWMqvqAuC2RZpsAt5VHRcCeyVZPax6JEmSNDq7jfG71wDX9Rxvb87dOJ5yJEnStNg8t41ztlw/7jImzvqHr+R1z33suMsAxhsy0+dc9W2YHEdnSJ21a9cOsyZJkjQCuxoS567tDJZuXLeqrZLUsnGGzO3Afj3H+wI39GtYVacCpwJs2LChbxCVJEntGXZP4a6GxI3rVrHpkDUcu9HOp0k1zpB5LnBikjOBjcDXqsqhckmSBjDpIXAphsTZN7SQmeQ9wFOBvZNsB14H3B+gqk4BzgOOBLYC3wBeNqxaJEka1LS862cI1KQbWsisqmOWuF7AK4b1/ZKk5aHtUDgt7/oZAjXpxjlcLknSkpYKkW2HQsOb1A5DpiRp4vQGy6VCpKFQmkyGTEnSRFgoWBoipelkyJQkjdRCw98GS2m2GDIlSX0Na5b1QsPfBktpthgyJUn30g2Xw5plbZiUlgdDpiTNuB3tkewNl4ZBSTvLkClJM2zz3DZOOvtyYPAeScOlpDYYMiVpBs0f8j75qAMNjZJGypApSTNg/pC4Q96Sxs2QKUkz4Jwt13PVjbezfvVKwHApafwMmZI0I9avXslZxx827jIkCTBkStLYtLkOZW8vpiRNAkOmJA3ZIDvc7Kr1q1ey6ZA1u3wfSWqLIVOSlrCrPY7ucCNpOTJkSlr2lgqRu9rjaJiUtBwZMiXNrEF7IJcKkYZESdpxhkxJU6utHkhDpCS1z5ApaSr0C5T2QErS5DJkSppIi+1g02WIlKTJZciUNJHcwUaSppshU9JE6fZgdgOmO9hI0nQyZEoaq8WGxV1cXJKmlyFT0lg5LC5Js8mQKWnkensvHRaXpNlkyJQ0Mt1w2Tsk7p7bkjSbDJmSRmLz3DZOOvtywCFxSVoODJmSWrfYwuknH3Wg4VKSlgFDpqTWzZ/MA/ZeStJyY8iU1Aon80iSet1v3AVImg3d3kvAyTySJHsyJbXH3ktJUpc9mZIkSWqdIVOSJEmtc7hc0g7ptzwRcJ/Z5JKk5c2QKWlJvcGyd7eeXk72kST1MmRK6muhYOl6l5KkQRgypWVooSHvXgZLSdKuMGRKy8QgQ969DJaSpF1hyJRmmEPekqRxMWRKM6x3D3GDpSRplAyZ0oxzFx5J0ji4GLs0ozbPbbtniFySpFGzJ1OaEfNnjHcDpmtXSpLGwZApzYje9y/B2eGSpPEyZEpTrLf3shswff9SkjQJDJnSlFloWSK3dZQkTRJDpjQFXO9SkjRtDJnSFHC9S0nStDFkShOuuxTRxnWrfN9SkjQ1DJnSBJm/DBG4FJEkaToZMqUJMn8ZInApIknSdDJkSmPmMkSSpFnktpLSmHV7LwGXIZIkzQx7MqUJYO+lJGnW2JMpjVF35rgkSbNmqCEzyeFJrk6yNclr+lx/SJIPJPlckiuTvGyY9UiTpvsupkPkkqRZM7SQmWQF8HbgCGA9cEyS9fOavQK4qqoOBp4K/FmS3YdVkzSJNq5b5cxxSdLMGWZP5qHA1qq6pqq+DZwJbJrXpoA9kwTYA7gNuGuINUmSJGkEhhky1wDX9Rxvb871+ivgR4AbgMuBX6uqu+ffKMlxSS5JcsnNN988rHolSZLUkmHOLk+fczXv+FnAFuAngUcCH03yiaq6/V4fqjoVOBVgw4YN8+8hTZV+62JKkjRrhhkytwP79RzvS6fHstfLgDdWVQFbk1wLPAa4aIh1SSPXGyy7s8k3rlvlupiSpJk1zJB5MXBAknXA9cCLgGPntdkGPB34RJIfAB4NXDPEmqSRWShYuk2kJGk5GFrIrKq7kpwInA+sAE6vqiuTnNBcPwV4PXBGksvpDK+/uqpuGVZN0ih0w6XBUpK0nA11x5+qOg84b965U3p+vgF45jBrkEZp89w2Tjr7cgCDpSRpWXNbSakF83svTz7qQMOlJGlZM2RKu8jeS0mS7suQKe2i7uQeey8lSfoeQ6a0E+avdenWkJIk3ZshU9oB/WaOu9alJEn3ZciUdsA5W66/p+fSdy8lSVqYIVMa0Oa5bcxdexsb163irOMPG3c5kiRNtPuNuwBpWnTfwXRoXJKkpdmTKTV6J/P04wQfSZIGZ0+m1Oi+b7kQJ/hIkjQ4ezIlfN9SkqS2GTK1rM1fksieSkmS2mHI1LLmkkSSJA2HIVPL3vrVKx0ilySpZU78kSRJUusMmZIkSWqdIVOSJEmt851MLTu9i65fdePtrF+9cswVSZI0e+zJ1LKyeW4bJ519+T1LFrnAuiRJw2FPppaVbg/myUcd6HJFkiQNkSFTM6vfXuTuPy5J0mg4XK6Z1W8vcofHJUkaDXsyNXO6PZjdST0utC5J0ugZMjUTeofGu5N6ultFSpKk0TNkaib09ly6D7kkSeNnyNTMcGhckqTJ4cQfSZIktc6QKUmSpNYZMiVJktQ6Q6YkSZJaZ8jU1Ns8t+2eZYskSdJkMGRq6nXXx3RNTEmSJodLGGkq9S6+7n7kkiRNHnsyNZV69yV3P3JJkiaPPZmaWi6+LknS5LInU5IkSa2zJ1NTpfsuZnefckmSNJkMmZoam+e2cdLZlwOwcd0q38OUJGmCGTI10XpnkXfXwjz5qAOdSS5J0oQzZGri9AuWG9etuqf30oApSdLkM2Rq4vS+c2mwlCRpOhkyNZFcnkiSpOnmEkaSJElqnSFTkiRJrTNkSpIkqXW+k6mJ4ULrkiTNDkOmxmqh5YpcaF2SpOk2cMhM8uCqunOYxWj5cbkiSZJm05IhM8mPAqcBewBrkxwMHF9VvzLs4jSbensvuwHT5YokSZotg0z8+QvgWcCtAFX1OeAnhlmUZld3//Hu0Pj61SsdGpckaQYNNFxeVdcl6T313eGUo1nV7b10/3FJkpaHQULmdc2QeSXZHfhV4PPDLUuzpvvupe9dSpK0PAwSMk8A3gqsAbYDHwF8H1MD2zy3jblrb2PjulW+eylJ0jIxSMh8dFW9uPdEkh8DPjmckjQr5g+R++6lJEnLxyATf/5ywHPSvfQOkfsOpiRJy8uCPZlJDgN+FNgnyat6Lq0EVgxy8ySH0xlqXwGcVlVv7NPmqcBbgPsDt1TVUwasXRPMIXJJkpa3xYbLd6ezNuZuwJ49528Hnr/UjZOsAN4OPIPOu5wXJzm3qq7qabMX8A7g8KraluRhO/wn0ETqroPpELkkScvTgiGzqj4OfDzJGVX1pZ2496HA1qq6BiDJmcAm4KqeNscC76uqbc133rQT36MJtXHdKofIJUlapgaZ+PONJG8CHgs8oHuyqn5yic+tAa7rOd4ObJzX5lHA/ZN8jE5v6Vur6l3zb5TkOOA4gLVrDS2SJEmTbpCJP+8G/gtYB/wB8EXg4gE+lz7nat7xbsATgGfT2VXo/yR51H0+VHVqVW2oqg377LPPAF8tSZKkcRokZD60qt4JfKeqPl5VLweeNMDntgP79RzvC9zQp82Hq+rOqroFuAA4eIB7S5IkaYINEjK/0/x+Y5JnJ3kcncC4lIuBA5Ksa3YKehFw7rw25wBPTrJbkgfRGU53NyFJkqQpN8g7mX+U5CHAb9BZH3Ml8OtLfaiq7kpyInA+nSWMTq+qK5Oc0Fw/pao+n+TDwGXA3XSWObpi5/4omgTdBdivuvF21q9eOe5yJEnSmCwZMqvqg82PXwOeBvfs+LOkqjoPOG/euVPmHb8JeNMg99Nk2zy3jZPOvhzgnj3KJUnS8rTYYuwrgBfSmSX+4aq6IslzgJOABwKPG02JmmTdnkvgnu0j3d1HkiQt1pP5TjoTdy4C3pbkS8BhwGuq6v0jqE0Tql+w3Lhu1T29lwZMSZK0WMjcABxUVXcneQBwC/DDVfXl0ZSmSdX7zqXBUpIk9bNYyPx2Vd0NUFX/k+QLBszlbf6kHvcklyRJC1ksZD4myWXNzwEe2RwHqKo6aOjVaaL0Bkwn9UiSpMUsFjJ/ZGRVaOJtntvG3LW3sXHdKnswJUnSkhYMmVX1pVEWosnWnehjD6YkSRrEIDv+SEBnBrkTfCRJ0iAG2fFHy5g7+EiSpJ0xUMhM8kBgbVVdPeR6NAEWWgfToXJJkjSoJUNmkucCbwZ2B9YlOQT4w6r66SHXpiHrDZO9XGBdkiTtqkF6Mn8fOBT4GEBVbUmy//BK0qgsNAxusJQkSbtqkJB5V1V9LcnQi9Houai6JEkahkFC5hVJjgVWJDkA+FXgU8MtS5IkSdNskCWMXgk8FvgWsBn4GvDrQ6xJI9BdXF2SJGkYBunJfHRV/S7wu8MuRqPj4uqSJGmYBgmZf55kNfBe4MyqunLINWmIete9dHF1SZI0LEuGzKp6WpIfBF4InJpkJXBWVf3R0KtTK1z3UpIkjdpAi7FX1ZeBtyX5D+C3gdcChswp0btUkcsTSZKkURhkMfYfAY4Gng/cCpwJ/MaQ61LLXKpIkiSN0iA9mX8HvAd4ZlXdMOR6JEmSNAMGeSfzSaMoRJIkSbNjwZCZ5B+r6oVJLgeq9xJQVXXQ0KuTJEnSVFqsJ/PXmt+fM4pCJEmSNDsW3PGnqm5sfvyVqvpS7y/gV0ZTniRJkqbRINtKPqPPuSPaLkSSJEmzY7F3Mn+ZTo/lDyW5rOfSnsAnh12YJEmSptdi72RuBj4E/DHwmp7zd1TVbUOtSpIkSVNtsZBZVfXFJK+YfyHJKoOmJEmSFrJUT+ZzgEvpLGGUnmsF/NAQ65IkSdIUWzBkVtVzmt/Xja4cSZIkzYIlZ5cn+bEkD25+/rkkf55k7fBLkyRJ0rQaZO/yvwYOTnIw8NvAO4F/AJ4yzMK0azbPbeOcLdcDcNWNt7N+9coxVyRJkpaTQdbJvKuqCtgEvLWq3kpnGSNNsHO2XM9VN94OwPrVK9l0yJoxVyRJkpaTQXoy70jyO8DPA09OsgK4/3DLUhvWr17JWccfNu4yJEnSMjRIT+bRwLeAl1fVl4E1wJuGWpUkSZKm2pIhswmW7wYekuQ5wP9U1buGXpkkSZKm1iCzy18IXAS8AHghMJfk+cMuTJIkSdNrkHcyfxd4YlXdBJBkH+BfgX8aZmHaOd1Z5c4olyRJ4zTIO5n36wbMxq0Dfk5j0BswnVEuSZLGZZCezA8nOR94T3N8NHDe8ErSrnJWuSRJGrclQ2ZV/VaSnwV+nM7+5adW1dlDr0ySJElTa8GQmeQA4M3AI4HLgd+squtHVZgkSZKm12LvVp4OfBB4HnAp8JcjqUiSJElTb7Hh8j2r6m+bn69O8plRFCRJkqTpt1jIfECSx9F5DxPggb3HVWXolCRJUl+LhcwbgT/vOf5yz3EBPzmsorRzNs9tY+7a29i4btW4S5EkScvcgiGzqp42ykK087oLsM9dexuA62NKkqSxG2SdTE2wzXPbOOnsywHYuG4Vmw5Zw7Eb1465KkmStNwZMqfcOVs6q0qdfNSBhktJkjQx3B5yBmxct8qAKUmSJsqSITMdP5fktc3x2iSHDr80SZIkTatBejLfARwGHNMc3wG8fWgVSZIkaeoN8k7mxqp6fJLPAlTVV5LsPuS6tITujPKrbryd9atXjrscSZKkexkkZH4nyQo6a2OSZB/g7qFWpUX1m1EuSZI0SQYZLn8bcDbwsCRvAP4TOHmQmyc5PMnVSbYmec0i7Z6Y5LtJnj9Q1ctc74zys44/zEk/kiRp4izZk1lV705yKfB0OltK/kxVfX6pzzW9n28HngFsBy5Ocm5VXdWn3Z8A5+9E/cuWM8olSdIkG2R2+VrgG8AHgHOBO5tzSzkU2FpV11TVt4EzgU192r0S+GfgpoGrliRJ0kQb5J3Mf6HzPmaABwDrgKuBxy7xuTXAdT3H24GNvQ2SrAGOorMP+hMXulGS44DjANauXZ69d92JPoCTfSRJ0sRbsiezqg6sqoOa3w+g00P5nwPcO/1uN+/4LcCrq+q7S9RwalVtqKoN++yzzwBfPXu6M8kB1q9e6WQfSZI00XZ4W8mq+kySBXsde2wH9us53he4YV6bDcCZSQD2Bo5McldVvX9H61oO1q9eyVnHHzbuMiRJkpa0ZMhM8qqew/sBjwduHuDeFwMHJFkHXA+8CDi2t0FVrev5njOADxowJUmSpt8gPZl79vx8F513NP95qQ9V1V1JTqQza3wFcHpVXZnkhOb6KTtRryRJkqbAoiGzWV5oj6r6rZ25eVWdB5w371zfcFlVL92Z75AkSdLkWXDiT5Ldmgk5jx9hPZIkSZoBi/VkXkQnYG5Jci7wXuDO7sWqet+Qa5MkSdKUGuSdzFXArXTWsuyul1mAIXMEuutjujamJEmaJouFzIc1M8uv4Hvhsmv+epcakt6A6dqYkiRpWiwWMlcAezDYouoaItfHlCRJ02axkHljVf3hyCqRJEnSzFhsW8l+PZiSJEnSkhYLmU8fWRWSJEmaKQuGzKq6bZSF6L42z21j7lofgyRJmj6L9WRqzM7Zcj2As8olSdLUGWSdTI1Y79qYG9et4tiNa8ddkiRJ0g6xJ3MCuTamJEmadvZkTijXxpQkSdPMnswJ42QfSZI0C+zJnBDd9zC7AdNhckmSNM0MmROid6LPpkPWONlHkiRNNUPmBPE9TEmSNCt8J1OSJEmtM2ROACf7SJKkWeNw+Rg52UeSJM0qQ+aYbJ7bxklnXw7gZB9JkjRzDJlj0t2X/OSjDjRcSpKkmWPIHDH3JZckScuBIXOE+g2RS5IkzSJD5gg5RC5JkpYLlzAaMYfIJUnScmDIlCRJUusMmZIkSWqdIVOSJEmtM2RKkiSpdYZMSZIktc6QKUmSpNa5TuYI9O7ys371ynGXI0mSNHT2ZI5Ab8B0lx9JkrQc2JM5IutXr+Ss4w8bdxmSJEkjYU+mJEmSWmfIlCRJUusMmZIkSWqdIVOSJEmtc+LPkHSXLQJcukiSJC079mQOSXfZIsCliyRJ0rJjT+YQuWyRJElaruzJlCRJUusMmZIkSWqdIVOSJEmtM2QOwea5bcxde9u4y5AkSRobQ+YQdJcucka5JElargyZQ7Jx3SqO3bh23GVIkiSNhSFTkiRJrTNkSpIkqXWGTEmSJLXOHX9a1N2v3L3KJUnScmdPZot6A6YzyyVJ0nJmT+Yu6vZeAvcETPcrlyRJy509mbuo23sJ2IMpSZLUGGpPZpLDgbcCK4DTquqN866/GHh1c/h14Jer6nPDrGkY7L2UJEm6t6H1ZCZZAbwdOAJYDxyTZP28ZtcCT6mqg4DXA6cOq55hcPtISZKk/oY5XH4osLWqrqmqbwNnApt6G1TVp6rqK83hhcC+Q6yndW4fKUmS1N8wQ+Ya4Lqe4+3NuYX8AvChfheSHJfkkiSX3HzzzS2WuOvcPlKSJOm+hhky0+dc9W2YPI1OyHx1v+tVdWpVbaiqDfvss0+LJUqSJGkYhjnxZzuwX8/xvsAN8xslOQg4DTiiqm4dYj2SJEkakWH2ZF4MHJBkXZLdgRcB5/Y2SLIWeB/w81X1hSHW0qrNc9s4+m8+fc/SRZIkSbq3ofVkVtVdSU4EzqezhNHpVXVlkhOa66cArwUeCrwjCcBdVbVhWDW1xZ19JEmSFjfUdTKr6jzgvHnnTun5+ReBXxxmDcPi2piSJEkLc8efHeTamJIkSUszZO4g18aUJElamiFzJ7g2piRJ0uKG+k7mrNg8t+2eHszuhB9JkiQtzJ7MAXRnkwPOKJckSRqAPZkDcja5JEnS4OzJlCRJUusMmZIkSWqdIVOSJEmtM2RKkiSpdYZMSZIktc6QKUmSpNYZMiVJktQ6Q6YkSZJaZ8iUJElS6wyZkiRJap0hU5IkSa0zZEqSJKl1hkxJkiS1zpApSZKk1hkyJUmS1DpDpiRJklpnyJQkSVLrdht3AZNs89w2ztlyPVfdeDvrV68cdzmSJElTw57MRfQGzE2HrBl3OZIkSVPDnswlrF+9krOOP2zcZUiSJE0VezIlSZLUOkOmJEmSWmfIlCRJUusMmZIkSWqdIVOSJEmtM2RKkiSpdYZMSZIktc6QKUmSpNYZMiVJktQ6Q6YkSZJaZ8iUJElS6wyZkiRJap0hU5IkSa0zZEqSJKl1hkxJkiS1zpApSZKk1hkyJUmS1DpDpiRJklpnyJQkSVLrDJmSJElqnSFTkiRJrTNkSpIkqXWGTEmSJLXOkClJkqTWGTIlSZLUOkOmJEmSWmfIlCRJUusMmZIkSWqdIVOSJEmtG2rITHJ4kquTbE3ymj7Xk+RtzfXLkjx+mPVIkiRpNIYWMpOsAN4OHAGsB45Jsn5esyOAA5pfxwF/Pax6JEmSNDrD7Mk8FNhaVddU1beBM4FN89psAt5VHRcCeyVZPcSaJEmSNALDDJlrgOt6jrc353a0jSRJkqbMbkO8d/qcq51oQ5Lj6Ayns3bt2l2vbEDrH75yZN8lSZI0S4YZMrcD+/Uc7wvcsBNtqKpTgVMBNmzYcJ8QOiyve+5jR/VVkiRJM2WYw+UXAwckWZdkd+BFwLnz2pwLvKSZZf4k4GtVdeMQa5IkSdIIDK0ns6ruSnIicD6wAji9qq5MckJz/RTgPOBIYCvwDeBlw6pHkiRJozPM4XKq6jw6QbL33Ck9PxfwimHWIEmSpNFzxx9JkiS1zpApSZKk1hkyJUmS1DpDpiRJklpnyJQkSVLrDJmSJElqnSFTkiRJrTNkSpIkqXWGTEmSJLXOkClJkqTWGTIlSZLUunS2D58eSW4GvjTCr9wbuGWE36d2+fyml89uuvn8ppvPb3qN+tk9oqr26Xdh6kLmqCW5pKo2jLsO7Ryf3/Ty2U03n9908/lNr0l6dg6XS5IkqXWGTEmSJLXOkLm0U8ddgHaJz296+eymm89vuvn8ptfEPDvfyZQkSVLr7MmUJElS6wyZkiRJap0hs5Hk8CRXJ9ma5DV9rifJ25rrlyV5/Djq1H0N8Oxe3Dyzy5J8KsnB46hT/S31/HraPTHJd5M8f5T1aXGDPL8kT02yJcmVST4+6hrV3wD/dj4kyQeSfK55di8bR526rySnJ7kpyRULXJ+IzGLIBJKsAN4OHAGsB45Jsn5esyOAA5pfxwF/PdIi1deAz+5a4ClVdRDweibopejlbsDn1233J8D5o61Qixnk+SXZC3gH8NNV9VjgBaOuU/c14N+9VwBXVdXBwFOBP0uy+0gL1ULOAA5f5PpEZBZDZsehwNaquqaqvg2cCWya12YT8K7quBDYK8nqUReq+1jy2VXVp6rqK83hhcC+I65RCxvk7x7AK4F/Bm4aZXFa0iDP71jgfVW1DaCqfIaTYZBnV8CeSQLsAdwG3DXaMtVPVV1A53ksZCIyiyGzYw1wXc/x9ubcjrbR6O3oc/kF4ENDrUg7Ysnnl2QNcBRwygjr0mAG+fv3KOD7k3wsyaVJXjKy6rSYQZ7dXwE/AtwAXA78WlXdPZrytIsmIrPsNuovnFDpc27+2k6DtNHoDfxckjyNTsj88aFWpB0xyPN7C/Dqqvpup0NFE2SQ57cb8ATg6cADgU8nubCqvjDs4rSoQZ7ds4AtwE8CjwQ+muQTVXX7kGvTrpuIzGLI7NgO7NdzvC+d/3Pb0TYavYGeS5KDgNOAI6rq1hHVpqUN8vw2AGc2AXNv4Mgkd1XV+0dSoRYz6L+dt1TVncCdSS4ADgYMmeM1yLN7GfDG6iyovTXJtcBjgItGU6J2wURkFofLOy4GDkiyrnmp+UXAufPanAu8pJmx9STga1V146gL1X0s+eySrAXeB/y8vScTZ8nnV1Xrqmr/qtof+CfgVwyYE2OQfzvPAZ6cZLckDwI2Ap8fcZ26r0Ge3TY6PdAk+QHg0cA1I61SO2siMos9mUBV3ZXkRDozV1cAp1fVlUlOaK6fApwHHAlsBb5B5//wNGYDPrvXAg8F3tH0ht1VVRvGVbO+Z8Dnpwk1yPOrqs8n+TBwGXA3cFpV9V12RaMz4N+91wNnJLmczvDrq6vqlrEVrXskeQ+dGf97J9kOvA64P0xWZnFbSUmSJLXO4XJJkiS1zpApSZKk1hkyJUmS1DpDpiRJklpnyJQkSVLrDJmSpl6S7ybZ0vNr/0Xafr2F7zsjybXNd30myWE7cY/Tkqxvfj5p3rVP7WqNzX26/12uSPKBJHst0f6QJEe28d2S5BJGkqZekq9X1R5tt13kHmcAH6yqf0ryTODNVXXQLtxvl2ta6r5J/h74QlW9YZH2LwU2VNWJbdciafmxJ1PSzEmyR5J/a3oZL0+yqU+b1Uku6Onpe3Jz/plJPt189r1Jlgp/FwA/3Hz2Vc29rkjy6825Byf5lySfa84f3Zz/WJINSd4IPLCp493Nta83v5/V27PY9KA+L8mKJG9KcnGSy5IcP8B/lk8Da5r7HJrkU0k+2/z+6GbXlz8Ejm5qObqp/fTmez7b77+jJC3EHX8kzYIHJtnS/Hwt8ALgqKq6PcnewIVJzq17D90cC5xfVW9IsgJ4UNP294Cfqqo7k7waeBWd8LWQ5wKXJ3kCnV01NtLZHWUuyceBHwJuqKpnAyR5SO+Hq+o1SU6sqkP63PtM4GjgvCYEPh34ZeAX6GwT98Qk3wd8MslHqurafgU2f76nA+9sTv0X8BPNri8/BZxcVc9L8lp6ejKTnAz8e1W9vBlqvyjJvzb7kEvSogyZkmbBN3tDWpL7Aycn+Qk6WxmuAX4A+HLPZy4GTm/avr+qtiR5CrCeTmgD2J1OD2A/b0rye8DNdELf04GzuwEsyfuAJwMfBt6c5E/oDLF/Ygf+XB8C3tYEycOBC6rqm80Q/UFJnt+0ewhwAJ2A3asbvvcHLgU+2tP+75McABTNdnR9PBP46SS/2Rw/AFiLe49LGoAhU9IsejGwD/CEqvpOki/SCUj3qKoLmhD6bOAfkrwJ+Arw0ao6ZoDv+K2q+qfuQdMjeB9V9YWml/NI4I+bHsfFekZ7P/s/ST4GPItOj+Z7ul8HvLKqzl/iFt+sqkOa3tMPAq8A3kZnT+r/qKqjmklSH1vg8wGeV1VXD1KvJPXynUxJs+ghwE1NwHwa8Ij5DZI8omnzt3SGkR8PXAj8WJLuO5YPSvKoAb/zAuBnms88GDgK+ESShwPfqKr/C7y5+Z75vtP0qPZzJp1h+CcD3VB5PvDL3c8keVTznX1V1deAXwV+s/nMQ4Drm8sv7Wl6B7Bnz/H5wCvTdOsmedxC3yFJ8xkyJc2idwMbklxCp1fzv/q0eSqwJclngecBb62qm+mErvckuYxO6HzMIF9YVZ8BzgAuAuaA06rqs8CBdN5l3AL8LvBHfT5+KnBZd+LPPB8BfgL416r6dnPuNOAq4DNJrgD+hiVGpppaPge8CPhTOr2qnwRW9DT7D2B9d+IPnR7P+ze1XdEcS9JAXMJIkiRJrbMnU5IkSa0zZEqSJKl1hkxJkiS1zpApSZKk1hkyJUmS1DpDpiRJklpnyJQkSVLr/j8iiWn76BN3KQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 792x468 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "clf_prob_pred = clf.predict_proba(X_test)[:, 1]\n",
    "\n",
    "fpr, tpr, threshold = roc_curve(y_test, clf_prob_pred)\n",
    "plt.figure(figsize=(11, 6.5))\n",
    "plt.plot(fpr, tpr)\n",
    "plt.title('ROC-curve (clf)')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC ROC score (clf): 0.85\n"
     ]
    }
   ],
   "source": [
    "print('AUC ROC score (clf): {:.2f}'.format(roc_auc_score(y_test, clf_prob_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix, classification_report, plot_confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.70      0.73       369\n",
      "           1       0.78      0.84      0.81       474\n",
      "\n",
      "    accuracy                           0.78       843\n",
      "   macro avg       0.77      0.77      0.77       843\n",
      "weighted avg       0.78      0.78      0.77       843\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, clf.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[257, 112],\n",
       "       [ 77, 397]], dtype=int64)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, clf.predict(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x206d114e700>"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEGCAYAAADxD4m3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAbAUlEQVR4nO3de5gV1Znv8e+vG2gIdwQUBRUNapAoOoZonCiaC8S5oDnjE5xMNIk55qK5OpPRPImXOJzkZDQ5mYyaUeNojJHBSUwwyWjU0WPMQRENXkCJrXhBiMhVQAS6+z1/VDVuoHt3lezN3rv693meenrvtatWvbt5+mWtWmtVKSIwMyuiploHYGZWLU5wZlZYTnBmVlhOcGZWWE5wZlZYfWodQKnmwQOjz8jhtQ7Dcui/or3WIVgOm7etZ2v769qdOqadNDBWr8n27/7I41vujIjpu3O+3VFXCa7PyOHsc/Hnax2G5fCOb6+tdQiWw7znb9ztOlavaWf+nftn2rd5zDMjd/uEu6GuEpyZ1b8AOuiodRiZOMGZWS5BsC0a49KEE5yZ5eYWnJkVUhC0N8gSTyc4M8utAyc4MyugANqd4MysqNyCM7NCCmCbr8GZWREF4S6qmRVUQHtj5DcnODPLJ1nJ0Bic4MwsJ9HObq3X32Oc4Mwsl2SQwQnOzAoomQfnBGdmBdXhFpyZFZFbcGZWWIFob5CnHTjBmVlu7qKaWSEFYms01zqMTJzgzCyXZKKvu6hmVlAeZDCzQooQ7eEWnJkVVIdbcGZWRMkgQ2OkjsaI0szqhgcZzKzQ2j0PzsyKqJFWMjRGlGZWVzqiKdNWjqT+kuZLekzSIkmXpuWXSHpZ0sJ0O6XkmAsltUpaImlaT3G6BWdmuSSL7SvSNtoCnBwRGyX1BR6Q9F/pZ9+LiMtLd5Y0EZgJHA7sC9wt6ZCIaO/uBE5wZpZLILZVYKlWRASwMX3bN93KPe1hBjA7IrYASyW1AlOAed0d4C6qmeUSAe3RlGkDRkpaULKdU1qXpGZJC4GVwF0R8VD60XmSHpd0vaThadl+wEslhy9Ly7rlBGdmOYmOjBuwKiKOKdmuKa0pItojYjIwFpgiaRJwNXAwMBlYAVyx/cS7Kvt8Lyc4M8slyNWCy1ZnxDrgPmB6RLySJr4O4FqSbigkLbZxJYeNBZaXq9cJzsxya6cp01aOpFGShqWvBwDvB56WNKZkt9OAJ9PXc4GZklokjQcmAPPLncODDGaWS6BK3fByDHCjpGaSxtaciPiVpJskTSZpLD4PfBogIhZJmgMsBtqAc8uNoIITnJnllDw2cPdTR0Q8DhzVRfnHyhwzC5iV9RxOcGaWkx/8bGYFFdDjKoV64QRnZrm5BWdmhRQht+DMrJiSQQY/VcvMCsnPZDCzgkoGGXwNzswKqlFueOkEZ2a5VHAlQ9U5wZlZbn7ojJkVUgRs63CCM7MCSrqoTnBmVlBeydBL9FmzlX2uW0rz+m0gWH/iKNZ9YG/2+sXLDL1/FW2Dk1/x6v+xH5uOGMbgeasZfsefth/fsmwzL148kS37v61WX6HX+dJXH2HKcX9i3boWPveJ9wPw5ycu46Mff4pxB2zgy589iWeWJHfJPurPXuHj5yyib98Otm1r4vofTuKxP4yuZfg152kiKUnTge8DzcB1EfHtap6vFqIJXv3IWLYcMBBtbueAby7m9YlDAFj7wb1ZO32fHfbfcNxebDhuLwD6LXudff+l1cltD7v7jgO4/baDOP9rj2wve2HpEP7pomP5/Pl/2GHf9etbuPRrx7Fm9QAOGL+ey77ze848/ZSdq+xl3EUlvYndlcAHSG41/LCkuRGxuFrnrIX2Yf1oH9YPgBjQzNYxA+izbmumYwc/tIYN7x5RzfCsC08+PpLR+2zaoeylF4d0ue9zrcO2v35h6RD69eugT9922rY1xlKlaulokC5qNdPwFKA1Ip6LiK3AbJLHfhVWn1VbaHnxdd44aBAAw+5ZyQEXLWLv65fStKltl/0Hz1/LhnfvtafDtLfo+BOX82zr0F6f3JJR1OZMW61VM8FlesSXpHM6HynWvmHTzh83DL3Rzr5XPsurZ4yjY0Az604azdL//U5euGQibUP7Muo/Xtph//7PbiT6NbF17IAaRWx57H/ga3zynCf5wRW73IC21+mc6Jtlq7VqJrhMj/iKiGs6HynWPHhgFcOporYO9r3yWV47dgQb/yy5ON0+tC80CZrE+hNH0X/pjsl78Hx3TxvFXqNe5xuXPcgV3zqGPy0fVOtw6kKOxwbWVDUTXO5HfDWkCPb59xfYOqY/66a9OaDQXHIdbtCja9myX0lLrSMYtGAtG6Y4wdW7gYO2cum35nHDtYez+ElfToA3R1EboQVXzVHUh4EJ6eO9XgZmAn9bxfPVRP9nNjJk3mq2jB3A/hcvApIpIYMfWkPLi5tBsG1kP14584Dtxwz44wbahvdj2+iWWoXdq331G/M5YvKrDBm6lR/f+ht+8u8T2fBaXz77xccYOnQrl3zr//Fc61C+8dU/569Oe45999vIzDOfZuaZTwPw9b8/nvXr+tf4W9RWrx9FjYg2SecBd5JME7k+IhZV63y18sYhg/nj9cfsUr7piGHdHrP5sCG89PWuR+2s+r5z2ZQuy+c9sMslYmbfdBizbzqs2iE1lAjR1tsTHEBE/Ab4TTXPYWZ7Xj10P7PwSgYzy8UrGcys0BolwTVGR9rM6kal5sFJ6i9pvqTHJC2SdGlaPkLSXZKeSX8OLznmQkmtkpZImtZTrE5wZpZbhebBbQFOjogjgcnAdEnHAhcA90TEBOCe9D2SJpLMxjgcmA5clS4J7ZYTnJnlEgFtHU2ZtvL1RETExvRt33QLkiWdN6blNwKnpq9nALMjYktELAVaSZaEdssJzsxyy9FFHdm5FDPdzimtR1KzpIXASuCuiHgI2DsiVgCkPzvvT5Vp+WcpDzKYWS45HzqzKiJ2nSjaWVdEOzBZ0jDgNkmTytSVaflnKbfgzCy3CGXastcX64D7SK6tvSJpDED6c2W6W+7ln05wZpZbJQYZJI1KW25IGgC8H3gamAucle52FvDL9PVcYKaklnQJ6ARgfrlzuItqZrlEVGwe3BjgxnQktAmYExG/kjQPmCPpbOBF4PTkvLFI0hxgMdAGnJt2cbvlBGdmOYn2Cjw2MCIeB3a5wV5ErAbe180xs4BZWc/hBGdmueW5vlZLTnBmlovXoppZcUVyHa4ROMGZWW71cDvyLJzgzCyXqNAgw57gBGdmubmLamaF5VFUMyukCCc4MyswTxMxs8LyNTgzK6RAdHgU1cyKqkEacE5wZpaTBxnMrNAapAnnBGdmuTV8C07SDyiTpyPiC1WJyMzqWgAdHQ2e4IAFeywKM2scATR6Cy4ibix9L2lgRGyqfkhmVu8aZR5cj5NZJB0naTHwVPr+SElXVT0yM6tfkXGrsSyz9f4PMA1YDRARjwEnVDEmM6tr2R4ZWA8DEZlGUSPiJWmHYMs+ycbMCq4OWmdZZElwL0l6DxCS+gFfIO2umlkvFBANMoqapYv6GeBcYD/gZWBy+t7Mei1l3GqrxxZcRKwCProHYjGzRtEgXdQso6gHSbpd0quSVkr6paSD9kRwZlanCjSK+lNgDjAG2Be4FbilmkGZWR3rnOibZStD0jhJ90p6StIiSV9Myy+R9LKkhel2SskxF0pqlbRE0rSeQs0yyKCIuKnk/U8knZfhODMrqApN9G0Dzo+IRyUNBh6RdFf62fci4vLSnSVNBGYCh5M0tu6WdEhEdDuro9xa1BHpy3slXQDMJsndHwF+/Va/kZkVQAVGUSNiBbAifb1B0lMkg5ndmQHMjogtwFJJrcAUYF53B5RrwT1CktA6v8mnS2MDLuvxG5hZISl7C26kpNJ17ddExDW71CcdCBwFPAQcD5wn6UySNfHnR8RakuT3YMlhyyifEMuuRR2f9RuYWS+SbwBhVUQcU24HSYOAnwFfiojXJF1N0oDqbEhdAXySruedlI0k00oGSZOAiUD/7bVG/DjLsWZWND0PIGSuSepLktxujoifA0TEKyWfXwv8Kn27DBhXcvhYYHm5+rNME7kY+EG6nQR8B/jr7F/BzAqnAtNElKz//BHwVER8t6R8TMlupwFPpq/nAjMltUgaD0wA5pc7R5YW3N8ARwJ/iIhPSNobuC7DcWZWVB0VqeV44GPAE5IWpmVfA86QNJkkRT5Pev0/IhZJmgMsJhmBPbfcCCpkS3CbI6JDUpukIcBKwBN9zXqrCt3wMiIeoOvrar8pc8wsYFbWc2RJcAskDQOuJRlZ3UgPzUIzK7Yco6g1lWUt6ufSlz+UdAcwJCIer25YZlbXGj3BSTq63GcR8Wh1QjIzq4xyLbgrynwWwMkVjoWW51/nkE/6WTeN5DfLF9Y6BMthyrS1Famn4buoEXHSngzEzBpEUJGlWnuCH/xsZvk1egvOzKw7Dd9FNTPrVoMkuCxLtSTp7yRdlL7fX9KU6odmZnWrQHf0vQo4Djgjfb8BuLJqEZlZXVNk32otSxf13RFxtKQ/AETE2vTxgWbWWxVoFHWbpGbSBqekUVRqqa2ZNaR6aJ1lkaWL+i/AbcBoSbOAB4D/VdWozKy+Ncg1uCxrUW+W9AjwPpKV/6dGhJ9sb9Zb1cn1tSx6THCS9gdeB24vLYuIF6sZmJnVsaIkOJInaHU+fKY/MB5YQvLoLjPrhdQgV+GzdFHfWfo+vcvIp7vZ3cysbuReyZA+pPVd1QjGzBpEUbqokr5S8rYJOBp4tWoRmVl9K9IgAzC45HUbyTW5n1UnHDNrCEVIcOkE30ER8Q97KB4zawSNnuAk9YmItnK3Ljez3kcUYxR1Psn1toWS5gK3Aps6P+x8CrWZ9TIFuwY3AlhN8gyGzvlwATjBmfVWBUhwo9MR1Cd5M7F1apCvZ2ZV0SAZoNxi+2ZgULoNLnnduZlZL1WJ+8FJGifpXklPSVok6Ytp+QhJd0l6Jv05vOSYCyW1SloiaVpPcZZrwa2IiG9m/L5m1ptUpgXXBpyfLh4YDDwi6S7g48A9EfFtSRcAFwD/KGkiMJNkmei+wN2SDomI9u5OUK4F1xh3tDOzPSuSUdQsW9lqIlZ0PkA+IjYATwH7ATOAG9PdbgROTV/PAGZHxJaIWAq0AmUfn1Auwb2vfHhm1mtlvx/cSEkLSrZzuqpO0oHAUcBDwN4RsQKSJAiMTnfbD3ip5LBlaVm3yj34eU3ZL2hmvVaOaSKrIuKYsnVJg0hWR30pIl6Tuu08dvVB2Uiy3NHXzGxHFbqjr6S+JMnt5pK5ta9IGpN+PgZYmZYvA8aVHD4WWF6ufic4M8sna3LreRRVwI+ApyLiuyUfzQXOSl+fBfyypHympBZJ44EJJAsSuuUHP5tZLqJiKxmOBz4GPCFpYVr2NeDbwBxJZwMvAqcDRMQiSXOAxSQjsOeWG0EFJzgzewsqkeAi4gG6n63R5SBnRMwCZmU9hxOcmeXXICsZnODMLD8nODMrpILdTcTMbEdOcGZWVEW44aWZWZfcRTWzYsq4SqEeOMGZWX5OcGZWRBVcyVB1TnBmlps6GiPDOcGZWT6+BmdmReYuqpkVlxOcmRWVW3BmVlxOcGZWSOGlWmZWUJ4HZ2bFFo2R4ZzgzCw3t+B6obEHv8HXfvjC9vf77L+Vm/55H95xzCbGHrwFgIFD2tn0WjOf+8ChtQqz19v6hjj/w29n29Ym2tvgvX+xnjP/4U88u6g/P7hgHJs3NbH32K3845UvMHBwB//98+HcetXo7ccvfao/V975Rw6etLmG36KGPNEXJF0P/CWwMiImVes89WTZs/23J66mpuDmRxfz+/8aym3Xjdq+zzkXLWfTBj+tsZb6tgTfufVZBgzsoG0bfOXUCbzr5Ne46utj+Z8XvcwRx23izltG8J9Xj+asr/6Jkz+8lpM/vBZIktslnxjfe5NbqlEGGar5l3YDML2K9de1ye/dyIoX+rHy5X4lpcEJf72Oe38xvGZxGUgwYGDyF9q2TbRvExIse7aFdx67CYCjTtjAA78etsux9/5iOFNPXbsnw61L6si21VrVElxE3A+sqVb99W7qjLXct1Mim/TuTax9tQ/Ll7bUKCrr1N4On33/oXzkiEkcdcIGDjv6dQ449A3m3TkEgN/9ahivLu+7y3H3zx3GSaeu28PR1pkgGWTIstVYzftKks6RtEDSgm1sqXU4FdGnbwfHfvA17r996A7lJ526jvt+Maw2QdkOmpvh6ruXcPMji1my8G08/3R/vvLdF7n9hpGcO+0QNm9sok+/Hf9An370bbQM6ODAw96oUdT1Q5Ftq7WaDzJExDXANQBDNKIOfiW7710nb6D1iQGsW/VmC6CpOTj+lPWcN31CDSOznQ0a2s6Rx23k4XsHc/pnX+Vbs58Dku7qQ/cM2WHf+345zN3TTg3yl1rzFlwRTT113S7d06Pfu4GXWltYtaJfN0fZnrJudTMb1zcDsGWzePR3gxn39i2sW5X8f9/RAT/9/t785cdWbz+moyPptk6dsa4WIdeVzom+lWjBSbpe0kpJT5aUXSLpZUkL0+2Uks8ulNQqaYmkaT3VX/MWXNG0DOjg6Pdu4PtfHbtD+Ykz3D2tF2te6cvlX9yfjg7R0QEn/NU6jv3Aa9x23Uhuv2EkAMd/aD0fnPnmJeQnHhzEyDHbGHPA1lqFXT8iKnnDyxuAfwV+vFP59yLi8tICSROBmcDhwL7A3ZIOiYj27iqv5jSRW4CpwEhJy4CLI+JH1TpfvdiyuYnTJ+06K+aKL+9fg2isKwdNfIOr7vrjLuWnfWoVp31qVZfHHPmejXz/V89UO7TGUaH8FhH3Szow4+4zgNkRsQVYKqkVmALM6+6AqiW4iDijWnWbWW3lGEAYKWlByftr0uvuPTlP0pnAAuD8iFgL7Ac8WLLPsrSsW+6imlk+AWTvoq6KiGNynuFq4LL0TJcBVwCfJLn811U03fIgg5nlFxm3t1J1xCsR0R4RHcC1JN1QSFps40p2HQssL1eXE5yZ5VbNeXCSxpS8PQ3oHGGdC8yU1CJpPDABmF+uLndRzSy3So2idjUYCUyVNJmkDfg88GmAiFgkaQ6wGGgDzi03ggpOcGaWVwXvJtLNYGS3sy0iYhYwK2v9TnBmlksy0bcxljI4wZlZfnVwp5AsnODMLDe34MysmHxHXzMrroquRa0qJzgzy89dVDMrJD/42cwKzS04MyusxshvTnBmlp86GqOP6gRnZvkEnuhrZsUkwhN9zazAnODMrLCc4MyskHwNzsyKzKOoZlZQ4S6qmRVU4ARnZgXWGD1UJzgzy8/z4MysuJzgzKyQIqC9MfqoTnBmlp9bcGZWWE5wZlZIATTIMxmaah2AmTWagOjItvVA0vWSVkp6sqRshKS7JD2T/hxe8tmFklolLZE0raf6neDMLJ8gGWTIsvXsBmD6TmUXAPdExATgnvQ9kiYCM4HD02OuktRcrnInODPLLyLb1mM1cT+wZqfiGcCN6esbgVNLymdHxJaIWAq0AlPK1e8EZ2b5ZU9wIyUtKNnOyVD73hGxIjlNrABGp+X7AS+V7LcsLeuWBxnMLKdci+1XRcQxFTqxug6me05wZpZPANW9XdIrksZExApJY4CVafkyYFzJfmOB5eUqchfVzPKr0DW4bswFzkpfnwX8sqR8pqQWSeOBCcD8chW5BWdmOVVuqZakW4CpJNfqlgEXA98G5kg6G3gROB0gIhZJmgMsBtqAcyOivVz9TnBmlk9AZJjjlqmqiDO6+eh93ew/C5iVtX4nODPLr0FWMjjBmVl+XotqZoUUUe1R1IpxgjOz/NyCM7NiCqK97OBl3XCCM7N8Guh2SU5wZpZfhaaJVJsTnJnlEkC4BWdmhRThFpyZFVejDDIo6mi4V9KrwAu1jqMKRgKrah2E5VLUf7MDImLU7lQg6Q6S308WqyJi5zv27jF1leCKStKCCt4Ty/YA/5sVg2+XZGaF5QRnZoXlBLdnXFPrACw3/5sVgK/BmVlhuQVnZoXlBGdmheUEV0WSpktaIqlV0gW1jsd6Jul6SSslPVnrWGz3OcFViaRm4ErgQ8BE4AxJE2sblWVwA1CzialWWU5w1TMFaI2I5yJiKzAbmFHjmKwHEXE/sKbWcVhlOMFVz37ASyXvl6VlZraHOMFVj7oo85wcsz3ICa56lgHjSt6PBZbXKBazXskJrnoeBiZIGi+pHzATmFvjmMx6FSe4KomINuA84E7gKWBORCyqbVTWE0m3APOAQyUtk3R2rWOyt85LtcyssNyCM7PCcoIzs8JygjOzwnKCM7PCcoIzs8JygmsgktolLZT0pKRbJb1tN+q6QdLfpK+vK3cjAElTJb3nLZzjeUm7PH2pu/Kd9tmY81yXSPr7vDFasTnBNZbNETE5IiYBW4HPlH6Y3sEkt4j4VEQsLrPLVCB3gjOrNSe4xvU74O1p6+peST8FnpDULOmfJT0s6XFJnwZQ4l8lLZb0a2B0Z0WS7pN0TPp6uqRHJT0m6R5JB5Ik0i+nrcf3Shol6WfpOR6WdHx67F6SfivpD5L+ja7X4+5A0i8kPSJpkaRzdvrsijSWeySNSssOlnRHeszvJB1Wkd+mFZKfbN+AJPUhuc/cHWnRFGBSRCxNk8T6iHiXpBbg95J+CxwFHAq8E9gbWAxcv1O9o4BrgRPSukZExBpJPwQ2RsTl6X4/Bb4XEQ9I2p9ktcY7gIuBByLim5L+AtghYXXjk+k5BgAPS/pZRKwGBgKPRsT5ki5K6z6P5GEwn4mIZyS9G7gKOPkt/BqtF3CCaywDJC1MX/8O+BFJ13F+RCxNyz8IHNF5fQ0YCkwATgBuiYh2YLmk/+6i/mOB+zvrioju7ov2fmCitL2BNkTS4PQcH06P/bWktRm+0xcknZa+HpfGuhroAP4jLf8J8HNJg9Lve2vJuVsynMN6KSe4xrI5IiaXFqR/6JtKi4DPR8SdO+13Cj3frkkZ9oHk0sZxEbG5i1gyr/2TNJUkWR4XEa9Lug/o383ukZ533c6/A7Pu+Bpc8dwJfFZSXwBJh0gaCNwPzEyv0Y0BTuri2HnAiZLGp8eOSMs3AINL9vstSXeRdL/J6cv7gY+mZR8ChvcQ61BgbZrcDiNpQXZqAjpboX9L0vV9DVgq6fT0HJJ0ZA/nsF7MCa54riO5vvZo+uCUfyNpqd8GPAM8AVwN/N+dD4yIV0mum/1c0mO82UW8HTitc5AB+AJwTDqIsZg3R3MvBU6Q9ChJV/nFHmK9A+gj6XHgMuDBks82AYdLeoTkGts30/KPAmen8S3Ct4G3Mnw3ETMrLLfgzKywnODMrLCc4MyssJzgzKywnODMrLCc4MyssJzgzKyw/j8ttIPSpOWrVAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_confusion_matrix(clf, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEWCAYAAACNJFuYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnt0lEQVR4nO3deZxWdd3/8dd7BkQUVBA1XBBULNESlczlvl1JTetnZhrmmvqjxSW37rQsSaPbNfu1aJqSmuSWeku4L3G7pCkgLqAIBipiAi4oxCLw+f1xvoNnrrnmmmtw5prrGt7PHucx53zPd7uYvD7z/Z5zvkcRgZmZWV5dR3fAzMyqj4ODmZk14eBgZmZNODiYmVkTDg5mZtaEg4OZASCpTpK/EwxwcDBbrUk6VNKjkmYB84FdOrpPVh0cHKzqSZopaamkPgXpkySFpP4d1LWaJukI4JfAOcBmEdEzIv7ewd2yKuHgYLViBnBEw4GkzwLdO647ncIvgMMj4onw07BWwMHBasWfgGNyx8cCN+QzSDpI0rOSPpD0hqQRuXPPSVogaZGkFWl/gaQfpfMh6VRJ/5Q0T9IlDfPvko6T9Hiurv9K+Yem4xGSbsyd75If0UjaUtIjkt5JdY+WtF5zH1TSbpKekTQ//dwtd26cpBNzx7Mk7VXQ7qbp+DpJP8/lvSed7yJpQ2BD4KTUp9cknZv7zHXp+DVJcyTdIGnddK5/Qz3p+HuSJktav/lfn9UaBwerFU8B60jaRlI98A3gxoI8C8kCyHrAQcB3JX0VICK2j4gewJeA2RHRI22/yJU/BBgC7AgcDBxf2AlJvYBTgfdb0XcB/w1sDGwDbAaMKJpR6g3cDfwaWJ9s2ufuT/rFmwLI53JJa6VtXWAAsCfZv9230vnj0rY3sAXQA/htkXqHAWcB+0fEO5+kj1ZdHBysljSMHr4IvAy8mT8ZEeMi4oWIWBERzwM3kX3pleuiiHg3Il4HfkVuGivnx8Aosou3ZYmI6RHxYEQsiYi5ZF/4zfXrIGBaRPwpIpZFxE1kn/UrrfgcjUgScDHw0yKnz4mIDyNiJnAZcHRKPxL4ZUT8MyIWkF2XGNYwWkgOAK4FvhQRs1a1f1adurScxaxq/Al4lOwv3RsKT0r6AnAhsB2wBtANuK0V9b+R23+N7C/9fP39gMOBbWk8xVVSmsL5NfCfQE+yP8reayb7xqntvNeATcptr4jDgXeAR3JpS3J1F2unsB+vkX1fbJRLuwaYSRbopn6C/lkV8sjBakZEvEZ2YfpA4I4iWf4MjCG782Zd4PdkUzrl2iy33w+YXXD+58DFEfFhK+qEbEopgM9FxDrAUSX6NRvYvCCtHwWjpFboClwA/LAg/W2yAJFvK99OYT/6ActSuQZHkE3vjZSU/7ezTsDBwWrNCcA+EbGwyLmewLsRsVjSzsA3W1n3DyT1Sl903wduyZ3bCvgCcNUq9LknsAB4X9ImwA9K5L0H2FrSN9OF428Ag4Cxq9AuZNNEf0/TbCtFxArgVrIv9p6SNgfO4OPrODcBp0saIKkH2Z1Nt0TEslw1j0XEi2SjolX5d7Eq5uBgNSUiXo2I8c2c/h5wvqQPyebXb21l9XcBE4BJZBeFr82d2wg4NyI+aqbsIenOoVlkUy0AT6afPyO7yD0/1Vts1ANAuqj7ZeBMsqmg/wK+HBHzctkuzrX1KeC2gnbzegE/aaa57wP/JhuNPUY28hqVzo3i42m8GcBi4JRm6vlvoK+kY5v7XFZ75NubzbJbWYGBETG9DeucGRH926o+s0ryyMGs/Tzechaz6uSRgxntM3Iwq2UODmZm1oSnlczMrAk/BNcJdOu5XqzdZ+OWM1rV2HRdrxlYa154buK8iNhgVcvXr7N5xLJFZeWNRXPvj4gDVrWttuDg0Ams3Wdjhp5XuMyQVbNLvzKoo7tgrbR5n+6FT663SixbRLdPH15W3sWTften5Vzty8HBzKwiBDX0oj0HBzOzShBQV9/RvSibg4OZWaWoNUt9dSwHBzOzivC0kpmZFeORg5mZNSI8cjAzs0LyyMHMzIrw3UpmZtaYL0ibmVkh4WklMzMrwiMHMzNrzNNKZmZWSEC9L0ibmVkhX3MwM7PGPK1kZmbFeORgZmZNeORgZmaNyMtnmJlZMV4+w8zMGvMFaTMzK8bTSmZm1ojf52BmZk15WsnMzIrxBWkzM2vC1xzMzKwReVrJzMyKqaGRQ+2EMTOzGieprK2MetaU9LSk5yRNlvSzlD5C0puSJqXtwFyZcyRNlzRV0v4tteGRg5lZBWRvCW2zkcMSYJ+IWCCpK/C4pHvTucsj4tJGbUuDgGHAtsDGwEOSto6I5c014JGDmVklSKiuvK0lkVmQDrumLUoUORi4OSKWRMQMYDqwc6k2HBzMzCqkFdNKfSSNz23Di9RVL2kSMAd4MCL+kU6dLOl5SaMk9UppmwBv5IrPSmnNcnAwM6uQVgSHeRExJLddXVhXRCyPiMHApsDOkrYDrgS2BAYDbwGXNTRdpDulRhoODmZmldJWF6TzIuJ9YBxwQES8nYLGCuAPfDx1NAvYLFdsU2B2qXodHMzMKkGt2FqqStpA0nppvzswFHhZUt9ctkOAF9P+GGCYpG6SBgADgadLteG7lczMKkC0flRQQl/gekn1ZH/k3xoRYyX9SdJgsimjmcC3ASJisqRbgSnAMuCkUncqgYODmVnF1NW1zWRNRDwP7FAk/egSZUYCI8ttw8HBzKxC2nDk0O4cHMzMKqHM6wnVwsHBzKxCPHIwM7NG2viCdLtzcDAzq5BylsaoFg4OZmaVIE8rmZlZEQ4OZmbWhIODmZk14gvSZmZWXO3EBgcHM7OKUNstn1EJDg5mZhXiaSUzM2uqdmKDg4N1nN5rdWX4bv1Zr3tXVkQwbto8Hpg6l0M+15c9t1qfDxcvA+C2SbN5fvYH7Nq/FwcO2mhl+c16deen97zM6+8t6qiPsNo555JbGPePKay/Xg/GXvMDAO793+f47Q0P8Orrc7jtt6fy2U9n75R5YsIrXHbN3Xz00XK6dq3nB8O/zK47DOzI7nc4jxwASQsiokfa7ws8DpwWEX9trzattiyP4KaJs3jt3UWs2aWO8w/8DC/+60MA7n9pDve+NKdR/idnvseTM98DYNP11uS0Pbd0YKiwr+0/hKO+ujs/vOimlWlb9/8UvxlxLOdd/pdGeXutszZXXnA8G/VZl1dmvMUJZ/+Bx275aaW7XDVW5S1vHandRw6SegL3ABc5MFje/EXLmL8oGx0sXraC2fMX06t717LK7tK/N0+lQGGV8/nPbcmsf73bKG3LzTcqmnfQwI/fXz+w/6dYunQZS5cuY401Vt8Ji1oKDu166VxSV+AOYEzhC7IlLZc0SdJ0SWNT2nWSvp72T5QUkvqk46MkPZ3KXJXegISkBbk6h0gaJ6l7yjdJ0lJJL6T9IQV9GCHprNzxWEl7pf39JD0paaKk2yT1KCi7paSJueOBkiak/Zm5fufr/Lmkk3Pl75M0QdJjkj6Tq+s6STNy/e+zKv/+taTP2muwee+1ePWdhQAM/fQG/PygbThxl36stUZ9k/xf2LwXT858t0m6Vaf7H3uebbbaZLUODJCtrVTOVg3a+76qUcCewE35xPTFvjAiBgMnFhaStCbwHWBOOt4G+AaweyqzHDiyuUYjYlFEDE55ZwN7p+Px5XQ6fRmfCwyNiB2B8cAZBW28CsxPr+QD+BZwXTn1J1cDp0TETsBZwBW5c/XAmbn+F+vjcEnjJY1f8mFt/wXdrUsdp+yxBaPHz2LxRyt4+JW5nHXXZH5y90u8v2gZ39xxk0b5t1h/LZYuW8Gb8xd3UI+tNabN/BeX/uEezj/90I7uSodrmFpqaasG7RnG1wZ6A8cBvwP2zZ3rDpT6L/sk4HrgzHS8L7AT8Ez6h+tOChxAd0mTcvW+1cp+ni7pqLQ/ALgU2AUYBDyR2lsDeLJI2WuAb0k6gyx47VxOg2kUshtwW+7/CN1yWVr69yGNxK4G6D1gUJTTbjWqF5y6xxY8OfNdxr/xPgAfpAvRAOOmz+OMvbdsVGaX/r14yqOGmvCvue9z8nnXcdEPh9Fv404/AC7NC++ttAQ4PCIWSjpG0pERMTqd25hm/iIG1gGOIPvybAgOAq6PiHOK5F+U/sImTRtd2sp+Xh4Rl6byY3PtPRgRR7RQ9nbgPOARYEJEvFNmm3XA+w39LqLUv0+ncsKumzN7/mLuy118Xrd7l5XXInbabD1mvf/xRWcBO/frxcgHX6l0V62VPliwiOE/vpYzTjiQnbYb0NHd6XACaig2tGtwWBYRC9P+ycBDksZGxHzgcOCJZsqdDlwSEUtzUfZh4C5Jl0fEHEm9gZ4R8Vo79f0p4HeStoqI6ZLWAjaNiEbfSBGxWNL9wJXACeVWHhEfpGsKh0XEbco+6Oci4jlJWwH9gSlt93Gq09YbrM1/bLE+r7+3iAsOzC653DZpNrv270W/XmsRwLyFS/jjP15fWebTG/Xg3X9/xNwFSzuo16u3M0beyNPPvcp78xeyx7ALOOXY/Viv51pc8Nv/4d35C/j2j69lmy035tqLhnPj/zzB67PnccXoh7hi9EMAjLrw/7J+r54d/Ck6SvVMGZWjIleH0hfsH4FfSJoK7A4c20x2ATcWlJ8i6VzgAUl1wEdkU0/tEhwiYq6k44CbJDVM95wLFPtzdTTwNeCBgvSxkpaRTU/9RtJ8oB9wcTp/JHBl+lxdgZslzQXuAoZHRKf/9ntl7kKOuXFik/TnZ3/QbJmX317A+fdPbc9uWQm//PFRRdO/+B+fbZL2vaOG8r2jhrZ3l2pKXZVcbC5HuwWHhmcccsc/yx3+Opc+DhiX9o8rKNM/t38LcEupdtIF572aq6NI2REFx1/O7T8CfL65sjn/AYyKiOXltJnLMwM4oMipbQvytViXmdUAeVpptSHpTmBLYJ+O7ouZVTfhkcNqIyIO6eg+mFntqKWRQ+2sH2tmVuPa6jkHSWumh4KfkzRZ0s9Sem9JD0qaln72ypU5R9lDx1Ml7d9SGw4OZmaVkK45lLOVYQmwT0RsDwwGDpC0C3A28HBEDCS7y/NsAEmDgGFk1zQPAK5oWGWiOQ4OZmYVIERdXV1ZW0si07B0UNe0BXAw2QPEpJ9fTfsHAzdHxJJ0M8x0Wnho18HBzKxCWjFy6NOwPE7ahjetS/VpdYg5ZA/t/gPYKCLeAkg/N0zZNwHeyBWfldKa5QvSZmYV0oqH4OZFxJBSGdLt84MlrQfcKWm7Uk0Xq6JU/R45mJlVQttec1gpIt4ne1bsAOBtZe/PaXiPTsO6NLOAzXLFNqWFJXocHMzMKiBbW6nN7lbaII0YkNQdGAq8DIzh49UnjiVbcYGUPkxSN0kDgIHA06Xa8LSSmVmFtOFzDn2B69MdR3XArRExVtKTwK2STgBeBw4DiIjJkm4lW7NtGXBSflWHYhwczMwqpK2ekI6I54EdiqS/Q+PXI+TPjQRGltuGg4OZWSX4fQ5mZlbI73MwM7Mi/D4HMzMrooZig4ODmVlFyEt2m5lZgYbnHGqFg4OZWYU4OJiZWRM1FBscHMzMKsUjBzMza2wVFtXrSA4OZmYVkL3sp3aig4ODmVmF1NXQ0MHBwcysQmooNjg4mJlVgrzwnpmZFVNDlxyaDw6SfkOJd4xGxKnt0iMzs06qs1yQHl+xXpiZdXIiu2OpVjQbHCLi+vyxpLUjYmH7d8nMrHOqoYEDdS1lkLSrpCnAS+l4e0lXtHvPzMw6E2XvcyhnqwYtBgfgV8D+wDsAEfEcsEc79snMrFOSytuqQVl3K0XEGwXRbHn7dMfMrHMSne8huDck7QaEpDWAU0lTTGZmVr5aulupnGml7wAnAZsAbwKD07GZmZWp3CmlahlctDhyiIh5wJEV6IuZWadWS9NK5dyttIWkv0qaK2mOpLskbVGJzpmZdSYqc6sG5Uwr/Rm4FegLbAzcBtzUnp0yM+uM2upWVkmbSfqbpJckTZb0/ZQ+QtKbkial7cBcmXMkTZc0VdL+LbVRzgVpRcSfcsc3Sjq5jHJmZpZkdyu1WXXLgDMjYqKknsAESQ+mc5dHxKWN2pYGAcOAbcn+yH9I0tYR0eydp6XWVuqddv8m6WzgZrK1lr4B3L2qn8jMbLWktnvZT0S8BbyV9j+U9BLZTUPNORi4OSKWADMkTQd2Bp5srkCpkcMEsmDQ8Gm+ne8bcEGLn8DMzFZqxdPPfSTl17e7OiKubqbO/sAOwD+A3YGTJR1Dtj7emRHxHlngeCpXbBalg0nJtZUGlPMJzMysZa2cVpoXEUNarFPqAdwOnBYRH0i6kuwP94Y/4C8Djqf4de5mV92GMp+QlrQdMAhYc2WtETeUU9bMzDJtuW6SpK5kgWF0RNwBEBFv587/ARibDmcBm+WKbwrMLlV/Obeyngf8Jm17AxcD/6f8j2BmZtB2t7IqizLXAi9FxC9z6X1z2Q4BXkz7Y4BhkrpJGgAMBJ4u1UY5I4evA9sDz0bEtyRtBFxTRjkzM0skqG+725V2B44GXpA0KaX9CDhC0mCyKaOZpGvFETFZ0q3AFLI7nU4qdacSlBccFkXECknLJK0DzAH8EJyZWSu11bRSRDxO8UHGPSXKjARGlttGOcFhvKT1gD+Q3cG0gBaGI2Zm1lQNrZ5R1tpK30u7v5d0H7BORDzfvt0yM+tchGpqbaVSD8HtWOpcRExsny6ZmXVCVbTiajlKjRwuK3EugH3auC+2igb0Xosbjmo2llsV6vV5r0CzOqqWV4CWo9RDcHtXsiNmZp2ZgPrOEBzMzKxt1dCL4BwczMwqxcHBzMwayV4BWjvRoZzlMyTpKEk/Tcf9JO3c/l0zM+tc6lTeVg3KeRPcFcCuwBHp+EPgd+3WIzOzTkoqb6sG5UwrfSEidpT0LEBEvCdpjXbul5lZpyKgS7V885ehnODwkaR60trfkjYAVrRrr8zMOqEaig1lBYdfA3cCG0oaSbZK67nt2iszs05G6iTLZzSIiNGSJgD7ko2MvhoRL7V7z8zMOpkaig0tBwdJ/YB/A3/Np0XE6+3ZMTOzzqZa7kQqRznTSneTXW8Q2WtCBwBTgW3bsV9mZp2KaNOX/bS7cqaVPps/Tqu1frvdemRm1hlV0TMM5Wj1E9IRMVHS59ujM2ZmnZnKekN0dSjnmsMZucM6YEdgbrv1yMysExKdb+TQM7e/jOwaxO3t0x0zs86r0wSH9PBbj4j4QYX6Y2bWadXSwnulXhPaJSKWlXpdqJmZlUeC+nJWs6sSpUYOT5NdX5gkaQxwG7Cw4WRE3NHOfTMz61Q61RPSQG/gHbJ3Rjc87xCAg4OZWZk60wXpDdOdSi/ycVBoEO3aKzOzTqiGBg4l3+dQD/RIW8/cfsNmZmZlE3Vlbi3WJG0m6W+SXpI0WdL3U3pvSQ9KmpZ+9sqVOUfSdElTJe3fUhulRg5vRcT55XxkMzMrTbTpyGEZcGZ6KLknMEHSg8BxwMMRcaGks4GzgR9KGgQMI1v2aGPgIUlbR8Ty5hooNXKooQGQmVmVE3SpU1lbSyLirYiYmPY/BF4CNgEOBq5P2a4Hvpr2DwZujoglETEDmA6UfN1zqeCwb4s9NDOzsjSMHMp8TWgfSeNz2/Bm65X6AzsA/wA2ioi3IAsgwIYp2ybAG7lis1Jas5qdVoqId1v8tGZmVrZW3Mo6LyKGtJRJUg+yFStOi4gPSjxkV+xEyRuLauiRDDOz2taKkUMZdakrWWAYnXvu7G1JfdP5vsCclD4L2CxXfFNgdqn6HRzMzCpAZF+45Wwt1pUNEa4FXoqIX+ZOjQGOTfvHAnfl0odJ6iZpADCQ7EHnZrV6yW4zM1sFatMnpHcHjgZekDQppf0IuBC4VdIJwOvAYQARMVnSrcAUsjudTip1pxI4OJiZVUT2hHTbBIeIeJzm7ygtejNRRIwERpbbhoODmVmF1NLzAQ4OZmYVUkvLZzg4mJlVhDrH+xzMzKztNNytVCscHMzMKqSzvc/BzMw+KXWS14SamVnb8bSSmZkV5ZGDmZk1UTuhwcHBzKwiBNR75GBmZoVqKDY4OJiZVYZQDU0sOTiYmVWIRw5mZtZIditr7UQHBwczs0poxVveqoGDg5lZhXj5DDMzayR72U9H96J8Dg5mZhXiu5XMzKyJGppVcnCw6jBt5tsc/6NRK49fm/0O5ww/iGdemMG0194GYP6CRazbozuP/fmcjurmaq3bGl24++rT6Na1C/Vd6hnz8LNcePU9bDdwEy47exg91urG62+9w/CfXM+HCxdz2AFDOOXooSvLb7vVxux59EW8+MqbHfgpOpZHDm1A0oKI6JH2+wKPA6dFxF87tmfWHgb232jll/7y5SsYdOCPOWjv7fnuN/demefcy+9gnR7dO6qLq70lS5dx8Hd/zcJFS+lSX8e915zBQ3+fwkU/OIyf/L87+fvE6Rz5lV045eh9+cXv7+a2+8Zz233jARi05caMvmz4ah4YauuaQ9WvICupJ3APcJEDw+rhf5+ZSv9NN6Bf394r0yKCOx+ayKH779SBPbOFi5YC0LVLPV271BMRbNVvQ/4+cToA455+ma/sPbhJuUP334nb759Qya5WH4m6MrdqUNXBQVJX4A5gTERcXXBuuaRJkqZLGpvSrpP09bR/oqSQ1CcdHyXp6VTmKkn1KX1Brs4hksZJ6p7yTZK0VNILaX9ILm+dpGmSNsgdT5fUp6AfZ0kakfaHSvpL2l9b0ihJz0h6VtLBubqPkzQ3tfluQ12rizsemNAkCPz92VfZcP2ebNlvww7qlQHU1YlHR5/NKw9cyLh/vMyEya/x8j/f4kt7fBaAg/fdkU026tWk3CFf3JHbHxhf6e5WHZW5VYOqDg7AKGBP4KZ8YvpiXxgRg4ETCwtJWhP4DjAnHW8DfAPYPZVZDhzZXKMRsSgiBqe8s4G90/H4XJ4VwI25eoYCz0XEvDI/24+BRyLi88DewCWS1k7n6oGbUvtjihWWNFzSeEnj586bW2aT1W/pR8u499EX+Oq+OzRKv/2B8Ry635BmSlmlrFgR7HHkhWx70LnsuO3mbLNlX04+fzQnHrYHf7vhv+ixVjc++mh5ozI7bbs5ixZ/xEuvvtVBva4O2bSSRw5tYW2gN3Ac8LuCc92BxSXKngRcDyxKx/sCOwHPSJqUjrdoqKthlACMbmUfRwHHpP3jgT+2oux+wNmp3XHAmkC/hj5R+vMREVdHxJCIGLJBnw1a0+eq9tDfp7D9ZzZjw/XXWZm2bNlyxv7tOQ754o4d2DPL+2DBIh6fMI19dx3EtNfe5tBTfsfex1zM7Q9MYMabjf9Y+dp+O3H7/R41gEcObWUJcHhE/Bn4SFL+L/2Nyf6iL2Yd4AjgqlyagOsbRgMR8emIGJHO5UcJzY4miomIN4C3Je0DfAG4txXFBRya61O/iHgpnSv1+Tq1v9w/nkP3azylNO7pqQzcfKOi0xVWOeuv12PlDQFrduvKXjt/mmkz36ZPrx5A9pazs47fnz/e/vjKMpI4eN8duP3B1fx6Q4M2ig5pSnqOpBdzaSMkvZmbEj8wd+6cNO09VdL+5XS1au9WApZFxMK0fzLwkKSxETEfOBx4oplypwOXRMTS3Cv5HgbuknR5RMyR1BvoGRGvtUE/ryGbXvpTRCxvKXPO/cApkk6JiJC0Q0Q8K6k78GXghDboW0359+KljHv6ZS7/0RGN0otdg7DK+1SfdbhixNHU19VRVyfufGgi9z/+It8ethcnfn0PAMaOm8Tovz61ssxuO2zF7Dnv89qb73RUt6tKG04ZXQf8FrihIP3yiLg0nyBpEDAM2JbsD8+HJG3d0vdVNQeHlSJiuqQ/Ar+QNBXYHTi2mewi+7LOl58i6VzgAUl1wEdkU09tERzGkE0nFU4pXSDpNGAToF7SUGBdYGrDeeBXwPPKothMsqBwL3BLRDzTBn2rKWutuQb/fOjiJulXjDi6A3pjhSZPn82eR13UJP2qm8dx1c3jipZ5YuI09jv+snbuWe1oq9AQEY9K6l9m9oOBmyNiCTBD0nRgZ+DJUoWqNjg0POOQO/5Z7vDXufRxZHP2RMRxBWX65/ZvAW4p1U664LxXc3U0Y3uyC9Ev58oc13z2lXkWAd8ukl7Yfot1mVmNKD869JGUv1BzdeEdm804WdIxwHjgzIh4j+wP1KdyeWaltJKq+ZpD1ZN0NnA74Ed2zayk7HJCef8D5jXccJK2cgLDlcCWwGDgLaBhyFYsJEVLlTk4fAIRcWFEbB4Rj7ec28xWa+l9DuVsqyIi3o6I5ek2+z+QTR1BNlLYLJd1U8q44cXBwcysQtrzVta0zFCDQ4CGO5nGAMMkdZM0ABgIPN1SfVV7zcHMrHMRaqO7lSTdRHZ9tI+kWcB5wF6SBpNNGc0kXdOMiMmSbgWmAMuAk8q5s9LBwcysQtrqTtaIOKJI8rUl8o8ERramDQcHM7MKqKann8vh4GBmVik1FB0cHMzMKsQv+zEzsyaqZMHVsjg4mJlVwid4hqEjODiYmVWIp5XMzKwR4ZGDmZkVUUOxwcHBzKxiaig6ODiYmVVItbwfuhwODmZmFVI7ocHBwcyscmooOjg4mJlVQMPLfmqFg4OZWSX4ITgzMyumhmKDg4OZWWW03ct+KsHBwcysQmooNjg4mJlVgl/2Y2ZmxdVQdHBwMDOrEN/KamZmTfiag5mZNSaoc3AwM7Omaic6ODiYmVWAX/ZjZmZF1VBsoK6jO2BmtrqQyttarkejJM2R9GIurbekByVNSz975c6dI2m6pKmS9i+nrw4OZmYVIqmsrQzXAQcUpJ0NPBwRA4GH0zGSBgHDgG1TmSsk1bfUgIODmVmFqMytJRHxKPBuQfLBwPVp/3rgq7n0myNiSUTMAKYDO7fUhoODmVkFlDullAYOfSSNz23Dy2hio4h4CyD93DClbwK8kcs3K6WV5AvSZmYV0oonpOdFxJA2a7apaKmQRw5mZpXSVvNKxb0tqS9A+jknpc8CNsvl2xSY3VJlDg5mZhXSvrGBMcCxaf9Y4K5c+jBJ3SQNAAYCT7dUmaeVzMwqQtS10VNwkm4C9iK7NjELOA+4ELhV0gnA68BhABExWdKtwBRgGXBSRCxvqQ0HBzOzCmjLJ6Qj4ohmTu3bTP6RwMjWtOFpJTMza8IjBzOzCvHaSmZm1oRf9mNmZo2VuW5StXBwMDOrAC/ZbWZmRXlayczMmvDIwczMmqih2ODgYGZWMTUUHRwczMwqQNBmy2dUgiJaXLnVqpykucBrHd2PdtAHmNfRnbBW6cy/s80jYoNVLSzpPrJ/n3LMi4jCN71VlIODVS1J49twTXurAP/OOg+vrWRmZk04OJiZWRMODlbNru7oDlir+XfWSfiag5mZNeGRg5mZNeHgYGZmTTg4rKYkLcjt95X0qqSvdGSf7JPx79TakoPDak5ST+Ae4KKI+GtH98c+Of9OrS04OKzGJHUF7gDGRMTVBeeWS5okabqksSntOklfT/snSgpJfdLxUZKeTmWuklSf0vN/zQ6RNE5S95RvkqSlkl5I+0MK+jBC0lm547GS9kr7+0l6UtJESbdJ6lFQdktJE3PHAyVNSPszc/3O1/lzSSfnyt8naYKkxyR9JlfXdZJm5Ppf7lOv7a6af6eS6iRNk7RB7ni6pD4F/ThL0oi0P1TSX9L+2pJGSXpG0rOSDs7VfZykuanNdxvqslXn4LB6GwXsCdyUT0xfAgsjYjBwYmEhSWsC3wHmpONtgG8Au6cyy4Ejm2s0IhZFxOCUdzawdzoeX06n05fXucDQiNgRGA+cUdDGq8B8SYNT0reA68qpP7kaOCUidgLOAq7InasHzsz1v5pU7e80IlYAN+bqGQo8FxHlLrfxY+CRiPg8sDdwiaS107l64KbU/pgy67MSvPDe6mttoDdwHPA7YN/cue7A4hJlTwKuB85Mx/sCOwHPKFtYrDvpSwboLmlSrt63WtnP0yUdlfYHAJcCuwCDgCdSe2sATxYpew3wLUlnkH3R7VxOg2kUshtwmz5eKK1bLktL/z4dpRZ+p6OAu4BfAccDf2xF2f2A/5MbTa4J9ANeonp/JzXLwWH1tQQ4PCIWSjpG0pERMTqd25jm/yJeBziC7Muz4YtEwPURcU6R/IvSX3OkKYZLW9nPyyPi0lR+bK69ByPiiBbK3g6cBzwCTIiId8pssw54v6HfRZT69+lIVf87jYg3JL0taR/gC5QYjRQh4NCImFrkXLX+TmqWp5VWX8siYmHaPxkYKWnddHw48EQz5U4Hfh0RS3NpDwNfl7QhgKTekjZvj04nTwG7S9oqtbeWpK0LM0XEYuB+4Epa8RdqRHwAzJB0WKpfkrZP+1sB/YEpn/RDtINa+Z1eQza9dGtELG9FufuBU5SGMpJ2SD+7A1+m+c9nq8AjByMipkv6I/ALSVOB3YFjm8kusv+w8+WnSDoXeEBSHfAR2TRFuywjHhFzJR0H3CSpYbrnXOCVItlHA18DHihIHytpGdn01G8kzSeborg4nT8SuDJ9rq7AzcqWRr8LGF7wRVp1qvx3OoYsWBcG7AsknQZsAtRLGgqsCzSMFC4gm456PgWImWRB4V7gloh4pg36ZomXz7BOLc1PrxsRP+novlgmTUVdHhH/2dF9seZ55GCdlqQ7gS2BfTq6L5aRdDbwXVp3rcE6gEcOZmbWhC9Im5lZEw4OZmbWhIODmZk14eBgq4XcukIvKluLaa1PUFd+HaBrJA0qkXcvSbutQhsr138qJ70gz4JS54vkb7SGlRk4ONjqo2Htn+2ApWTrCK3UsKhca0XEiRFR6oG4vciePDarKQ4Otjp6DNgq/VX/N0l/Bl6QVC/pkrTq5/OSvg0rn5D+raQpku4GNmyoSNmKpEPS/gHKVol9TtLDkvqTBaHT06jlPyVtIOn21MYzknZPZdeX9ICy1UavInswrSRJ/6Ns1djJkoYXnLss9eVhfbwKarMrzZoV8nMOtlqR1AX4EnBfStoZ2C4iZqQv2PkR8fn05PUTkh4AdgA+DXwW2Ihs6YxRBfVuAPwB2CPV1Tsi3pX0e2BBbn2oP5M9APa4pH5kS0JsQ7YG1OMRcb6kg4BGX/bNOD610Z1sgbzb0/pRawMTI+JMST9NdZ9MttLsdyJimqQvkK0062dArCgHB1td5FcSfQy4lmy65+mImJHS9wM+p4/fBbAuMBDYg2w56OXAbEmPFKl/F+DRhroi4t1m+jEUGKSPV3tdR9nLefYgW+aDiLhb0ntlfKZTJR2S9jdLfX0HWAHcktJvBO5QyyvNmjXi4GCri5UriTZIX5IL80lk73C4vyDfgUBLT4uqjDyQTeXuGhGLivSl7CdSlb2gaGiq69+SxpEtYV1M0PJKs2aN+JqD2cfuB76r7G1qSNpa2ctkHgWGpWsSfcleNFPoSWBPSQNS2d4p/UOgZy7fA2RTPKR8g9Puo6QlJSR9CejVQl/XBd5LgeEzZCOXBnVAw+jnm2TTVc2uNGtWjIOD2ceuIbueMFHSi8BVZKPrO4FpwAtky3//b2HBiJhLdp3gDknP8fG0zl+BQxouSAOnAkPSBe8pfHzX1M+APZS92nQ/4PUW+nof0EXS82SrlT6VO7cQ2FbZa1H3Ac5P6UcCJ6T+TQYOxqwZXlvJzMya8MjBzMyacHAwM7MmHBzMzKwJBwczM2vCwcHMzJpwcDAzsyYcHMzMrIn/D5gH7SORFoMgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "disp = plot_confusion_matrix(clf, X_test, y_test, display_labels=['Клиент не ушел', 'Клиент ушел'], cmap=plt.cm.Blues)\n",
    "disp.ax_.set_title(\"Матрица ошибок\")\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}