05-model-evaluation-and-improvement.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_blobs\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set score: 0.88\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n  \"this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# create a synthetic dataset\n",
    "X, y = make_blobs(random_state=0)\n",
    "# split data and labels into a training and a test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
    "# instantiate a model and fit it to the training set\n",
    "logreg = LogisticRegression().fit(X_train, y_train)\n",
    "# evaluate the model on the test set\n",
    "print(\"Test set score: {:.2f}\".format(logreg.score(X_test, y_test)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation scores: 0.9600000000000002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n  \"this warning.\", FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n  \"this warning.\", FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n  \"this warning.\", FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n  \"this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "iris = load_iris()\n",
    "logreg = LogisticRegression()\n",
    "\n",
    "scores = cross_val_score(logreg, iris.data, iris.target,cv=5)\n",
    "print(\"Cross-validation scores: {}\".format(scores.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0. 0. 0.]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "kf=KFold(n_splits=3)\n",
    "score=cross_val_score(logreg,iris.data,iris.target,cv=kf)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.9  0.96 0.96]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n  \"this warning.\", FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n  \"this warning.\", FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n  \"this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "kf=KFold(n_splits=3,shuffle=True,random_state=0)\n",
    "score=cross_val_score(logreg,iris.data,iris.target,cv=kf)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {'C': [0.001, 0.01, 0.1, 1, 10, 100],\n",
    "              'gamma': [0.001, 0.01, 0.1, 1, 10, 100]}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ML\\lib\\site-packages\\sklearn\\model_selection\\_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9736842105263158"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import train_test_split\n",
    "grid=GridSearchCV(SVC(),param_grid,cv=5)\n",
    "X_train,X_test,y_train,y_test=train_test_split(iris.data,iris.target,random_state=0)\n",
    "grid.fit(X_train,y_train)\n",
    "grid.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 100, 'gamma': 0.01}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9732142857142857"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=100, cache_size=200, class_weight=None, coef0=0.0,\n  decision_function_shape='ovr', degree=3, gamma=0.01, kernel='rbf',\n  max_iter=-1, probability=False, random_state=None, shrinking=True,\n  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mean_fit_time  std_fit_time  mean_score_time  std_score_time param_C  \\\n0       0.007181      0.012401         0.001804        0.002612   0.001   \n1       0.000998      0.000631         0.000598        0.000488   0.001   \n2       0.000798      0.000399         0.000199        0.000399   0.001   \n3       0.000591      0.000483         0.000392        0.000480   0.001   \n4       0.000598      0.000488         0.000600        0.000490   0.001   \n\n  param_gamma                        params  split0_test_score  \\\n0       0.001  {'C': 0.001, 'gamma': 0.001}              0.375   \n1        0.01   {'C': 0.001, 'gamma': 0.01}              0.375   \n2         0.1    {'C': 0.001, 'gamma': 0.1}              0.375   \n3           1      {'C': 0.001, 'gamma': 1}              0.375   \n4          10     {'C': 0.001, 'gamma': 10}              0.375   \n\n   split1_test_score  split2_test_score       ...         mean_test_score  \\\n0           0.347826           0.363636       ...                0.366071   \n1           0.347826           0.363636       ...                0.366071   \n2           0.347826           0.363636       ...                0.366071   \n3           0.347826           0.363636       ...                0.366071   \n4           0.347826           0.363636       ...                0.366071   \n\n   std_test_score  rank_test_score  split0_train_score  split1_train_score  \\\n0        0.011371               22            0.363636            0.370787   \n1        0.011371               22            0.363636            0.370787   \n2        0.011371               22            0.363636            0.370787   \n3        0.011371               22            0.363636            0.370787   \n4        0.011371               22            0.363636            0.370787   \n\n   split2_train_score  split3_train_score  split4_train_score  \\\n0            0.366667            0.366667            0.362637   \n1            0.366667            0.366667            0.362637   \n2            0.366667            0.366667            0.362637   \n3            0.366667            0.366667            0.362637   \n4            0.366667            0.366667            0.362637   \n\n   mean_train_score  std_train_score  \n0          0.366079         0.002852  \n1          0.366079         0.002852  \n2          0.366079         0.002852  \n3          0.366079         0.002852  \n4          0.366079         0.002852  \n\n[5 rows x 22 columns]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ML\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n  warnings.warn(*warn_args, **warn_kwargs)\nD:\\ML\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n  warnings.warn(*warn_args, **warn_kwargs)\nD:\\ML\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n  warnings.warn(*warn_args, **warn_kwargs)\nD:\\ML\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n  warnings.warn(*warn_args, **warn_kwargs)\nD:\\ML\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n  warnings.warn(*warn_args, **warn_kwargs)\nD:\\ML\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n  warnings.warn(*warn_args, **warn_kwargs)\nD:\\ML\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "# convert to Dataframe\n",
    "results = pd.DataFrame(grid.cv_results_)\n",
    "# show the first 5 rows\n",
    "print(results.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'results' is not defined",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-19-7d3af14c260d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mget_ipython\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun_line_magic\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'matplotlib'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'inline'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[0mscores\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresults\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmean_test_score\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m6\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m6\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;31m# plot the mean cross-validation scores\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'results' is not defined"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "import mglearn\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "scores = np.array(results.mean_test_score).reshape(6, 6)\n",
    "\n",
    "# plot the mean cross-validation scores\n",
    "mglearn.tools.heatmap(scores, xlabel='gamma', xticklabels=param_grid['gamma'],\n",
    "                      ylabel='C', yticklabels=param_grid['C'], cmap=\"viridis\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation scores:  [0.96666667 1.         0.96666667 0.96666667 1.        ]\nMean cross-validation score:  0.9800000000000001\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'C': [0.001, 0.01, 0.1, 1, 10, 100],\n",
    "              'gamma': [0.001, 0.01, 0.1, 1, 10, 100]}\n",
    "scores = cross_val_score(GridSearchCV(SVC(), param_grid, cv=5),\n",
    "                         iris.data, iris.target, cv=5)\n",
    "print(\"Cross-validation scores: \", scores)\n",
    "print(\"Mean cross-validation score: \", scores.mean())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_digits\n",
    "\n",
    "digits = load_digits()\n",
    "y = digits.target == 9\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    digits.data, y, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dummy score: 0.80\nlogreg score: 0.98\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.dummy import DummyClassifier\n",
    "\n",
    "dummy = DummyClassifier().fit(X_train, y_train)\n",
    "pred_dummy = dummy.predict(X_test)\n",
    "print(\"dummy score: {:.2f}\".format(dummy.score(X_test, y_test)))\n",
    "\n",
    "logreg = LogisticRegression(C=0.1).fit(X_train, y_train)\n",
    "pred_logreg = logreg.predict(X_test)\n",
    "print(\"logreg score: {:.2f}\".format(logreg.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix:\n[[401   2]\n [  8  39]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1bbd3bbc4e0>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAAD8CAYAAABJsn7AAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEmxJREFUeJzt3Xu4VXWdx/H395wDircwNRMOCRVeMi85aajhCJqJSVhjBpNahOJjWt4qdZoZo6lnsqbMpstE6qSN4SWvOVZkymiNAoqIIPpIZnJEQzIv6Axw9vnNH2fZ7Ohc9sF9zu/sxfvlsx72Xmudtb+PD8+H7/Ndv7VPpJSQJA28ptwFSNLmygCWpEwMYEnKxACWpEwMYEnKxACWpEwMYEnqQUQ0R8QDEXFr8X5MRMyPiMci4pqIGFrs36J4v6I4Prq3axvAktSzM4HlVe8vAi5OKY0F/gjMKPbPAP6YUnorcHFxXo8MYEnqRkS0Au8DLi3eBzAR+HFxyhXAscXrKcV7iuOHF+d3q6XeBW9sw5rHfdROf2HYiPG5S9Ag1L7+qR4DqxZ9yZwhO765t8/7BvBZYNvi/Q7A8yml9uJ9GzCyeD0SWAmQUmqPiBeK89d0d3E7YEmbrYiYGRH3VW0zq44dA6xOKd1f/SNdXCbVcKxL/d4BS9KA6qjUfGpKaTYwu5vDhwDvj4ijgS2B7ejsiIdHREvRBbcCq4rz24BRQFtEtACvA57r6fPtgCWVS6W99q0HKaULUkqtKaXRwFTgjpTSR4A7geOK0z4K3Fy8vqV4T3H8jtTLt53ZAUsqlZQ6+vsjzgOujogvAg8AlxX7LwN+GBEr6Ox8p/Z2IQNYUrl01D+AU0rzgHnF68eBA7s453+BD/XlugawpHLp/w64bgxgSeXSh5twuRnAksrFDliS8ki9rG4YTAxgSeXSDzfh+osBLKlcHEFIUibehJOkTOyAJSkTb8JJUibehJOkPFJyBixJeTgDlqRMHEFIUiZ2wJKUSWVD7gpqZgBLKhdHEJKUiSMIScrEDliSMjGAJSmP5E04ScrEGbAkZeIIQpIysQOWpEzsgCUpEztgScqk3S9kl6Q87IAlKRNnwJKUiR2wJGViByxJmdgBS1ImroKQpExSyl1BzQxgSeXiDFiSMmmgAG7KXYAk1VXqqH3rQURsGRELIuLBiFgWEbOK/VdFxKMRsTQiLo+IIcX+iIhvRsSKiFgSEfv3VqoBLKlcKpXat56tAyamlPYF9gOOiohxwFXAHsDewDDg5OL8ScDYYpsJfLe3D3AEIalc6jSCSCklYG3xdkixpZTSba+eExELgNbi7RTgyuLn7o2I4RGxS0rp6e4+ww5YUrl0dNS+9SIimiNiMbAa+EVKaX7VsSHAicDPil0jgZVVP95W7OuWASypXPowA46ImRFxX9U2888ulVIlpbQfnV3ugRHx9qrD3wHuSindXbyPrqrpqVRHEJJKJXXUvg44pTQbmF3Dec9HxDzgKGBpRFwI7AScWnVaGzCq6n0rsKqn69oBSyqXOo0gImKniBhevB4GHAE8EhEnA+8FpqX0Z0spbgFOKlZDjANe6Gn+C3bAksqm99UNtdoFuCIimulsVq9NKd0aEe3A74B7IgLghpTSF4DbgKOBFcArwPTePsAAllQu9VsFsQR4Rxf7u8zNYvXD6X35DEcQdVapVDjuY6fzic9cCEDbqmeYdspZHP3hGZz7D//Mhg0bALhv8UN8aPoZ7Hvo+5h75909XVIl1do6gtvnXsdDS+bx4OI7+OQZM3KXVA51XAXR3wzgOvuP627mzaPf9Kf3F3/3ck788LHcds1lbLftNlx/688B2GXnN/DFz53L0e+ZkKtUZdbe3s5nPjuLvfc5jEPePZnTTvsYe+45NndZjS+l2rfMeg3giNgjIs4rHrG7pHi950AU12ieWf0sd/33Av5m8nsBSCkx//4HOfKw8QBMOfoI7rjrHgBG7rIzu791DE3R1coVbQ6eeWY1DyxeCsDatS/zyCOPMXLEGzNXVQJl6YAj4jzgajrXty0AFhav50TE+f1fXmO56JLvcc4nZhDR+b/1+RdeZNtttqalpRmAnXfakdXP/iFniRqkdt21lf32fTvzFzyQu5TG15Fq3zLr7SbcDGCvlNKG6p0R8XVgGfDl/iqs0cz79Xxev/1w9tpjLAsWLQE6O+CNhR2vNrL11ltx7TXf55xPX8hLL63t/QfUs/qtguh3vQVwBzCCziUX1XYpjnWpeJpkJsB3vvZFTj5p2mupsSE8sORh5v3qXu6+ZyHr1m/g5Zdf4aJLvsdLa1+mvb1CS0szv392DTvt+PrcpWoQaWlp4bprvs+cOTdy000/zV1OKaRBMFqoVW8BfBbwy4h4jP9/xvlNwFuBM7r7oeqnSzaseTx/nz8Azj5tOmef1rnsb8GiJfxgzvVc9PnzOOfvv8TceXdz9BGHcfNttzNx/EGZK9Vg8v3ZX2P5Iyv4xiW9PoylWg2C0UKtepwBp5R+BuwGzAJ+DswFPg/sXhxTL84+7eNcefWNTDr+47zw4ot88JgjAXho+aMcfuwJzL3zbmZ95V+Z8pFTe7mSyuaQgw/gxBOOY8KEg7lv4VzuWziXSUdNzF1W46vT9wEPhOhqTllPm0sHrL4ZNmJ87hI0CLWvf+o13yR5+QsfqTlztv7Hq7LelPFJOEnl0l6em3CS1FgGwWihVgawpHJpoJtwBrCkUinTMjRJaix2wJKUiQEsSZmU6FFkSWooffmdcLkZwJLKxQCWpExcBSFJmdgBS1ImBrAk5ZEqjiAkKQ87YEnKw2VokpSLASxJmTTOCNgAllQuqb1xEtgAllQujZO/BrCkcvEmnCTlYgcsSXnYAUtSLnbAkpRHas9dQe0MYEml0kC/lZ6m3AVIUl119GHrQUSMiog7I2J5RCyLiDM3Ov7piEgRsWPxPiLimxGxIiKWRMT+vZVqByypVOrYAbcD56aUFkXEtsD9EfGLlNLDETEKeA/wZNX5k4CxxfYu4LvFn92yA5ZUKqmj9q3H66T0dEppUfH6JWA5MLI4fDHwWaB6ycUU4MrU6V5geETs0tNn2AFLKpVUibpfMyJGA+8A5kfE+4GnUkoPRvzZZ40EVla9byv2Pd3ddQ1gSaXSlxFERMwEZlbtmp1Smr3ROdsA1wNn0TmW+BxwZFeX66qcnj7fAJZUKqmj9g64CNvZ3R2PiCF0hu9VKaUbImJvYAzwavfbCiyKiAPp7HhHVf14K7Cqp893BiypVOo1A47OhL0MWJ5S+jpASumhlNIbUkqjU0qj6Qzd/VNKzwC3ACcVqyHGAS+klLodP4AdsKSSSaluM+BDgBOBhyJicbHv71JKt3Vz/m3A0cAK4BVgem8fYABLKpV6LUNLKf2Krue61eeMrnqdgNP78hkGsKRS6eiHVRD9xQCWVCp9uQmXmwEsqVQMYEnKJDXO1wEbwJLKxQ5YkjKp4zK0fmcASyqViqsgJCkPO2BJysQZsCRl4ioIScrEDliSMql0NM6XPBrAkkrFEYQkZdLhKghJysNlaJKUiSOIKtu0/nV/f4Qa0K7b7Zy7BJWUIwhJysRVEJKUSQNNIAxgSeXiCEKSMnEVhCRlUqdfijwgDGBJpZJ6/k3yg4oBLKlU2h1BSFIedsCSlIkzYEnKxA5YkjKxA5akTCp2wJKURwP9RiIDWFK5dNgBS1IefhmPJGXiTThJyqQjHEFIUhaV3AX0QeN8dbwk1aAjat96ExGXR8TqiFi60f5PRsSjEbEsIr5Stf+CiFhRHHtvb9e3A5ZUKnVeBfED4FvAla/uiIgJwBRgn5TSuoh4Q7H/bcBUYC9gBHB7ROyWUuq2KbcDllQqqQ9br9dK6S7guY12nwZ8OaW0rjhndbF/CnB1SmldSum3wArgwJ6ubwBLKpV6jiC6sRswPiLmR8R/RcQBxf6RwMqq89qKfd1yBCGpVPqyDC0iZgIzq3bNTinN7uXHWoDtgXHAAcC1EfFm6HL20WOjbQBLKpVKHzrbImx7C9yNtQE3pJQSsCAiOoAdi/2jqs5rBVb1dCFHEJJKpaMP2ya6CZgIEBG7AUOBNcAtwNSI2CIixgBjgQU9XcgOWFKp1PNJuIiYAxwG7BgRbcCFwOXA5cXStPXAR4tueFlEXAs8DLQDp/e0AgIMYEklU89fCZdSmtbNoRO6Of9LwJdqvb4BLKlU/C4IScqkkR5FNoAllYpfyC5JmTiCkKRMDGBJysTfiCFJmTgDlqRMXAUhSZl0NNAQwgCWVCrehJOkTBqn/zWAJZWMHbAkZdIejdMDG8CSSqVx4tcAllQyjiAkKROXoUlSJo0TvwawpJJxBCFJmVQaqAc2gCWVih2wJGWS7IAlKQ87YP2FT33yZKZPn0pKsHTZI5xyyrmsW7cud1kaYEO3GMrVP7mUoUOH0tzSzM9+8ksuuejfOGj8AZw/6yyGDhnC0geXc/6ZX6BSaaQvVhw8GmkZWlPuAjYHI0a8kdNPn85BBx/D/n91BM1NTRx//Ptzl6UM1q9bzwkfOJVjDpvK5MOmcejEg9j/gH346rdmceYpFzBp/PE81fY0H5x6TO5SG1bqw5abATxAmltaGDZsS5qbm9lqq2E8/fTvc5ekTF55+X8AaBnSQsuQFiqVDtav28ATv3kSgF/Nm89Rxxyes8SG1k6qecttkwM4IqbXs5AyW7XqGb5x8fdY8di9/O6J+3nhxZe4/fa7cpelTJqamvjJnXNYsPx2fj1vPg8uWkrLkBb23m9PACZNPpxdRu6cucrGlfrwX26vpQOe1d2BiJgZEfdFxH2VytrX8BHlMHz46zhm8pHsvsfBjB7zTrbeaiumTftA7rKUSUdHB5MnTOOQfY5i3/33Yrc93sKZp1zA5/7p09ww90peXvsK7e3OfzdVRx+23Hq8CRcRS7o7BHT7T3RKaTYwG2CLLUfl/2cms4kT380TT6xkzZrnALjp5p9y0Lh3MmfOjZkrU04vvbiWe399P4cefjCXfvuHTJ08A4B3HzaO0W95U+bqGtdg6Gxr1VsHvDNwEjC5i+0P/Vtaeaxc+RTvOvAdDBu2JQATJhzCI488lrkq5fD6HYaz7XbbALDFlltwyKHv4jePPcEOO24PwNChQzj1Ux9jzhXX5yyzoZWmAwZuBbZJKS3e+EBEzOuXikpo4cLF3HDjbcy/96e0t1dY/OBSLr3sR7nLUgY77bwTX/3WLJqbm2lqCv7z5l9w59y7Of/zZzHhyPE0NQVX/fuPuefuhblLbViV1DgdcKR+LtYRhLrSus1OuUvQIPSbNYvitV7jb3f9QM2Z86Pf3fiaP++18EEMSaXSSDNgA1hSqQyG2W6tDGBJpdJIjyIbwJJKpZFGED6KLKlUKinVvPUmIs6OiGURsTQi5kTElhExJiLmR8RjEXFNRAzd1FoNYEml0kGqeetJRIwEPgW8M6X0dqAZmApcBFycUhoL/BGYsam1GsCSSqXOD2K0AMMiogXYCngamAj8uDh+BXDsptZqAEsqlb58GU/199YU28w/XSelp4B/AZ6kM3hfAO4Hnk8ptRentQEjN7VWb8JJKpW+rIKo/t6ajUXE9sAUYAzwPHAdMKmry/S9yk4GsKRSqePTvUcAv00pPQsQETcABwPDI6Kl6IJbgVWb+gGOICSVSoVU89aLJ4FxEbFVRARwOPAwcCdwXHHOR4GbN7VWA1hSqdRrFURKaT6dN9sWAQ/RmZezgfOAcyJiBbADcNmm1uoIQlKp1PMLxlJKFwIXbrT7ceDAelzfAJZUKj6KLEmZNNKjyAawpFJppC9kN4AllYojCEnKxACWpEz6+9es1ZMBLKlU7IAlKRNXQUhSJpXUOL8VzgCWVCrOgCUpE2fAkpSJM2BJyqTDEYQk5WEHLEmZuApCkjJxBCFJmTiCkKRM7IAlKRM7YEnKpJIquUuomQEsqVR8FFmSMvFRZEnKxA5YkjJxFYQkZeIqCEnKxEeRJSkTZ8CSlIkzYEnKxA5YkjJxHbAkZWIHLEmZuApCkjLxJpwkZeIIQpIy8Uk4ScrEDliSMmmkGXA00r8WjS4iZqaUZueuQ4OLfy82X025C9jMzMxdgAYl/15spgxgScrEAJakTAzggeWcT13x78VmyptwkpSJHbAkZWIAD5CIOCoiHo2IFRFxfu56lF9EXB4RqyNiae5alIcBPAAiohn4NjAJeBswLSLelrcqDQI/AI7KXYTyMYAHxoHAipTS4yml9cDVwJTMNSmzlNJdwHO561A+BvDAGAmsrHrfVuyTtBkzgAdGdLHP5SfSZs4AHhhtwKiq963Aqky1SBokDOCBsRAYGxFjImIoMBW4JXNNkjIzgAdASqkdOAP4ObAcuDaltCxvVcotIuYA9wC7R0RbRMzIXZMGlk/CSVImdsCSlIkBLEmZGMCSlIkBLEmZGMCSlIkBLEmZGMCSlIkBLEmZ/B+6L5Jk7SDOfgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "import seaborn as sns\n",
    "confusion = confusion_matrix(y_test, pred_logreg)\n",
    "print(\"Confusion matrix:\\n{}\".format(confusion))\n",
    "sns.heatmap(confusion,fmt='d',annot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8863636363636364"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "f1_score(y_test,pred_logreg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n\n    not nine       0.98      1.00      0.99       403\n        nine       0.95      0.83      0.89        47\n\n   micro avg       0.98      0.98      0.98       450\n   macro avg       0.97      0.91      0.94       450\nweighted avg       0.98      0.98      0.98       450\n\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, pred_logreg,\n",
    "                            target_names=[\"not nine\", \"nine\"]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "X, y = make_blobs(n_samples=(400, 50), cluster_std=[7.0, 2],\n",
    "                  random_state=22)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
    "svc = SVC(gamma=.05).fit(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n\n           0       0.89      0.97      0.93        96\n           1       0.67      0.35      0.46        17\n\n   micro avg       0.88      0.88      0.88       113\n   macro avg       0.78      0.66      0.70       113\nweighted avg       0.86      0.88      0.86       113\n\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(svc.predict(X_test),y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pre_lower_threshold=svc.decision_function(X_test)>-0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n\n           0       1.00      0.82      0.90       104\n           1       0.32      1.00      0.49         9\n\n   micro avg       0.83      0.83      0.83       113\n   macro avg       0.66      0.91      0.69       113\nweighted avg       0.95      0.83      0.87       113\n\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,y_pre_lower_threshold))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "precison,recall,thresholds=precision_recall_curve(y_test,svc.decision_function(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1bbd3aadeb8>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VNXZwPHfk52QQCAJaxISZCdh3zTIoqColIigaF2KCL5WsFYr1draWqqvS61vq7gXcKkiAkUiLriwCQgSCDsCAUISwpIVsi8z5/1jwhhCICFkMjPJ8/185sPMPefe++QyuU/uueeeI8YYlFJKKQAPZweglFLKdWhSUEopZadJQSmllJ0mBaWUUnaaFJRSStlpUlBKKWWnSUEppZSdJgWllFJ2mhSUUkrZeTk7gEsVEhJiIiMjnR2GUkq5la1bt2YaY0Jrqud2SSEyMpKEhARnh6GUUm5FRI7Wpp42HymllLLTpKCUUspOk4JSSik7t7unoFRTUVZWRlpaGsXFxc4ORbkRPz8/wsLC8Pb2rtP6mhSUclFpaWkEBgYSGRmJiDg7HOUGjDFkZWWRlpZGVFRUnbbhsOYjEZkvIqdEZPcFykVEXhGRJBHZKSIDHBWLUu6ouLiY4OBgTQiq1kSE4ODgy7q6dOSVwrvAXOD9C5TfAHSteA0F3qj4VylV4VISQkpKCvHx8eTm5hIUFERcXBzh4eEOjE65osv9I8JhScEYs05EIi9SJQ5439jmA90kIkEi0t4Yc9wR8ZwpLuP11Ye4ZUBHurUNdMQulHKKEydOMGvWLJYtW4bVarUvf/jhh5k4cSJz586lXbt2ToxQuRNn9j7qCKRW+pxWsew8InK/iCSISEJGRkaddrb/RB5vrj3EsdyiOq2vlCs6ceIEsbGxLF26FE9PTyZNmsSTTz7JpEmT8PDwYOnSpcTGxnLy5MlL3nZubi6vv/66/fOaNWsYP358fYYPwLvvvsusWbMuaZ3IyEgyMzPPW/7000/z0ksv1VdoTZIzk0J11zimuorGmLeNMYOMMYNCQ2t8Srtah07lA/DIou1kF5TWaRtKuZpZs2Zx+PBhBgwYwOHDh1myZAnPPvssS5Ys4ciRI/blM2fOvORtV00KtWWxWC55HVdVXl7u7BAanDOTQhpQucEzDEh31M6GdQ4GILewjDX7T/HjkWy2Hs2h3GKtYU2lXFNKSgrLli3D29ub5cuXExYWdk55WFgYn376KV5eXixbtozU1NQLbKl6TzzxBIcOHaJfv37Mnj0bgPz8fCZPnkyPHj248847sbX+2v5ynzNnDsOHD2fx4sUcOnSIcePGMXDgQK6++mp++uknABYvXkx0dDR9+/ZlxIgR9n2lp6czbtw4unbtyu9//3v78oULFxITE0N0dDSPP/54tXE+++yzdO/enTFjxrB///5q6/Tr18/+atasGWvXrqWgoIBp06YxePBg+vfvz/LlywHblcutt97KL37xC6677jqMMcyePZvo6GhiYmJYtGjRJR1Ht2OMcdgLiAR2X6DsJuBLbFcMw4Afa7PNgQMHmrr65zcHTKfHV5zzev+H5DpvTylH2rt370XLX331VQOYSZMmXbTepEmTDGDmzp17Sfs/cuSI6d27t/3z6tWrTYsWLUxqaqqxWCxm2LBh5vvvvzfGGNOpUyfzwgsv2Otec8015sCBA8YYYzZt2mRGjx5tjDEmOjrapKWlGWOMycnJMcYYs2DBAhMVFWVyc3NNUVGRiYiIMCkpKebYsWMmPDzcnDp1ypSVlZnRo0ebZcuW2feXkZFhEhISTHR0tCkoKDCnT582V1xxhfn73/9+wZ8pPj7eDB8+3JSWlpo//OEP5oMPPrDH0rVrV5Ofn28WLFhgOnbsaLKysowxxixZssSMGTPGlJeXmxMnTpjw8HCTnp5+SceyoVX33QESTC3OsQ670SwiC4FRQIiIpAF/AbwrEtGbwBfAjUASUAjc66hYzvqfkZ0ZHNUKY2Df8TM88/k+/u+bA+w5dhqAVs19eOy67nh6aBdA5fpyc3MB6N69+0XrdevWDYCcnJzL3ueQIUPsVyT9+vUjOTmZ4cOHAzBlyhTAdjWxceNGbr31Vvt6JSUlAMTGxjJ16lRuu+02brnlFnv5tddeS8uWLQHo1asXR48eJSsri1GjRnG2yfjOO+9k3bp13Hzzzfb1vv/+eyZOnIi/vz8AEyZMuGDsBw8eZPbs2axatQpvb2++/vpr4uPj7fcgiouLSUlJAWDs2LG0bt0agPXr13PHHXfg6elJ27ZtGTlyJFu2bLnovtyZI3sf3VFDuQEuvaHzMvh5e3LVFSEAtG/px382HSW7oJTV+0+RX1xOQamF+4ZHERLg25BhKVUnQUFBABdsMjnrwIEDALRq1eqy9+nr+/Pvhqen5zlt7s2bNwfAarUSFBTE9u3bz1v/zTffZPPmzXz++ef069fPXqe67RpT7S3G89SmC2ZBQQG33XYb77zzDh06dABsrSRLly49L6lu3rzZ/rOcrdeUNNmxjzqHBrBm9mh2Pn09m58cw4R+ti/KVc+t4h9fX/yXTClXMGHCBDw8PIiPjyctLa3aOqmpqSxfvhwPD49L/ss2MDCQvLy8S46rRYsWREVFsXjxYsB2Ut2xYwcAhw4dYujQocyZM4eQkJCL3ucYOnQoa9euJTMzE4vFwsKFCxk5cuQ5dUaMGMGyZcsoKioiLy+Pzz77rNpt3Xvvvdx7771cffXV9mXXX389r776qv2kn5iYWO26I0aMYNGiRVgsFjIyMli3bh1Dhgyp/QFxM002KVR159BO/M+IzpRarKw9ULdur0o1pIiICCZOnEhZWRlxcXHnnWBTU1O5+eabKS8vZ+LEiZf8IFtwcDCxsbFER0fbbzTX1ocffsi8efPo27cvvXv3tt/EnT17tv3G8YgRI+jbt+8Ft9G+fXuee+45Ro8eTd++fRkwYABxcXHn1BkwYABTpkyhX79+TJo06ZyT/llHjx5lyZIlzJ8/336zOSEhgaeeeoqysjL69OlDdHQ0Tz31VLVxTJw4kT59+tC3b1+uueYaXnzxxUb93Ie426XRoEGDjKMm2ckrLiPm6a8BOPLcjTq8gHKqffv20bNnz4vWOfucwuHDh/Hy8iIuLo5u3bpx4MABli9fTnl5OZ07d2bjxo20bdu2gSJXzlbdd0dEthpjBtW0rg6IV0mgnze9O7RgT/oZ7nvvwonHx9ODP97Uk/DW/g0YnVLna9euHRs2bLA/0bx06VJ7mYeHB5MmTeK1117ThKBqTZNCFTf1aY/FasjIK6m2PKewlLScIsZFtyPQ7+fD5+Plgb+PHk7V8Nq1a8eSJUtITU0lPj6enJwcWrVqxYQJE3TsI3XJtPnoEv37+8M88/m+85Z7ewpfPnw1XdrouEqqftSm+Uip6mjzUQO6uX9HfLw8sFh/Tqbf7TvF+qRMlm475rTurJ1a+zOmlzYRKKUujyaFSxQS4Ms9V0aesyynoJT1SZm8seaQc4ICAn292PXX6522f6VU46BJoR48MrYb913d2Tk7N3D1i6soKbdy0yvfn1fcMagZb909UHtSKaVqRZNCPRARWjar23yo9eHWQeEczSo4b/mmw9kkZxaw/2QeHvWUFPy8PIkI1l5X6vJcddVVbNy48YLlN954Ix999JH9qW1XM3XqVMaPH8/kyZMZNWoUL730EoMG1dhc7xY0KTQCT43vVe3y6L+spKDUwrh/nn8FcTk+mjHUPlyIUhaLBU9Pz0ta52IJAeCLL764nJCqVV5ejpeXe5zy6nJM64t7HCFVJx/NGEpqdv1NKjRv/WG2peSyPDH9nKe+x/VuR/+Iyx9XR7mW5ORkxo0bx9ChQ0lMTKRbt268//77+Pv7ExkZybRp0/j666+ZNWsWgwcPZubMmWRkZODv788777xDjx49OHnyJA888ACHDx8G4I033uCqq64iICCA/Px8jh8/zpQpUzhz5gzl5eW88cYbXH311URGRpKQkEBISAgvv/wy8+fPB2D69On89re/JTk5mRtuuIHhw4ezceNGOnbsyPLly2nWrNk5P8PUqVNp3bo1iYmJDBgwgDlz5vDQQw+xa9cuysvLefrpp4mLi8NisfD444+zcuVKRIQZM2bw0EMPMWfOHD777DOKioq46qqreOutt2rdFLtlyxYefvhhCgoK8PX15bvvvmPp0qUkJCQwd+5cAMaPH89jjz3GqFGjCAgI4NFHH2XlypWMHz+eXbt28cknnwC2CY7+8Y9/8Nlnn/H111/zl7/8hZKSEq644goWLFhAQEBAff23a1JozPqEBdEnrP4uvzccymRP+hk+3X4MAGOg1GIlv7hck4KD/fWzPexNP1Ov2+zVoQV/+UXvi9bZv38/8+bNIzY2lmnTpvH666/z2GOPAeDn58f69esB2yinb775Jl27dmXz5s08+OCDrFq1it/85jeMHDmSZcuWYbFYyM/PP2f7H330Eddffz1//OMfsVgsFBYWnlO+detWFixYwObNmzHGMHToUEaOHEmrVq04ePAgCxcu5J133uG2225j6dKl3HXXXef9DAcOHODbb7/F09OTJ598kmuuuYb58+eTm5vLkCFDGDNmDO+//z5HjhwhMTERLy8vsrOzAdskRn/+858BuPvuu1mxYgW/+MUvajy2paWlTJkyhUWLFjF48GDOnDlzXsKqqqCggOjoaObMmWN/Er2goIDmzZuzaNEipkyZQmZmJs888wzffvstzZs354UXXuDll1+2x1gfNCmoWvvfiTH878QY++clW9N4bPEOlm9P5+u9lz7do7vq3aEF797beAdEqyw8PJzY2FgA7rrrLl555RV7UqjNUNmrVq3i/fffB2yjn54dHvuswYMHM23aNMrKyrj55pvp16/fOeXr169n4sSJ9lFLb7nlFr7//nsmTJhAVFSUvf7AgQNJTk6u9me49dZb7U0xFxou+9tvv+WBBx6wNy+dHTZ79erVvPjiixQWFpKdnU3v3r1rlRT2799P+/btGTx4MGAbJLAmZ6dTBfDy8mLcuHF89tlnTJ48mc8//5wXX3yRtWvXsnfvXvv/SWlpKVdeeWWN274UmhRUnQ3q1Iq7h3Wi3OpeD0DWVbnFyuKtaZwuKmvwfdf0F72jVG0qqfy5NkNl12TEiBGsW7eOzz//nLvvvpvZs2dzzz332Msv9nBt1eG2i4qqbyqtOgx2dcNlG2PO+1mLi4t58MEHSUhIIDw8nKeffpri4uJa/VzVbQ9sJ3ur9efZHitvz8/P75z7CFOmTOG1116jdevWDB48mMDAQIwxjB07loULF9YqjrrQpKDqLDKkOX+7OdrZYTSYpVvTWLw1jZmjujg7lAaTkpLCDz/8wJVXXsnChQvtE+pUVnmo7FtvvRVjDDt37qRv375ce+21vPHGG/z2t7/FYrFQUFBwzl/NR48epWPHjsyYMYOCggK2bdt2TlIYMWIEU6dO5YknnsAYw7Jly/jggw/q/POcHS771VdfRURITEykf//+XHfddbz55puMGjXK3nzk4WEbRDokJIT8/HyWLFnC5MmTa7WfHj16kJ6ezpYtWxg8eDB5eXk0a9aMyMhIXn/9daxWK8eOHePHH3+84DZGjRrFfffdxzvvvGO/Khs2bBgzZ84kKSmJLl26UFhYSFpamn0ipfqgQ2crVUt//WwPAH3DXbObpCP07NmT9957jz59+pCdnc2vf/3rautdaKjsf/3rX6xevZqYmBgGDhzInj17zllvzZo19OvXj/79+7N06VIefvjhc8oHDBjA1KlTGTJkCEOHDmX69On079+/zj/PhYbLnj59OhEREfYhss92h50xYwYxMTHcfPPN9qag2vDx8WHRokU89NBD9O3bl7Fjx1JcXExsbCxRUVHExMTw2GOPMWDAgAtuw9PTk/Hjx/Pll18yfvx4AEJDQ3n33Xe544476NOnD8OGDbPPf11fdOwjpWop8onPAVjx0HCiO7asofblc/bYR8nJyYwfP57du3c7LQZVN5cz9pFeKShVS93b2gY7HP/qeno+9RX7T1z6rGRKuTpNCkrV0u+u68bD13YlrFUzisosbEjKpLjM4uywHCYyMlKvEpogTQpK1dJ1vdvxyNhuNK+YN2POir18u8+xXXHdrXlXOd/lfme095FSl+i/D17F9PcS+OFwFvPWH2Hhjynn1Zl+dWdGd29zWfvx8/MjKyuL4OBgHdBQ1YoxhqysLPz8/Oq8DU0KSl2i5r5eDO8aQpnF1t+8pOznfueZ+SUkZxUS0zGIKzsH4+dd9/FrwsLCSEtLIyMjo+bKSlXw8/MjLCyszutr7yOl6tGcz/Yyf8MRAPpHBLHswVgnR6SUjc68ppQTTL86igBfT15ZlcTpwjJeX5MEgLeHB5MGhtG6uY+TI1Tq4jQpKFWPOgQ1Y2T3Nryx9hCHMwt48av99rK2Lf2Y0LeDE6NTqmbafKSUA5SWW7FW/G49+OE2Vv10ik7B/vh4Vt/hb8rgcKY7a/Y+1SRo85FSTuTj9fPJ//rebfHzrj4ZZOSVsCU5hx+PZDOiWyhd2wRoTyPlVHqloJQT/fv7wzzz+T775wVTBzO6x+V1ZVWqOjrMhVJu4K5hnVgwdTBd29hmzsoqKHVyRKqpc2hSEJFxIrJfRJJE5IlqyiNEZLWIJIrIThG50ZHxKOVq/Lw9Gd2jDVeE2pLCV7uPOzki1dQ5LCmIiCfwGnAD0Au4Q0SqzjD/J+ATY0x/4HbgdUfFo5QrGxfdDoB1BzPpN+drvtylyUE5hyOvFIYAScaYw8aYUuBjIK5KHQOcnXGjJZDuwHiUclkDO7Vi+vAourcNJLewjC3JOWw8lIm1icxqp1yHI5NCRyC10ue0imWVPQ3cJSJpwBfAQw6MRymXFd7anz+N70VEsD8A8zcc4ZfvbGZLcraTI1NNjSOTQnX96qr+2XMH8K4xJgy4EfhARM6LSUTuF5EEEUnQcWBUY/b3yX2InxVLaKBt/uGkjHwnR6SaGkcmhTQgvNLnMM5vHroP+ATAGPMD4AeEVN2QMeZtY8wgY8yg0NBQB4WrlPP5+3jRJyyIARG2KT+/23fKyRGppsaRSWEL0FVEokTEB9uN5PgqdVKAawFEpCe2pKCXAqrJu7ZnWwBW/XSK3cdOOzka1ZQ4LCkYY8qBWcBKYB+2XkZ7RGSOiEyoqPY7YIaI7AAWAlONuz1Np5QDVJ6LISO/xImRqKbGocNcGGO+wHYDufKyP1d6vxfQsYWVqiI00JfOoc05nFHAq98dvOwJe5SqLX2iWSkX9cDIKwC4Y0iEkyNRTYkmBaVcUEm5hVdXHaRn+xZMGlD3WbSUulSaFJRyQR9uSiE1u4gnbuiBh4eOmqoajiYFpVzMmeIyXl11kNguwYzoel4PbaUcSpOCUi7mrbWHyCks44lxPXVuBdXgNCko5UJOnC5m3vojTOjbgZiwls4ORzVBOvOaUi7kn98eoLjMyugeoSRUGvcoyN+bLm0CnRiZaio0KSjlQlbstA2Z/ciiHeeVff/70YS39m/okFQTo0lBKRcSPyuWY7lF9s9FpRYeWphI93aBdAxq5sTIVFOhSUEpF9I5NIDOFbOwAbz63UFKyq388cae2jVVNQi90ayUC3tz7SEAfrXgR2KeXsmPR3R+BeVYeqWglAt7/IYeHMspYsXO45w4U0zr5t7ODkk1cnqloJQLu+fKSCYNDOPEmWLuHBqhPZCUw2lSUMrFvfjVfixWw0/H85j+3hZW7jnh7JBUI6ZJQSkX1zm0Ob07tCAtp5Bv951iR2qus0NSjZjeU1DKxT15Y0+MMdzxziYKyyzcP6IzACkpKcTHx5Obm0tQUBBxcXGEh4fXsDWlLk6TglJu4KvdJ9h0OJvR3UPJysxg+qMPs2zZMqxWq73Oww8/zMSJE5k7dy7t2rVzYrTKnWlSUMoN7Dt+BoDV+zNYO+9ZjmyIx9vbm4kTJ9K9e3f279/P8uXLWbp0KYmJiWzcuJG2bds6OWrljjQpKOUGHhnbjWXbj5GaXURx1zF0jrmJrl27YPXzZcqE3vQJCyItLY24uDi2bdvGzJkzWbJkibPDVm5IbzQr5QZEhNiI5hQd2oK1IJshfXqQWWhlW0ouu46dJulUHsU+Lfnn/I/x8m3GsmXLSE1NdXbYyg1pUlDKTYRlbuHUkr8y3LqLhb8ega+X7df3j8t2M+bldYx5eR13LzxAzH3PY7VaiY+Pd3LEyh1p85FSbiI319YVtXv37gC8c88gDmXk28u/3XeS5dvTkcBQvEM6kZOT45Q4lXvTpKCUmwgKCgJg//79APTq0IJeHVrYy0/llbBi53GyrC3w7x5Lq1atnBKncm/afKSUm5gwYQIeHh7Ex8eTlpZ2Xvm02Eii2/ljLTpDwfbPmTBhghOiVO5Ok4JSbiIiIoKJEydSVlZGXFzceTeS5323ix3pBeSsfY+4cWP0QTZVJ9p8pJQbmTt3LomJiWzbto3OnTsTFxdHt27d2JuUzNa2N1J++hSheUm89toGZ4eq3JReKSjlRtq1a8eGDRuYNGkSVquVpUuX8txzz7Eu0x/P5q0IbhXEDX/5D//ccJLU7EJnh6vckF4pKOVm2rVrx5IlS0hNTSU+Pp6cnBz2eERxxHjjGRTJdwdzKC7L4rre7XROZ3XJNCko5abCw8OZOXPmOctO5RVz7T/WMiCiFaO6hTopMuXOtPlIqUbkbyv2kVdczqBOrfh0+zG+2n2cMou15hWVqqBXCko1IusPZgDwyqok+7IPpw8ltkuIs0JSbsahVwoiMk5E9otIkog8cYE6t4nIXhHZIyIfOTIepRq7NbNHs+axUXw6M5bmPp7EdGzJ0KjWzg5LuRGHXSmIiCfwGjAWSAO2iEi8MWZvpTpdgT8AscaYHBFp46h4lGoKWjbzpmUzb36/ZAfF5VaenxSDl6e2Eqvac+S3ZQiQZIw5bIwpBT4G4qrUmQG8ZozJATDGnHJgPEo1CRuSMvkkIY37R3Smd4eWzg5HuRlHJoWOQOVHLtMqllXWDegmIhtEZJOIjKtuQyJyv4gkiEhCRkaGg8JVyv0Vl1l4ctkuPATaBPpSUm5xdkjKzTgyKUg1y0yVz15AV2AUcAfwbxEJOm8lY942xgwyxgwKDdVudkpdSE5hKZl5JVgN/PWzvaRmFzk7JOVmHJkU0oDKg6+EAenV1FlujCkzxhwB9mNLEkqpOmjfshkfzRiGp4cQ168DXdoEODsk5WYcmRS2AF1FJEpEfIDbgaqzfnwKjAYQkRBszUmHHRiTUo1amcXKH/67Cy8PYepVkc4OR7khhyUFY0w5MAtYCewDPjHG7BGROSJydkzflUCWiOwFVgOzjTFZjopJqcZuS3I2e4+foaTcysTXN5KcWeDskJSbEWOqNvO7tkGDBpmEhARnh6GUS7JaDZ8kpPLEf3cxrHNrPpo+DA+P6m7vqaZGRLYaYwbVVE87MCvVyCxLPEaArxd/n9xXE4K6ZJoUlGpE4neks/lINsVlFu6et5nTRWXODkm5mYs+0Swij16s3Bjzcv2Go5S6HH3CWjKxf0eWJR4jM7+UXWmnie0SjIheMajaqelKIbCGl1LKhXQODeCqK4IByC8p5655m0k6le/kqJQ7ueiVgjHmrw0ViFKqfkzs3xED/H7JTsb2aqvPKqhLUlPz0SsXKzfG/KZ+w1FKXa4yi+HtdYcJCfDh2ZujKbcawOCtA+OpWqhplNStDRKFUqrefLAp2d5kNOR/v7Mv/9vN0dw9rJOzwlJuoqbmo/caKhClVP0Y17s9ZRaDMQZjYN6GIxSUlNM//LxhxZQ6T63mUxCRUOBxoBfgd3a5MeYaB8WllKqjiGB/Zo7uAsD7PySTW1jGn27qSXRHHUZb1ay2jYwfYhuqIgr4K5CMbWwjpZSL2pN+mmdW7OOaHm24b3iUs8NRbqK2SSHYGDMPKDPGrDXGTAOGOTAupdRlKCgp56GPEmnV3JuXbu2rzymoWqvtdJxnH4s8LiI3YRsCO8wxISmlLocxhj99upvkrAI+mjGM1s19nB2SciO1TQrPiEhL4HfAq0AL4BGHRaWUqrOPt6SyLPEYXdoE8OORbH48kl0v2x3UqRVXdQmpl20p11WrpGCMWVHx9jQV8x8opVxTQnIOAEmn8nn5mwP1tt3R3UM1KTQBtRo6W0TeAx42xuRWfG4F/KPi3kKD0qGzlbo4YwzWehwR/+GPE1mx8zitm/vQspn3OWUPjOzMlMER9bcz5TC1HTq7ts1Hfc4mBABjTI6I9K9zdEophxERPOvxvvKIrqF4VLlRvflIFifPlODn7Vl/O1IuobZJwUNEWhljcgBEpPUlrKuUcmO3DQ7ntsE/T7e+K+00X+w6zk0x7ZnQt4MTI1OOUNsT+z+AjSKyBDDAbcCzDotKKeWSCkvLefjjREIDfXl2YrR2dW2Eanuj+X0RSQCuAQS4xRiz16GRKaVczt9W7ONIVgEfTh9KkL92dW2MLqUJqDVQYIxZICKhIhJljDniqMCUUq4lMSWHhT+mADD9vYt39hjeJYS376nxnqZyQbUd++gvwCCgO7AA8Ab+A8Q6LjSllCuJCmnOrNFdKCm3XLBOYkouCUdziGjt34CRqfpU2yuFiUB/YBuAMSZdRHTmNaWakCB/Hx67vvsFy0/lFXPDP7+nR7vAi9ZTrq22Yx+VGtsDDQZARJo7LiSllLuxWg2zF+8kv6ScV+/or11V3VhtrxQ+EZG3gCARmQFMA/7tuLCUUu5kwcZk1h7I4IkbetA+qBn5JeUXrOvtKfh6adJwVbV6ohlARMYC12HrfbTSGPONIwO7EH2iWSnXUlhaTr+/fkOpxVqr+j6eHqyZPYoOQc0cHJmqrL6faKYiCXxTsXFPEbnTGPPhZcSolGoEmnl78vykGLLySy9aLyO/hLfXHaZbuwBCA30bKDp1qS6aFESkBTAT6AjEY0sKM4HZwHZsk+8opZowEeGWARcfSd8Yw9QFW/Dz9uCfU/rj7Vnb25mqodV0pfABkAP8AEzHlgx8gDhjzHYHx6aUaiSWbjvG2gMZBPp58bvFOwC4pnsbHh7T1cmRqapqSgp3SItlAAAV+UlEQVSdjTExACLybyATiDDG5Dk8MqVUo9G+pR+juodiDBzJLCAlu5A+Ome0S6opKZydcQ1jjEVEjmhCUEpdqtguIcR2CSErv4Qb/vU9nUOb84cbezg7LFWNmhr2+orImYpXHtDn7HsROVPTxkVknIjsF5EkEXniIvUmi4gREX0uXqlGymo1/G7xDnKLyph7xwD8fXSgZVd00f8VY0ydOxOLiCfwGjAWSAO2iEh81YH0Kp6M/g2wua77Ukq5vvkbjrBmfwZz4nrTq0MLZ4ejLsCRXQCGAEnGmMPGmFLgYyCumnp/A14Eih0Yi1LKiXak5vLCVz9xXa+23D2sk7PDURfhyKTQEUit9DmtYpldxext4ZXmgFZKNTLZBaX8+j9b8fQQbhnQkU2Hs9mbXmPrs3ISRzbqVTf7hv3xaRHxAP4PmFrjhkTuB+4HiIjQ+WCVcif/3ZZG+mlbQ8AD/9lmX5741FhaNdc5GVyNI5NCGhBe6XMYkF7pcyAQDaypmL2pHRAvIhOMMeeMY2GMeRt4G2zDXDgwZqVUPbtrWCdiOrbEamDdwQzeWHOIQD8v/veLfQA08/Hkd2O709Lf28mRKnBsUtgCdBWRKOAYcDvwy7OFxpjTQMjZzyKyBnisakJQSrk3P29PhnYOBuBYbhEdg5phjOH7g5mcOFOMl4dwb2yUJgUX4bCkYIwpF5FZwErAE5hvjNkjInOABGNMvKP2rZRyTZMHhjF5oG1IjJe/OcAr3x3kz7/oRVSIjsbvKhzaUdgY8wXwRZVlf75A3VGOjEUp5Tq+3nOCV747yOSBYdobycXoqFRKqQaVdCqfRz/ZQZ+wljxzczQV9xSVi9CkoJRqMKeLyvifDxLw8fLgjbsG6gxtLkiTglKqQZSWW/n1f7ZyNKuQ1345gI46yY5L0sFHlFIOZ4zhyWW72Hgoi5du7cuVVwQ7OyR1AZoUlFIO99rqJJZsTaNrmwByCkp5Z91hvD2FWwaG0cJPu6K6Ek0KSimHW7nnJAAHT+XzbMVDawCdQwMY0S3UWWGpamhSUEo53LIHr6K43ArA/PVHePmbA9w9rBNXdw2pYU3V0DQpKKUczsvTgwBPDz7bkc7L3xwgpmNLpgwO56cTF56zKzTQl5AA3waMUoEmBaVUAzHG8NTy3QDsOnaa8a+uv2j9ti182fzkmIYITVWiSUEp1SBEhPfuHcLx00UXrTd3dRK7j53hgZFXNFBkqjJNCkqpBtM3PIi+4UEXLJ+3/gi7j51hWmwU98ZGNWBk6ix9eE0p5RJW7Eznmc/3Mq53O/54U09nh9NkaVJQSjndqp9O8tuPtzOoUyv+eXs/PD10PCRn0aSglHKqjYcyeeA/2+jZvgXzpg7W8ZCcTO8pKKWc5sDJPGa8l0BpuZUgf2+e+nS3vaxrmwBmXdPVidE1TZoUlFJOk1NQSruWflishtTsQlKzC8nMLyW/pJw+YS01KTiBJgWllNMM7RzMd78bZf+8LSWHu/+9mc6hzZn3q8HOC6wJ03sKSimXsDMtl1/N/5HQQF8WzhhGaKA+zewMmhSUUk7345Fs7nxnMy2befPRjGG0beHn7JCaLE0KSimnWrP/FPfM30xoC18++Z8r6aCT7ziV3lNQSjnNip3pPLJoO93aBvL+tCEE6wB4TqdJQSnlFO9uOMJfV+xlUKdWzJs6WCfbcRGaFJRSDe65L/fx1trD+Hl7MDQqmA9+OGovC/D14o4hEfh4aeu2M2hSUEo1KKvV8Nn2dACKy6zMXZ10TrmnhzCmV1s66r0Fp9CkoJRqUB4ewvrHr6HcauzL9qSfZsb7Wykps/D6XQM0ITiRJgWlVIPz8BB8Kga9+27fSR5amEhQM28+nH4V3dsFOjm6pk2TglLKKYwxvL7mEC99vZ/oDi2Z96tBtNHnE5xOk4JSqsEZY3hoYSIrdh7H00Po0S6Q16rcW+jSJoC7r4x0ToBNmCYFpVSDK7MYDp7MJ8jf1g31m30nATAGTheVAdCrfQtNCk6gSUEp1eB8vDxY+ciIc5YVl1n406e7WbI1jdHdQ/nnlP5Oiq5p06SglHK69Nwi7v8ggd3HztA3PIgpgyPYfCTrnDrNfb24snMwHjorm0M5NCmIyDjgX4An8G9jzPNVyh8FpgPlQAYwzRhz9LwNKaUatXc3JrP72BkAdqTm8sB/tlZb79tHR9CljfZOciQxxtRcqy4bFvEEDgBjgTRgC3CHMWZvpTqjgc3GmEIR+TUwyhgz5WLbHTRokElISHBIzEop5ygpt3DwZP55y3MKS3ngg60UlFoI9PUitktItet7e3nw++u7E97a39Ghui0R2WqMGVRTPUdeKQwBkowxhysC+hiIA+xJwRizulL9TcBdDoxHKeWifL08ie7Y8rzlaTmFdG0bSFGpBYAjmQXnlJ84U8zpojJ8vTz4nxGdCW+QaBs3RyaFjkBqpc9pwNCL1L8P+LK6AhG5H7gfICIior7iU0q5uLBW/nw6M7basqVb0/jDsl10DGrGW3cPrDapqEvnyKRQ3d2gatuqROQuYBAwsrpyY8zbwNtgaz6qrwCVUu6ntNzKc1/uY8GGZK7sHMzcX/bXIbfrkSOTQhqcczUXBqRXrSQiY4A/AiONMSUOjEcp5eYOnszjt4u2syf9DNNio3jyxh54eepoqvXJkUlhC9BVRKKAY8DtwC8rVxCR/sBbwDhjzCkHxqKUcmNWq+Hdjck8/9VPBPp68c49gxjbq62zw2qUHJYUjDHlIjILWImtS+p8Y8weEZkDJBhj4oG/AwHAYhEBSDHGTHBUTEop93P8dBGPLd7BhqQsru3Rhucn9SE0UJuLHMWhzykYY74Avqiy7M+V3o9x5P6VUu7LYjX8Z9NRXlq5H4sxPH9LDFMGh1PxB6RyEH2iWSnlctJyCnnww23sTDsNQFy/DqTnFvF/3xyotr6Plwd3Du1Eq+Y+DRlmo6RJQSnlcnaknmZv+hnOXhTE7zivjwpgG0DvrCFRwQyJat0A0TVumhSUUi7npj7tualP+wuW5xSU8uLKn1j4Yyqdgv15YVIfTQj1RJOCUsptGGP4JCGV57/8iZzCMvqFB/HI2G54iPDjkewLruchEBPWEl8vzwaM1j1pUlBKuY39J/N4fOku++ftqbn8av6PtVr3z+N7MW14lKNCazQ0KSil3Eb3toF8OjOWgpLyGutuPZrD3NVJlJZbuX1wOLcN1pGRakOTglLKbYgI/cKDLlonK7+E57/8icVb0+jQ0o/nJvVhZLfQBorQ/WlSUEo1CsVlFt7/IZm5q5IoLLXwwMgr+M21XfD30dPcpdCjpZRya+UWK8u3p/PyNwc4llvEyG6h/OmmnnRtq5Px1IUmBaWUWyqzWFmWeIzXVyeRnFVITMeW/H1yH666wEQ8qnY0KSil3EpecRmLE9KYt/4Ix3KL6N2hBW/eNYDrerXT+ZvrgSYFpZRbSM4s4N2NySzZmkZ+STkDO7Xibzf3ZnT3NjoeUj3SpKCUclnFZRa+2n2CTxJS2XgoC29P4aaY9twbG0XfGnohqbrRpKCUcilWq2FrSg6fJh4jfkc6ecXlhLduxiNjunHHkHDatPBzdoiNmiYFpZTTWa2GbSk5rNh5nC93H+fkmRJ8vTy4MaY9tw4KY1hUsN4vaCCaFJRSTpV0Ko975v1I+unic5a3a+nHgZN5PPv5vjptd2L/jky/unN9hNikaFJQSjmVr5cn/SNa0avcUqf1jYGDp/JJyS60L2vl702Ar57e6kKPmlLKqcJb+/PanQMuaR2r1bA9LZeVe07wzZ6TpGQX4uUhjO7RhkkDwrimRxt8vDwcFHHjpklBKeUWTheVselwFusOZPDN3pOcyivBy0O48opg7o2N5MaY9gQH6NzNl0uTglLKJRWXWdh6NIcNSZlsOJTFrrRcrAb8fTwZ2S2U63u3Y3T3NrT093Z2qI2KJgWllNMZY0g/Xcz2lFy2p+aQmJLLzmOnKS234uVhGxl11jVdib0imP4RrbRpyIE0KSilGlxWfgk/nchje2qu/ZWRVwKAj5cH0R1acM+wTlzVJZghUcF607gB6ZFWSjnMmeIyDp7MY/+JfA6czGP/iTwOnMwjq6DUXicqpDlXdwmhX0QQ/cKD6NGuhV4JOJEmBaXUZSkus5CWU0hKdiEpWYWkZBdxODOfAyfyznn2oLmPJ13bBjKmZ1u6tQukW9sAYjq2JMjfx4nRq6o0KSilLsoYQ2Z+KSnZhaRm207+R7N+fn/izLkPnTXz9iQypDlDolrTrV0g3dsG0q1tIB2DmulTyW5Ak4JS6qJ+v2Qni7em1bp+UZmFfcfPsO/4GQdGdS4fLw8+vn8YAyJaNdg+GytNCkqpi5o0MIwOQc0cvp8yi5XM/BIy8ko4lVfCidPF59x7qMrfx5NOwc2JDPanS5sArggNcHiMTYEmBaXURQ3rHMywzsF1WtcYw5nicjLyijmVZzvhn32dsv9bTEZeCTmFZeetf7YpKjLY/+d/g5sTFdKc0EBfnUfBATQpKKUuqtxiJa+43PYqKbO/z6/03vYqq1heTk5hqf3kX1JuPW+bPl4etAn0JTTQl6iK+w9tAv0IDfQlNMCXNi18advCjzZ64m9wmhSUaoTKLVZKLVZKymz/nj1p55ecewKv9gRfUukEX1xOUVnNA9X5eHoQ6OdFgJ8XgX5etGzmzeDI1vYT/9mX7bMfLfy89GTvohyaFERkHPAvwBP4tzHm+SrlvsD7wEAgC5hijEl2ZExKOYrFaigpt1BabqW03EpJxcv2vmJ5pRN15eWV6/5c5+eyytssLbdSYrFSUmaptJ3KdSxYTe3jbu7jSaCft/2k3rKZN2GtmhHoazvBB/rZRhw9+z7Q7/zlft6ejjuwqkE5LCmIiCfwGjAWSAO2iEi8MWZvpWr3ATnGmC4icjvwAjDFUTEpVZkxhk+3H+NoVuF5J9WqJ/BzTtrllmpO4FYsl3ImdiBPD6GZlwc+Xh74VvwbcIETfICvZ626iRaXWyjOt5CRX9IAP4Gqjo+nB7cODHf4WE+OvFIYAiQZYw4DiMjHQBxQOSnEAU9XvF8CzBURMca4xm+XatQsVsOzn+8jM//CPVzckcVqKLJaatXso9xLgK8Xtw+JcOg+HJkUOgKplT6nAUMvVMcYUy4ip4FgINOBcSkFgJenB5v+cC3lLvIXvlIXkpFXwtUvrm6Q76ojk0J116RVf6La1EFE7gfuB4iIcGyWVE2Ll6cHXtocrlxcCz9vboxpR0Rrf4fvy5FJIQ0Ir/Q5DEi/QJ00EfECWgLZVTdkjHkbeBtg0KBB+medUqpJaenvzet3DmyQfTlyKMItQFcRiRIRH+B2IL5KnXjgVxXvJwOr9H6CUko5j8OuFCruEcwCVmLrkjrfGLNHROYACcaYeGAe8IGIJGG7QrjdUfEopZSqmUOfUzDGfAF8UWXZnyu9LwZudWQMSimlak9nslBKKWWnSUEppZSdJgWllFJ2mhSUUkrZaVJQSillJ+72WICIZABHnR1HHYSgw3dUpsfjfHpMzqXH43yXc0w6GWNCa6rkdknBXYlIgjFmkLPjcBV6PM6nx+RcejzO1xDHRJuPlFJK2WlSUEopZadJoeG87ewAXIwej/PpMTmXHo/zOfyY6D0FpZRSdnqloJRSyk6TQj0TkXEisl9EkkTkiWrKHxWRvSKyU0S+E5FOzoizodR0PCrVmywiRkQafW+T2hwTEbmt4nuyR0Q+augYG1ItfmciRGS1iCRW/N7c6Iw4G4qIzBeRUyKy+wLlIiKvVByvnSIyoF4DMMboq55e2IYIPwR0BnyAHUCvKnVGA/4V738NLHJ23M48HhX1AoF1wCZgkLPjdvYxAboCiUCris9tnB23k4/H28CvK973ApKdHbeDj8kIYACw+wLlNwJfYpu5chiwuT73r1cK9WsIkGSMOWyMKQU+BuIqVzDGrDbGFFZ83IRtRrrGqsbjUeFvwItAcUMG5yS1OSYzgNeMMTkAxphTDRxjQ6rN8TBAi4r3LTl/BsdGxRizjmpmoKwkDnjf2GwCgkSkfX3tX5NC/eoIpFb6nFax7ELuw5bxG6saj4eI9AfCjTErGjIwJ6rNd6Qb0E1ENojIJhEZ12DRNbzaHI+ngbtEJA3b/CwPNUxoLutSzzOXxKGT7DRBUs2yart3ichdwCBgpEMjcq6LHg8R8QD+D5jaUAG5gNp8R7ywNSGNwnYl+b2IRBtjch0cmzPU5njcAbxrjPmHiFyJbbbGaGOM1fHhuaRan2fqQq8U6lcaEF7pcxjVXOqKyBjgj8AEY0xJA8XmDDUdj0AgGlgjIsnY2kfjG/nN5tp8R9KA5caYMmPMEWA/tiTRGNXmeNwHfAJgjPkB8MM2BlBTVavzTF1pUqhfW4CuIhIlIj7Y5pyOr1yhornkLWwJoTG3FUMNx8MYc9oYE2KMiTTGRGK7xzLBGJPgnHAbRI3fEeBTbB0SEJEQbM1Jhxs0yoZTm+ORAlwLICI9sSWFjAaN0rXEA/dU9EIaBpw2xhyvr41r81E9MsaUi8gsYCW2XhXzjTF7RGQOkGCMiQf+DgQAi0UEIMUYM8FpQTtQLY9Hk1LLY7ISuE5E9gIWYLYxJst5UTtOLY/H74B3ROQRbM0kU01FN5zGSEQWYms6DKm4j/IXwBvAGPMmtvsqNwJJQCFwb73uvxEfW6WUUpdIm4+UUkrZaVJQSillp0lBKaWUnSYFpZRSdpoUlFJK2WlSUE2SiFhEZLuI7BaRxSLiXw/bHCQir1ykvIOILLnc/SjlSNolVTVJIpJvjAmoeP8hsNUY83KlcsH2+9FUh1JQTZReKSgF3wNdRCRSRPaJyOvANiBcRK4TkR9EZFvFFcXZRDJYRDaKyA4R+VFEAkVklIisqCgfWXElsr1iHoDAiu3vrij3E5EFIrKrovzsE8xTReS/IvKViBwUkReddExUE6VJQTVpIuIF3ADsqljUHduwxP2BAuBPwBhjzAAgAXi0YjiGRcDDxpi+wBigqMqmHwNmGmP6AVdXUz4TwBgTg23At/dExK+irB8wBYgBpohIOEo1EE0KqqlqJiLbsZ3oU4B5FcuPVoxRD7YB+noBGyrq/grohC1xHDfGbAEwxpwxxpRX2f4G4GUR+Q0QVE35cOCDivV/Ao5iG+MI4LuKcaGKgb0V+1SqQejYR6qpKqr4K96uYiyqgsqLgG+MMXdUqdeHGoYqNsY8LyKfYxujZlPFyLiVJxGqbvjjsyqPnGtBf09VA9IrBaUubBMQKyJdAETEX0S6AT8BHURkcMXywIpmKDsRucIYs8sY8wK2q5EeVba9Drizom43IALbENlKOZUmBaUuwBiTgW0CoIUishNbkuhRMW3kFOBVEdkBfINtOOfKflvR3XUHtvsJVWfYex3wFJFd2O5PTG3kc2soN6FdUpVSStnplYJSSik7TQpKKaXsNCkopZSy06SglFLKTpOCUkopO00KSiml7DQpKKWUstOkoJRSyu7/AZCxDMtUFNQkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Use more data points for a smoother curve\n",
    "X, y = make_blobs(n_samples=(4000, 500), cluster_std=[7.0, 2], random_state=22)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
    "svc = SVC(gamma=.05).fit(X_train, y_train)\n",
    "precision, recall, thresholds = precision_recall_curve(\n",
    "    y_test, svc.decision_function(X_test))\n",
    "# find threshold closest to zero\n",
    "close_zero = np.argmin(np.abs(thresholds))\n",
    "plt.plot(precision[close_zero], recall[close_zero], 'o', markersize=10,\n",
    "         label=\"threshold zero\", fillstyle=\"none\", c='k', mew=2)\n",
    "\n",
    "plt.plot(precision, recall, label=\"precision recall curve\")\n",
    "plt.xlabel(\"Precision\")\n",
    "plt.ylabel(\"Recall\")\n",
    "plt.legend(loc=\"best\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1bbd3998cf8>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XdYVFf6wPHvYWjSQbACoiI2FFQsiS0azZpoJESNJT2bsru6m7IxZTdmUza/9B7TTKJpJkZdo0bTo9HEaMSS2KIiKiAiHekwzPn9cXEEAUGdYSjv53nuMzP3nnvvOyPOO+fce85RWmuEEEIIACdHByCEEKLpkKQghBDCSpKCEEIIK0kKQgghrCQpCCGEsJKkIIQQwkqSghBCCCtJCkIIIawkKQghhLBydnQA5yowMFCHhYU5OgwhhGhWtm3blqm1DqqvXLNLCmFhYcTHxzs6DCGEaFaUUkcbUk6aj4QQQlhJUhBCCGElSUEIIYRVs7umIISoX3l5OSkpKZSUlDg6FNHI3N3dCQ4OxsXF5bz2l6QgRAuUkpKCt7c3YWFhKKUcHY5oJFprsrKySElJoWvXrud1DLs1Hyml3lNKpSuldtexXSmlXlFKJSilfldKDbRXLEK0NiUlJbRt21YSQiujlKJt27YXVEO05zWFRcCEs2y/HOhRudwOvGHHWIRodSQhtE4X+u9ut+YjrfUGpVTYWYrEAh9oYz7QzUopP6VUR631cbsElLSZvL3fsfWEIrpXOIHtOoJHIHi0BY8AMJ1f+5sQQrQkjrym0BlIrvI6pXJdjaSglLodozZBaGjo+Z0teQu+m59lHMDhWra7+1YmiMpE4dn2jNenEkjl4uYN8ktMCNHCODIp1PaNqmsrqLV+G3gbICYmptYy9Rp+J68WXsb7P+xgSLsKXp7cBZfSLCjKgsLKx6IsKMqEkylw/DfjeUVZ7cczuVZPErUlDuvrQKmNCCGaBUcmhRQgpMrrYCDVnicc07cTz/+QyNp0mGHpjbuHCTcfJ/p19sXJqZYcpTWUFZyRODIrX2dWSSRZlUkkC0py6w6gztrImTWSAOO11EZEM1ZYWMg111xDSkoKFRUVzJ07lzVr1vDZZ58BsH79ep5//nlWr17NV199xb/+9S8qKioIDAzk+++/d3D0rZcjk8IqYI5S6lNgKJBnt+sJlSI7+zJzSCif/JrEDe/9al3/7o0xXNq7fc0dlDK+mN28wT+sYSepKIfinCpJI7P2pJIntRHROB5dvYe9qSdtesw+nXz4z5V9z1rmq6++olOnTqxZswaAvLw85s2bR2FhIZ6enixZsoTp06eTkZHBbbfdxoYNG+jatSvZ2dk2jVWcG7slBaXUJ8AlQKBSKgX4D+ACoLV+E1gLXAEkAEXAzfaKpap/XdGLSf07AvBzQiavrz/EC98e4Nu9JwAIC/TkL6O7n/8JTC7g1c5YGsLWtRE331quh0htRDS+fv36ce+993L//fczadIkRo4cyYQJE1i9ejVTp05lzZo1PPPMM6xfv55Ro0ZZ76sPCAhwcOStmz3vPppZz3YNzLbX+evi7e7C8PBAANycnVi+PYXjeSVkFpSSW1SOUnDHqG6Ndztfk6qNBNR9Yb3qa6mNNCv1/aK3l4iICLZt28batWt58MEHueyyy5g+fTrz588nICCAwYMH4+3tjdZabp9tQpTx3dx8xMTEaHsNnX39u1vYeDATV2cnHpvclxlDzvNOp6bmVG2kMBOKss9eGzn1+rxrI7Wsc/OR2kgj27dvH71793ZoDKmpqQQEBODu7s7nn3/OokWLWL58Od27d2fw4MFMmzaNa665hoyMDAYOHFit+UhqCxemtn9/pdQ2rXVMffvKMBdV/GV0d4L9Pfjk1yS2Hc1pOUmham0koIFd321ZG3FyqdlsVbX24dUeQoaCdy3XdUSztWvXLubOnYuTkxMuLi688cYbmEwmJk2axKJFi3j//fcBCAoK4u233+bqq6/GYrHQrl07vv32WwdH33pJTeEM+9Py+dNLG3BzdmL/fy+323lanBq1kTpqImerjQRGQNhI6DrSePQMdMx7aQGaQk1BOI7UFGyoZwdvXE1OlJot3LJoa53lfNyd+W9cP7zc5CMELqw2kpsER34ylt+XQPy7xvZ2fcA35OzHOJ84L5pjJB4hRA3yjVaLWUND2ZyYRUZ+aa3b006WkJFfyp9HdCMkoI11vbuLCXcXU2OF2fxVvVMrOAZG3GUkitSdcGQDHPkZCk7Y9pxZCcajJAUhaiVJoRaPTD773RoP/m8Xn/yaxJWv/VRtvbe7M/EPjcPNWRLDeTO5QMhgYxn5T9sff80/YecnYC4DZ1fbH1+IZk6Swnm4Y1Q3Itp7VVu3eEsSB9ML+PCXow67vW5YtwD6dvJ1yLmbje5jYes7kLxFagtC1EKSwnkIC/Tk5sDq7eZbErM5mF7Af9fsc1BU8Ke+7Xnr+nqvI7VuYSNBmeDQD5IUqkhKSmLVqlXk5ubi5+dHbGwsISE2vp4jmgVJCjYy/9qBFJSaHXLuP46fZPrbm9l2NIeJr2yssX1yVCfuuJBe2i2Juw+EDDFqCwnfGeu6j4Xxjzo2LgdJS0tjzpw5rFixAovFYl1/5513EhcXx2uvvUaHDh0cGKFobJIUbMTkpPBt45ievt2CvJgc1YmisupJqdRsYePBTAI8XRnTq4HDbjRAkJcb/p7NuD1+5D9ha+UdTkmbjFpDK0wKaWlpDB8+nMTERFxcXIiLi6Nnz57s37+flStXsnz5cnbs2MGmTZto3/7c+pDk5uayePFi/va3vwHG4HfPPfccX3zxhU3fw6JFi4iPj+e1115r8D5hYWHEx8cTGFj9ludHHnkELy8v7r33XpvG2NxIUmgBgrzdeGXmgBrrtx3NZuPBTDYezOSyFzfY7HyBXq7EPzTeZsdrdD3GGwvAi5EQ1NOx8TjInDlzSExMZODAgaxcuZLg4GDrtpSUFGJjY9m+fTuzZ89m2bJl53Ts3NxcXn/9dWtSaKiKigpMppZxo4bZbMbZufl9xTa/iEWDRYf4s/DmwRSVVtjkeOUVFuYu+w2Lhie/PH3tpI2LiZsv7oqvRzMbE6msEPKSIfAGR0fS6JKSklixYgUuLi41EgJAcHAwn3/+Od26dWPFihUkJyef0zWGBx54gEOHDhEdHc348eOZOHEiBQUFTJ06ld27dzNo0CA++ugjlFKEhYVxyy238M033zBnzhwGDx7M7NmzycjIwMPDgwULFtCrVy+WLl3Ko48+islkwtfXlw0bjB86qampTJgwgUOHDhEXF8czzzwDwCeffML//d//obVm4sSJPP300zXifOKJJ/jggw8ICQkhKCiIQYMG1SgTHR1tfb5//36++uorYmJi+Pvf/86uXbswm8088sgjxMbGsmjRItasWUNJSQmFhYV8//333HfffXz55ZcopXjooYeYPn16teOfOcT4vHnz8PHxYeHChY4ZZlxr3ayWQYMGaeEYBSXl+uInv9cR/15rXbo9uEZ3uf8L/evhLEeHd+6O7dD6Pz5a717h6Ehsbu/evWfd/uqrr2pAT5ky5azlpkyZogH92muvndP5Dx8+rPv27Wt9vW7dOu3j46OTk5N1RUWFHjZsmN64caPWWusuXbrop59+2lp27Nix+sCBA1prrTdv3qzHjBmjtdY6MjJSp6SkaK21zsnJ0VprvXDhQt21a1edm5uri4uLdWhoqE5KStLHjh3TISEhOj09XZeXl+sxY8boFStWWM+XkZGh4+PjdWRkpC4sLNR5eXm6e/fu+tlnn63zPa1atUqPGDFCl5WV6QcffFB/+OGH1lh69OihCwoK9MKFC3Xnzp11Vpbx/2HZsmV63Lhx2mw267S0NB0SEqJTU1OrHXfZsmX61ltvtb7Ozc3V5eXlOiQkRBcUFGittf7LX/6iP/zwQ52enq6Dg4N1YmKi1lpbz3Om2v79gXjdgO9YqSmIBvN0c+bnB8ZWW3dqEMG/frSt2Y10eZllI08AM1Zkc+jz7xq8n7OT4vlrori4e/MdhiM31xhipGfPszedRUREAJCTk3PB5xwyZIi1RhIdHc2RI0cYMWIEgPXXc0FBAZs2bWLatGnW/UpLjU6kw4cP56abbuKaa67h6quvtm6/9NJL8fU1bsXu06cPR48eJSsri0suuYSgoCAArr32WjZs2MBVV11l3W/jxo3ExcXh4eEBwOTJk+uM/eDBg8ydO5cffvgBFxcXvvnmG1atWsVzzz0HQElJCUlJSQCMHz/eOqDfTz/9xMyZMzGZTLRv357Ro0ezdevWaueqbYhxwGHDjEtSEBdk+uAQgv09HB3GeYnJcoZjENspjyKT8R+60NmX420i6txn+9Ec9p/Ix6mZJcAz+fn5AUZzyNkcOHAAAH9//ws+p5ubm/W5yWTCbD59Y4SnpycAFosFPz8/du7cWWP/N998ky1btrBmzRqio6OtZWo7rm7gmG4N+SFzqnlnwYIFdOrUCTBaWJYvX14jqW7ZssX6Xk6Vq09tQ4w//PDDjhtmvCHViaa0SPORsJnflxrNR2cuyVtrLV5mrtAXP/m9jpv/k7ZYLI0c7Lmpr/no6NGj2snJSbu4uOjk5ORayyQlJWlnZ2ft5OSkk5KSzun8mZmZOjQ01Pp63bp1euLEidbXs2fP1gsXLtRan27OOeWiiy7Sn332mdZaa4vFonfu3Km11johIcFaJjo6Wu/YsUMvXLhQz54927p+4sSJet26dTo1NVWHhobqjIwMbTab9aWXXqo///zzaufbtm2b7tevny4qKtInT57U4eHhtTYfTZs2Tb/88svV1j344IN69uzZ1r+D7du3a611jXiWL1+uL7vsMm02m3V6eroODQ3Vx48fr3asY8eO6eLiYq211itWrNCxsbFaa63NZrPu0qWLnjp1ql6yZInWWjdK85GTfVOOEE1Y3zi47Qe45WtjuWmtMaz3D4/XWnz1b6kcyy0mrK1ns2sqO1NoaChxcXGUl5cTGxtLcnJyte3JyclcddVVmM1m4uLizrkjW9u2bRk+fDiRkZHMnTv3nPb9+OOPeffdd4mKiqJv376sXLkSgLlz59KvXz8iIyMZNWoUUVFRdR6jY8eOPPnkk4wZM4aoqCgGDhxIbGxstTIDBw5k+vTpREdHM2XKFGuzTVVHjx5l2bJlvPfee0RHRxMdHU18fDzz5s2jvLyc/v37ExkZybx582qNIy4ujv79+xMVFcXYsWN55plnavT72LVrF0OGDCE6OponnniChx56CMA6zPiXX37JpEmTgOrDjEdFRdW4aG0LMnS2EFV99S/YugDmZdTY9PmOY9y1ZCeXR3bgjetq3qXSlDRk6Oyq/RScnZ2JjY0lIiKCAwcOsHLlSsxmM926dTuvfgrCsS5k6GypKQhRlbuvMVnQI77G8usC66ZTlYMvd6cR9sAa/r1il4OCtI0OHTrw888/M2XKFCwWC8uXL+fJJ59k+fLlWCwWpkyZIgmhFZILzUJUNeBa49vfUgG/zIe001/8F3cP5N7LIjiUUciKHcc4mF7AvuMn6d3Rx4EBX5gOHTqwbNkykpOTWbVqFTk5Ofj7+zN58mQZ+6iVkqQgRFW+wTD6PuP570ugvNi6KcjbjTlje7BiRwordhzj18PZ3Lfsd1b/fYSDgrWdkJAQZs+e7egwRBMgSUGIurh4wK7PIPcoXPEsdDQubMYNCMbd2cRfP95O2skSrn1nc41dQ/w9eGpK/8aOWIgLJtcUhKhL9EzjMXkL5KVU29SzgzfDw9vSJcCD0nJLteWXQ1l8uTuNnMIyBwQtxIWRmoIQdRn2N9j+gTH/QsTl1TZ1C/Li41uH1dglNbeYi5/6gbzicgb+91uW/eUiBnWxfa9TIexFagpC1GX3/yDzAFxyPzg17L9Kex933rh2IN0CPdEavtyVxuvrE3h9fQKbDmXaOeCmbenSpfTu3ZsxY8Y4OhRxFlJTEKIuGX8Yj7/MB99QCK6/b4LJSXF5v44s3HSExMxC3vnpsHXbwFA//ve35jte0oXQWrNgwQJef/11SQpNnCQFIeoy5t8Q0A2+fxTeGQuBEaCc4KI5MPD6s+766W3DKKswZjL75VAWNy/aSkJ6AeNf+LHW8m29XHnvpsF4uLac/5JHjhzh8ssvZ8yYMbzxxhuA0UN48uTJPPvssw6OTtSl5fwFCmFrTk5Gv4U+k43aQvpeSN0JPz4N0deetUnJyUnh7mRMFhPezourojtZk8SZ1v2RwdGsIvakGn0evNxs/N/yyweq9bewiQ794PKn6i22f/9+Fi5cyOuvv84ll1zCc889R0yMzCPelElSEKI+bt5wyQPG893LYdktcPhH6N6wZpCQAA9emlFzZjyAkvIKes37CoBpb/7CuN7teefGlvOl2aVLF4YNq3lBXjRdkhSEOBc9JxpDYXz7MBy42GhSGnQTOJ3fFJLuLiZWzxnB2t3HeWP9IXYk5VBSXoG7iw2npGzAL3p7qTqMtGge7Hr3kVJqglJqv1IqQSn1QC3bQ5VS65RSO5RSvyulrrBnPEJcMBd3uOjvRoe2nYthzT3w4VVw8vh5H7JfsC9Duhq3rWYVlpFXXG6raIU4Z3arKSilTMB8YDyQAmxVSq3SWu+tUuwh4DOt9RtKqT7AWiDMXjEJYROj5xqL1rDjI/jyPni5P7i0qb38RXNOD51Rh0sigvB0NVFYVsH4F37kou5teev6ltOMJJoPezYfDQEStNaJAEqpT4FYoGpS0MCp0cR8gVQ7xiOEbSll3IUUMhS2vw8Wc80yB76GfavrTQpKKe4eH8G+4/ks357CoYxCvt93ggGh/gR4utrpDdhXWFgYu3fvtr5ev36944IRDWbPpNAZqDpzRwow9IwyjwDfKKX+DngC4+wYjxD2ERQBf3qi9m2unvDTS5DwHYSf/c/71pHd2JSQyfLtKSSkF/Dn9+O5ekBnXpgebYeghaidPZNCbVNTnTmjz0xgkdb6eaXURcCHSqlIrXW1e/eUUrcDt4MxY5QQzcbwu+DAN7DkBugxvvo2jwAYO894rHRxeCDf3D2KZdtSeHtDIpsTsygsNeNp69tUhaiDPS80pwBVB2QPpmbz0J+BzwC01r8A7kCNLp9a67e11jFa65igoCA7hSuEHbj7wHXLIDgG0vdVX7a9D9/WnMYxor03MV38AUjNK6G4vKKxoxatmD1/fmwFeiilugLHgBnArDPKJAGXAouUUr0xkkLNeRCFaM68O8CNq2qu//Y/8PNLEDUTwqrPyTC2Vzvr87d+PMS/J/axd5RCAHasKWitzcAc4GtgH8ZdRnuUUo8ppSZXFvsncJtS6jfgE+Am3dwmjRbifI2+H/xCYfVdsG0R5KdZN5mcFNMGBQOwIykXcx29oW3FbDbz0ksv8dJLL2E213LBXLQadm2o1FqvxbjNtOq6h6s83wsMt2cMQjRZrh4w6UX4ZCasvhMGXAex8wHjbqQe7b0AiD+aw9HsIroHedktlMWLF3P33XcDEBAQwA033GC3c4mmTYbOFsKRwsfBvQeN+RoSqw+WN2toFwAiO/vQLdB+PYPNZjOPP/649fXjjz9+wbWF3NxcXn/9devr9evXM2nSpAs6Zm0WLVrEnDlzzmmfsLAwMjNrDmP+yCOP8Nxzz9VYX1payvTp0wkPD2fo0KEcOXKkzuP269eP6OjoBo/vVFpayrhx44iOjmbJkiXn9D7sRZKCEI7m7A4n9oBXu2qrF2xIBODx2EiUqu1mPttYvHgxCQkJhIeHEx4eTkJCAosXL76gY56ZFBqqoqLpXVR/99138ff3JyEhgbvvvpv777+/zrLr1q1j586dxMfH13tcs9nMjh07KC8vZ+fOnUyfPt2WYZ83SQpCONqvb0NeElxqbVklI7+UBRsTuaJfBwaE+tvt1FVrCfPmzWPePONuqAutLTzwwAMcOnSI6Oho5s6dC0BBQQFTp06lV69eXHvttZy6fBgWFsZjjz3GiBEjWLp0KYcOHWLChAkMGjSIkSNH8scfxrwWS5cuJTIykqioKEaNGmU9V2pqKhMmTKBHjx7cd9/pToKffPIJ/fr1IzIyss4v8ieeeIKePXsybtw49u/fX2uZlStXcuONNwIwdepUvv/+e8730udNN93EPffcw5gxY7jtttu47rrr2LlzJ9HR0Rw6dOi8jmlzWutmtQwaNEgL0WIUZmn9ZIjWH06ptvqhFbt0twfX6MSMgvM67N69extU7v3339eADg8P1+Xl5bq8vFyHh4drQL///vvndW6ttT58+LDu27ev9fW6deu0j4+PTk5O1hUVFXrYsGF648aNWmutu3Tpop9++mlr2bFjx+oDBw5orbXevHmzHjNmjNZa68jISJ2SkqK11jonJ0drrfXChQt1165ddW5uri4uLtahoaE6KSlJHzt2TIeEhOj09HRdXl6ux4wZo1esWGE9X0ZGho6Pj9eRkZG6sLBQ5+Xl6e7du+tnn322xnvp27evTk5Otr7u1q2bzsjIqFEuLCxMDxgwQA8cOFC/9dZbtX4uN954o544caI2m83Wz2XixIkN/FQbrrZ/fyBeN+A7VmoKQjjSxuehNB/GP2pddTizkE9+TWLmkBC6NtK1hHnz5uHs7Iyzs7PNagtnGjJkCMHBwTg5OREdHV2tbf5U00lBQQGbNm1i2rRpREdHc8cdd3D8uDHY4PDhw7nppptYsGBBtWamSy+9FF9fX9zd3enTpw9Hjx5l69atXHLJJQQFBeHs7My1117Lhg0bqsWzceNG4uLi8PDwwMfHh8mTJ1MbXUutoLbmvJ9//pnt27fz5ZdfMn/+/BrnO2XatGmYTDYcBdfGJCkI4Sg5R4ymo+hZ0L6vdfWzX/+Bq7MTd14aYdfTV72WMGvW6S5Es2bNstm1harc3Nysz00mU7WEc2qIbYvFgp+fHzt37rQu+/btA+DNN9/kv//9L8nJyURHR5OVlVXncWv7Iq9NQ67VBAcHk5xsjNhjNpvJy8sjICCgRrlOnToB0K5dO+Li4vj1119rPV5TH05ckoIQjvLtf8DJ2Zj2s9KOpBzW7kpjTM92HM0qJP5INvFHstl2NJsys+36KtRWSzjFFrUFb29v8vPzz3k/Hx8funbtytKlSwHjV/pvv/0GwKFDhxg6dCiPPfYYgYGB1i/q2gwdOpQff/yRzMxMKioq+OSTTxg9enS1MqNGjWLFihUUFxeTn5/P6tWraz3W5MmTef/99wFYtmwZY8eOrZFMCgsLre+3sLCQb775hsjIyHN+/02BJAUhHKE0H/auNCbpaXP6V+faXUZTyZpdx5n65i/WZcobvzB/XYLNTl9XLeGUC60ttG3bluHDhxMZGWm90NxQH3/8Me+++y5RUVH07duXlStXAjB37lzrheNRo0YRFRVV5zE6duzIk08+yZgxY4iKimLgwIHExsZWKzNw4ECmT59OdHQ0U6ZMYeTIkbUe689//jNZWVmEh4fzwgsv8NRTxqRFqampXHGFMQXMiRMnGDFiBFFRUQwZMoSJEycyYcKEc3rfTYVqaDWrqYiJidENud1LiCZvy1vGXAydB0GH/gCUWzTpRRaS+/6N8jbGMGALNh5m48EMvvj7CPp28m3Qofft20fv3r1r3WY2m+nduzcJCQ1LMuHh4ezbt69abUI0bbX9+yultmmt6+1AIf/KQjjK0DugjT98/xj8sQYAF6BzYTqdO4VC/7kkZxex+VAWVw8IbnBCqM+pWkJDnaotSC/n1kFqCkI0Ne+Mg5St4OzOKz0/4IVtZlxMCpOTYnpMCI/G1t9Wfbaagmj5LqSmINcUhGhqJjxtPJpLGBsdwR2jujGyRxAl5ZZzmlehuf3gE7Zxof/ukhSEaGoKThiPE54iskdXHri8F3nF5QR6ufHXS7o36BDu7u5kZWVJYmhltNZkZWXh7u5+3seQawpCNCXmUvj6XxDUCwbfCsA3e0+w7WgOnf3acPeSnQT7e/DI5L5nPUxwcDApKSlkZMj0JK2Nu7s7wcHB572/JAUhmpJf5kPOYbh+BZhcAPBt40L/YF/KKzTf7UunnbdbvUnBxcWFrl27NkbEooWRpCBEU3Ey1Rj2oudE6D7WunpYt7asmjOC5dtS+OfS37hvQi/rtqSkJFatWkVubi5+fn7ExsYSEhJS29GFaBBJCkI0Fd88BBXl8KcnamwqKDXz9Fd/4OXmTO+O3qSlpTFnzhxWrFiBxXK6p/Odd95JXFwcr732Gh06dGjM6EULIUlBiKbi4HfQayIE1Gz2ScsrIbeonLIKC08t/4VNL80mMTERFxcX4uLi6NmzJ/v372flypUsX76cHTt2sGnTJtq3b++ANyKaM7n7SIimostFcKz2Pjjh7byYMzYcgDVv/R+JiYn4hkQwet6njJ3zFE888QTLli3j8OHDDBw4kMTERGbPnt2Y0YsWQpKCEE1BUTakxINP3XeNRIX4MTiwgty9P+Nkcibm1ic5WOjG1sPZJKTnk5CeT4mrL+99vARnZ2dWrFhx1kHjhKiNJAUhmoLvH4XiHLjimTqLjI4IYrR7ElpbiLsqliuGGXcgrdufwbgXNliXaR8dZNKVk7FYLKxataqx3oFoIeSaghCOlhIP2xYZz939zlo0NzcXgJ49e/Lnkd2I6ODNqf5puUVlzFu5B1eTEyZ/Y2z/nJwce0UtWihJCkI42qkezAAJ30LMLXUW9fMzksb+/fvxbePCpP6drNsSMwrw93Ahp6icH7f+DoC/v/3mdxYtkzQfCeFg5vA/8VLOpby0uRRzaO1j+p8yefJknJycWLVqFSkpKdW2dQvy4t4/9cR8MoOcfZtwcnKqc4pJIeoiSUEIB1u8eDF3v7KCu78uZfFXv5y1bGhoKHFxcZSXlxMbG1vtQnJWQSn/XbKRojVPUWE2ExcXJx3ZxDmTobOFcCCz2UzvnhEkJB4GGjahTVpaGsOHDycxMRFnZ2diY2OJiIhg+Q9bObB1HVgq6Natm/RTENXI0NlCNAOLFy8mIfEw4QFOhLf3bND0lx06dODnn39mypQpWCwWli9fzpNPPsmBLd+B1vQYOo7x9y9gR4bt5nQWrYfUFIRwkKp5pVufAAAgAElEQVTTYr4/1Q/MJdz4eck5TX+ZnJzMqlWr2HM4le8OFeDT8yIsnm3JLizj6gGdeWF6dCO8E9EcyHScQjRxp6bFDA8PZ9ZAM7TtzuO7j57T9JchISE1ei7f8WE8Px7I4O7xEfYKXbRg0nwkhAOYzWYef/xxAOY99BDOXgE4u3kwb948AB5//HHMZvM5H/e7vSf4es8JYroEEH80mxU7UjhxssSmsYuWTZKCEA5QrZZw7bXQJxaObGRW7GWEh4c36NpCbX5KyLQ+3r3kN+5e8hsvfXfQ1uGLFsyuSUEpNUEptV8plaCUeqCOMtcopfYqpfYopc79f4EQzUy1WsK8eca1g75Xg7bgvO0d5j30EHB+tYWHJvZm/b2XsP7eSxjZIxBnJ8WNF3ex+XsQLZfdkoJSygTMBy4H+gAzlVJ9zijTA3gQGK617gvcZa94hGgqqtUSZs0yVrbva0yus+FZZjl/RXj3budVW3A2OREW6ElSdhEbD2byl9Hd6dXBxw7vQrRUdrv7SCl1EfCI1vpPla8fBNBaP1mlzDPAAa31Ow09rtx9JJqzqnccNcS53Il0SlGZmT+9tAEXJyfW3jkSdxfT+YYrWpCm0E+hM1B13N6UynVVRQARSqmflVKblVITajuQUup2pVS8UipeJiIXzdmpWkJDnU9t4eXvDpKcXUzvjj5k5Jeea4iilbNnUlC1rDuzWuIM9AAuAWYC7yilagwTqbV+W2sdo7WOCQoKsnmgQjSWG264Aa113cuuZej/+KAPb7Sua8itqVXtP5EPwJpdx1m2LaWe0kJUZ8+kkAJUHXglGEitpcxKrXW51vowsB8jSQjROu3+H3h1gNCLzvsQb10/iF4dvGnr6cqNF4fZLjbRKtgzKWwFeiiluiqlXIEZwJkzfnwOjAFQSgViNCcl2jEmIZqukpNw8FvoexU4nf91gHc2HuaPtHyuHdYFLzfpnyrOjd2SgtbaDMwBvgb2AZ9prfcopR5TSp0az/drIEsptRdYB8zVWmfZKyYhmrRj8VBRCt0uuaDDLNp0BIBXvj/IWz8euuCwROti134KWuu1WusIrXV3rfUTlese1lqvqnyutdb3aK37aK37aa0/tWc8QjRpnQaAyQ0Orbugw3x861A6+7WhjYuJK6M61b+DEFVIj2Yhmor8NFAKygov6DBbj2RzLLeYf13Ri7BATxsFJ1oLSQpCNAXlxbD0ZnDzhksfPu/DFJSaeWLNPgBe/O4gX+46bqsIRStx1qtQSql7zrZda/2CbcMRopX66gHI2AfX/Q+8z39inDYuJm4d2Y0PfjlCdmEZ+0/kc3F4IL5tXGwXq2jR6qspeNezCCEuVPZh2LYILpoD4Zde0KFMToq/ju7OyeJyAF767iDv/nTYBkGK1uKsNQWt9aONFYgQrdaxbcZj/+k2OVwbVxPf3D2a697ZQom5gllDQm1yXNE61Nd89MrZtmut/2HbcIRohY7/BiZXCOpls0Ou2JFC2skSXpk5gLZerpRXWHB2UihV20ADQpxWX8+WbY0ShRCtWfo+COgGzq42Odyx3GJeX2/0T/jHJzus6y/u3pbFtw2zyTlEy1Vf89H7jRWIEK1Wh0g49INxS6p3hws+XHtvN/4zqQ8FpcZcDBsOZvLr4Wxiuvhf8LFFy9egPvBKqSDgfox5EdxPrddaj7VTXEK0HsoJdAXkHLVJUnA2OXHT8K4AHMks5M0fE4np4s8/LpVhxUT9GtpP4WOMoSq6Ao8CRzDGNhJCXIg9n8PG5yFqJoQMsemhy8wW/vHpDkxOipdnDsDZJN2SRP0a+lfSVmv9LlCutf5Ra30LII2TQlyI1J2w4i8QPAQmvWT0ZrahZ7/+g99T8nh6Sn86+7Wx6bFFy9XQIRTLKx+PK6UmYgyBHWyfkIRoBfLT4JOZ4BkIMz4GF/f69zkH3+87wYKNh7nhoi5MiLzwJinRejQ0KfxXKeUL/BN4FfAB7rZbVEK0ZOXF8OksKMmFP38DXu1sevhjucXc89lvODspvN2deeX7gzY5bqCXGzOHhMhtrS1cg5KC1vqLyqd5VM5/IIQ4D1rDqr8bHdamfwQd+tn8FPvTTpJfUo5Fw/x1ths6293FiasHdpY5n1u4ht599D5wp9Y6t/K1P/B85bUFIURDbXwedi2FsfOg95V2OcXYXu05+MQVNjveih3HuHfpbygUl7+8sdq2wWH+PDM1ymbnEo7X0Oaj/qcSAoDWOkcpNcBOMQnRMu1bDT88Dv2mwch/2vVUJifbNfFEtPfiquhOWKrMsH44s5Bdx/IYGCp9H1qahiYFJ6WUv9Y6B0ApFXAO+woh0nbB/+6AzoNg8qs2v9PInvoH+/HSjNO/AQtLzUx8ZSOd/drw8JV9HBiZsIeGfrE/D2xSSi0DNHAN8ITdohKiJSnMgsUzoI0fzFgMLs379tDHVu/laHYRn942TIbkboEaeqH5A6VUPDAWUMDVWuu9do1MiJZi4/OQnwq3/WCTHsuO9NXu4yyJT2b2mO4M7dbW0eEIOziXJqAAoFBrvVApFaSU6qq1loHahTib/DSIfxf6zzDmYG7GKiyaB/+3C4B3fzrMwp+P1FnW38OVr+8ehZebtDI3Nw29++g/QAzQE1gIuAAfAcPtF5oQLcBPL0FFOYye6+hILpiTgr+P7cHxvOI6y6SdLGX1b6l0D3LFQ25dbZYamsbjgAHAdgCtdapSSmZeE+JsTh6H+PeMcY0Cujk6mgumlOKWEV3r3G6xaK57dwttXEy8PCMaJxveASUaT0OTQpnWWiulNIBSytOOMQnRMvzyGlSUQuZ++PTa6ts6RsHo+xwTl528vTGRTYeyeHpKP7oFeTk6HHGeGpoUPlNKvQX4KaVuA24B3rFfWEK0AH6h0D7SGNYi58jp9SV58McX0G9qi6hBAPyekstzX+9nTM8grujX0TqXQ21MStHGVZqWmiqlta6/FKCUGg9chnH30dda62/tGVhdYmJidHx8vCNOLYRtnEyFF/vC8Ltg3H8cHY1NTH/rF7Yczm5w+TevGyQD9TUypdQ2rXVMfeUafGtAZRL4tvLgJqXUtVrrjy8gRiFaJ59O0OMy2PERjLwH3Jr/5bm7x0ewKyXvrGXMFs2rPxzE5KToF+zbSJGJc3XWpKCU8gFmA52BVRhJYTYwF9iJMfmOEOJcDfsrfBALrwyES+6HgTeCqfl2BBvWrS3D6um38NoPBykqq+DlGdEyv0MTdtbmI6XUSiAH+AW4FPAHXDEGx9vZKBGeQZqPRIuRvBW+fRiSNoFPMHi3r749qBdc+XKzThanHMoo4E8vbkADkZ2NWkKwXxtenTlA7lJqJLZqPuqmte5XecB3gEwgVGudb4MYhWjdQgbDzWvhwFew/UPjTqVTzKWw82PjQvVFf3NcjDbi5uzE2F7tKDVbyC4sY9exPMwVluY0BFSrUV9SODXjGlrrCqXUYUkIQtiQUtDzcmOpSmv4eCqsf9K4S8nGE/E0tmB/D96+IYYKi+a6d071ZRggE/Y0QfXN0RyllDpZueQD/U89V0qdrO/gSqkJSqn9SqkEpdQDZyk3VSmllVL1Vm2EaBWUgglPG7ezfveIo6OxmTfWJ/BLYhaPTu5LeDvpy9AUnTUpaK1NWmufysVba+1c5bnP2fZVSpmA+cDlQB9gplKqxji7lT2j/wFsOf+3IUQLFBhuNB3t/Ni4/tDMbTuazYvfHeTKqE5Mi5Ep3puq+moKF2IIkKC1TtRalwGfArG1lHsceAYosWMsQjRPo+aCZxB817z7M+QUlvGPT3bSyc+dJ+IipdmoCbNnUugMJFd5nVK5zqpy9raQKnNACyGqyk2G4hzw6+LoSM5bhUVz15KdHMst5rqhXdhz7CTbjmZjrrA4OjRRC3uOa1vbTwHr/a9KKSfgReCmeg+k1O3A7QChoaE2Ck+IJs5igdX/ADcfuOxxR0dz3vamnuTHAxkAPPnlH9b179wQw7g+7evaTTiIPZNCChBS5XUwkFrltTcQCayvrEp2AFYppSZrrat1RNBavw28DUY/BTvGLETTEf8upGyFuLfAM9DR0Zy3yM4+fD57OMVlFRzPK+afS39Da1j1Wypf70nDqXL01Z4dmn/P7pbAnklhK9BDKdUVOAbMAGad2qi1zgOsf+lKqfXAvWcmBCFapZOp8N2j0O0S6D/d0dFcEKUU0SF+AOw+lkeXAA/KzBbij2RzIr+UCotmeI9ASQpNhN2SgtbarJSaA3wNmID3tNZ7lFKPAfFa61X2OrcQzd7auWAxw6QXaUk9vCI7+7J+7hgA1v2Rzi3vbyVuQGeu7N/RwZGJU+w6V57Wei2w9ox1D9dR9hJ7xiJEs7FvtTG09piHWszQ2mc6klnIPz7dQe8OPvxfXD+5G6kJkQlUhWhqDnxlPK5/EvavhdCLIHSYsTTzns0AhaVmbv8wHpOT4q3rB8ncCk2MJAUhmprLn4XesZC8GZI2w9Z3YPN8Y1tAd/APO/cmpf7Tof81Ng/1XFksmvuW/U5CegHv3zKEkAAPR4ckziBJQYimxtUDIi4zFjAGxzv+GyT9YiSJghMNO46lAo5XDmbc5WL7xHqOnvtmP2t2HefBy3sxskeQo8MRtZCkIERT5+wGIUOMZfidDd8vfiF8cRdEXwsj7rFffA306a9JvL7+EDOHhHL7qJZ5raQlsGePZiGEI/UYD/5dYc/ncOQnh4byc0Im//58N15uzoQGePDOxsMs2JDIkcxCh8YlapKaghAtlW8wTP8I3hwOOxdD15EOC+W7fSeosGgKSs08/dXpXs2FZWbuGhfhsLhETZIUhGjJNr8OTi4w4m6HhvHwpD7887KeAPx6OIvbP9hGVIgffxnd3aFxiZokKQjRUiVtMYbdHn4XBDn217hSCi83Zw6cyOeuT3fi5e7MPy+L4PBZmo+83Z0J9pe7kxqbJAUhWqIKM6z9J/h0NobfbiKe+eoPTpaYAZi1oP4pVH7916W083G3d1iiCkkKQrRE8e9B2i6Ytgjcms4MZw9P6svUQWefYGf178dZ8/txrh7QmSBvt0aKTJwiSUGIlqYgA374rzGYXp+rHB1NNaFtPQhtW3eT0KaETL7Zk8bF3dvy1JT+MvyFA8gtqUK0ND+9AKV5MObfzWowvX3HT3LHR9voGujJG9cNwtVZvp4cQWoKQrQ0YSONoTFWzobxj4HJ5fQ2z3bQvi84Na3xhhIzCrj+3S14uTnz3k2D8W3jUv9Owi4kKQjR0vS6Aq5fAZ9eC5/MqLndzdcYXC9sOHQZDh2jqieORpaSU8R172xBa/jo1qFyx5GDSVIQoiUKGwF/3w7Zh06v0xpyjxq9m49ugoNfG+tdPI0hNIb9FSL+1KhhFpaaue6dLaTmldCvsy+vfH/Qus3b3ZmHJvbB3aVp1WpaOkkKQrRUnm2NparQoadHS80/AUmb4MjPcPAbWHwN9J4ME54C7w62jUU51Xp9o6isAi93Z8LaepBfUs5vybnkl5jJKizDt40Lcy/rJUmhkSmtm9eUxzExMTo+XmbsFMKmzGXwy6vw4zNgLrH98b07woDrYeAN4BdSZ7ETJ0uY/tYvZBWUsfi2YfQL9rV9LK2UUmqb1jqm3nKSFIQQVtmJsGeFMey2rWgNx+Lh4LdGbaHHZcatsj6djEmDvNqDux8ZheXMePsX0vJK+PDWoQwM9bddDEKSghCiick5Cts/gB0f1pgTQjs5k6V9OGHxpXNwF/yCgsEryEgYnkGnk4dnELTxb1a32jYVkhSEEE1TRTlkH4bCdCg4QWZaCmt+2Yl3RQ5jg8GvIgcK0o3tFnPN/Z1cjCThGQTt+kD4pdBtTM3rJ6KahiYFudAshGhcJhdjgL6gCPak5nHj57+i6cMHtw/Br1OVawgWC5TkGgmi4AQUZlR/np9mzGf922JAQacBED7OWIIHg5N0fjsfkhSEEA6xKSGTOz7ahpebMx/dOpTuQWeM0eTkBB4BxtKuV+0HsVRA6k5I+A4OfQ8bn4MNz8CklyDmZvu/iRZIkoIQotGt+f04dy3ZQXmF5sqoTny1O826zeSkmDoomECvBgyG52SC4EHGEj0L3r0MdAV0H2vH6Fs2SQpCiEa3Zlcq5RXG9czFW5JqbO8e5MX4Pu0bfsDCLPgwDsoK4ea14N/FVqG2OpIUhBCNbv6sgdakAJBZUMrtH8azJ/Uk8yb2ObeEUJwLH0+BvGRjeI8OkXaIuPWQpCCEaHRKKVydjdtKE9LzuWnhVjILSnnrukFc1vccelMXZcOHV8GJvTD9Q+hysZ0ibj0kKQghHOa7vSe4e8lO3FxMLLn9IqJC/Bq+c2EmfHAVZB6AGYsh4jL7BdqKSFIQQjjEuz8d5vEv9gIwtFsA/9uewv+2p1i3+7Rx4R+X9sDFVMutpSdT4cOrIecwzPzE6KsgbEKSghDCIXal5OLnYQzZHX80h/ijOQDkl5ipsGi83Z25ZXhX/D1dq++YvBWWXGtcVJ71GXQb3diht2iSFIQQDvHSjAHVXmuteXtDIk9/9Qc92nnx1vWDaiaEHR/BF3eDT2e4YSW0692IEbcOkhSEEA5XXFbBvUt/Y82u43TwcWf2mHAS0gtISC8AQFnM9Nr1DCEH3jfmnp660OjUJmzOrklBKTUBeBkwAe9orZ86Y/s9wK2AGcgAbtFaH7VnTEKIpmfd/nTW7DoOQNrJEu5astO6zY98XnN5hRDTHg51v4Hus14Ek/yetRe7DYinlDIBB4DxQAqwFZiptd5bpcwYYIvWukgp9VfgEq319LMdVwbEE6Ll0Vpz4EQB5RWWautdsv7Aa8X1BFqymGe+heyIa3CuZUwjpeDWkd0Y1EWG265LUxgQbwiQoLVOrAzoUyAWsCYFrfW6KuU3A9fZMR4hRBOllKJnB+/qK/d9gf7iDvKUK/d7Pck+516QXVytSE5RGen5pSgFV0Z1asSIWy57JoXOQHKV1ynA0LOU/zPwZW0blFK3A7cDhIaG2io+IURTdWIPLLkWBfj5deGlDj+CbyL4Bhszt/mGsC3Xk9uXH8XbzZkXp0cz7lx6QYs62TMp1DYLRq1tVUqp64AYoNZ7y7TWbwNvg9F8ZKsAhRBNVEA3+NP/QVYC5CZD1iFIXA9lBdYig4CfccXJPwTX+FBICAHf0CqJI9i4S8nk4rC30RzZMymkAFUnYw0GUs8spJQaB/wbGK21LrVjPEKI5sKlDVw0u/o6rcnMOMFrK9Zx7OgBxnQoZUq4xrXgmJE49u8xJuapSjkZ80P7BoNvyOlk4Rt6+rnbGc1WrZw9k8JWoIdSqitwDJgBzKpaQCk1AHgLmKC1Tq95CCGEMHy7L50Hlu8iv9SPB664gRkXh+HkdEaDRHkx5B0zBsfLSzaSRV6K8TxlK+z9vOZsbu5+4BcKE5+HkCGN94aaKLslBa21WSk1B/ga45bU97TWe5RSjwHxWutVwLOAF7BUGXOuJmmtJ9srJiFE81NYaubxL/by6dZk+nT04dMZ0fRoX8eve5c2EBhuLLWxVBgzt+VWJo2kzbB1AWSWUHuLd+sjczQLIZqs7/ae4D+r9pCaV8xfR3fnrnERuDrbaJrNjP3GHAyl+TDzUwgbbpvjNlFN4ZZUIYQ4LyXlFdz16U6+2mPMyDY8vC3OTorXfjhY+w5KcfWAzoQFejbsBGm74f0roTgHJr8K7fvYKPLmT5KCEKLJSc4uYuPBDFRli86mQ1lsOpRVo1zVho4OPu4NTwrJW6A423i+ao6xuPtBQFfw73r60T/MeO7dyZgzuhWQ5iMhRLNTUl7BWz8mMn99Ap6uJh6Z3JfJUZ1Q6hyuC5TmQ84RY8k+bAzDfeoxN9mY6/kUkxv0mgjTFtr6rTQaaT4SQrRIGw5kMG/lbo5mFREa4MEDl/ci0MuNrUdyzrpfn04+eLlV+cpz84YO/YzlTBVm40J0diJ8eZ/RX8L7HGaEa8YkKQghmo0ys4VbP4inzGyMkZSUXcTfPt7eoH1vuKgLj8U2cP5mk7Nxm+pPLxgJYehfjc50rYAkBSFEs+Hq7MQXfx9BRn79/VwTMwp47psD5BWXM7ZXO+aMqeM21bqsvhN2fAiBERA6FJJ/BZ+O4NUBnF3r37+ZkqQghGhWItp7E1FXPwWMuRle/eEgCzYm4u5i4tmp/Zk6KPjcrjeA0XxkcjXmgF56U/VtHoFGT2nvDkaiOPXcu1PlY0fwDGqWF6clKQghWgSLRfP5zmM89/V+UvNKuHpgZx68vDdB3m7nd8AbVhq3NxVlQ/7x08vJU8/TID8V0n6HgnRqDO3m6g23fQ9BPS/4vTUmSQpCiGZNa82Gg5k89eUf7Dt+kn6dfXlxejRDu7W98IMrBZ5tjaXDWa5HVJQbiSH/uJEkvrgH3H2M2kIzI0lBCNEsaa354Y90Xv0hgZ3JuYQEtOHlGdFc2b9TzTGR7M3kAr6djSTw9b/B2R1mLG6WU4ZKUhBCNCtlZgtrdx1nwcZE9qSeJNi/Df8X148pgzrj5mxybHDfPgzJm6FdH9i7Eo5shDb+lUtAlef+TfZitSQFIUSzkFVQyuItSXy4+Sjp+aV0C/Tk2an9uWpAZ1xMTeSCrrsPBPaEwgzY9ErNEVmrcvWqniQ8AmpPIKfWewQazVh2Jj2ahRBNVoVFs/FgBkvjU/h27wnKKiyMigji5uFhjO4R1PjNROdCa2NSoKJsY4yl4lOPOVCUc/r5qfVFVbZX7U1d1cxPoefl5xWO9GgWQjRbB0/ks3JnKsu3p3A8rwR/DxdmDQ3lumGhhLdrJpPiKGX0mnbzBv8uDd9Payg9WSWBZBtDcay5B07WmKfM5iQpCCGahIT0fL74/Thrdx3nwIkCnBSMighi3qQ+XNq7neOvFzQWpcDd11j8w4x1+SeMpNAIJCkIIRyqsNTMrHe28FtybrX1gV5uZOSXMn9dAvPXJZzzcQeHBfDI5L62CrPVkKQghHAoZ5OiZ3svgrzO/26cY7kl7Dt+0vq6jYuJsb3a2SK8VkeSghDCodycTTwzNeqc9tFak5BewNd70vh6zwlrQhjaNYCpg4K5vF/H6iOiigaTT00I0SyUlFew7WgOGw5m8O2eEyRmFgIQHeLH/RN6MbFfR0Lbejg4yuZPkoIQokmqsGh2Hcvj54RMfk7IJP5oDmVmC85OimHd2nLz8DDG9+lAB193R4faokhSEEI0CTmFZexMzmVHcq7xmJRDfonR+atXB2+uH9aFEeGBDO4aIE1DdiSfrBCi0RWUmjlwIp/fKxPAzuRcjmQVAeCkjOGxJ/XvyEXdA7m4e1sCvc5zpFNxziQpCCHspqS8gsSMQg6cyGf/iXwOpBmPKTnF1jLtvN0YEOrH9MGhRIf40T/YF0+pCTiMfPJCiAtSYdGk5haTnF1EUnYRR7OLOJJpJIIjWUVUWIyhdFxMiu5BXgwM9WfmkFAi2nvTt5MPHX3dz30CHGE3khSEEPUqKDWTlFVEUnYhSae+/LOKSM4uIiWnGLPl9Bhqzk6KkAAPerTz4op+HYlo702vDt6EBXo2nYHrRJ0kKQghzuqL31OZs3hHg8ubLZrDmYUczizkm70n7BjZaUrB89OiuHpgcKOcryWTpCCEOKuBof7cMz7C2gxkLxatySosIyO/lPT8Uk7klZB2sqTO8q4mJ0IC2tA10JOwtp4MDmt+E9o0RZIUhBBn1cmvDf+4tMd5719cVkF6fon1y954PPN1KVkFpZyZd1xMRlNU17aehAV6EtbWo/LRk05+bTA15aGzmylJCkKIs7JYNAVlZgpKzOSXmMkvKSe/tMrzklPbjOf5pWbyisvJrPyyLyitOdGMyUkR6OVKO2932vu406+zL0HebrTzdiPI240gb3faebvJF78DSFIQogWyWDRlFRZKzRZKzRUUl1WQX2LmZEl5tS/3gsov95NnvM6vUq6gzEx9c3E5KfB2d8HLzRlvd2d82rjQu5MPo7zcaOfjRjtv92pf+gEerk17gpxWzK5JQSk1AXgZMAHvaK2fOmO7G/ABMAjIAqZrrY/YMyYh7EXr01/EZZVLqfWxotq6UrPFKFteQVlF9bJVy1v3r7BQWm6pLFtR6zmqHq+8ouHt/27OTni7O+Pt7oK3uzNebs4EBnpaX3u7nd7m7e6Cl3vlF7+7M15uxnoPV5PcVtpC2C0pKKVMwHxgPJACbFVKrdJa761S7M9AjtY6XCk1A3gamG6vmIQ4U3J2ESt3Hqvy5V3zi7zal+4ZX+7VnldYHP12rNycnXB1djIeTU60cTVZv9h9qvyi93J3xtW5/ttEyy0WsovKyC4qa4ToxZnalGZyM8b1mTZ2Ppc9awpDgAStdSKAUupTIBaomhRigUcqny8DXlNKKd3cJo4WzdYPf6Tz3DcHHB2GzZ1KbvmODkTYRBC53OwOu1NPMtjO57JnUugMJFd5nQIMrauM1tqslMoD2gKZdoxLCKsbLw5j+uAQR4chxFllpSXDu8Ztu/Zmz6RQWwPjme+oIWVQSt0O3A4QGhp64ZEJUYW7SyuZ+1c0W16enmz3GoVH++52P5c9k0IKUPUnWDCQWkeZFKWUM+ALZJ95IK3128DbADExMdK0JIRoVXwDghh47+pGOZc9ByLZCvRQSnVVSrkCM4BVZ5RZBdxY+Xwq8INcTxBCCMexW02h8hrBHOBrjFtS39Na71FKPQbEa61XAe8CHyqlEjBqCDPsFY8QQoj62bWfgtZ6LbD2jHUPV3leAkyzZwxCCCEaTsaxFUIIYSVJQQghhJUkBSGEEFaSFIQQQlhJUhBCCGGlmlu3AKVUBnDU0XGcp0BkCI+q5POoTj6P6uTzqOlCPpMuWuug+go1u6TQnCml4rXWMY6Oo6mQz6M6+Tyqk8+jpsb4TKT5SAghhJUkBSGEEFaSFBrX244OoImRz6M6+Tyqk8+jJrt/JnJNQQghhJXUFIQQQlhJUhUhhJQAAAWISURBVLADpdQEpdR+pVSCUuqBWrbfo5Taq5T6XSn1vVKqiyPibCz1fR5Vyk1VSmmlVIu+46Qhn4dS6prKv5E9SqnFjR1jY2rA/5dQpdQ6pdSOyv8zVzgizsailHpPKZWulNpdx3allHql8vP6XSk10KYBaK1lseGCMUz4IaAb4Ar8BvQ5o8wYwKPy+V+BJY6O25GfR2U5b2ADsBmIcXTcDv776AHsAPwrX7dzdNwO/jzeBv5a+bwPcMTRcdv5MxkFDAR217H9CuBLjJkrhwFbbHl+qSnY3hAgQWudqLUuAz4FYqsW0Fqv01oXVb7cjDErXUtV7+dR6XHgGaCkMYNzgIZ8HrcB87XWOQBa6/RGjrExNeTz0IBP5XNfas7g2KJorTdQywyUVcQCH2jDZsBPKdXRVueXpGB7nYHkKq9TKtfV5c8YWb+lqvfzUEoNAEK01l80ZmAO0pC/jwggQin1s1Jqs1JqQqNF9//t3V2IVVUYxvH/UxYmDnThlWQZmVloWSgUFQqJUBfeiiQ10aV9f1wVFF31QQYFUkRfREgZUUNBIkEkpqSJNmpGUSmCFxJkZBZpTxdruTkMo3O0OefYnOd3NbPXnrUXi332e9baa97Vfe30x5PASkkHKPuz3Nudpp21TvcZc1o6uslOn9Iox0Zd4iVpJbAAWNTRFvXWKftD0jnAC8BgtxrUY+3cH5MoU0iLKaPIjZLm2v61w23rhXb6YwXwpu3nJd1A2a1xru1/Ot+8s1Lbz5gzkZHC+DsAzGj5/SJGGe5KWgI8Biyz/VeX2tYLY/XHADAX+FzSz5Q50qEJ/LK5nfvjAPCR7b9t/wR8RwkSE1E7/XE38B6A7c3AZEoOoH7V1jPmTCUojL+twOWSLpV0PmXf6aHWE+p0ySuUgDCR54thjP6wfdj2NNszbc+kvGNZZntbb5rbcWPeH8CHlMUISJpGmU76saut7J52+mM/cAuApCspQeFQV1t5dhkC7qirkK4HDts+OF6VZ/ponNk+JukeYD1lZcXrtndLegrYZnsIeA6YCqyTBLDf9rKeNbqD2uyPvtFmf6wHlkraAxwHHrX9S+9a3Tlt9sfDwKuSHqRMkwy6LsOZiCStpUwdTqvvUZ4AzgOw/TLlvcptwA/AH8Bd43r9Cdy3ERFxmjJ9FBERjQSFiIhoJChEREQjQSEiIhoJChER0UhQiL4k6bikHZJ2SVonaco41LlA0ounKJ8u6f3/ep2ITsqS1OhLkn63PbX+/A7wte3VLeWifD76NZVC9KmMFCJgIzBL0kxJ30paA2wHZkhaKmmzpO11RHEikCyU9KWknZK+kjQgabGkj2v5ojoS2VH3ARio9e+q5ZMlvSFpuJaf+A/mQUkfSPpU0veSnu1Rn0SfSlCIviZpEnArMFwPXUFJS3wtcAR4HFhi+zpgG/BQTcfwLnC/7WuAJcDREVU/AqyyPR+4eZTyVQC251ESvr0laXItmw8sB+YByyXNIKJLEhSiX10gaQflQb8feK0e31dz1ENJzncVsKmeeydwCSVwHLS9FcD2b7aPjah/E7Ba0n3AhaOU3wS8Xf9+L7CPkuMI4LOaE+pPYE+9ZkRXJPdR9Kuj9Vt8o+ahOtJ6CNhge8WI865mjFTFtp+W9AklR82WmhW3dQOh0dIfn9CaNfc4+ZxGF2WkEHFyW4AbJc0CkDRF0mxgLzBd0sJ6fKBOQzUkXWZ72PYzlNHInBF1fwHcXs+dDVxMSZEd0VMJChEnYfsQZfOftZK+oQSJOXXbyOXAS5J2Ahso6ZxbPVCXu+6kvE8YubveGuBcScOU9xODE3xfjfifyJLUiIhoZKQQERGNBIWIiGgkKERERCNBISIiGgkKERHRSFCIiIhGgkJERDQSFCIiovEvnse84dYJS6gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier(n_estimators=100, random_state=0, max_features=2)\n",
    "rf.fit(X_train, y_train)\n",
    "\n",
    "# RandomForestClassifier has predict_proba, but not decision_function\n",
    "precision_rf, recall_rf, thresholds_rf = precision_recall_curve(\n",
    "    y_test, rf.predict_proba(X_test)[:, 1])\n",
    "\n",
    "plt.plot(precision, recall, label=\"svc\")\n",
    "\n",
    "plt.plot(precision[close_zero], recall[close_zero], 'o', markersize=10,\n",
    "         label=\"threshold zero svc\", fillstyle=\"none\", c='k', mew=2)\n",
    "\n",
    "plt.plot(precision_rf, recall_rf, label=\"rf\")\n",
    "\n",
    "close_default_rf = np.argmin(np.abs(thresholds_rf - 0.5))\n",
    "plt.plot(precision_rf[close_default_rf], recall_rf[close_default_rf], '^', c='k',\n",
    "         markersize=10, label=\"threshold 0.5 rf\", fillstyle=\"none\", mew=2)\n",
    "plt.xlabel(\"Precision\")\n",
    "plt.ylabel(\"Recall\")\n",
    "plt.legend(loc=\"best\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.659559370099291 0.6663140319391998\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import average_precision_score\n",
    "average_pre1=average_precision_score(y_test,rf.predict_proba(X_test)[:,1])\n",
    "average_pre2=average_precision_score(y_test,svc.decision_function(X_test))\n",
    "print(average_pre1,average_pre2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n  FutureWarning)\nD:\\ML\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.953\nConfusion matrix:\n[[37  0  0  0  0  0  0  0  0  0]\n [ 0 39  0  0  0  0  2  0  2  0]\n [ 0  0 41  3  0  0  0  0  0  0]\n [ 0  0  1 43  0  0  0  0  0  1]\n [ 0  0  0  0 38  0  0  0  0  0]\n [ 0  1  0  0  0 47  0  0  0  0]\n [ 0  0  0  0  0  0 52  0  0  0]\n [ 0  1  0  1  1  0  0 45  0  0]\n [ 0  3  1  0  0  0  0  0 43  1]\n [ 0  0  0  1  0  1  0  0  1 44]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    digits.data, digits.target, random_state=0)\n",
    "lr = LogisticRegression().fit(X_train, y_train)\n",
    "pred = lr.predict(X_test)\n",
    "print(\"Accuracy: {:.3f}\".format(accuracy_score(y_test, pred)))\n",
    "print(\"Confusion matrix:\\n{}\".format(confusion_matrix(y_test, pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ML\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1943: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grid-Search with accuracy\nBest parameters: {'gamma': 0.0001}\nBest cross-validation score (accuracy)): 0.970\nTest set AUC: 0.992\nTest set accuracy: 0.973\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    digits.data, digits.target == 9, random_state=0)\n",
    "\n",
    "# we provide a somewhat bad grid to illustrate the point:\n",
    "param_grid = {'gamma': [0.0001, 0.01, 0.1, 1, 10]}\n",
    "# using the default scoring of accuracy:\n",
    "grid = GridSearchCV(SVC(), param_grid=param_grid)\n",
    "grid.fit(X_train, y_train)\n",
    "print(\"Grid-Search with accuracy\")\n",
    "print(\"Best parameters:\", grid.best_params_)\n",
    "print(\"Best cross-validation score (accuracy)): {:.3f}\".format(grid.best_score_))\n",
    "print(\"Test set AUC: {:.3f}\".format(\n",
    "    roc_auc_score(y_test, grid.decision_function(X_test))))\n",
    "print(\"Test set accuracy: {:.3f}\".format(grid.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ML\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1943: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\nGrid-Search with AUC\nBest parameters: {'gamma': 0.01}\nBest cross-validation score (AUC): 0.997\nTest set AUC: 1.000\nTest set accuracy: 1.000\n"
     ]
    }
   ],
   "source": [
    "# using AUC scoring instead:\n",
    "grid = GridSearchCV(SVC(), param_grid=param_grid, scoring=\"roc_auc\")\n",
    "grid.fit(X_train, y_train)\n",
    "print(\"\\nGrid-Search with AUC\")\n",
    "print(\"Best parameters:\", grid.best_params_)\n",
    "print(\"Best cross-validation score (AUC): {:.3f}\".format(grid.best_score_))\n",
    "print(\"Test set AUC: {:.3f}\".format(\n",
    "    roc_auc_score(y_test, grid.decision_function(X_test))))\n",
    "print(\"Test set accuracy: {:.3f}\".format(grid.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}