07_1_Hyperparameter_manual.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "19793f1f-c150-45dc-ad73-28177998e4dc",
   "metadata": {},
   "source": [
    "## 데이터 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "adabe6e4-58fa-42eb-8e64-9e05e9dd7b4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df=pd.read_csv('https://raw.githubusercontent.com/ralbu85/DataScience_2022S/master/data/auto.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8e2da77a-ba54-4f13-8aed-f01152264675",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cylinders</th>\n",
       "      <th>displacement</th>\n",
       "      <th>horsepower</th>\n",
       "      <th>weight</th>\n",
       "      <th>acceleration</th>\n",
       "      <th>model_year</th>\n",
       "      <th>origin</th>\n",
       "      <th>mpg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8</td>\n",
       "      <td>307.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>3504.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>350.0</td>\n",
       "      <td>165.0</td>\n",
       "      <td>3693.0</td>\n",
       "      <td>11.5</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>318.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>3436.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>304.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>3433.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>302.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>3449.0</td>\n",
       "      <td>10.5</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "      <td>17.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>387</th>\n",
       "      <td>4</td>\n",
       "      <td>140.0</td>\n",
       "      <td>86.0</td>\n",
       "      <td>2790.0</td>\n",
       "      <td>15.6</td>\n",
       "      <td>82</td>\n",
       "      <td>1</td>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>388</th>\n",
       "      <td>4</td>\n",
       "      <td>97.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>2130.0</td>\n",
       "      <td>24.6</td>\n",
       "      <td>82</td>\n",
       "      <td>2</td>\n",
       "      <td>44.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>389</th>\n",
       "      <td>4</td>\n",
       "      <td>135.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>2295.0</td>\n",
       "      <td>11.6</td>\n",
       "      <td>82</td>\n",
       "      <td>1</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>390</th>\n",
       "      <td>4</td>\n",
       "      <td>120.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>2625.0</td>\n",
       "      <td>18.6</td>\n",
       "      <td>82</td>\n",
       "      <td>1</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>391</th>\n",
       "      <td>4</td>\n",
       "      <td>119.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>2720.0</td>\n",
       "      <td>19.4</td>\n",
       "      <td>82</td>\n",
       "      <td>1</td>\n",
       "      <td>31.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>392 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     cylinders  displacement  horsepower  weight  acceleration  model_year  \\\n",
       "0            8         307.0       130.0  3504.0          12.0          70   \n",
       "1            8         350.0       165.0  3693.0          11.5          70   \n",
       "2            8         318.0       150.0  3436.0          11.0          70   \n",
       "3            8         304.0       150.0  3433.0          12.0          70   \n",
       "4            8         302.0       140.0  3449.0          10.5          70   \n",
       "..         ...           ...         ...     ...           ...         ...   \n",
       "387          4         140.0        86.0  2790.0          15.6          82   \n",
       "388          4          97.0        52.0  2130.0          24.6          82   \n",
       "389          4         135.0        84.0  2295.0          11.6          82   \n",
       "390          4         120.0        79.0  2625.0          18.6          82   \n",
       "391          4         119.0        82.0  2720.0          19.4          82   \n",
       "\n",
       "     origin   mpg  \n",
       "0         1  18.0  \n",
       "1         1  15.0  \n",
       "2         1  18.0  \n",
       "3         1  16.0  \n",
       "4         1  17.0  \n",
       "..      ...   ...  \n",
       "387       1  27.0  \n",
       "388       2  44.0  \n",
       "389       1  32.0  \n",
       "390       1  28.0  \n",
       "391       1  31.0  \n",
       "\n",
       "[392 rows x 8 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "861a9699-c6b2-49c9-8dc7-6eb94f006973",
   "metadata": {},
   "source": [
    "## 데이터 나누기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4175ecb6-cde8-4a1a-9ce3-09489b072098",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df=pd.get_dummies(df,columns=['origin'])\n",
    "y=df['mpg']\n",
    "X=df.drop(columns=['mpg'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df7c23d6-6ad2-497a-a196-8da2aa55a5fd",
   "metadata": {},
   "source": [
    "## 교차검증"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8e544389-620b-4e98-804f-8f2968f891b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': array([0.0028553 , 0.00196171, 0.00190687, 0.00192261, 0.00190735]),\n",
       " 'score_time': array([0.00177574, 0.00157762, 0.00164819, 0.00159502, 0.00152898]),\n",
       " 'test_neg_mean_squared_error': array([-13.88489386, -10.31628049, -11.17414797,  -8.97219426,\n",
       "        -12.18829307]),\n",
       " 'train_neg_mean_squared_error': array([ -9.93622668, -10.94725059, -10.62272588, -11.11879969,\n",
       "        -10.45256934]),\n",
       " 'test_r2': array([0.76432928, 0.77496584, 0.82889072, 0.86687802, 0.81284998]),\n",
       " 'train_r2': array([0.83731874, 0.83020708, 0.82151804, 0.81174759, 0.82463489])}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import KFold, cross_validate\n",
    "\n",
    "reg= LinearRegression()\n",
    "kfold = KFold(5,shuffle=True)\n",
    "result=cross_validate(estimator=reg, X=X, y=y, cv=kfold, scoring=['neg_mean_squared_error','r2'], return_train_score=True)\n",
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41a46b50-8d06-47a3-9897-25332c267105",
   "metadata": {},
   "source": [
    "## 하이퍼패러미터 변경(차수 K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "0c9cd99a-78cb-4b3d-8697-750ef5d4e478",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 train -10.603577243981722 0.8254812886644662\n",
      "1 test -11.411635706286182 0.81198093672582\n",
      "2 train -12.495316646462372 0.7947941035264618\n",
      "2 test -25.383526960444993 0.5762603237277467\n",
      "3 train -3.3867599948503164 0.944214356799338\n",
      "3 test -18262.579493294812 -394.33279205024206\n",
      "4 train -0.23191616238540158 0.9961889558558588\n",
      "4 test -116717.17103776161 -1882.7260784282275\n"
     ]
    }
   ],
   "source": [
    "kfold = KFold(5,shuffle=True)\n",
    "for i in range(1,5):\n",
    "    transformer = PolynomialFeatures(degree=i, interaction_only=False)\n",
    "    X_=transformer.fit_transform(X) #데이터에 차수 추가\n",
    "    reg= LinearRegression()    \n",
    "    result=cross_validate(estimator=reg, X=X_, y=y, cv=kfold, scoring=['neg_mean_squared_error','r2'], return_train_score=True)\n",
    "    print(i,'train',result['train_neg_mean_squared_error'].mean(), result['train_r2'].mean())\n",
    "    print(i,'test',result['test_neg_mean_squared_error'].mean(), result['test_r2'].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "faa96e97-fe3e-451e-9e9e-b5536fe1285c",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "b82ae222-b401-4392-bf8c-f6493a7fd6fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.44629638,  0.02286369, -0.01762971, -0.00671945,  0.07651421,\n",
       "        0.77615517, -2.56966676,  0.        ,  0.16691861])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import Ridge, Lasso\n",
    "reg = Lasso(alpha=0.01)\n",
    "reg.fit(X,y)\n",
    "reg.coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30252c89-3c0f-4d13-b477-f8e578a159c1",
   "metadata": {},
   "source": [
    "## Lambda의 크기에 따른 계수의 변화를 살펴보자"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c8d16d00-b133-4808-ac66-efde5ba94f29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 [-0.48970942  0.02397864 -0.01818346 -0.00671038  0.07910304  0.77702694\n",
      " -2.76337047 -0.13336811  0.08985776]\n",
      "0.0001 [-0.48927812  0.02396754 -0.01817795 -0.00671047  0.07907707  0.77701821\n",
      " -2.6293992   0.          0.22266311]\n",
      "0.001 [-0.4853459   0.02386686 -0.0181282  -0.0067113   0.07884366  0.77693967\n",
      " -2.62396889  0.          0.21759332]\n",
      "0.01 [-0.44629638  0.02286369 -0.01762971 -0.00671945  0.07651421  0.77615517\n",
      " -2.56966676  0.          0.16691861]\n",
      "0.1 [-0.06660805  0.0126583  -0.01331772 -0.00673194  0.05328912  0.76251936\n",
      " -1.8486989   0.          0.        ]\n",
      "1 [-0.          0.         -0.00734394 -0.00646937  0.          0.66308442\n",
      " -0.          0.          0.        ]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 864x576 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.figure(figsize=(12,8))\n",
    "lambdas = [0,0.0001,0.001,0.01,0.1,1]\n",
    "for i in lambdas:\n",
    "    reg = Lasso(alpha=i)\n",
    "    reg.fit(X,y)\n",
    "    print(i, reg.coef_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5ad92412-8829-4d0a-bdf4-bee21ae78a29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 train -10.624943723771423 0.8250576243665207\n",
      "0 test -11.21925019070598 0.813454554854224\n",
      "0.001 train -10.620527006157564 0.825147611809555\n",
      "0.001 test -11.23574532721657 0.8137264490227409\n",
      "0.01 train -10.612339518808165 0.8251440463176248\n",
      "0.01 test -11.29767833712374 0.8107973651854292\n",
      "0.1 train -10.761675967779825 0.822619660347095\n",
      "0.1 test -11.348198460238105 0.8083323211338123\n",
      "0.5 train -11.630788556471327 0.808560737984612\n",
      "0.5 test -11.986698781556473 0.802152288128758\n",
      "1 train -11.691459585895021 0.8073484751826163\n",
      "1 test -12.057468542892227 0.795984989758138\n",
      "10 train -18.017024898571727 0.7029458828695535\n",
      "10 test -18.41815129226664 0.6885933992720603\n",
      "100 train -18.68068965013027 0.6925099043382046\n",
      "100 test -18.771463503708016 0.6897321621401196\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "## 하이퍼패러미터 변경(다항식 추가)\n",
    "kfold = KFold(5,shuffle=True)\n",
    "lambdas = [0,0.001,0.01,0.1,0.5,1,10,100]\n",
    "for l in lambdas:    \n",
    "    reg= Lasso(alpha=l)\n",
    "    result=cross_validate(estimator=reg, X=X, y=y, cv=kfold, scoring=['neg_mean_squared_error','r2'], return_train_score=True)\n",
    "    print(l,'train',result['train_neg_mean_squared_error'].mean(), result['train_r2'].mean())\n",
    "    print(l,'test',result['test_neg_mean_squared_error'].mean(), result['test_r2'].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "999a3238-e7da-4bfb-a426-669a6b0d63bf",
   "metadata": {},
   "source": [
    "## 다항식 변경 + hyperparameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "446fa74f-08c7-4d60-8035-d00378471073",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 0 test -11.63148527908695 0.8056687503746932\n",
      "1 0.001 test -11.138428605710804 0.8144368786264593\n",
      "1 0.01 test -11.549574406940783 0.8054561475252108\n",
      "1 0.05 test -11.119504263638106 0.8155569623433042\n",
      "1 0.1 test -11.200069250289378 0.8161722772313936\n",
      "1 0.15 test -11.43423340019818 0.8083366088973218\n",
      "1 0.2 test -11.557466328486244 0.8061169051476436\n",
      "2 0 test -8.576015406754212 0.8548652554393769\n",
      "2 0.001 test -8.114738685782813 0.860315503686085\n",
      "2 0.01 test -8.528231706447212 0.8559216901566019\n",
      "2 0.05 test -8.057953107286519 0.8665203943751448\n",
      "2 0.1 test -8.85364004753627 0.8444393041703198\n",
      "2 0.15 test -7.899586590121771 0.8668743720868713\n",
      "2 0.2 test -8.095252912446265 0.8635662852321735\n",
      "3 0 test -7.4302476856288235 0.8781159761084449\n",
      "3 0.001 test -7.370654769332288 0.8789208439755655\n",
      "3 0.01 test -7.683707120426044 0.8718186984132344\n",
      "3 0.05 test -8.056370320277136 0.8664261950384619\n",
      "3 0.1 test -7.6199888420107085 0.8734452091164002\n",
      "3 0.15 test -7.709105091974524 0.8726574685241811\n",
      "3 0.2 test -7.711470800904581 0.870785524488797\n",
      "4 0 test -8.293625525444686 0.8631374924152091\n",
      "4 0.001 test -7.867044335038874 0.8669128466357332\n",
      "4 0.01 test -8.931462703861218 0.8544621912620209\n",
      "4 0.05 test -8.29645553413842 0.8592152113485648\n",
      "4 0.1 test -7.888505933585594 0.8673906461054575\n",
      "4 0.15 test -7.967021843579158 0.8668229750025258\n",
      "4 0.2 test -7.743421812055111 0.8703768635109416\n"
     ]
    }
   ],
   "source": [
    "# 1. 공정한 비교가 아님 (제대로 하려면 동일한 데이터셋으로 쪼개진 상태를 통제하면서 수행해야 하므로 반복이 더 추가됨)\n",
    "# 2. 데이터의 결과물을 정리하기가 어려움\n",
    "# 3. For loop를 써서 지저분해보임\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from sklearn.linear_model import Lasso\n",
    "kfold = KFold(5,shuffle=True)\n",
    "for i in range(1,5):\n",
    "    for j in [0,0.001,0.01,0.05,0.1,0.15,0.2]:\n",
    "        transformer = PolynomialFeatures(degree=i, interaction_only=False)\n",
    "        X_=transformer.fit_transform(X)\n",
    "        reg= Lasso(alpha=j)    \n",
    "        result=cross_validate(estimator=reg, X=X_, y=y, cv=kfold, scoring=['neg_mean_squared_error','r2'], return_train_score=True)\n",
    "    #     print(i,'train',result['train_neg_mean_squared_error'].mean(), result['train_r2'].mean())\n",
    "        print(i,j,'test',result['test_neg_mean_squared_error'].mean(), result['test_r2'].mean())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}