Permutation Acc Test (RF,LSTM).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f5d6a399",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 10216546920150213332\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7ccea782",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n",
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np, os\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm.notebook import tqdm\n",
    "import seaborn as sns\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(action='ignore')\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, LSTM\n",
    "from tensorflow.keras.layers import SimpleRNN\n",
    "from tensorflow.keras.layers import Dropout, InputLayer\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.regularizers import l2\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "\n",
    "print(tf.__version__)\n",
    "print(keras.__version__)\n",
    "\n",
    "#한글설정\n",
    "import matplotlib.font_manager as fm\n",
    "\n",
    "font_dirs = ['/usr/share/fonts/truetype/nanum', ]\n",
    "font_files = fm.findSystemFonts(fontpaths=font_dirs)\n",
    "\n",
    "for font_file in font_files:\n",
    "    fm.fontManager.addfont(font_file)\n",
    "    \n",
    "# 한글 출력을 위해서 폰트 옵션을 설정합니다.\n",
    "# \"axes.unicode_minus\" : 마이너스가 깨질 것을 방지\n",
    "\n",
    "sns.set(font=\"NanumBarunGothic\", \n",
    "        rc={\"axes.unicode_minus\":False},\n",
    "        style='darkgrid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "495c99c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5f122728",
   "metadata": {},
   "outputs": [],
   "source": [
    "aa = pd.read_csv('./data/lstm_feature_importance_fold_1.csv')\n",
    "baseline = aa[aa['feature']=='BASELINE'].baseline_mae.values[0]\n",
    "aa['baseline_mae'] = aa['baseline_mae'].fillna(baseline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "37ceb9b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "aa['diff'] = aa['mae'] - aa['baseline_mae']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "92cc41ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "aaa = aa.sort_values('diff', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6b6676a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>baseline_mae</th>\n",
       "      <th>mae</th>\n",
       "      <th>diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.478122</td>\n",
       "      <td>0.479388</td>\n",
       "      <td>0.001266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>63323026201</td>\n",
       "      <td>0.478122</td>\n",
       "      <td>0.478833</td>\n",
       "      <td>0.000711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>51079000220</td>\n",
       "      <td>0.478122</td>\n",
       "      <td>0.478641</td>\n",
       "      <td>0.000519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>472500360</td>\n",
       "      <td>0.478122</td>\n",
       "      <td>0.478635</td>\n",
       "      <td>0.000513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>574705050</td>\n",
       "      <td>0.478122</td>\n",
       "      <td>0.478522</td>\n",
       "      <td>0.000400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4065</th>\n",
       "      <td>50902</td>\n",
       "      <td>0.478122</td>\n",
       "      <td>0.477465</td>\n",
       "      <td>-0.000657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4066</th>\n",
       "      <td>50912</td>\n",
       "      <td>0.478122</td>\n",
       "      <td>0.476552</td>\n",
       "      <td>-0.001570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4067</th>\n",
       "      <td>51006</td>\n",
       "      <td>0.478122</td>\n",
       "      <td>0.476292</td>\n",
       "      <td>-0.001830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4068</th>\n",
       "      <td>51277</td>\n",
       "      <td>0.478122</td>\n",
       "      <td>0.475999</td>\n",
       "      <td>-0.002123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4069</th>\n",
       "      <td>BASELINE</td>\n",
       "      <td>0.478122</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4070 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          feature  baseline_mae       mae      diff\n",
       "0               0      0.478122  0.479388  0.001266\n",
       "1     63323026201      0.478122  0.478833  0.000711\n",
       "2     51079000220      0.478122  0.478641  0.000519\n",
       "3       472500360      0.478122  0.478635  0.000513\n",
       "4       574705050      0.478122  0.478522  0.000400\n",
       "...           ...           ...       ...       ...\n",
       "4065        50902      0.478122  0.477465 -0.000657\n",
       "4066        50912      0.478122  0.476552 -0.001570\n",
       "4067        51006      0.478122  0.476292 -0.001830\n",
       "4068        51277      0.478122  0.475999 -0.002123\n",
       "4069     BASELINE      0.478122       NaN       NaN\n",
       "\n",
       "[4070 rows x 4 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aaa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "f31fc995",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['63323026201',\n",
       " '51079000220',\n",
       " '472500360',\n",
       " '574705050',\n",
       " '55390000401',\n",
       " '51221',\n",
       " '51222',\n",
       " '182050789',\n",
       " '121065721',\n",
       " '37205036174']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "per_list = aaa['feature'][1:11].to_list()\n",
    "per_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "8beedb2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3832, 3045, 2287, 2388, 3456, 174, 175, 1803, 1589, 2877]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "COLS = list(pd.read_csv('/project/LSH/total_data_7727.csv')['ITEMID'].sort_values().unique())\n",
    "#COLS에서의 index 추출 (COLS의 index가 곧 3차원데이터 feature의 index이기 때문)\n",
    "\n",
    "per_list_index = [COLS.index(int(i)) for i in per_list]\n",
    "per_list_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "db730032",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "정확도 : 0.6330097087378641\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "random.seed(42)\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn import metrics \n",
    "\n",
    "\n",
    "path = '/project/LSH/'\n",
    "X =  np.load(path + 'x_(7727,4068).npy')[:,per_list_index]\n",
    "y = np.load(path + 'y_(7727,1).npy')\n",
    "\n",
    "idx = list(range(len(X)))\n",
    "random.shuffle(idx)\n",
    "\n",
    "i = round(X.shape[0]*0.8)\n",
    "X_train, y_train = X[idx[:i],:], y[idx[:i]]\n",
    "X_test, y_test = X[idx[i:],:], y[idx[i:]]\n",
    "\n",
    "X_train.shape, y_train.shape, X_test.shape, y_test.shape\n",
    "\n",
    "model = RandomForestClassifier()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "pred = model.predict(X_test)\n",
    "\n",
    "print(f'정확도 : {metrics.accuracy_score(y_test, pred)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4142d0ec",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "ac5078c9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/500\n",
      "37/37 [==============================] - 4s 47ms/step - loss: 0.6759 - acc: 0.5984 - val_loss: 0.6660 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.66601, saving model to ./model/rf_top10_seed42-01-0.6660.hdf5\n",
      "Epoch 2/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6732 - acc: 0.6066 - val_loss: 0.6662 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.66601\n",
      "Epoch 3/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6740 - acc: 0.6083 - val_loss: 0.6655 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.66601 to 0.66545, saving model to ./model/rf_top10_seed42-03-0.6655.hdf5\n",
      "Epoch 4/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6709 - acc: 0.6098 - val_loss: 0.6651 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.66545 to 0.66508, saving model to ./model/rf_top10_seed42-04-0.6651.hdf5\n",
      "Epoch 5/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6703 - acc: 0.6091 - val_loss: 0.6646 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.66508 to 0.66459, saving model to ./model/rf_top10_seed42-05-0.6646.hdf5\n",
      "Epoch 6/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6706 - acc: 0.6117 - val_loss: 0.6603 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.66459 to 0.66031, saving model to ./model/rf_top10_seed42-06-0.6603.hdf5\n",
      "Epoch 7/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6586 - acc: 0.6122 - val_loss: 0.6444 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.66031 to 0.64440, saving model to ./model/rf_top10_seed42-07-0.6444.hdf5\n",
      "Epoch 8/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6530 - acc: 0.6260 - val_loss: 0.6394 - val_acc: 0.6578\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.64440 to 0.63944, saving model to ./model/rf_top10_seed42-08-0.6394.hdf5\n",
      "Epoch 9/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6475 - acc: 0.6268 - val_loss: 0.6363 - val_acc: 0.6546\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.63944 to 0.63630, saving model to ./model/rf_top10_seed42-09-0.6363.hdf5\n",
      "Epoch 10/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6470 - acc: 0.6318 - val_loss: 0.6342 - val_acc: 0.6533\n",
      "\n",
      "Epoch 00010: val_loss improved from 0.63630 to 0.63417, saving model to ./model/rf_top10_seed42-10-0.6342.hdf5\n",
      "Epoch 11/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6439 - acc: 0.6335 - val_loss: 0.6337 - val_acc: 0.6591\n",
      "\n",
      "Epoch 00011: val_loss improved from 0.63417 to 0.63372, saving model to ./model/rf_top10_seed42-11-0.6337.hdf5\n",
      "Epoch 12/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6432 - acc: 0.6344 - val_loss: 0.6327 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.63372 to 0.63267, saving model to ./model/rf_top10_seed42-12-0.6327.hdf5\n",
      "Epoch 13/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6436 - acc: 0.6363 - val_loss: 0.6324 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00013: val_loss improved from 0.63267 to 0.63238, saving model to ./model/rf_top10_seed42-13-0.6324.hdf5\n",
      "Epoch 14/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6415 - acc: 0.6363 - val_loss: 0.6313 - val_acc: 0.6636\n",
      "\n",
      "Epoch 00014: val_loss improved from 0.63238 to 0.63131, saving model to ./model/rf_top10_seed42-14-0.6313.hdf5\n",
      "Epoch 15/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6415 - acc: 0.6426 - val_loss: 0.6307 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.63131 to 0.63066, saving model to ./model/rf_top10_seed42-15-0.6307.hdf5\n",
      "Epoch 16/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6416 - acc: 0.6419 - val_loss: 0.6304 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00016: val_loss improved from 0.63066 to 0.63045, saving model to ./model/rf_top10_seed42-16-0.6304.hdf5\n",
      "Epoch 17/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6422 - acc: 0.6426 - val_loss: 0.6299 - val_acc: 0.6617\n",
      "\n",
      "Epoch 00017: val_loss improved from 0.63045 to 0.62995, saving model to ./model/rf_top10_seed42-17-0.6299.hdf5\n",
      "Epoch 18/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6407 - acc: 0.6462 - val_loss: 0.6299 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00018: val_loss improved from 0.62995 to 0.62994, saving model to ./model/rf_top10_seed42-18-0.6299.hdf5\n",
      "Epoch 19/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6404 - acc: 0.6467 - val_loss: 0.6296 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00019: val_loss improved from 0.62994 to 0.62964, saving model to ./model/rf_top10_seed42-19-0.6296.hdf5\n",
      "Epoch 20/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6400 - acc: 0.6465 - val_loss: 0.6290 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00020: val_loss improved from 0.62964 to 0.62901, saving model to ./model/rf_top10_seed42-20-0.6290.hdf5\n",
      "Epoch 21/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6383 - acc: 0.6495 - val_loss: 0.6291 - val_acc: 0.6617\n",
      "\n",
      "Epoch 00021: val_loss did not improve from 0.62901\n",
      "Epoch 22/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6384 - acc: 0.6475 - val_loss: 0.6291 - val_acc: 0.6591\n",
      "\n",
      "Epoch 00022: val_loss did not improve from 0.62901\n",
      "Epoch 23/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6396 - acc: 0.6480 - val_loss: 0.6289 - val_acc: 0.6591\n",
      "\n",
      "Epoch 00023: val_loss improved from 0.62901 to 0.62892, saving model to ./model/rf_top10_seed42-23-0.6289.hdf5\n",
      "Epoch 24/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6393 - acc: 0.6514 - val_loss: 0.6283 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00024: val_loss improved from 0.62892 to 0.62830, saving model to ./model/rf_top10_seed42-24-0.6283.hdf5\n",
      "Epoch 25/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6389 - acc: 0.6503 - val_loss: 0.6286 - val_acc: 0.6611\n",
      "\n",
      "Epoch 00025: val_loss did not improve from 0.62830\n",
      "Epoch 26/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6366 - acc: 0.6531 - val_loss: 0.6285 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00026: val_loss did not improve from 0.62830\n",
      "Epoch 27/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6370 - acc: 0.6531 - val_loss: 0.6281 - val_acc: 0.6617\n",
      "\n",
      "Epoch 00027: val_loss improved from 0.62830 to 0.62809, saving model to ./model/rf_top10_seed42-27-0.6281.hdf5\n",
      "Epoch 28/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6351 - acc: 0.6525 - val_loss: 0.6278 - val_acc: 0.6585\n",
      "\n",
      "Epoch 00028: val_loss improved from 0.62809 to 0.62782, saving model to ./model/rf_top10_seed42-28-0.6278.hdf5\n",
      "Epoch 29/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6357 - acc: 0.6538 - val_loss: 0.6279 - val_acc: 0.6617\n",
      "\n",
      "Epoch 00029: val_loss did not improve from 0.62782\n",
      "Epoch 30/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6375 - acc: 0.6506 - val_loss: 0.6279 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00030: val_loss did not improve from 0.62782\n",
      "Epoch 31/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6355 - acc: 0.6536 - val_loss: 0.6278 - val_acc: 0.6636\n",
      "\n",
      "Epoch 00031: val_loss improved from 0.62782 to 0.62779, saving model to ./model/rf_top10_seed42-31-0.6278.hdf5\n",
      "Epoch 32/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6371 - acc: 0.6551 - val_loss: 0.6280 - val_acc: 0.6591\n",
      "\n",
      "Epoch 00032: val_loss did not improve from 0.62779\n",
      "Epoch 33/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6359 - acc: 0.6521 - val_loss: 0.6282 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00033: val_loss did not improve from 0.62779\n",
      "Epoch 34/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6356 - acc: 0.6536 - val_loss: 0.6279 - val_acc: 0.6591\n",
      "\n",
      "Epoch 00034: val_loss did not improve from 0.62779\n",
      "Epoch 35/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6356 - acc: 0.6564 - val_loss: 0.6282 - val_acc: 0.6578\n",
      "\n",
      "Epoch 00035: val_loss did not improve from 0.62779\n",
      "Epoch 36/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6361 - acc: 0.6549 - val_loss: 0.6279 - val_acc: 0.6552\n",
      "\n",
      "Epoch 00036: val_loss did not improve from 0.62779\n",
      "Epoch 37/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6341 - acc: 0.6585 - val_loss: 0.6281 - val_acc: 0.6552\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00037: val_loss did not improve from 0.62779\n",
      "Epoch 38/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6336 - acc: 0.6557 - val_loss: 0.6279 - val_acc: 0.6546\n",
      "\n",
      "Epoch 00038: val_loss did not improve from 0.62779\n",
      "Epoch 39/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6354 - acc: 0.6544 - val_loss: 0.6283 - val_acc: 0.6559\n",
      "\n",
      "Epoch 00039: val_loss did not improve from 0.62779\n",
      "Epoch 40/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6341 - acc: 0.6594 - val_loss: 0.6281 - val_acc: 0.6552\n",
      "\n",
      "Epoch 00040: val_loss did not improve from 0.62779\n",
      "Epoch 41/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6349 - acc: 0.6540 - val_loss: 0.6280 - val_acc: 0.6552\n",
      "\n",
      "Epoch 00041: val_loss did not improve from 0.62779\n",
      "Epoch 42/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6336 - acc: 0.6534 - val_loss: 0.6284 - val_acc: 0.6527\n",
      "\n",
      "Epoch 00042: val_loss did not improve from 0.62779\n",
      "Epoch 43/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6355 - acc: 0.6506 - val_loss: 0.6283 - val_acc: 0.6552\n",
      "\n",
      "Epoch 00043: val_loss did not improve from 0.62779\n",
      "Epoch 44/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6343 - acc: 0.6553 - val_loss: 0.6283 - val_acc: 0.6539\n",
      "\n",
      "Epoch 00044: val_loss did not improve from 0.62779\n",
      "Epoch 45/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6321 - acc: 0.6575 - val_loss: 0.6293 - val_acc: 0.6578\n",
      "\n",
      "Epoch 00045: val_loss did not improve from 0.62779\n",
      "Epoch 46/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6340 - acc: 0.6536 - val_loss: 0.6281 - val_acc: 0.6559\n",
      "\n",
      "Epoch 00046: val_loss did not improve from 0.62779\n",
      "Epoch 47/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6343 - acc: 0.6596 - val_loss: 0.6287 - val_acc: 0.6565\n",
      "\n",
      "Epoch 00047: val_loss did not improve from 0.62779\n",
      "Epoch 48/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6336 - acc: 0.6570 - val_loss: 0.6288 - val_acc: 0.6565\n",
      "\n",
      "Epoch 00048: val_loss did not improve from 0.62779\n",
      "Epoch 49/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6334 - acc: 0.6609 - val_loss: 0.6286 - val_acc: 0.6578\n",
      "\n",
      "Epoch 00049: val_loss did not improve from 0.62779\n",
      "Epoch 50/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6343 - acc: 0.6514 - val_loss: 0.6290 - val_acc: 0.6565\n",
      "\n",
      "Epoch 00050: val_loss did not improve from 0.62779\n",
      "Epoch 51/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6313 - acc: 0.6557 - val_loss: 0.6285 - val_acc: 0.6585\n",
      "\n",
      "Epoch 00051: val_loss did not improve from 0.62779\n",
      "Epoch 52/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6314 - acc: 0.6551 - val_loss: 0.6296 - val_acc: 0.6578\n",
      "\n",
      "Epoch 00052: val_loss did not improve from 0.62779\n",
      "Epoch 53/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6333 - acc: 0.6531 - val_loss: 0.6293 - val_acc: 0.6591\n",
      "\n",
      "Epoch 00053: val_loss did not improve from 0.62779\n",
      "Epoch 54/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6315 - acc: 0.6579 - val_loss: 0.6289 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00054: val_loss did not improve from 0.62779\n",
      "Epoch 55/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6323 - acc: 0.6536 - val_loss: 0.6288 - val_acc: 0.6598\n",
      "\n",
      "Epoch 00055: val_loss did not improve from 0.62779\n",
      "Epoch 56/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6299 - acc: 0.6547 - val_loss: 0.6298 - val_acc: 0.6591\n",
      "\n",
      "Epoch 00056: val_loss did not improve from 0.62779\n",
      "Epoch 57/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6318 - acc: 0.6566 - val_loss: 0.6295 - val_acc: 0.6617\n",
      "\n",
      "Epoch 00057: val_loss did not improve from 0.62779\n",
      "Epoch 58/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6317 - acc: 0.6549 - val_loss: 0.6295 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00058: val_loss did not improve from 0.62779\n",
      "Epoch 59/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6315 - acc: 0.6596 - val_loss: 0.6302 - val_acc: 0.6598\n",
      "\n",
      "Epoch 00059: val_loss did not improve from 0.62779\n",
      "Epoch 60/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6301 - acc: 0.6557 - val_loss: 0.6300 - val_acc: 0.6611\n",
      "\n",
      "Epoch 00060: val_loss did not improve from 0.62779\n",
      "Epoch 61/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6302 - acc: 0.6596 - val_loss: 0.6300 - val_acc: 0.6636\n",
      "\n",
      "Epoch 00061: val_loss did not improve from 0.62779\n",
      "Epoch 62/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6319 - acc: 0.6503 - val_loss: 0.6301 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00062: val_loss did not improve from 0.62779\n",
      "Epoch 63/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.6322 - acc: 0.6570 - val_loss: 0.6303 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00063: val_loss did not improve from 0.62779\n",
      "Epoch 64/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.6314 - acc: 0.6525 - val_loss: 0.6299 - val_acc: 0.6604\n",
      "Restoring model weights from the end of the best epoch.\n",
      "\n",
      "Epoch 00064: val_loss did not improve from 0.62779\n",
      "Epoch 00064: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f4c281a86d0>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random    \n",
    "seed_num = 42\n",
    "random.seed(seed_num)\n",
    "path = '/project/LSH/'\n",
    "X = np.load(path + 'x_(7727,10,4068).npy')[:,:,per_list_index]\n",
    "y = np.load(path + 'y_(7727,1).npy')\n",
    "\n",
    "idx = list(range(len(X)))\n",
    "random.shuffle(idx)\n",
    "\n",
    "\n",
    "i = round(X.shape[0]*0.8)\n",
    "X_train, y_train = X[idx[:i],:,:], y[idx[:i]]\n",
    "X_test, y_test = X[idx[i:],:,:], y[idx[i:]]\n",
    "\n",
    "X_train.shape, y_train.shape, X_test.shape, y_test.shape\n",
    "\n",
    "# ---------------------\n",
    "seed_num = 42 \n",
    "# ---------------------\n",
    "tf.random.set_seed(seed_num)\n",
    "\n",
    "lstm = Sequential()\n",
    "lstm.add(InputLayer(input_shape=(X_train.shape[1],X_train.shape[2])))\n",
    "lstm.add(LSTM(units=128, activation='hard_sigmoid', return_sequences=True))\n",
    "lstm.add(LSTM(units=64, activation='hard_sigmoid', return_sequences=True))\n",
    "lstm.add(Dropout(0.2))\n",
    "lstm.add(LSTM(units=64, activation='hard_sigmoid', return_sequences=True))\n",
    "lstm.add(LSTM(units=32, activation='hard_sigmoid', return_sequences=False))\n",
    "lstm.add(Dropout(0.2))\n",
    "lstm.add(Dense(units=1, activation='sigmoid'))\n",
    "\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "import os\n",
    "\n",
    "MODEL_SAVE_FOLDER_PATH = './model/'\n",
    "if not os.path.exists(MODEL_SAVE_FOLDER_PATH):\n",
    "    os.mkdir(MODEL_SAVE_FOLDER_PATH)\n",
    "\n",
    "model_path = MODEL_SAVE_FOLDER_PATH + 'rf_top10_seed42-{epoch:02d}-{val_loss:.4f}.hdf5'\n",
    "\n",
    "cb_checkpoint = ModelCheckpoint(filepath=model_path, monitor='val_loss',\n",
    "                                verbose=1, save_best_only=True, overwrite=True)\n",
    "\n",
    "early_stop = EarlyStopping(monitor='val_acc', patience=50, verbose=1, restore_best_weights=True)\n",
    "lstm.compile(optimizer= keras.optimizers.Adam(learning_rate = 0.001), loss = \"binary_crossentropy\", metrics=['acc'])\n",
    "lstm.fit(X_train, y_train, validation_split=0.25, batch_size=128, epochs=500,  callbacks=[early_stop,cb_checkpoint], shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "fb3eae4d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'model/rf_top10_seed42-31-0.6278.hdf5'"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "paths = sorted(Path(MODEL_SAVE_FOLDER_PATH).iterdir(), key=os.path.getmtime)[-1]\n",
    "best_model_path = str(paths)\n",
    "best_model_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "25a987a5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "정확도 :0.6466019417475728, seed_num = 0\n",
      "정확도 :0.6576051779935275, seed_num = 1\n",
      "정확도 :0.6647249190938511, seed_num = 2\n",
      "정확도 :0.6498381877022654, seed_num = 3\n",
      "정확도 :0.6401294498381876, seed_num = 4\n",
      "정확도 :0.6699029126213593, seed_num = 5\n",
      "정확도 :0.6537216828478964, seed_num = 6\n",
      "정확도 :0.6466019417475728, seed_num = 7\n",
      "정확도 :0.6621359223300971, seed_num = 8\n",
      "정확도 :0.6627831715210356, seed_num = 9\n",
      "정확도 :0.6517799352750809, seed_num = 10\n",
      "정확도 :0.6673139158576051, seed_num = 11\n",
      "정확도 :0.6550161812297735, seed_num = 12\n",
      "정확도 :0.6666666666666666, seed_num = 13\n",
      "정확도 :0.6627831715210356, seed_num = 14\n",
      "정확도 :0.6660194174757281, seed_num = 15\n",
      "정확도 :0.6724919093851133, seed_num = 16\n",
      "정확도 :0.6459546925566343, seed_num = 17\n",
      "정확도 :0.6550161812297735, seed_num = 18\n",
      "정확도 :0.6504854368932039, seed_num = 19\n",
      "정확도 :0.6621359223300971, seed_num = 20\n",
      "정확도 :0.6466019417475728, seed_num = 21\n",
      "정확도 :0.6588996763754046, seed_num = 22\n",
      "정확도 :0.6530744336569579, seed_num = 23\n",
      "정확도 :0.6498381877022654, seed_num = 24\n",
      "정확도 :0.6634304207119741, seed_num = 25\n",
      "정확도 :0.6491909385113268, seed_num = 26\n",
      "정확도 :0.6440129449838188, seed_num = 27\n",
      "정확도 :0.6517799352750809, seed_num = 28\n",
      "정확도 :0.6498381877022654, seed_num = 29\n",
      "정확도 :0.6537216828478964, seed_num = 30\n",
      "정확도 :0.6453074433656958, seed_num = 31\n",
      "정확도 :0.6550161812297735, seed_num = 32\n",
      "정확도 :0.6407766990291263, seed_num = 33\n",
      "정확도 :0.6595469255663431, seed_num = 34\n",
      "정확도 :0.6711974110032363, seed_num = 35\n",
      "정확도 :0.6576051779935275, seed_num = 36\n",
      "정확도 :0.6453074433656958, seed_num = 37\n",
      "정확도 :0.6711974110032363, seed_num = 38\n",
      "정확도 :0.6362459546925566, seed_num = 39\n",
      "정확도 :0.6491909385113268, seed_num = 40\n",
      "정확도 :0.6614886731391586, seed_num = 41\n",
      "정확도 :0.654368932038835, seed_num = 42\n",
      "정확도 :0.6414239482200648, seed_num = 43\n",
      "정확도 :0.6491909385113268, seed_num = 44\n",
      "정확도 :0.6614886731391586, seed_num = 45\n",
      "정확도 :0.6498381877022654, seed_num = 46\n",
      "정확도 :0.6731391585760518, seed_num = 47\n",
      "정확도 :0.6783171521035599, seed_num = 48\n",
      "정확도 :0.655663430420712, seed_num = 49\n",
      "평균 정확도 : 0.6557281553398058\n"
     ]
    }
   ],
   "source": [
    "best_model = keras.models.load_model(best_model_path)\n",
    "result = 0\n",
    "for seed in range(0, 50):\n",
    "    random.seed(seed)\n",
    "\n",
    "    X = np.load(path + 'x_(7727,10,4068).npy')[:,:,per_list_index]\n",
    "    y = np.load(path + 'y_(7727,1).npy')\n",
    "    \n",
    "    idx = list(range(len(X)))\n",
    "    random.shuffle(idx)\n",
    "    i = round(X.shape[0]*0.8)\n",
    "    X_train, y_train = X[idx[:i],:,:], y[idx[:i]]\n",
    "    X_test, y_test = X[idx[i:],:,:], y[idx[i:]]\n",
    "\n",
    "    pred = best_model.predict(X_test)\n",
    "    pred[pred>0.5]=1\n",
    "    pred[pred<=0.5]=0\n",
    "    acc = metrics.accuracy_score(y_test, pred)\n",
    "    result += acc\n",
    "    print(f'정확도 :{acc}, seed_num = {seed}')\n",
    "print(f'평균 정확도 : {result/50}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d03769bb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}