10. RF versus LSTM with top features.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "849f5135",
   "metadata": {},
   "source": [
    "**각 Top Features만을 사용한 RandomForestClassifier와 LSTM의 성능 비교**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "6bc40b07",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n",
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np, os\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(action='ignore')\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, LSTM\n",
    "from tensorflow.keras.layers import SimpleRNN\n",
    "from tensorflow.keras.layers import Dropout, InputLayer\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.regularizers import l2\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "\n",
    "print(tf.__version__)\n",
    "print(keras.__version__)\n",
    "\n",
    "#한글설정\n",
    "import matplotlib.font_manager as fm\n",
    "\n",
    "font_dirs = ['/usr/share/fonts/truetype/nanum', ]\n",
    "font_files = fm.findSystemFonts(fontpaths=font_dirs)\n",
    "\n",
    "for font_file in font_files:\n",
    "    fm.fontManager.addfont(font_file)\n",
    "    \n",
    "# 한글 출력을 위해서 폰트 옵션을 설정합니다.\n",
    "# \"axes.unicode_minus\" : 마이너스가 깨질 것을 방지\n",
    "\n",
    "sns.set(font=\"NanumBarunGothic\", \n",
    "        rc={\"axes.unicode_minus\":False},\n",
    "        style='darkgrid')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbed9615",
   "metadata": {},
   "source": [
    "# RandomForest FI Top10 list\n",
    "1. Feature Importance를 구하여 Top Features 추출\n",
    "2. Top Features만을 사용한 데이터로 RandomForestClassifier 학습 후 정확도 측정"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9708332d",
   "metadata": {},
   "outputs": [],
   "source": [
    "COLS = list(pd.read_csv('/project/LSH/total_data_7727.csv')['ITEMID'].sort_values().unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "b8477470",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "random.seed(42)\n",
    "path = '/project/LSH/'\n",
    "x =  np.load(path + 'x_(7727,4068).npy')\n",
    "y = np.load(path + 'y_(7727,1).npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f5ef7b00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7727, 4068)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "0743afcd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "정확도 : 1.0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn import metrics \n",
    "np.random.seed(42)\n",
    "model = RandomForestClassifier()\n",
    "model.fit(x, y)\n",
    "\n",
    "pred = model.predict(x)\n",
    "\n",
    "print(f'정확도 : {metrics.accuracy_score(y, pred)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "f5f8b650",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.40546352e-03, 8.98727598e-04, 3.07665371e-03, ...,\n",
       "       3.64771738e-04, 9.96495876e-05, 4.21937811e-05])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "importances = model.feature_importances_\n",
    "importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "cca7c24b",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = []\n",
    "for f in range(x.shape[1]):\n",
    "    result.append({'feature' : COLS[f], 'importances' : importances[f]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "5f1fa573",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importances</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>51277</td>\n",
       "      <td>0.013739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>51006</td>\n",
       "      <td>0.009766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>50912</td>\n",
       "      <td>0.009121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>50983</td>\n",
       "      <td>0.007452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>50862</td>\n",
       "      <td>0.007445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2281</th>\n",
       "      <td>472120006</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1604</th>\n",
       "      <td>131541070</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3217</th>\n",
       "      <td>51079079420</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1042</th>\n",
       "      <td>62401002</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3339</th>\n",
       "      <td>53169010260</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4068 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          feature  importances\n",
       "206         51277     0.013739\n",
       "121         51006     0.009766\n",
       "56          50912     0.009121\n",
       "103         50983     0.007452\n",
       "20          50862     0.007445\n",
       "...           ...          ...\n",
       "2281    472120006     0.000000\n",
       "1604    131541070     0.000000\n",
       "3217  51079079420     0.000000\n",
       "1042     62401002     0.000000\n",
       "3339  53169010260     0.000000\n",
       "\n",
       "[4068 rows x 2 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_fi = pd.DataFrame(result).sort_values('importances', ascending=False)\n",
    "rf_fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "31fb8980",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[51277,\n",
       " 51006,\n",
       " 50912,\n",
       " 50983,\n",
       " 50862,\n",
       " 63323026201,\n",
       " 51003,\n",
       " 904224461,\n",
       " 50882,\n",
       " 50868,\n",
       " 51275,\n",
       " 50813,\n",
       " 227194,\n",
       " 51244,\n",
       " 409606211,\n",
       " 51274,\n",
       " 50971,\n",
       " 245004101,\n",
       " 51256,\n",
       " 51079000220,\n",
       " 51493,\n",
       " 55390000401,\n",
       " 50960,\n",
       " 50902,\n",
       " 51265,\n",
       " 51248,\n",
       " 224275,\n",
       " 51249,\n",
       " 50893,\n",
       " 50970]"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_feature = rf_fi.feature[:30].to_list()\n",
    "rf_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "4072025d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[206,\n",
       " 121,\n",
       " 56,\n",
       " 103,\n",
       " 20,\n",
       " 3832,\n",
       " 119,\n",
       " 2611,\n",
       " 34,\n",
       " 26,\n",
       " 204,\n",
       " 8,\n",
       " 383,\n",
       " 184,\n",
       " 2195,\n",
       " 203,\n",
       " 95,\n",
       " 1903,\n",
       " 193,\n",
       " 3045,\n",
       " 278,\n",
       " 3456,\n",
       " 86,\n",
       " 48,\n",
       " 197,\n",
       " 186,\n",
       " 305,\n",
       " 187,\n",
       " 41,\n",
       " 94]"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#COLS에서의 index 추출 (COLS의 index가 곧 3차원데이터 feature의 index이기 때문)\n",
    "\n",
    "rf_feature_index = [COLS.index(i) for i in rf_feature]\n",
    "rf_feature_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "819e5d80",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51277\n",
      "51006\n",
      "50912\n",
      "50983\n",
      "50862\n",
      "63323026201\n",
      "51003\n",
      "904224461\n",
      "50882\n",
      "50868\n",
      "51275\n",
      "50813\n",
      "227194\n",
      "51244\n",
      "409606211\n",
      "51274\n",
      "50971\n",
      "245004101\n",
      "51256\n",
      "51079000220\n",
      "51493\n",
      "55390000401\n",
      "50960\n",
      "50902\n",
      "51265\n",
      "51248\n",
      "224275\n",
      "51249\n",
      "50893\n",
      "50970\n"
     ]
    }
   ],
   "source": [
    "#검증\n",
    "for i in rf_feature_index:\n",
    "    print(COLS[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01368690",
   "metadata": {},
   "source": [
    "# LSTM FI Top10 list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "b00e0dbd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[409606211,\n",
       " 74606211,\n",
       " 67434504,\n",
       " 54817525,\n",
       " 10019055302,\n",
       " 227194,\n",
       " 54858516,\n",
       " 74706811,\n",
       " 45006701,\n",
       " 4003822,\n",
       " 228125,\n",
       " 50889,\n",
       " 63323016501,\n",
       " 58177025504,\n",
       " 182138167,\n",
       " 51479005520,\n",
       " 51079007520,\n",
       " 51200,\n",
       " 406112101,\n",
       " 25021011210,\n",
       " 63323016101,\n",
       " 58468002101,\n",
       " 781155613,\n",
       " 224270,\n",
       " 245021201,\n",
       " 51079008620,\n",
       " 50383068304,\n",
       " 172572810,\n",
       " 63323030201,\n",
       " 63323038810]"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_fi = pd.read_csv('./data/method1_score.csv')\n",
    "lstm_fi['abs_score'] = lstm_fi['score'].apply(lambda x:abs(x))\n",
    "lstm_fi = lstm_fi.sort_values('abs_score', ascending=False)\n",
    "lstm_feature = lstm_fi.feature[:30].to_list()\n",
    "lstm_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "7086cb1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2195,\n",
       " 1276,\n",
       " 1081,\n",
       " 982,\n",
       " 2744,\n",
       " 383,\n",
       " 1011,\n",
       " 1306,\n",
       " 845,\n",
       " 469,\n",
       " 392,\n",
       " 37,\n",
       " 3820,\n",
       " 3557,\n",
       " 1815,\n",
       " 3283,\n",
       " 3070,\n",
       " 162,\n",
       " 2110,\n",
       " 2860,\n",
       " 3818,\n",
       " 3567,\n",
       " 2547,\n",
       " 301,\n",
       " 1915,\n",
       " 3076,\n",
       " 2988,\n",
       " 1722,\n",
       " 3841,\n",
       " 3852]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#COLS에서의 index 추출 (COLS의 index가 곧 3차원데이터 feature의 index이기 때문)\n",
    "\n",
    "lstm_feature_index = [COLS.index(i) for i in lstm_feature]\n",
    "lstm_feature_index"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86681d50",
   "metadata": {},
   "source": [
    "# With RF FI Top10"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f97ca843",
   "metadata": {},
   "source": [
    "## RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "8023f301",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((6182, 30), (6182,), (1545, 30), (1545,))"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = '/project/LSH/'\n",
    "X =  np.load(path + 'x_(7727,4068).npy')[:,rf_feature_index]\n",
    "y = np.load(path + 'y_(7727,1).npy')\n",
    "\n",
    "idx = list(range(len(X)))\n",
    "random.shuffle(idx)\n",
    "\n",
    "i = round(X.shape[0]*0.8)\n",
    "X_train, y_train = X[idx[:i],:], y[idx[:i]]\n",
    "X_test, y_test = X[idx[i:],:], y[idx[i:]]\n",
    "\n",
    "X_train.shape, y_train.shape, X_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "b5f304b0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "정확도 : 0.7242718446601941\n"
     ]
    }
   ],
   "source": [
    "model = RandomForestClassifier()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "pred = model.predict(X_test)\n",
    "\n",
    "print(f'정확도 : {metrics.accuracy_score(y_test, pred)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8df509ad",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e4162b96",
   "metadata": {},
   "source": [
    "## LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "4ec5f4da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((6182, 10, 30), (6182,), (1545, 10, 30), (1545,))"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random    \n",
    "seed_num = 42\n",
    "random.seed(seed_num)\n",
    "path = '/project/LSH/'\n",
    "X = np.load(path + 'x_(7727,10,4068).npy')[:,:,rf_feature_index]\n",
    "y = np.load(path + 'y_(7727,1).npy')\n",
    "\n",
    "idx = list(range(len(X)))\n",
    "random.shuffle(idx)\n",
    "\n",
    "\n",
    "i = round(X.shape[0]*0.8)\n",
    "X_train, y_train = X[idx[:i],:,:], y[idx[:i]]\n",
    "X_test, y_test = X[idx[i:],:,:], y[idx[i:]]\n",
    "\n",
    "X_train.shape, y_train.shape, X_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "77e8c165",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/500\n",
      "37/37 [==============================] - 5s 54ms/step - loss: 0.6758 - acc: 0.5984 - val_loss: 0.6659 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.66589, saving model to ./model/rf_top10_seed42-01-0.6659.hdf5\n",
      "Epoch 2/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.6728 - acc: 0.6070 - val_loss: 0.6655 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.66589 to 0.66547, saving model to ./model/rf_top10_seed42-02-0.6655.hdf5\n",
      "Epoch 3/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6680 - acc: 0.6091 - val_loss: 0.6442 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.66547 to 0.64418, saving model to ./model/rf_top10_seed42-03-0.6442.hdf5\n",
      "Epoch 4/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6175 - acc: 0.6450 - val_loss: 0.5856 - val_acc: 0.6986\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.64418 to 0.58557, saving model to ./model/rf_top10_seed42-04-0.5856.hdf5\n",
      "Epoch 5/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5778 - acc: 0.7073 - val_loss: 0.5618 - val_acc: 0.7089\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.58557 to 0.56182, saving model to ./model/rf_top10_seed42-05-0.5618.hdf5\n",
      "Epoch 6/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5683 - acc: 0.7146 - val_loss: 0.5548 - val_acc: 0.7180\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.56182 to 0.55484, saving model to ./model/rf_top10_seed42-06-0.5548.hdf5\n",
      "Epoch 7/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5609 - acc: 0.7196 - val_loss: 0.5505 - val_acc: 0.7186\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.55484 to 0.55051, saving model to ./model/rf_top10_seed42-07-0.5505.hdf5\n",
      "Epoch 8/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5567 - acc: 0.7198 - val_loss: 0.5465 - val_acc: 0.7147\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.55051 to 0.54648, saving model to ./model/rf_top10_seed42-08-0.5465.hdf5\n",
      "Epoch 9/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5503 - acc: 0.7209 - val_loss: 0.5448 - val_acc: 0.7186\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.54648 to 0.54478, saving model to ./model/rf_top10_seed42-09-0.5448.hdf5\n",
      "Epoch 10/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5458 - acc: 0.7291 - val_loss: 0.5424 - val_acc: 0.7160\n",
      "\n",
      "Epoch 00010: val_loss improved from 0.54478 to 0.54244, saving model to ./model/rf_top10_seed42-10-0.5424.hdf5\n",
      "Epoch 11/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5453 - acc: 0.7282 - val_loss: 0.5410 - val_acc: 0.7193\n",
      "\n",
      "Epoch 00011: val_loss improved from 0.54244 to 0.54097, saving model to ./model/rf_top10_seed42-11-0.5410.hdf5\n",
      "Epoch 12/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5396 - acc: 0.7353 - val_loss: 0.5401 - val_acc: 0.7173\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.54097 to 0.54012, saving model to ./model/rf_top10_seed42-12-0.5401.hdf5\n",
      "Epoch 13/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5396 - acc: 0.7295 - val_loss: 0.5376 - val_acc: 0.7225\n",
      "\n",
      "Epoch 00013: val_loss improved from 0.54012 to 0.53756, saving model to ./model/rf_top10_seed42-13-0.5376.hdf5\n",
      "Epoch 14/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5341 - acc: 0.7334 - val_loss: 0.5373 - val_acc: 0.7186\n",
      "\n",
      "Epoch 00014: val_loss improved from 0.53756 to 0.53727, saving model to ./model/rf_top10_seed42-14-0.5373.hdf5\n",
      "Epoch 15/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5366 - acc: 0.7345 - val_loss: 0.5356 - val_acc: 0.7238\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.53727 to 0.53555, saving model to ./model/rf_top10_seed42-15-0.5356.hdf5\n",
      "Epoch 16/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5346 - acc: 0.7321 - val_loss: 0.5356 - val_acc: 0.7206\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.53555\n",
      "Epoch 17/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5342 - acc: 0.7364 - val_loss: 0.5343 - val_acc: 0.7199\n",
      "\n",
      "Epoch 00017: val_loss improved from 0.53555 to 0.53434, saving model to ./model/rf_top10_seed42-17-0.5343.hdf5\n",
      "Epoch 18/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5293 - acc: 0.7381 - val_loss: 0.5349 - val_acc: 0.7212\n",
      "\n",
      "Epoch 00018: val_loss did not improve from 0.53434\n",
      "Epoch 19/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5303 - acc: 0.7330 - val_loss: 0.5337 - val_acc: 0.7199\n",
      "\n",
      "Epoch 00019: val_loss improved from 0.53434 to 0.53369, saving model to ./model/rf_top10_seed42-19-0.5337.hdf5\n",
      "Epoch 20/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5297 - acc: 0.7349 - val_loss: 0.5320 - val_acc: 0.7219\n",
      "\n",
      "Epoch 00020: val_loss improved from 0.53369 to 0.53196, saving model to ./model/rf_top10_seed42-20-0.5320.hdf5\n",
      "Epoch 21/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5272 - acc: 0.7392 - val_loss: 0.5316 - val_acc: 0.7180\n",
      "\n",
      "Epoch 00021: val_loss improved from 0.53196 to 0.53163, saving model to ./model/rf_top10_seed42-21-0.5316.hdf5\n",
      "Epoch 22/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5263 - acc: 0.7347 - val_loss: 0.5310 - val_acc: 0.7193\n",
      "\n",
      "Epoch 00022: val_loss improved from 0.53163 to 0.53099, saving model to ./model/rf_top10_seed42-22-0.5310.hdf5\n",
      "Epoch 23/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5276 - acc: 0.7386 - val_loss: 0.5313 - val_acc: 0.7193\n",
      "\n",
      "Epoch 00023: val_loss did not improve from 0.53099\n",
      "Epoch 24/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5229 - acc: 0.7422 - val_loss: 0.5308 - val_acc: 0.7193\n",
      "\n",
      "Epoch 00024: val_loss improved from 0.53099 to 0.53085, saving model to ./model/rf_top10_seed42-24-0.5308.hdf5\n",
      "Epoch 25/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5236 - acc: 0.7399 - val_loss: 0.5295 - val_acc: 0.7186\n",
      "\n",
      "Epoch 00025: val_loss improved from 0.53085 to 0.52952, saving model to ./model/rf_top10_seed42-25-0.5295.hdf5\n",
      "Epoch 26/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5192 - acc: 0.7422 - val_loss: 0.5312 - val_acc: 0.7212\n",
      "\n",
      "Epoch 00026: val_loss did not improve from 0.52952\n",
      "Epoch 27/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5194 - acc: 0.7433 - val_loss: 0.5290 - val_acc: 0.7199\n",
      "\n",
      "Epoch 00027: val_loss improved from 0.52952 to 0.52896, saving model to ./model/rf_top10_seed42-27-0.5290.hdf5\n",
      "Epoch 28/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5173 - acc: 0.7455 - val_loss: 0.5284 - val_acc: 0.7212\n",
      "\n",
      "Epoch 00028: val_loss improved from 0.52896 to 0.52839, saving model to ./model/rf_top10_seed42-28-0.5284.hdf5\n",
      "Epoch 29/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5171 - acc: 0.7427 - val_loss: 0.5277 - val_acc: 0.7225\n",
      "\n",
      "Epoch 00029: val_loss improved from 0.52839 to 0.52766, saving model to ./model/rf_top10_seed42-29-0.5277.hdf5\n",
      "Epoch 30/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5196 - acc: 0.7457 - val_loss: 0.5272 - val_acc: 0.7245\n",
      "\n",
      "Epoch 00030: val_loss improved from 0.52766 to 0.52717, saving model to ./model/rf_top10_seed42-30-0.5272.hdf5\n",
      "Epoch 31/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5152 - acc: 0.7491 - val_loss: 0.5258 - val_acc: 0.7186\n",
      "\n",
      "Epoch 00031: val_loss improved from 0.52717 to 0.52583, saving model to ./model/rf_top10_seed42-31-0.5258.hdf5\n",
      "Epoch 32/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5171 - acc: 0.7446 - val_loss: 0.5266 - val_acc: 0.7251\n",
      "\n",
      "Epoch 00032: val_loss did not improve from 0.52583\n",
      "Epoch 33/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5158 - acc: 0.7422 - val_loss: 0.5251 - val_acc: 0.7206\n",
      "\n",
      "Epoch 00033: val_loss improved from 0.52583 to 0.52512, saving model to ./model/rf_top10_seed42-33-0.5251.hdf5\n",
      "Epoch 34/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5134 - acc: 0.7504 - val_loss: 0.5260 - val_acc: 0.7232\n",
      "\n",
      "Epoch 00034: val_loss did not improve from 0.52512\n",
      "Epoch 35/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5148 - acc: 0.7461 - val_loss: 0.5243 - val_acc: 0.7225\n",
      "\n",
      "Epoch 00035: val_loss improved from 0.52512 to 0.52433, saving model to ./model/rf_top10_seed42-35-0.5243.hdf5\n",
      "Epoch 36/500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5125 - acc: 0.7489 - val_loss: 0.5250 - val_acc: 0.7219\n",
      "\n",
      "Epoch 00036: val_loss did not improve from 0.52433\n",
      "Epoch 37/500\n",
      "37/37 [==============================] - 2s 46ms/step - loss: 0.5078 - acc: 0.7496 - val_loss: 0.5258 - val_acc: 0.7251\n",
      "\n",
      "Epoch 00037: val_loss did not improve from 0.52433\n",
      "Epoch 38/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5096 - acc: 0.7481 - val_loss: 0.5263 - val_acc: 0.7277\n",
      "\n",
      "Epoch 00038: val_loss did not improve from 0.52433\n",
      "Epoch 39/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5086 - acc: 0.7511 - val_loss: 0.5251 - val_acc: 0.7245\n",
      "\n",
      "Epoch 00039: val_loss did not improve from 0.52433\n",
      "Epoch 40/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5067 - acc: 0.7457 - val_loss: 0.5247 - val_acc: 0.7245\n",
      "\n",
      "Epoch 00040: val_loss did not improve from 0.52433\n",
      "Epoch 41/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.5081 - acc: 0.7500 - val_loss: 0.5234 - val_acc: 0.7283\n",
      "\n",
      "Epoch 00041: val_loss improved from 0.52433 to 0.52342, saving model to ./model/rf_top10_seed42-41-0.5234.hdf5\n",
      "Epoch 42/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5031 - acc: 0.7496 - val_loss: 0.5236 - val_acc: 0.7290\n",
      "\n",
      "Epoch 00042: val_loss did not improve from 0.52342\n",
      "Epoch 43/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5058 - acc: 0.7496 - val_loss: 0.5246 - val_acc: 0.7264\n",
      "\n",
      "Epoch 00043: val_loss did not improve from 0.52342\n",
      "Epoch 44/500\n",
      "37/37 [==============================] - 2s 42ms/step - loss: 0.5032 - acc: 0.7511 - val_loss: 0.5242 - val_acc: 0.7277\n",
      "\n",
      "Epoch 00044: val_loss did not improve from 0.52342\n",
      "Epoch 45/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4997 - acc: 0.7573 - val_loss: 0.5259 - val_acc: 0.7238\n",
      "\n",
      "Epoch 00045: val_loss did not improve from 0.52342\n",
      "Epoch 46/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5018 - acc: 0.7535 - val_loss: 0.5256 - val_acc: 0.7212\n",
      "\n",
      "Epoch 00046: val_loss did not improve from 0.52342\n",
      "Epoch 47/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4980 - acc: 0.7552 - val_loss: 0.5256 - val_acc: 0.7264\n",
      "\n",
      "Epoch 00047: val_loss did not improve from 0.52342\n",
      "Epoch 48/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4997 - acc: 0.7509 - val_loss: 0.5248 - val_acc: 0.7264\n",
      "\n",
      "Epoch 00048: val_loss did not improve from 0.52342\n",
      "Epoch 49/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4962 - acc: 0.7550 - val_loss: 0.5279 - val_acc: 0.7251\n",
      "\n",
      "Epoch 00049: val_loss did not improve from 0.52342\n",
      "Epoch 50/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4992 - acc: 0.7543 - val_loss: 0.5294 - val_acc: 0.7238\n",
      "\n",
      "Epoch 00050: val_loss did not improve from 0.52342\n",
      "Epoch 51/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4946 - acc: 0.7535 - val_loss: 0.5287 - val_acc: 0.7251\n",
      "\n",
      "Epoch 00051: val_loss did not improve from 0.52342\n",
      "Epoch 52/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4957 - acc: 0.7558 - val_loss: 0.5264 - val_acc: 0.7251\n",
      "\n",
      "Epoch 00052: val_loss did not improve from 0.52342\n",
      "Epoch 53/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4958 - acc: 0.7532 - val_loss: 0.5259 - val_acc: 0.7283\n",
      "\n",
      "Epoch 00053: val_loss did not improve from 0.52342\n",
      "Epoch 54/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4925 - acc: 0.7526 - val_loss: 0.5260 - val_acc: 0.7264\n",
      "\n",
      "Epoch 00054: val_loss did not improve from 0.52342\n",
      "Epoch 55/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4919 - acc: 0.7489 - val_loss: 0.5272 - val_acc: 0.7309\n",
      "\n",
      "Epoch 00055: val_loss did not improve from 0.52342\n",
      "Epoch 56/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4876 - acc: 0.7578 - val_loss: 0.5274 - val_acc: 0.7245\n",
      "\n",
      "Epoch 00056: val_loss did not improve from 0.52342\n",
      "Epoch 57/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4902 - acc: 0.7558 - val_loss: 0.5274 - val_acc: 0.7245\n",
      "\n",
      "Epoch 00057: val_loss did not improve from 0.52342\n",
      "Epoch 58/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4880 - acc: 0.7582 - val_loss: 0.5282 - val_acc: 0.7270\n",
      "\n",
      "Epoch 00058: val_loss did not improve from 0.52342\n",
      "Epoch 59/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4893 - acc: 0.7597 - val_loss: 0.5298 - val_acc: 0.7277\n",
      "\n",
      "Epoch 00059: val_loss did not improve from 0.52342\n",
      "Epoch 60/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4849 - acc: 0.7597 - val_loss: 0.5300 - val_acc: 0.7251\n",
      "\n",
      "Epoch 00060: val_loss did not improve from 0.52342\n",
      "Epoch 61/500\n",
      "37/37 [==============================] - 2s 50ms/step - loss: 0.4872 - acc: 0.7595 - val_loss: 0.5303 - val_acc: 0.7264\n",
      "\n",
      "Epoch 00061: val_loss did not improve from 0.52342\n",
      "Epoch 62/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4885 - acc: 0.7543 - val_loss: 0.5293 - val_acc: 0.7251\n",
      "\n",
      "Epoch 00062: val_loss did not improve from 0.52342\n",
      "Epoch 63/500\n",
      "37/37 [==============================] - 2s 51ms/step - loss: 0.4849 - acc: 0.7601 - val_loss: 0.5313 - val_acc: 0.7232\n",
      "\n",
      "Epoch 00063: val_loss did not improve from 0.52342\n",
      "Epoch 64/500\n",
      "37/37 [==============================] - 2s 53ms/step - loss: 0.4865 - acc: 0.7588 - val_loss: 0.5296 - val_acc: 0.7219\n",
      "\n",
      "Epoch 00064: val_loss did not improve from 0.52342\n",
      "Epoch 65/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4817 - acc: 0.7604 - val_loss: 0.5331 - val_acc: 0.7251\n",
      "\n",
      "Epoch 00065: val_loss did not improve from 0.52342\n",
      "Epoch 66/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4789 - acc: 0.7614 - val_loss: 0.5350 - val_acc: 0.7245\n",
      "\n",
      "Epoch 00066: val_loss did not improve from 0.52342\n",
      "Epoch 67/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4784 - acc: 0.7610 - val_loss: 0.5345 - val_acc: 0.7245\n",
      "\n",
      "Epoch 00067: val_loss did not improve from 0.52342\n",
      "Epoch 68/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4780 - acc: 0.7638 - val_loss: 0.5339 - val_acc: 0.7238\n",
      "\n",
      "Epoch 00068: val_loss did not improve from 0.52342\n",
      "Epoch 69/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4800 - acc: 0.7640 - val_loss: 0.5351 - val_acc: 0.7232\n",
      "\n",
      "Epoch 00069: val_loss did not improve from 0.52342\n",
      "Epoch 70/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4778 - acc: 0.7638 - val_loss: 0.5347 - val_acc: 0.7251\n",
      "\n",
      "Epoch 00070: val_loss did not improve from 0.52342\n",
      "Epoch 71/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4757 - acc: 0.7619 - val_loss: 0.5401 - val_acc: 0.7206\n",
      "\n",
      "Epoch 00071: val_loss did not improve from 0.52342\n",
      "Epoch 72/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4756 - acc: 0.7597 - val_loss: 0.5355 - val_acc: 0.7245\n",
      "\n",
      "Epoch 00072: val_loss did not improve from 0.52342\n",
      "Epoch 73/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4747 - acc: 0.7638 - val_loss: 0.5380 - val_acc: 0.7238\n",
      "\n",
      "Epoch 00073: val_loss did not improve from 0.52342\n",
      "Epoch 74/500\n",
      "37/37 [==============================] - 2s 48ms/step - loss: 0.4728 - acc: 0.7634 - val_loss: 0.5379 - val_acc: 0.7219\n",
      "\n",
      "Epoch 00074: val_loss did not improve from 0.52342\n",
      "Epoch 75/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4756 - acc: 0.7703 - val_loss: 0.5381 - val_acc: 0.7238\n",
      "\n",
      "Epoch 00075: val_loss did not improve from 0.52342\n",
      "Epoch 76/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4703 - acc: 0.7670 - val_loss: 0.5399 - val_acc: 0.7238\n",
      "\n",
      "Epoch 00076: val_loss did not improve from 0.52342\n",
      "Epoch 77/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4703 - acc: 0.7664 - val_loss: 0.5396 - val_acc: 0.7225\n",
      "\n",
      "Epoch 00077: val_loss did not improve from 0.52342\n",
      "Epoch 78/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4731 - acc: 0.7653 - val_loss: 0.5381 - val_acc: 0.7238\n",
      "\n",
      "Epoch 00078: val_loss did not improve from 0.52342\n",
      "Epoch 79/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4691 - acc: 0.7681 - val_loss: 0.5436 - val_acc: 0.7251\n",
      "\n",
      "Epoch 00079: val_loss did not improve from 0.52342\n",
      "Epoch 80/500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4703 - acc: 0.7705 - val_loss: 0.5399 - val_acc: 0.7206\n",
      "\n",
      "Epoch 00080: val_loss did not improve from 0.52342\n",
      "Epoch 81/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4710 - acc: 0.7649 - val_loss: 0.5384 - val_acc: 0.7212\n",
      "\n",
      "Epoch 00081: val_loss did not improve from 0.52342\n",
      "Epoch 82/500\n",
      "37/37 [==============================] - 2s 45ms/step - loss: 0.4675 - acc: 0.7642 - val_loss: 0.5401 - val_acc: 0.7212\n",
      "\n",
      "Epoch 00082: val_loss did not improve from 0.52342\n",
      "Epoch 83/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4634 - acc: 0.7692 - val_loss: 0.5436 - val_acc: 0.7206\n",
      "\n",
      "Epoch 00083: val_loss did not improve from 0.52342\n",
      "Epoch 84/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4658 - acc: 0.7709 - val_loss: 0.5441 - val_acc: 0.7199\n",
      "\n",
      "Epoch 00084: val_loss did not improve from 0.52342\n",
      "Epoch 85/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4627 - acc: 0.7720 - val_loss: 0.5431 - val_acc: 0.7173\n",
      "\n",
      "Epoch 00085: val_loss did not improve from 0.52342\n",
      "Epoch 86/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4626 - acc: 0.7683 - val_loss: 0.5438 - val_acc: 0.7154\n",
      "\n",
      "Epoch 00086: val_loss did not improve from 0.52342\n",
      "Epoch 87/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4643 - acc: 0.7720 - val_loss: 0.5433 - val_acc: 0.7167\n",
      "\n",
      "Epoch 00087: val_loss did not improve from 0.52342\n",
      "Epoch 88/500\n",
      "37/37 [==============================] - 2s 48ms/step - loss: 0.4617 - acc: 0.7711 - val_loss: 0.5456 - val_acc: 0.7173\n",
      "\n",
      "Epoch 00088: val_loss did not improve from 0.52342\n",
      "Epoch 89/500\n",
      "37/37 [==============================] - 2s 48ms/step - loss: 0.4594 - acc: 0.7716 - val_loss: 0.5461 - val_acc: 0.7167\n",
      "\n",
      "Epoch 00089: val_loss did not improve from 0.52342\n",
      "Epoch 90/500\n",
      "37/37 [==============================] - 2s 48ms/step - loss: 0.4594 - acc: 0.7729 - val_loss: 0.5481 - val_acc: 0.7180\n",
      "\n",
      "Epoch 00090: val_loss did not improve from 0.52342\n",
      "Epoch 91/500\n",
      "37/37 [==============================] - 2s 49ms/step - loss: 0.4566 - acc: 0.7729 - val_loss: 0.5507 - val_acc: 0.7167\n",
      "\n",
      "Epoch 00091: val_loss did not improve from 0.52342\n",
      "Epoch 92/500\n",
      "37/37 [==============================] - 2s 49ms/step - loss: 0.4594 - acc: 0.7739 - val_loss: 0.5481 - val_acc: 0.7173\n",
      "\n",
      "Epoch 00092: val_loss did not improve from 0.52342\n",
      "Epoch 93/500\n",
      "37/37 [==============================] - 2s 47ms/step - loss: 0.4596 - acc: 0.7729 - val_loss: 0.5474 - val_acc: 0.7147\n",
      "\n",
      "Epoch 00093: val_loss did not improve from 0.52342\n",
      "Epoch 94/500\n",
      "37/37 [==============================] - 2s 47ms/step - loss: 0.4583 - acc: 0.7731 - val_loss: 0.5531 - val_acc: 0.7154\n",
      "\n",
      "Epoch 00094: val_loss did not improve from 0.52342\n",
      "Epoch 95/500\n",
      "37/37 [==============================] - 2s 48ms/step - loss: 0.4557 - acc: 0.7765 - val_loss: 0.5470 - val_acc: 0.7154\n",
      "\n",
      "Epoch 00095: val_loss did not improve from 0.52342\n",
      "Epoch 96/500\n",
      "37/37 [==============================] - 2s 47ms/step - loss: 0.4537 - acc: 0.7739 - val_loss: 0.5499 - val_acc: 0.7147\n",
      "\n",
      "Epoch 00096: val_loss did not improve from 0.52342\n",
      "Epoch 97/500\n",
      "37/37 [==============================] - 2s 48ms/step - loss: 0.4512 - acc: 0.7804 - val_loss: 0.5517 - val_acc: 0.7160\n",
      "\n",
      "Epoch 00097: val_loss did not improve from 0.52342\n",
      "Epoch 98/500\n",
      "37/37 [==============================] - 2s 49ms/step - loss: 0.4521 - acc: 0.7774 - val_loss: 0.5520 - val_acc: 0.7141\n",
      "\n",
      "Epoch 00098: val_loss did not improve from 0.52342\n",
      "Epoch 99/500\n",
      "37/37 [==============================] - 2s 49ms/step - loss: 0.4517 - acc: 0.7701 - val_loss: 0.5542 - val_acc: 0.7147\n",
      "\n",
      "Epoch 00099: val_loss did not improve from 0.52342\n",
      "Epoch 100/500\n",
      "37/37 [==============================] - 2s 47ms/step - loss: 0.4518 - acc: 0.7739 - val_loss: 0.5537 - val_acc: 0.7154\n",
      "\n",
      "Epoch 00100: val_loss did not improve from 0.52342\n",
      "Epoch 101/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4480 - acc: 0.7791 - val_loss: 0.5589 - val_acc: 0.7173\n",
      "\n",
      "Epoch 00101: val_loss did not improve from 0.52342\n",
      "Epoch 102/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4505 - acc: 0.7776 - val_loss: 0.5607 - val_acc: 0.7173\n",
      "\n",
      "Epoch 00102: val_loss did not improve from 0.52342\n",
      "Epoch 103/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4469 - acc: 0.7813 - val_loss: 0.5591 - val_acc: 0.7147\n",
      "\n",
      "Epoch 00103: val_loss did not improve from 0.52342\n",
      "Epoch 104/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4460 - acc: 0.7800 - val_loss: 0.5656 - val_acc: 0.7173\n",
      "\n",
      "Epoch 00104: val_loss did not improve from 0.52342\n",
      "Epoch 105/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.4465 - acc: 0.7791 - val_loss: 0.5640 - val_acc: 0.7173\n",
      "Restoring model weights from the end of the best epoch.\n",
      "\n",
      "Epoch 00105: val_loss did not improve from 0.52342\n",
      "Epoch 00105: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7efed63be100>"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ---------------------\n",
    "seed_num = 42 \n",
    "# ---------------------\n",
    "tf.random.set_seed(seed_num)\n",
    "\n",
    "lstm = Sequential()\n",
    "lstm.add(InputLayer(input_shape=(X_train.shape[1],X_train.shape[2])))\n",
    "lstm.add(LSTM(units=128, activation='hard_sigmoid', return_sequences=True))\n",
    "lstm.add(LSTM(units=64, activation='hard_sigmoid', return_sequences=True))\n",
    "lstm.add(Dropout(0.2))\n",
    "lstm.add(LSTM(units=64, activation='hard_sigmoid', return_sequences=True))\n",
    "lstm.add(LSTM(units=32, activation='hard_sigmoid', return_sequences=False))\n",
    "lstm.add(Dropout(0.2))\n",
    "lstm.add(Dense(units=1, activation='sigmoid'))\n",
    "\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "import os\n",
    "\n",
    "MODEL_SAVE_FOLDER_PATH = './model/'\n",
    "if not os.path.exists(MODEL_SAVE_FOLDER_PATH):\n",
    "    os.mkdir(MODEL_SAVE_FOLDER_PATH)\n",
    "\n",
    "model_path = MODEL_SAVE_FOLDER_PATH + 'rf_top10_seed42-{epoch:02d}-{val_loss:.4f}.hdf5'\n",
    "\n",
    "cb_checkpoint = ModelCheckpoint(filepath=model_path, monitor='val_loss',\n",
    "                                verbose=1, save_best_only=True)\n",
    "\n",
    "early_stop = EarlyStopping(monitor='val_acc', patience=50, verbose=1, restore_best_weights=True)\n",
    "lstm.compile(optimizer= keras.optimizers.Adam(learning_rate = 0.001), loss = \"binary_crossentropy\", metrics=['acc'])\n",
    "lstm.fit(X_train, y_train, validation_split=0.25, batch_size=128, epochs=500,  callbacks=[early_stop,cb_checkpoint], shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "96118b00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'model/rf_top10_seed42-41-0.5234.hdf5'"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "paths = sorted(Path(MODEL_SAVE_FOLDER_PATH).iterdir(), key=os.path.getmtime)[-1]\n",
    "best_model_path = str(paths)\n",
    "best_model_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "a37818d6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "정확도 :0.7294498381877023, seed_num = 0\n",
      "정확도 :0.7385113268608414, seed_num = 1\n",
      "정확도 :0.7449838187702266, seed_num = 2\n",
      "정확도 :0.7385113268608414, seed_num = 3\n",
      "정확도 :0.7475728155339806, seed_num = 4\n",
      "정확도 :0.7527508090614887, seed_num = 5\n",
      "정확도 :0.7495145631067961, seed_num = 6\n",
      "정확도 :0.7378640776699029, seed_num = 7\n",
      "정확도 :0.7624595469255664, seed_num = 8\n",
      "정확도 :0.7385113268608414, seed_num = 9\n",
      "정확도 :0.7443365695792881, seed_num = 10\n",
      "정확도 :0.7572815533980582, seed_num = 11\n",
      "정확도 :0.7488673139158576, seed_num = 12\n",
      "정확도 :0.7411003236245954, seed_num = 13\n",
      "정확도 :0.7475728155339806, seed_num = 14\n",
      "정확도 :0.7495145631067961, seed_num = 15\n",
      "정확도 :0.7592233009708738, seed_num = 16\n",
      "정확도 :0.7443365695792881, seed_num = 17\n",
      "정확도 :0.7488673139158576, seed_num = 18\n",
      "정확도 :0.7417475728155339, seed_num = 19\n",
      "정확도 :0.7333333333333333, seed_num = 20\n",
      "정확도 :0.7352750809061489, seed_num = 21\n",
      "정확도 :0.7346278317152104, seed_num = 22\n",
      "정확도 :0.7391585760517799, seed_num = 23\n",
      "정확도 :0.7288025889967638, seed_num = 24\n",
      "정확도 :0.7404530744336569, seed_num = 25\n",
      "정확도 :0.7288025889967638, seed_num = 26\n",
      "정확도 :0.7346278317152104, seed_num = 27\n",
      "정확도 :0.7449838187702266, seed_num = 28\n",
      "정확도 :0.7423948220064724, seed_num = 29\n",
      "정확도 :0.7663430420711974, seed_num = 30\n",
      "정확도 :0.7372168284789644, seed_num = 31\n",
      "정확도 :0.7294498381877023, seed_num = 32\n",
      "정확도 :0.7456310679611651, seed_num = 33\n",
      "정확도 :0.7566343042071197, seed_num = 34\n",
      "정확도 :0.7404530744336569, seed_num = 35\n",
      "정확도 :0.7585760517799353, seed_num = 36\n",
      "정확도 :0.7540453074433657, seed_num = 37\n",
      "정확도 :0.7385113268608414, seed_num = 38\n",
      "정확도 :0.7417475728155339, seed_num = 39\n",
      "정확도 :0.7411003236245954, seed_num = 40\n",
      "정확도 :0.7346278317152104, seed_num = 41\n",
      "정확도 :0.7456310679611651, seed_num = 42\n",
      "정확도 :0.7514563106796116, seed_num = 43\n",
      "정확도 :0.7262135922330097, seed_num = 44\n",
      "정확도 :0.7514563106796116, seed_num = 45\n",
      "정확도 :0.7385113268608414, seed_num = 46\n",
      "정확도 :0.7546925566343042, seed_num = 47\n",
      "정확도 :0.7611650485436893, seed_num = 48\n",
      "정확도 :0.7443365695792881, seed_num = 49\n",
      "평균 정확도 : 0.7440647249190938\n"
     ]
    }
   ],
   "source": [
    "best_model = keras.models.load_model(best_model_path)\n",
    "result = 0\n",
    "for seed in range(0, 50):\n",
    "    random.seed(seed)\n",
    "\n",
    "    X = np.load(path + 'x_(7727,10,4068).npy')[:,:,rf_feature_index]\n",
    "    y = np.load(path + 'y_(7727,1).npy')\n",
    "    \n",
    "    idx = list(range(len(x)))\n",
    "    random.shuffle(idx)\n",
    "    i = round(X.shape[0]*0.8)\n",
    "    X_train, y_train = X[idx[:i],:,:], y[idx[:i]]\n",
    "    X_test, y_test = X[idx[i:],:,:], y[idx[i:]]\n",
    "\n",
    "    pred = best_model.predict(X_test)\n",
    "    pred[pred>0.5]=1\n",
    "    pred[pred<=0.5]=0\n",
    "    acc = metrics.accuracy_score(y_test, pred)\n",
    "    result += acc\n",
    "    print(f'정확도 :{acc}, seed_num = {seed}')\n",
    "print(f'평균 정확도 : {result/50}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98b51981",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "d0c6976d",
   "metadata": {},
   "source": [
    "# With LSTM FI Top10"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab5308ea",
   "metadata": {},
   "source": [
    "## RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "de7e432f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((6182, 30), (6182,), (1545, 30), (1545,))"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = '/project/LSH/'\n",
    "X =  np.load(path + 'x_(7727,4068).npy')[:,lstm_feature_index]\n",
    "y = np.load(path + 'y_(7727,1).npy')\n",
    "\n",
    "idx = list(range(len(X)))\n",
    "random.shuffle(idx)\n",
    "\n",
    "i = round(X.shape[0]*0.8)\n",
    "X_train, y_train = X[idx[:i],:], y[idx[:i]]\n",
    "X_test, y_test = X[idx[i:],:], y[idx[i:]]\n",
    "\n",
    "X_train.shape, y_train.shape, X_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "d3237355",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "정확도 : 0.6608414239482201\n"
     ]
    }
   ],
   "source": [
    "model = RandomForestClassifier()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "pred = model.predict(X_test)\n",
    "\n",
    "print(f'정확도 : {metrics.accuracy_score(y_test, pred)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbe4350c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "361bec0a",
   "metadata": {},
   "source": [
    "## LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "c6a2fbe0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((6182, 10, 30), (6182,), (1545, 10, 30), (1545,))"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random    \n",
    "seed_num = 42\n",
    "random.seed(seed_num)\n",
    "path = '/project/LSH/'\n",
    "X = np.load(path + 'x_(7727,10,4068).npy')[:,:,lstm_feature_index]\n",
    "y = np.load(path + 'y_(7727,1).npy')\n",
    "\n",
    "idx = list(range(len(X)))\n",
    "random.shuffle(idx)\n",
    "\n",
    "\n",
    "i = round(X.shape[0]*0.8)\n",
    "X_train, y_train = X[idx[:i],:,:], y[idx[:i]]\n",
    "X_test, y_test = X[idx[i:],:,:], y[idx[i:]]\n",
    "\n",
    "X_train.shape, y_train.shape, X_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "831aca8f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/500\n",
      "37/37 [==============================] - 6s 54ms/step - loss: 0.6759 - acc: 0.5984 - val_loss: 0.6660 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.66603, saving model to ./model/lstm_top10_seed42-01-0.6660.hdf5\n",
      "Epoch 2/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6732 - acc: 0.6066 - val_loss: 0.6662 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.66603\n",
      "Epoch 3/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6740 - acc: 0.6085 - val_loss: 0.6655 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.66603 to 0.66551, saving model to ./model/lstm_top10_seed42-03-0.6655.hdf5\n",
      "Epoch 4/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6711 - acc: 0.6100 - val_loss: 0.6653 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.66551 to 0.66526, saving model to ./model/lstm_top10_seed42-04-0.6653.hdf5\n",
      "Epoch 5/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6706 - acc: 0.6091 - val_loss: 0.6652 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.66526 to 0.66516, saving model to ./model/lstm_top10_seed42-05-0.6652.hdf5\n",
      "Epoch 6/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6721 - acc: 0.6113 - val_loss: 0.6651 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.66516 to 0.66514, saving model to ./model/lstm_top10_seed42-06-0.6651.hdf5\n",
      "Epoch 7/500\n",
      "37/37 [==============================] - 1s 40ms/step - loss: 0.6710 - acc: 0.6111 - val_loss: 0.6651 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.66514\n",
      "Epoch 8/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6714 - acc: 0.6111 - val_loss: 0.6652 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.66514\n",
      "Epoch 9/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6702 - acc: 0.6113 - val_loss: 0.6651 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.66514 to 0.66511, saving model to ./model/lstm_top10_seed42-09-0.6651.hdf5\n",
      "Epoch 10/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6699 - acc: 0.6115 - val_loss: 0.6651 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00010: val_loss improved from 0.66511 to 0.66509, saving model to ./model/lstm_top10_seed42-10-0.6651.hdf5\n",
      "Epoch 11/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6697 - acc: 0.6115 - val_loss: 0.6650 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00011: val_loss improved from 0.66509 to 0.66497, saving model to ./model/lstm_top10_seed42-11-0.6650.hdf5\n",
      "Epoch 12/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6692 - acc: 0.6115 - val_loss: 0.6648 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.66497 to 0.66479, saving model to ./model/lstm_top10_seed42-12-0.6648.hdf5\n",
      "Epoch 13/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6689 - acc: 0.6117 - val_loss: 0.6643 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00013: val_loss improved from 0.66479 to 0.66431, saving model to ./model/lstm_top10_seed42-13-0.6643.hdf5\n",
      "Epoch 14/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6642 - acc: 0.6115 - val_loss: 0.6566 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00014: val_loss improved from 0.66431 to 0.65657, saving model to ./model/lstm_top10_seed42-14-0.6566.hdf5\n",
      "Epoch 15/500\n",
      "37/37 [==============================] - 2s 43ms/step - loss: 0.6456 - acc: 0.6124 - val_loss: 0.6410 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.65657 to 0.64101, saving model to ./model/lstm_top10_seed42-15-0.6410.hdf5\n",
      "Epoch 16/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6275 - acc: 0.6206 - val_loss: 0.6341 - val_acc: 0.6287\n",
      "\n",
      "Epoch 00016: val_loss improved from 0.64101 to 0.63409, saving model to ./model/lstm_top10_seed42-16-0.6341.hdf5\n",
      "Epoch 17/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.6115 - acc: 0.6331 - val_loss: 0.6246 - val_acc: 0.6313\n",
      "\n",
      "Epoch 00017: val_loss improved from 0.63409 to 0.62465, saving model to ./model/lstm_top10_seed42-17-0.6246.hdf5\n",
      "Epoch 18/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5989 - acc: 0.6365 - val_loss: 0.6298 - val_acc: 0.6326\n",
      "\n",
      "Epoch 00018: val_loss did not improve from 0.62465\n",
      "Epoch 19/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5953 - acc: 0.6389 - val_loss: 0.6279 - val_acc: 0.6345\n",
      "\n",
      "Epoch 00019: val_loss did not improve from 0.62465\n",
      "Epoch 20/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5945 - acc: 0.6370 - val_loss: 0.6141 - val_acc: 0.6481\n",
      "\n",
      "Epoch 00020: val_loss improved from 0.62465 to 0.61414, saving model to ./model/lstm_top10_seed42-20-0.6141.hdf5\n",
      "Epoch 21/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5863 - acc: 0.6484 - val_loss: 0.6191 - val_acc: 0.6475\n",
      "\n",
      "Epoch 00021: val_loss did not improve from 0.61414\n",
      "Epoch 22/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5849 - acc: 0.6503 - val_loss: 0.6194 - val_acc: 0.6507\n",
      "\n",
      "Epoch 00022: val_loss did not improve from 0.61414\n",
      "Epoch 23/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5825 - acc: 0.6542 - val_loss: 0.6180 - val_acc: 0.6527\n",
      "\n",
      "Epoch 00023: val_loss did not improve from 0.61414\n",
      "Epoch 24/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5793 - acc: 0.6547 - val_loss: 0.6159 - val_acc: 0.6527\n",
      "\n",
      "Epoch 00024: val_loss did not improve from 0.61414\n",
      "Epoch 25/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5776 - acc: 0.6544 - val_loss: 0.6151 - val_acc: 0.6552\n",
      "\n",
      "Epoch 00025: val_loss did not improve from 0.61414\n",
      "Epoch 26/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5751 - acc: 0.6629 - val_loss: 0.6138 - val_acc: 0.6565\n",
      "\n",
      "Epoch 00026: val_loss improved from 0.61414 to 0.61377, saving model to ./model/lstm_top10_seed42-26-0.6138.hdf5\n",
      "Epoch 27/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5718 - acc: 0.6674 - val_loss: 0.6109 - val_acc: 0.6578\n",
      "\n",
      "Epoch 00027: val_loss improved from 0.61377 to 0.61089, saving model to ./model/lstm_top10_seed42-27-0.6109.hdf5\n",
      "Epoch 28/500\n",
      "37/37 [==============================] - 2s 43ms/step - loss: 0.5729 - acc: 0.6661 - val_loss: 0.6057 - val_acc: 0.6572\n",
      "\n",
      "Epoch 00028: val_loss improved from 0.61089 to 0.60571, saving model to ./model/lstm_top10_seed42-28-0.6057.hdf5\n",
      "Epoch 29/500\n",
      "37/37 [==============================] - 2s 43ms/step - loss: 0.5724 - acc: 0.6672 - val_loss: 0.6097 - val_acc: 0.6565\n",
      "\n",
      "Epoch 00029: val_loss did not improve from 0.60571\n",
      "Epoch 30/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5718 - acc: 0.6702 - val_loss: 0.6032 - val_acc: 0.6591\n",
      "\n",
      "Epoch 00030: val_loss improved from 0.60571 to 0.60316, saving model to ./model/lstm_top10_seed42-30-0.6032.hdf5\n",
      "Epoch 31/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5678 - acc: 0.6721 - val_loss: 0.6032 - val_acc: 0.6585\n",
      "\n",
      "Epoch 00031: val_loss did not improve from 0.60316\n",
      "Epoch 32/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5710 - acc: 0.6648 - val_loss: 0.5996 - val_acc: 0.6591\n",
      "\n",
      "Epoch 00032: val_loss improved from 0.60316 to 0.59956, saving model to ./model/lstm_top10_seed42-32-0.5996.hdf5\n",
      "Epoch 33/500\n",
      "37/37 [==============================] - 2s 43ms/step - loss: 0.5686 - acc: 0.6657 - val_loss: 0.5993 - val_acc: 0.6611\n",
      "\n",
      "Epoch 00033: val_loss improved from 0.59956 to 0.59934, saving model to ./model/lstm_top10_seed42-33-0.5993.hdf5\n",
      "Epoch 34/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5682 - acc: 0.6663 - val_loss: 0.5985 - val_acc: 0.6598\n",
      "\n",
      "Epoch 00034: val_loss improved from 0.59934 to 0.59855, saving model to ./model/lstm_top10_seed42-34-0.5985.hdf5\n",
      "Epoch 35/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5685 - acc: 0.6704 - val_loss: 0.5976 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00035: val_loss improved from 0.59855 to 0.59763, saving model to ./model/lstm_top10_seed42-35-0.5976.hdf5\n",
      "Epoch 36/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5688 - acc: 0.6702 - val_loss: 0.5988 - val_acc: 0.6591\n",
      "\n",
      "Epoch 00036: val_loss did not improve from 0.59763\n",
      "Epoch 37/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5669 - acc: 0.6719 - val_loss: 0.5972 - val_acc: 0.6591\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00037: val_loss improved from 0.59763 to 0.59715, saving model to ./model/lstm_top10_seed42-37-0.5972.hdf5\n",
      "Epoch 38/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5673 - acc: 0.6771 - val_loss: 0.5972 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00038: val_loss did not improve from 0.59715\n",
      "Epoch 39/500\n",
      "37/37 [==============================] - 2s 43ms/step - loss: 0.5664 - acc: 0.6695 - val_loss: 0.5968 - val_acc: 0.6598\n",
      "\n",
      "Epoch 00039: val_loss improved from 0.59715 to 0.59684, saving model to ./model/lstm_top10_seed42-39-0.5968.hdf5\n",
      "Epoch 40/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5658 - acc: 0.6739 - val_loss: 0.5966 - val_acc: 0.6591\n",
      "\n",
      "Epoch 00040: val_loss improved from 0.59684 to 0.59661, saving model to ./model/lstm_top10_seed42-40-0.5966.hdf5\n",
      "Epoch 41/500\n",
      "37/37 [==============================] - 2s 43ms/step - loss: 0.5668 - acc: 0.6689 - val_loss: 0.5958 - val_acc: 0.6611\n",
      "\n",
      "Epoch 00041: val_loss improved from 0.59661 to 0.59576, saving model to ./model/lstm_top10_seed42-41-0.5958.hdf5\n",
      "Epoch 42/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5650 - acc: 0.6743 - val_loss: 0.5957 - val_acc: 0.6611\n",
      "\n",
      "Epoch 00042: val_loss improved from 0.59576 to 0.59570, saving model to ./model/lstm_top10_seed42-42-0.5957.hdf5\n",
      "Epoch 43/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5666 - acc: 0.6717 - val_loss: 0.5953 - val_acc: 0.6611\n",
      "\n",
      "Epoch 00043: val_loss improved from 0.59570 to 0.59533, saving model to ./model/lstm_top10_seed42-43-0.5953.hdf5\n",
      "Epoch 44/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5653 - acc: 0.6730 - val_loss: 0.5946 - val_acc: 0.6617\n",
      "\n",
      "Epoch 00044: val_loss improved from 0.59533 to 0.59465, saving model to ./model/lstm_top10_seed42-44-0.5946.hdf5\n",
      "Epoch 45/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5640 - acc: 0.6769 - val_loss: 0.5953 - val_acc: 0.6617\n",
      "\n",
      "Epoch 00045: val_loss did not improve from 0.59465\n",
      "Epoch 46/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5666 - acc: 0.6747 - val_loss: 0.5941 - val_acc: 0.6617\n",
      "\n",
      "Epoch 00046: val_loss improved from 0.59465 to 0.59412, saving model to ./model/lstm_top10_seed42-46-0.5941.hdf5\n",
      "Epoch 47/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5642 - acc: 0.6708 - val_loss: 0.5945 - val_acc: 0.6611\n",
      "\n",
      "Epoch 00047: val_loss did not improve from 0.59412\n",
      "Epoch 48/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5645 - acc: 0.6728 - val_loss: 0.5943 - val_acc: 0.6611\n",
      "\n",
      "Epoch 00048: val_loss did not improve from 0.59412\n",
      "Epoch 49/500\n",
      "37/37 [==============================] - 2s 43ms/step - loss: 0.5646 - acc: 0.6715 - val_loss: 0.5946 - val_acc: 0.6617\n",
      "\n",
      "Epoch 00049: val_loss did not improve from 0.59412\n",
      "Epoch 50/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5662 - acc: 0.6721 - val_loss: 0.5934 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00050: val_loss improved from 0.59412 to 0.59341, saving model to ./model/lstm_top10_seed42-50-0.5934.hdf5\n",
      "Epoch 51/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5637 - acc: 0.6752 - val_loss: 0.5944 - val_acc: 0.6611\n",
      "\n",
      "Epoch 00051: val_loss did not improve from 0.59341\n",
      "Epoch 52/500\n",
      "37/37 [==============================] - 2s 43ms/step - loss: 0.5634 - acc: 0.6715 - val_loss: 0.5935 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00052: val_loss did not improve from 0.59341\n",
      "Epoch 53/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5651 - acc: 0.6739 - val_loss: 0.5927 - val_acc: 0.6611\n",
      "\n",
      "Epoch 00053: val_loss improved from 0.59341 to 0.59273, saving model to ./model/lstm_top10_seed42-53-0.5927.hdf5\n",
      "Epoch 54/500\n",
      "37/37 [==============================] - 2s 43ms/step - loss: 0.5636 - acc: 0.6767 - val_loss: 0.5934 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00054: val_loss did not improve from 0.59273\n",
      "Epoch 55/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5641 - acc: 0.6734 - val_loss: 0.5932 - val_acc: 0.6611\n",
      "\n",
      "Epoch 00055: val_loss did not improve from 0.59273\n",
      "Epoch 56/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5634 - acc: 0.6739 - val_loss: 0.5931 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00056: val_loss did not improve from 0.59273\n",
      "Epoch 57/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5644 - acc: 0.6739 - val_loss: 0.5936 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00057: val_loss did not improve from 0.59273\n",
      "Epoch 58/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5621 - acc: 0.6732 - val_loss: 0.5940 - val_acc: 0.6611\n",
      "\n",
      "Epoch 00058: val_loss did not improve from 0.59273\n",
      "Epoch 59/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5663 - acc: 0.6700 - val_loss: 0.5927 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00059: val_loss improved from 0.59273 to 0.59266, saving model to ./model/lstm_top10_seed42-59-0.5927.hdf5\n",
      "Epoch 60/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5636 - acc: 0.6743 - val_loss: 0.5934 - val_acc: 0.6643\n",
      "\n",
      "Epoch 00060: val_loss did not improve from 0.59266\n",
      "Epoch 61/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5637 - acc: 0.6749 - val_loss: 0.5950 - val_acc: 0.6656\n",
      "\n",
      "Epoch 00061: val_loss did not improve from 0.59266\n",
      "Epoch 62/500\n",
      "37/37 [==============================] - 2s 43ms/step - loss: 0.5671 - acc: 0.6732 - val_loss: 0.5952 - val_acc: 0.6649\n",
      "\n",
      "Epoch 00062: val_loss did not improve from 0.59266\n",
      "Epoch 63/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5647 - acc: 0.6732 - val_loss: 0.5965 - val_acc: 0.6649\n",
      "\n",
      "Epoch 00063: val_loss did not improve from 0.59266\n",
      "Epoch 64/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5664 - acc: 0.6676 - val_loss: 0.5982 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00064: val_loss did not improve from 0.59266\n",
      "Epoch 65/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5653 - acc: 0.6700 - val_loss: 0.6016 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00065: val_loss did not improve from 0.59266\n",
      "Epoch 66/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5644 - acc: 0.6704 - val_loss: 0.6040 - val_acc: 0.6591\n",
      "\n",
      "Epoch 00066: val_loss did not improve from 0.59266\n",
      "Epoch 67/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5646 - acc: 0.6719 - val_loss: 0.6027 - val_acc: 0.6598\n",
      "\n",
      "Epoch 00067: val_loss did not improve from 0.59266\n",
      "Epoch 68/500\n",
      "37/37 [==============================] - 2s 43ms/step - loss: 0.5648 - acc: 0.6715 - val_loss: 0.6033 - val_acc: 0.6617\n",
      "\n",
      "Epoch 00068: val_loss did not improve from 0.59266\n",
      "Epoch 69/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5641 - acc: 0.6698 - val_loss: 0.5992 - val_acc: 0.6643\n",
      "\n",
      "Epoch 00069: val_loss did not improve from 0.59266\n",
      "Epoch 70/500\n",
      "37/37 [==============================] - 2s 43ms/step - loss: 0.5639 - acc: 0.6730 - val_loss: 0.5981 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00070: val_loss did not improve from 0.59266\n",
      "Epoch 71/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5622 - acc: 0.6764 - val_loss: 0.6007 - val_acc: 0.6643\n",
      "\n",
      "Epoch 00071: val_loss did not improve from 0.59266\n",
      "Epoch 72/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5647 - acc: 0.6689 - val_loss: 0.6001 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00072: val_loss did not improve from 0.59266\n",
      "Epoch 73/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5645 - acc: 0.6687 - val_loss: 0.6002 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00073: val_loss did not improve from 0.59266\n",
      "Epoch 74/500\n",
      "37/37 [==============================] - 2s 43ms/step - loss: 0.5630 - acc: 0.6739 - val_loss: 0.5982 - val_acc: 0.6617\n",
      "\n",
      "Epoch 00074: val_loss did not improve from 0.59266\n",
      "Epoch 75/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5635 - acc: 0.6743 - val_loss: 0.5974 - val_acc: 0.6643\n",
      "\n",
      "Epoch 00075: val_loss did not improve from 0.59266\n",
      "Epoch 76/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5631 - acc: 0.6747 - val_loss: 0.5994 - val_acc: 0.6636\n",
      "\n",
      "Epoch 00076: val_loss did not improve from 0.59266\n",
      "Epoch 77/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5633 - acc: 0.6704 - val_loss: 0.5979 - val_acc: 0.6643\n",
      "\n",
      "Epoch 00077: val_loss did not improve from 0.59266\n",
      "Epoch 78/500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5626 - acc: 0.6730 - val_loss: 0.5981 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00078: val_loss did not improve from 0.59266\n",
      "Epoch 79/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5608 - acc: 0.6721 - val_loss: 0.5978 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00079: val_loss did not improve from 0.59266\n",
      "Epoch 80/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5628 - acc: 0.6717 - val_loss: 0.6002 - val_acc: 0.6636\n",
      "\n",
      "Epoch 00080: val_loss did not improve from 0.59266\n",
      "Epoch 81/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5633 - acc: 0.6713 - val_loss: 0.5954 - val_acc: 0.6643\n",
      "\n",
      "Epoch 00081: val_loss did not improve from 0.59266\n",
      "Epoch 82/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5611 - acc: 0.6745 - val_loss: 0.5985 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00082: val_loss did not improve from 0.59266\n",
      "Epoch 83/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5594 - acc: 0.6739 - val_loss: 0.5997 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00083: val_loss did not improve from 0.59266\n",
      "Epoch 84/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5625 - acc: 0.6708 - val_loss: 0.5971 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00084: val_loss did not improve from 0.59266\n",
      "Epoch 85/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5620 - acc: 0.6762 - val_loss: 0.6002 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00085: val_loss did not improve from 0.59266\n",
      "Epoch 86/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5598 - acc: 0.6758 - val_loss: 0.5997 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00086: val_loss did not improve from 0.59266\n",
      "Epoch 87/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5603 - acc: 0.6756 - val_loss: 0.5972 - val_acc: 0.6643\n",
      "\n",
      "Epoch 00087: val_loss did not improve from 0.59266\n",
      "Epoch 88/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5599 - acc: 0.6756 - val_loss: 0.6000 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00088: val_loss did not improve from 0.59266\n",
      "Epoch 89/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5598 - acc: 0.6713 - val_loss: 0.5999 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00089: val_loss did not improve from 0.59266\n",
      "Epoch 90/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5613 - acc: 0.6717 - val_loss: 0.5977 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00090: val_loss did not improve from 0.59266\n",
      "Epoch 91/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5592 - acc: 0.6769 - val_loss: 0.5973 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00091: val_loss did not improve from 0.59266\n",
      "Epoch 92/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5597 - acc: 0.6745 - val_loss: 0.5988 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00092: val_loss did not improve from 0.59266\n",
      "Epoch 93/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5613 - acc: 0.6747 - val_loss: 0.5995 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00093: val_loss did not improve from 0.59266\n",
      "Epoch 94/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5611 - acc: 0.6736 - val_loss: 0.5980 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00094: val_loss did not improve from 0.59266\n",
      "Epoch 95/500\n",
      "37/37 [==============================] - 2s 44ms/step - loss: 0.5601 - acc: 0.6756 - val_loss: 0.5986 - val_acc: 0.6649\n",
      "\n",
      "Epoch 00095: val_loss did not improve from 0.59266\n",
      "Epoch 96/500\n",
      "37/37 [==============================] - 1s 40ms/step - loss: 0.5602 - acc: 0.6747 - val_loss: 0.5963 - val_acc: 0.6649\n",
      "\n",
      "Epoch 00096: val_loss did not improve from 0.59266\n",
      "Epoch 97/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.5576 - acc: 0.6758 - val_loss: 0.5975 - val_acc: 0.6649\n",
      "\n",
      "Epoch 00097: val_loss did not improve from 0.59266\n",
      "Epoch 98/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.5589 - acc: 0.6782 - val_loss: 0.5972 - val_acc: 0.6649\n",
      "\n",
      "Epoch 00098: val_loss did not improve from 0.59266\n",
      "Epoch 99/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.5581 - acc: 0.6758 - val_loss: 0.5983 - val_acc: 0.6649\n",
      "\n",
      "Epoch 00099: val_loss did not improve from 0.59266\n",
      "Epoch 100/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.5593 - acc: 0.6771 - val_loss: 0.5992 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00100: val_loss did not improve from 0.59266\n",
      "Epoch 101/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.5585 - acc: 0.6780 - val_loss: 0.6008 - val_acc: 0.6624\n",
      "\n",
      "Epoch 00101: val_loss did not improve from 0.59266\n",
      "Epoch 102/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.5595 - acc: 0.6782 - val_loss: 0.5971 - val_acc: 0.6636\n",
      "\n",
      "Epoch 00102: val_loss did not improve from 0.59266\n",
      "Epoch 103/500\n",
      "37/37 [==============================] - 1s 38ms/step - loss: 0.5578 - acc: 0.6780 - val_loss: 0.6003 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00103: val_loss did not improve from 0.59266\n",
      "Epoch 104/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.5607 - acc: 0.6754 - val_loss: 0.5997 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00104: val_loss did not improve from 0.59266\n",
      "Epoch 105/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.5586 - acc: 0.6762 - val_loss: 0.5988 - val_acc: 0.6630\n",
      "\n",
      "Epoch 00105: val_loss did not improve from 0.59266\n",
      "Epoch 106/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.5587 - acc: 0.6747 - val_loss: 0.5987 - val_acc: 0.6643\n",
      "\n",
      "Epoch 00106: val_loss did not improve from 0.59266\n",
      "Epoch 107/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.5575 - acc: 0.6780 - val_loss: 0.5994 - val_acc: 0.6643\n",
      "\n",
      "Epoch 00107: val_loss did not improve from 0.59266\n",
      "Epoch 108/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.5573 - acc: 0.6773 - val_loss: 0.5990 - val_acc: 0.6643\n",
      "\n",
      "Epoch 00108: val_loss did not improve from 0.59266\n",
      "Epoch 109/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.5576 - acc: 0.6771 - val_loss: 0.5975 - val_acc: 0.6649\n",
      "\n",
      "Epoch 00109: val_loss did not improve from 0.59266\n",
      "Epoch 110/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.5564 - acc: 0.6749 - val_loss: 0.6019 - val_acc: 0.6643\n",
      "\n",
      "Epoch 00110: val_loss did not improve from 0.59266\n",
      "Epoch 111/500\n",
      "37/37 [==============================] - 1s 37ms/step - loss: 0.5576 - acc: 0.6764 - val_loss: 0.5971 - val_acc: 0.6649\n",
      "Restoring model weights from the end of the best epoch.\n",
      "\n",
      "Epoch 00111: val_loss did not improve from 0.59266\n",
      "Epoch 00111: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7efed4031250>"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ---------------------\n",
    "seed_num = 42 \n",
    "# ---------------------\n",
    "tf.random.set_seed(seed_num)\n",
    "\n",
    "lstm = Sequential()\n",
    "lstm.add(InputLayer(input_shape=(X_train.shape[1],X_train.shape[2])))\n",
    "lstm.add(LSTM(units=128, activation='hard_sigmoid', return_sequences=True))\n",
    "lstm.add(LSTM(units=64, activation='hard_sigmoid', return_sequences=True))\n",
    "lstm.add(Dropout(0.2))\n",
    "lstm.add(LSTM(units=64, activation='hard_sigmoid', return_sequences=True))\n",
    "lstm.add(LSTM(units=32, activation='hard_sigmoid', return_sequences=False))\n",
    "lstm.add(Dropout(0.2))\n",
    "lstm.add(Dense(units=1, activation='sigmoid'))\n",
    "\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "import os\n",
    "\n",
    "MODEL_SAVE_FOLDER_PATH = './model/'\n",
    "if not os.path.exists(MODEL_SAVE_FOLDER_PATH):\n",
    "    os.mkdir(MODEL_SAVE_FOLDER_PATH)\n",
    "\n",
    "model_path = MODEL_SAVE_FOLDER_PATH + 'lstm_top10_seed42-{epoch:02d}-{val_loss:.4f}.hdf5'\n",
    "\n",
    "cb_checkpoint = ModelCheckpoint(filepath=model_path, monitor='val_loss',\n",
    "                                verbose=1, save_best_only=True, overwrite=True,)\n",
    "\n",
    "early_stop = EarlyStopping(monitor='val_acc', patience=50, verbose=1, restore_best_weights=True)\n",
    "lstm.compile(optimizer= keras.optimizers.Adam(learning_rate = 0.001), loss = \"binary_crossentropy\", metrics=['acc'])\n",
    "lstm.fit(X_train, y_train, validation_split=0.25, batch_size=128, epochs=500,  callbacks=[early_stop,cb_checkpoint], shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "0d244d58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'model/lstm_top10_seed42-59-0.5927.hdf5'"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "paths = sorted(Path(MODEL_SAVE_FOLDER_PATH).iterdir(), key=os.path.getmtime)[-1]\n",
    "best_model_path = str(paths)\n",
    "best_model_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "300a7adf",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "정확도 :0.6550161812297735, seed_num = 0\n",
      "정확도 :0.6511326860841424, seed_num = 1\n",
      "정확도 :0.655663430420712, seed_num = 2\n",
      "정확도 :0.6427184466019418, seed_num = 3\n",
      "정확도 :0.6608414239482201, seed_num = 4\n",
      "정확도 :0.6595469255663431, seed_num = 5\n",
      "정확도 :0.6595469255663431, seed_num = 6\n",
      "정확도 :0.6362459546925566, seed_num = 7\n",
      "정확도 :0.6647249190938511, seed_num = 8\n",
      "정확도 :0.654368932038835, seed_num = 9\n",
      "정확도 :0.6362459546925566, seed_num = 10\n",
      "정확도 :0.6686084142394823, seed_num = 11\n",
      "정확도 :0.6498381877022654, seed_num = 12\n",
      "정확도 :0.6491909385113268, seed_num = 13\n",
      "정확도 :0.6498381877022654, seed_num = 14\n",
      "정확도 :0.6653721682847896, seed_num = 15\n",
      "정확도 :0.6750809061488673, seed_num = 16\n",
      "정확도 :0.6498381877022654, seed_num = 17\n",
      "정확도 :0.6563106796116505, seed_num = 18\n",
      "정확도 :0.6414239482200648, seed_num = 19\n",
      "정확도 :0.6660194174757281, seed_num = 20\n",
      "정확도 :0.6608414239482201, seed_num = 21\n",
      "정확도 :0.6427184466019418, seed_num = 22\n",
      "정확도 :0.656957928802589, seed_num = 23\n",
      "정확도 :0.6537216828478964, seed_num = 24\n",
      "정확도 :0.6588996763754046, seed_num = 25\n",
      "정확도 :0.6336569579288026, seed_num = 26\n",
      "정확도 :0.6614886731391586, seed_num = 27\n",
      "정확도 :0.6537216828478964, seed_num = 28\n",
      "정확도 :0.6472491909385113, seed_num = 29\n",
      "정확도 :0.6588996763754046, seed_num = 30\n",
      "정확도 :0.654368932038835, seed_num = 31\n",
      "정확도 :0.6588996763754046, seed_num = 32\n",
      "정확도 :0.6621359223300971, seed_num = 33\n",
      "정확도 :0.6640776699029126, seed_num = 34\n",
      "정확도 :0.6627831715210356, seed_num = 35\n",
      "정확도 :0.6498381877022654, seed_num = 36\n",
      "정확도 :0.6504854368932039, seed_num = 37\n",
      "정확도 :0.6420711974110033, seed_num = 38\n",
      "정확도 :0.6731391585760518, seed_num = 39\n",
      "정확도 :0.6407766990291263, seed_num = 40\n",
      "정확도 :0.6563106796116505, seed_num = 41\n",
      "정확도 :0.6375404530744336, seed_num = 42\n",
      "정확도 :0.6368932038834951, seed_num = 43\n",
      "정확도 :0.6401294498381876, seed_num = 44\n",
      "정확도 :0.6653721682847896, seed_num = 45\n",
      "정확도 :0.6524271844660194, seed_num = 46\n",
      "정확도 :0.6699029126213593, seed_num = 47\n",
      "정확도 :0.6802588996763754, seed_num = 48\n",
      "정확도 :0.6608414239482201, seed_num = 49\n",
      "평균 정확도 : 0.6546796116504855\n"
     ]
    }
   ],
   "source": [
    "best_model = keras.models.load_model('./model/lstm_top10_seed42-84-0.6057.hdf5')\n",
    "result = 0\n",
    "for seed in range(0, 50):\n",
    "    random.seed(seed)\n",
    "    X = np.load(path + 'x_(7727,10,4068).npy')[:,:,lstm_feature_index]\n",
    "    y = np.load(path + 'y_(7727,1).npy')\n",
    "    \n",
    "    idx = list(range(len(x)))\n",
    "    random.shuffle(idx)\n",
    "    i = round(X.shape[0]*0.8)\n",
    "    X_train, y_train = X[idx[:i],:,:], y[idx[:i]]\n",
    "    X_test, y_test = X[idx[i:],:,:], y[idx[i:]]\n",
    "\n",
    "    pred = best_model.predict(X_test)\n",
    "    pred[pred>0.5]=1\n",
    "    pred[pred<=0.5]=0\n",
    "    acc = metrics.accuracy_score(y_test, pred)\n",
    "    result += acc\n",
    "    print(f'정확도 :{acc}, seed_num = {seed}')\n",
    "print(f'평균 정확도 : {result/50}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e24f24d2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d5c2e96",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b679e184",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab910b7e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "356.949px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}