Data_prep/data_processing_for_ANN_180227.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data processing for tensorflow\n",
    "## from newly prepared data from rserver by incheol.\n",
    "### revising data\n",
    "\n",
    "* 2019-3-14\n",
    "\n",
    "1. data load\n",
    "2. making one hot coding y-label\n",
    "3. minmax scaling\n",
    "4. save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_path = './divided_revised_data/'\n",
    "save_path = './revised_data_for_ANN/'\n",
    "data_file_list = os.listdir(data_path)\n",
    "# data_name_list = [names[0:-4] for names in data_file_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_name_list = [names[0:-4] for names in data_file_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 74 entries, 0 to 73\n",
      "Data columns (total 44 columns):\n",
      "Unnamed: 0                                74 non-null int64\n",
      "adaptation                                74 non-null float64\n",
      "avg_isi                                   74 non-null float64\n",
      "f_i_curve_slope                           74 non-null float64\n",
      "fast_trough_t_long_square                 74 non-null float64\n",
      "fast_trough_t_ramp                        74 non-null float64\n",
      "fast_trough_t_short_square                74 non-null float64\n",
      "fast_trough_v_long_square                 74 non-null float64\n",
      "fast_trough_v_ramp                        74 non-null float64\n",
      "fast_trough_v_short_square                74 non-null float64\n",
      "latency                                   74 non-null float64\n",
      "peak_t_long_square                        74 non-null float64\n",
      "peak_t_ramp                               74 non-null float64\n",
      "peak_t_short_square                       74 non-null float64\n",
      "peak_v_long_square                        74 non-null float64\n",
      "peak_v_ramp                               74 non-null float64\n",
      "peak_v_short_square                       74 non-null float64\n",
      "ri                                        74 non-null float64\n",
      "sag                                       74 non-null float64\n",
      "threshold_i_long_square                   74 non-null float64\n",
      "threshold_i_ramp                          74 non-null float64\n",
      "threshold_i_short_square                  74 non-null float64\n",
      "threshold_t_long_square                   74 non-null float64\n",
      "threshold_t_ramp                          74 non-null float64\n",
      "threshold_t_short_square                  74 non-null float64\n",
      "threshold_v_long_square                   74 non-null float64\n",
      "threshold_v_ramp                          74 non-null float64\n",
      "threshold_v_short_square                  74 non-null float64\n",
      "trough_t_long_square                      74 non-null float64\n",
      "trough_t_ramp                             74 non-null float64\n",
      "trough_t_short_square                     74 non-null float64\n",
      "trough_v_long_square                      74 non-null float64\n",
      "trough_v_ramp                             74 non-null float64\n",
      "trough_v_short_square                     74 non-null float64\n",
      "upstroke_downstroke_ratio_long_square     74 non-null float64\n",
      "upstroke_downstroke_ratio_ramp            74 non-null float64\n",
      "upstroke_downstroke_ratio_short_square    74 non-null float64\n",
      "vrest                                     74 non-null float64\n",
      "tau                                       74 non-null float64\n",
      "firing_rate                               74 non-null float64\n",
      "height_short                              74 non-null float64\n",
      "height_long                               74 non-null float64\n",
      "height_ramp                               74 non-null float64\n",
      "L_tg                                      74 non-null object\n",
      "dtypes: float64(42), int64(1), object(1)\n",
      "memory usage: 25.5+ KB\n"
     ]
    }
   ],
   "source": [
    "data_frame = pd.read_csv(data_path+'Ltest.csv')\n",
    "data_frame.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* For ANN, we have to separate the input feature file and output labeled file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for file_name in data_file_list:\n",
    "    if file_name[0] == 'B':\n",
    "        data_frame = pd.read_csv(data_path+file_name)\n",
    "        data_frameX = data_frame.drop(['Unnamed: 0', 'binary_neuron'], axis = 1)\n",
    "        data_frameX = np.array(data_frameX)\n",
    "        np.savetxt(save_path + file_name[0:-4] + 'X.csv', data_frameX, delimiter = ',')\n",
    "        data_frameY = data_frame['binary_neuron']\n",
    "        data_frameY = np.array(data_frameY.str.contains('E').astype(int))\n",
    "        data_frameY = np.eye(2)[data_frameY]\n",
    "        np.savetxt(save_path + file_name[0:-4] + 'Y.csv', data_frameY, delimiter = ',')\n",
    "    elif file_name[0] == 'I':\n",
    "        data_frame = pd.read_csv(data_path+file_name)\n",
    "        data_frameX = data_frame.drop(['Unnamed: 0', 'transgenic_line'], axis = 1)\n",
    "        data_frameX = np.array(data_frameX)\n",
    "        np.savetxt(save_path + file_name[0:-4] + 'X.csv', data_frameX, delimiter = ',')\n",
    "        data_frameY = data_frame['transgenic_line'].factorize(sort = True)\n",
    "        data_frameY_np = np.array(data_frameY[0].astype(int))\n",
    "        data_frameY = np.eye(len(data_frameY[1]))[data_frameY_np]\n",
    "        np.savetxt(save_path + file_name[0:-4] + 'Y.csv', data_frameY, delimiter = ',')\n",
    "    elif file_name[0] == 'E':\n",
    "        data_frame = pd.read_csv(data_path+file_name)\n",
    "        data_frameX = data_frame.drop(['Unnamed: 0', 'transgenic_line'], axis = 1)\n",
    "        data_frameX = np.array(data_frameX)\n",
    "        np.savetxt(save_path + file_name[0:-4] + 'X.csv', data_frameX, delimiter = ',')\n",
    "        data_frameY = data_frame['transgenic_line'].factorize(sort = True)\n",
    "        data_frameY_np = np.array(data_frameY[0].astype(int))\n",
    "        data_frameY = np.eye(len(data_frameY[1]))[data_frameY_np]\n",
    "        np.savetxt(save_path + file_name[0:-4] + 'Y.csv', data_frameY, delimiter = ',')\n",
    "    elif file_name[0] == 'L':\n",
    "        data_frame = pd.read_csv(data_path+file_name)\n",
    "        data_frameX = data_frame.drop(['Unnamed: 0', 'L_tg'], axis = 1)\n",
    "        data_frameX = np.array(data_frameX)\n",
    "        np.savetxt(save_path + file_name[0:-4] + 'X.csv', data_frameX, delimiter = ',')\n",
    "        data_frameY = data_frame['L_tg'].factorize(sort = True)\n",
    "        data_frameY_np = np.array(data_frameY[0].astype(int))\n",
    "        data_frameY = np.eye(len(data_frameY[1]))[data_frameY_np]\n",
    "        np.savetxt(save_path + file_name[0:-4] + 'Y.csv', data_frameY, delimiter = ',')\n",
    "    \n",
    "       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['EtestY.csv',\n",
       " 'Itest_longY.csv',\n",
       " 'BtestX.csv',\n",
       " 'BtestY.csv',\n",
       " 'Btest_longX.csv',\n",
       " 'Btest_longY.csv',\n",
       " 'Btest_rampX.csv',\n",
       " 'Btest_rampY.csv',\n",
       " 'Btest_shortX.csv',\n",
       " 'Btest_shortY.csv',\n",
       " 'BtrainX.csv',\n",
       " 'BtrainY.csv',\n",
       " 'Btrain_longX.csv',\n",
       " 'Btrain_longY.csv',\n",
       " 'Btrain_rampX.csv',\n",
       " 'Btrain_rampY.csv',\n",
       " 'Btrain_shortX.csv',\n",
       " 'Btrain_shortY.csv',\n",
       " 'EtestX.csv',\n",
       " 'Etest_longX.csv',\n",
       " 'Etest_longY.csv',\n",
       " 'Etest_rampX.csv',\n",
       " 'Etest_rampY.csv',\n",
       " 'Etest_shortX.csv',\n",
       " 'Etest_shortY.csv',\n",
       " 'EtrainX.csv',\n",
       " 'EtrainY.csv',\n",
       " 'Etrain_longX.csv',\n",
       " 'Etrain_longY.csv',\n",
       " 'Etrain_rampX.csv',\n",
       " 'Etrain_rampY.csv',\n",
       " 'Etrain_shortX.csv',\n",
       " 'Etrain_shortY.csv',\n",
       " 'ItestX.csv',\n",
       " 'ItestY.csv',\n",
       " 'Itest_longX.csv',\n",
       " 'Itest_rampX.csv',\n",
       " 'Itest_rampY.csv',\n",
       " 'Itest_shortX.csv',\n",
       " 'Itest_shortY.csv',\n",
       " 'ItrainX.csv',\n",
       " 'ItrainY.csv',\n",
       " 'Itrain_longX.csv',\n",
       " 'Itrain_longY.csv',\n",
       " 'Itrain_rampX.csv',\n",
       " 'Itrain_rampY.csv',\n",
       " 'Itrain_shortX.csv',\n",
       " 'Itrain_shortY.csv',\n",
       " 'LtestX.csv',\n",
       " 'LtestY.csv',\n",
       " 'Ltest_longX.csv',\n",
       " 'Ltest_longY.csv',\n",
       " 'Ltest_rampX.csv',\n",
       " 'Ltest_rampY.csv',\n",
       " 'Ltest_shortX.csv',\n",
       " 'Ltest_shortY.csv',\n",
       " 'LtrainX.csv',\n",
       " 'LtrainY.csv',\n",
       " 'Ltrain_longX.csv',\n",
       " 'Ltrain_longY.csv',\n",
       " 'Ltrain_rampX.csv',\n",
       " 'Ltrain_rampY.csv',\n",
       " 'Ltrain_shortX.csv',\n",
       " 'Ltrain_shortY.csv']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensordata_file_list = os.listdir(save_path)\n",
    "tensordata_file_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* For ANN traning, we make minmax-scaled input data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "minmax_path = './minmax_scaled_revised_data_for_ANN/'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for file_name in tensordata_file_list:\n",
    "    if file_name[-5] == 'X' and file_name[1:3] == 'tr':\n",
    "        train_name = file_name\n",
    "        test_name = file_name[0] + 'test'+ file_name[6:]\n",
    "        train_dataX = np.loadtxt(save_path + train_name, delimiter = ',')\n",
    "        test_dataX = np.loadtxt(save_path + test_name, delimiter = ',')\n",
    "        minmax_scale = preprocessing.MinMaxScaler().fit(train_dataX)\n",
    "        train_dataX_minmax = minmax_scale.transform(train_dataX)\n",
    "        test_dataX_minmax = minmax_scale.transform(test_dataX)\n",
    "        np.savetxt(minmax_path + train_name[0:-4] + '.csv', \n",
    "                    train_dataX_minmax, delimiter = ',')\n",
    "        np.savetxt(minmax_path + test_name[0:-4] + '.csv', \n",
    "                    test_dataX_minmax, delimiter = ',')\n",
    "    elif file_name[-5] == 'Y':\n",
    "        dataY = np.loadtxt(save_path + file_name, delimiter = ',')\n",
    "        np.savetxt(minmax_path + file_name, dataY, delimiter = ',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EtestY.csv\n",
      "6\n",
      "Itest_longY.csv\n",
      "8\n",
      "BtestY.csv\n",
      "2\n",
      "Btest_longY.csv\n",
      "2\n",
      "Btest_rampY.csv\n",
      "2\n",
      "Btest_shortY.csv\n",
      "2\n",
      "BtrainY.csv\n",
      "2\n",
      "Btrain_longY.csv\n",
      "2\n",
      "Btrain_rampY.csv\n",
      "2\n",
      "Btrain_shortY.csv\n",
      "2\n",
      "Etest_longY.csv\n",
      "6\n",
      "Etest_rampY.csv\n",
      "6\n",
      "Etest_shortY.csv\n",
      "6\n",
      "EtrainY.csv\n",
      "6\n",
      "Etrain_longY.csv\n",
      "6\n",
      "Etrain_rampY.csv\n",
      "6\n",
      "Etrain_shortY.csv\n",
      "6\n",
      "ItestY.csv\n",
      "8\n",
      "Itest_rampY.csv\n",
      "8\n",
      "Itest_shortY.csv\n",
      "8\n",
      "ItrainY.csv\n",
      "8\n",
      "Itrain_longY.csv\n",
      "8\n",
      "Itrain_rampY.csv\n",
      "8\n",
      "Itrain_shortY.csv\n",
      "8\n",
      "LtestY.csv\n",
      "5\n",
      "Ltest_longY.csv\n",
      "5\n",
      "Ltest_rampY.csv\n",
      "5\n",
      "Ltest_shortY.csv\n",
      "5\n",
      "LtrainY.csv\n",
      "5\n",
      "Ltrain_longY.csv\n",
      "5\n",
      "Ltrain_rampY.csv\n",
      "5\n",
      "Ltrain_shortY.csv\n",
      "5\n"
     ]
    }
   ],
   "source": [
    "for file_name in tensordata_file_list:\n",
    "    if file_name[-5] == 'Y':\n",
    "        train_name = file_name\n",
    "        train_dataY = np.loadtxt(save_path + train_name, delimiter = ',')\n",
    "        print(train_name)\n",
    "        print(train_dataY.shape[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EtestY.csv\n",
      "6\n",
      "Itest_longY.csv\n",
      "8\n",
      "BtestY.csv\n",
      "2\n",
      "Btest_longY.csv\n",
      "2\n",
      "Btest_rampY.csv\n",
      "2\n",
      "Btest_shortY.csv\n",
      "2\n",
      "BtrainY.csv\n",
      "2\n",
      "Btrain_longY.csv\n",
      "2\n",
      "Btrain_rampY.csv\n",
      "2\n",
      "Btrain_shortY.csv\n",
      "2\n",
      "Etest_longY.csv\n",
      "6\n",
      "Etest_rampY.csv\n",
      "6\n",
      "Etest_shortY.csv\n",
      "6\n",
      "EtrainY.csv\n",
      "6\n",
      "Etrain_longY.csv\n",
      "6\n",
      "Etrain_rampY.csv\n",
      "6\n",
      "Etrain_shortY.csv\n",
      "6\n",
      "ItestY.csv\n",
      "8\n",
      "Itest_rampY.csv\n",
      "8\n",
      "Itest_shortY.csv\n",
      "8\n",
      "ItrainY.csv\n",
      "8\n",
      "Itrain_longY.csv\n",
      "8\n",
      "Itrain_rampY.csv\n",
      "8\n",
      "Itrain_shortY.csv\n",
      "8\n",
      "LtestY.csv\n",
      "5\n",
      "Ltest_longY.csv\n",
      "5\n",
      "Ltest_rampY.csv\n",
      "5\n",
      "Ltest_shortY.csv\n",
      "5\n",
      "LtrainY.csv\n",
      "5\n",
      "Ltrain_longY.csv\n",
      "5\n",
      "Ltrain_rampY.csv\n",
      "5\n",
      "Ltrain_shortY.csv\n",
      "5\n"
     ]
    }
   ],
   "source": [
    "minmax_file_list = os.listdir(minmax_path)\n",
    "for file_name in minmax_file_list:\n",
    "    if file_name[-5] == 'Y':\n",
    "        train_name = file_name\n",
    "        train_dataY = np.loadtxt(save_path + train_name, delimiter = ',')\n",
    "        print(train_name)\n",
    "        print(train_dataY.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "notmerged_data_path = './divided_revised_notmerged_data/'\n",
    "notmerged_save_path = './revised_notmerged_data_for_ANN/'\n",
    "data_file_list = os.listdir(notmerged_data_path)\n",
    "# data_name_list = [names[0:-4] for names in data_file_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_name_list = [names[0:-4] for names in data_file_list]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* For ANN, we have to separate the input feature file and output labeled file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for file_name in data_file_list:\n",
    "    if file_name[0] == 'B':\n",
    "        data_frame = pd.read_csv(notmerged_data_path+file_name)\n",
    "        data_frameX = data_frame.drop(['Unnamed: 0', 'binary_neuron'], axis = 1)\n",
    "        data_frameX = np.array(data_frameX)\n",
    "        np.savetxt(notmerged_save_path + file_name[0:-4] + 'X.csv', data_frameX, delimiter = ',')\n",
    "        data_frameY = data_frame['binary_neuron']\n",
    "        data_frameY = np.array(data_frameY.str.contains('E').astype(int))\n",
    "        data_frameY = np.eye(2)[data_frameY]\n",
    "        np.savetxt(notmerged_save_path + file_name[0:-4] + 'Y.csv', data_frameY, delimiter = ',')\n",
    "    elif file_name[0] == 'I':\n",
    "        data_frame = pd.read_csv(notmerged_data_path+file_name)\n",
    "        data_frameX = data_frame.drop(['Unnamed: 0', 'transgenic_line'], axis = 1)\n",
    "        data_frameX = np.array(data_frameX)\n",
    "        np.savetxt(notmerged_save_path + file_name[0:-4] + 'X.csv', data_frameX, delimiter = ',')\n",
    "        data_frameY = data_frame['transgenic_line'].factorize(sort = True)\n",
    "        data_frameY_np = np.array(data_frameY[0].astype(int))\n",
    "        data_frameY = np.eye(len(data_frameY[1]))[data_frameY_np]\n",
    "        np.savetxt(notmerged_save_path + file_name[0:-4] + 'Y.csv', data_frameY, delimiter = ',')\n",
    "    elif file_name[0] == 'E':\n",
    "        data_frame = pd.read_csv(notmerged_data_path+file_name)\n",
    "        data_frameX = data_frame.drop(['Unnamed: 0', 'transgenic_line'], axis = 1)\n",
    "        data_frameX = np.array(data_frameX)\n",
    "        np.savetxt(notmerged_save_path + file_name[0:-4] + 'X.csv', data_frameX, delimiter = ',')\n",
    "        data_frameY = data_frame['transgenic_line'].factorize(sort = True)\n",
    "        data_frameY_np = np.array(data_frameY[0].astype(int))\n",
    "        data_frameY = np.eye(len(data_frameY[1]))[data_frameY_np]\n",
    "        np.savetxt(notmerged_save_path + file_name[0:-4] + 'Y.csv', data_frameY, delimiter = ',')\n",
    "    elif file_name[0] == 'L':\n",
    "        data_frame = pd.read_csv(notmerged_data_path+file_name)\n",
    "        data_frameX = data_frame.drop(['Unnamed: 0', 'L_tg'], axis = 1)\n",
    "        data_frameX = np.array(data_frameX)\n",
    "        np.savetxt(notmerged_save_path + file_name[0:-4] + 'X.csv', data_frameX, delimiter = ',')\n",
    "        data_frameY = data_frame['L_tg'].factorize(sort = True)\n",
    "        data_frameY_np = np.array(data_frameY[0].astype(int))\n",
    "        data_frameY = np.eye(len(data_frameY[1]))[data_frameY_np]\n",
    "        np.savetxt(notmerged_save_path + file_name[0:-4] + 'Y.csv', data_frameY, delimiter = ',')\n",
    "    \n",
    "       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['EtestX.csv',\n",
       " 'EtestY.csv',\n",
       " 'Etest_longX.csv',\n",
       " 'Etest_longY.csv',\n",
       " 'Etest_rampX.csv',\n",
       " 'Etest_rampY.csv',\n",
       " 'Etest_shortX.csv',\n",
       " 'Etest_shortY.csv',\n",
       " 'EtrainX.csv',\n",
       " 'EtrainY.csv',\n",
       " 'Etrain_longX.csv',\n",
       " 'Etrain_longY.csv',\n",
       " 'Etrain_rampX.csv',\n",
       " 'Etrain_rampY.csv',\n",
       " 'Etrain_shortX.csv',\n",
       " 'Etrain_shortY.csv',\n",
       " 'ItestX.csv',\n",
       " 'ItestY.csv',\n",
       " 'Itest_longX.csv',\n",
       " 'Itest_longY.csv',\n",
       " 'Itest_rampX.csv',\n",
       " 'Itest_rampY.csv',\n",
       " 'Itest_shortX.csv',\n",
       " 'Itest_shortY.csv',\n",
       " 'ItrainX.csv',\n",
       " 'ItrainY.csv',\n",
       " 'Itrain_longX.csv',\n",
       " 'Itrain_longY.csv',\n",
       " 'Itrain_rampX.csv',\n",
       " 'Itrain_rampY.csv',\n",
       " 'Itrain_shortX.csv',\n",
       " 'Itrain_shortY.csv']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensordata_file_list = os.listdir(notmerged_save_path)\n",
    "tensordata_file_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* For ANN traning, we make minmax-scaled input data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "minmax_path = './minmax_scaled_revised_notmerged_data_for_ANN/'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for file_name in tensordata_file_list:\n",
    "    if file_name[-5] == 'X' and file_name[1:3] == 'tr':\n",
    "        train_name = file_name\n",
    "        test_name = file_name[0] + 'test'+ file_name[6:]\n",
    "        train_dataX = np.loadtxt(notmerged_save_path + train_name, delimiter = ',')\n",
    "        test_dataX = np.loadtxt(notmerged_save_path + test_name, delimiter = ',')\n",
    "        minmax_scale = preprocessing.MinMaxScaler().fit(train_dataX)\n",
    "        train_dataX_minmax = minmax_scale.transform(train_dataX)\n",
    "        test_dataX_minmax = minmax_scale.transform(test_dataX)\n",
    "        np.savetxt(minmax_path + train_name[0:-4] + '.csv', \n",
    "                    train_dataX_minmax, delimiter = ',')\n",
    "        np.savetxt(minmax_path + test_name[0:-4] + '.csv', \n",
    "                    test_dataX_minmax, delimiter = ',')\n",
    "    elif file_name[-5] == 'Y':\n",
    "        dataY = np.loadtxt(notmerged_save_path + file_name, delimiter = ',')\n",
    "        np.savetxt(minmax_path + file_name, dataY, delimiter = ',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EtestY.csv\n",
      "8\n",
      "Etest_longY.csv\n",
      "8\n",
      "Etest_rampY.csv\n",
      "8\n",
      "Etest_shortY.csv\n",
      "8\n",
      "EtrainY.csv\n",
      "9\n",
      "Etrain_longY.csv\n",
      "9\n",
      "Etrain_rampY.csv\n",
      "9\n",
      "Etrain_shortY.csv\n",
      "9\n",
      "ItestY.csv\n",
      "9\n",
      "Itest_longY.csv\n",
      "9\n",
      "Itest_rampY.csv\n",
      "9\n",
      "Itest_shortY.csv\n",
      "9\n",
      "ItrainY.csv\n",
      "9\n",
      "Itrain_longY.csv\n",
      "9\n",
      "Itrain_rampY.csv\n",
      "9\n",
      "Itrain_shortY.csv\n",
      "9\n"
     ]
    }
   ],
   "source": [
    "for file_name in tensordata_file_list:\n",
    "    if file_name[-5] == 'Y':\n",
    "        train_name = file_name\n",
    "        train_dataY = np.loadtxt(notmerged_save_path + train_name, delimiter = ',')\n",
    "        print(train_name)\n",
    "        print(train_dataY.shape[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EtestY.csv\n",
      "8\n",
      "Etest_longY.csv\n",
      "8\n",
      "Etest_rampY.csv\n",
      "8\n",
      "Etest_shortY.csv\n",
      "8\n",
      "EtrainY.csv\n",
      "9\n",
      "Etrain_longY.csv\n",
      "9\n",
      "Etrain_rampY.csv\n",
      "9\n",
      "Etrain_shortY.csv\n",
      "9\n",
      "ItestY.csv\n",
      "9\n",
      "Itest_longY.csv\n",
      "9\n",
      "Itest_rampY.csv\n",
      "9\n",
      "Itest_shortY.csv\n",
      "9\n",
      "ItrainY.csv\n",
      "9\n",
      "Itrain_longY.csv\n",
      "9\n",
      "Itrain_rampY.csv\n",
      "9\n",
      "Itrain_shortY.csv\n",
      "9\n"
     ]
    }
   ],
   "source": [
    "minmax_file_list = os.listdir(minmax_path)\n",
    "for file_name in minmax_file_list:\n",
    "    if file_name[-5] == 'Y':\n",
    "        train_name = file_name\n",
    "        train_dataY = np.loadtxt(notmerged_save_path + train_name, delimiter = ',')\n",
    "        print(train_name)\n",
    "        print(train_dataY.shape[1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "523px",
    "left": "1550px",
    "right": "20px",
    "top": "120px",
    "width": "343px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}