LSTM_V5.0.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基于LSTM深度神经网络的股指预测模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from pandas import Series,DataFrame\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import datetime\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib import learn\n",
    "from tensorflow.python.ops import rnn, rnn_cell\n",
    "import matplotlib.pyplot as plt\n",
    "import cPickle\n",
    "import gzip\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 提取主力合约"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第一个合约交割日时间： 2016-05-20 14:51:00.000000\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "u'2016-05-20 14:51:00.000000' is not in list",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-bf48666aaa38>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0;32mprint\u001b[0m \u001b[0;34m\"第一个合约交割日时间：\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmain_last_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m         \u001b[0mmain_last_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnewtime_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmain_last_time\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# 返回上一个合约的交割日在当前合约中的索引\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m         \u001b[0;32mprint\u001b[0m \u001b[0;34m\"上个合约交割时间所在当前合约的索引：\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmain_last_index\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: u'2016-05-20 14:51:00.000000' is not in list"
     ]
    }
   ],
   "source": [
    "path = '/home/chocolate/LSTM-source/Minute_data_IF/'\n",
    "new_path = '/home/chocolate/LSTM-source/Main_product/'\n",
    "file_list = os.listdir(path)\n",
    "for i in range(len(file_list)):\n",
    "    train_data = pd.read_csv(path + file_list[i],encoding=\"gbk\")\n",
    "    train_data = train_data.sort_values(by='Time')  # 对原数据重新按时间排序，因为有些数据顺序被打乱了。\n",
    "#     train_data = train_data[:-5] # 最后5分钟的数据不可取，去除\n",
    "    newtime = pd.to_datetime(train_data['Time']) # 将时间转换为datetime格式\n",
    "    newtime_list = list(train_data['Time']) # 将时间Series格式转换为list\n",
    "    main_month = newtime.iloc[-1].month # 提取主力合约的月份\n",
    "    if i == 0:\n",
    "        for j in range(len(train_data)):\n",
    "            if newtime.iloc[j].month != main_month:\n",
    "                 train_data.drop(j,inplace=True) # 如果不是主力合约的数据直接删除\n",
    "        train_data.to_csv(new_path + file_list[i].split('.')[0]+ '_main' + '.csv',sep=',') # 将提取的主力合约另存为csv文件\n",
    "        main_last_time = train_data['Time'].iloc[-10] # 取上一个合约最后一个交割日的时间\n",
    "        print \"第一个合约交割日时间：\",main_last_time\n",
    "    else:\n",
    "        main_last_index = newtime_list.index(main_last_time) # 返回上一个合约的交割日在当前合约中的索引\n",
    "        print \"上个合约交割时间所在当前合约的索引：\",main_last_index\n",
    "        for k in range(len(train_data)):\n",
    "            if k< main_last_index:\n",
    "                train_data.drop(k,inplace=True) # 如果不是主力合约的数据直接删除\n",
    "        print \"当前主力合约起始时间：\",train_data['Time'].iloc[1]\n",
    "        train_data.to_csv(new_path + file_list[i].split('.')[0]+ '_main' + '.csv',sep=',') # 将提取的主力合约另存为csv文件\n",
    "        main_last_time = train_data['Time'].iloc[-10] # 取上一个合约最后一个交割日的时间\n",
    "        print \"当前合约交割日时间：\",main_last_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第一个合约交割日时间： 2016-05-20 14:46:00.000000\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "u'2016-05-20 14:46:00.000000' is not in list",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-7591827afe57>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0;32mprint\u001b[0m \u001b[0;34m\"第一个合约交割日时间：\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmain_last_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m         \u001b[0mmain_last_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnewtime_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmain_last_time\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# 返回上一个合约的交割日在当前合约中的索引\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m         \u001b[0;32mprint\u001b[0m \u001b[0;34m\"上个合约交割时间所在当前合约的索引：\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmain_last_index\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: u'2016-05-20 14:46:00.000000' is not in list"
     ]
    }
   ],
   "source": [
    "# 提取训练集的主力合约\n",
    "trainset_path = '/home/chocolate/LSTM-source/Minute_data_IF/' # 预处理后的合约\n",
    "trainset_main_path = '/home/chocolate/LSTM-source/Main_product/'\n",
    "file_list = os.listdir(trainset_path)\n",
    "for i in range(len(file_list)):\n",
    "    train_data = pd.read_csv(trainset_path + file_list[i],header=0, sep=',',encoding=\"gbk\")\n",
    "    train_data = train_data.sort_values(by='Time')  # 对原数据重新按时间排序，因为有些数据顺序被打乱了。\n",
    "    train_data = train_data[:-5] # 最后5分钟的数据不可取，去除\n",
    "    newtime = pd.to_datetime(train_data['Time']) # 将时间转换为datetime格式\n",
    "    newtime_list = list(train_data['Time']) # 将时间Series格式转换为list\n",
    "    main_month = newtime.iloc[-1].month # 提取主力合约的月份\n",
    "    if i == 0:\n",
    "        for j in range(len(train_data)):\n",
    "            if newtime.iloc[j].month != main_month:\n",
    "                 train_data.drop(j,inplace=True) # 如果不是主力合约的数据直接删除\n",
    "        train_data.to_csv(trainset_main_path + file_list[i].split('.')[0]+ '_main' + '.csv',sep=',') # 将提取的主力合约另存为csv文件\n",
    "        main_last_time = train_data['Time'].iloc[-10] # 取上一个合约最后一个交割日的时间\n",
    "        print \"第一个合约交割日时间：\",main_last_time\n",
    "    else:\n",
    "        main_last_index = newtime_list.index(main_last_time) # 返回上一个合约的交割日在当前合约中的索引\n",
    "        print \"上个合约交割时间所在当前合约的索引：\",main_last_index\n",
    "        for k in range(len(train_data)):\n",
    "            if k< main_last_index:\n",
    "                train_data.drop(k,inplace=True) # 如果不是主力合约的数据直接删除\n",
    "        print \"当前主力合约起始时间：\",train_data['Time'].iloc[1]\n",
    "        train_data.to_csv(trainset_main_path + file_list[i].split('.')[0]+ '_main' + '.csv',sep=',') # 将提取的主力合约另存为csv文件\n",
    "        main_last_time = train_data['Time'].iloc[-10] # 取上一个合约最后一个交割日的时间\n",
    "        print \"当前合约交割日时间：\",main_last_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Int64Index([2381], dtype='int64')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception KeyboardInterrupt in 'zmq.backend.cython.message.Frame.__dealloc__' ignored\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "indices are out-of-bounds",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-59-dbe1ad8f6455>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0midx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Time'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misnull\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m \u001b[0;31m# data = data.sort_values(by='Time')  # 对原数据重新按时间排序，因为有些数据顺序被打乱了。\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python2.7/dist-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   1961\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mSeries\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mIndex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1962\u001b[0m             \u001b[0;31m# either boolean or fancy integer index\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1963\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1964\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1965\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_frame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python2.7/dist-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m_getitem_array\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2006\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2007\u001b[0m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_convert_to_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2008\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2009\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2010\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python2.7/dist-packages/pandas/core/generic.pyc\u001b[0m in \u001b[0;36mtake\u001b[0;34m(self, indices, axis, convert, is_copy)\u001b[0m\n\u001b[1;32m   1369\u001b[0m         new_data = self._data.take(indices,\n\u001b[1;32m   1370\u001b[0m                                    \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_block_manager_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1371\u001b[0;31m                                    convert=True, verify=True)\n\u001b[0m\u001b[1;32m   1372\u001b[0m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_constructor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__finalize__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1373\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python2.7/dist-packages/pandas/core/internals.pyc\u001b[0m in \u001b[0;36mtake\u001b[0;34m(self, indexer, axis, verify, convert)\u001b[0m\n\u001b[1;32m   3617\u001b[0m         \u001b[0mn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3618\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3619\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmaybe_convert_indices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3620\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3621\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mverify\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python2.7/dist-packages/pandas/core/indexing.pyc\u001b[0m in \u001b[0;36mmaybe_convert_indices\u001b[0;34m(indices, n)\u001b[0m\n\u001b[1;32m   1748\u001b[0m     \u001b[0mmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mindices\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mindices\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1749\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1750\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mIndexError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"indices are out-of-bounds\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1751\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1752\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: indices are out-of-bounds"
     ]
    }
   ],
   "source": [
    "path = '/home/chocolate/LSTM-source/Minute_data_IC/'\n",
    "new_path = '/home/chocolate/LSTM-source/Main_product/'\n",
    "file_list = os.listdir(path)\n",
    "data = pd.DataFrame()\n",
    "for i in range(len(file_list)):\n",
    "    data_per = pd.read_csv(path + file_list[i],header=0, sep=',',encoding=\"gbk\")\n",
    "#     print data_per['Time']\n",
    "    data = data.append(data_per)\n",
    "#     print data_per.shape\n",
    "data['Time'] = pd.to_datetime(data['Time']) \n",
    "data = data.sort_values(by='Time')  # 对原数据重新按时间排序，因为有些数据顺序被打乱了。\n",
    "\n",
    "dataVolume = data.groupby('Time')[['Volume']].max()\n",
    "data1 = pd.merge(data,dataVolume,left_on='Time',right_index=True,how='right')\n",
    "data1 = data1.drop_duplicates('Time')\n",
    "data1.drop('Volume_x',axis=1,inplace=True)\n",
    "data1.index = range(len(data1))\n",
    "data1 = DataFrame(data1,columns=['Time','Latestprice','MaxPrice','MinPrice','Stockup','Volume_y','First.Latestprice','Last.Buy1price','Last.Buy1quantity','Last.Sell1price','Last.Sell1quantity'])\n",
    "data1.to_csv(new_path + 'IC_main.csv')\n",
    "\n",
    "# 提取完之后，手动将列名”Volume_y“改为”Volume“\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          Date                Time Latestprice MaxPrice  \\\n",
      "2381  2015-05-28T05:41:00.000Z 2015-05-28 13:41:00      3301.6   3306.4   \n",
      "\n",
      "     MinPrice Stockup     Turnover Position Volume First.Latestprice  \\\n",
      "2381   3301.6       2  2.08219e+07     5807     21            3306.2   \n",
      "\n",
      "     Last.Buy1price Last.Buy1quantity Last.Sell1price Last.Sell1quantity  \n",
      "2381         3300.8                 1            3302                  6  \n",
      "                          Date                Time Latestprice MaxPrice  \\\n",
      "2382  2015-05-28T05:42:00.000Z 2015-05-28 13:42:00      3300.2   3304.2   \n",
      "\n",
      "     MinPrice Stockup     Turnover Position Volume First.Latestprice  \\\n",
      "2382     3298       5  2.67357e+07     5812     27              3302   \n",
      "\n",
      "     Last.Buy1price Last.Buy1quantity Last.Sell1price Last.Sell1quantity  \n",
      "2382         3300.6                 1          3302.4                  1  \n"
     ]
    }
   ],
   "source": [
    "# 在处理IH合约的时候，某条数据日期格式有错误，找出并删除，重新保存\n",
    "path = '/home/chocolate/LSTM-source/Minute_data_IH/'\n",
    "new_path = '/home/chocolate/LSTM-source/Main_product/'\n",
    "file_list = os.listdir(path)\n",
    "data = pd.DataFrame()\n",
    "for i in range(len(file_list)):\n",
    "    data_per = pd.read_csv(path + file_list[i],header=0, sep=',',encoding=\"gbk\")\n",
    "#     print data_per['Time']\n",
    "    data = data.append(data_per)\n",
    "data['Time'] = pd.to_datetime(data['Time'], dayfirst=True, errors='coerce') \n",
    "idx = data[data['Time'].isnull()].index\n",
    "print idx\n",
    "print data.iloc[idx]\n",
    "# print data.iloc[idx + 1]\n",
    "data = data.drop(data.iloc[idx])\n",
    "\n",
    "data['Time'] = pd.to_datetime(data['Time']) \n",
    "data = data.sort_values(by='Time')  # 对原数据重新按时间排序，因为有些数据顺序被打乱了。\n",
    "\n",
    "dataVolume = data.groupby('Time')[['Volume']].max()\n",
    "data1 = pd.merge(data,dataVolume,left_on='Time',right_index=True,how='right')\n",
    "data1 = data1.drop_duplicates('Time')\n",
    "data1.drop('Volume_x',axis=1,inplace=True)\n",
    "data1.index = range(len(data1))\n",
    "data1 = DataFrame(data1,columns=['Time','Latestprice','MaxPrice','MinPrice','Stockup','Volume_y','First.Latestprice','Last.Buy1price','Last.Buy1quantity','Last.Sell1price','Last.Sell1quantity'])\n",
    "data1.to_csv(new_path + 'IH_main.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据清洗"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 读取已经提取好的主力合约文件\n",
    "\n",
    "# 选择2016年1-8月的主力合约数据作为训练集，在主力合约中进行提取\n",
    "# 删除非正常交易时间的数据，交易时间为上午9：30-11：30，下午1:00到3:00\n",
    "# 因为每天开盘的前10分钟和收盘的最后10分钟数据波动较大，暂时在训练时去掉这些数据\n",
    "\n",
    "# 一分钟之内，最大股价和最低股价相差超过50点的，那么应该尽量删掉这些数据。\n",
    "path = '/home/chocolate/LSTM-source/Main_product/'\n",
    "new_path = '/home/chocolate/LSTM-source/Main_product_cleaned/'\n",
    "file_list = os.listdir(path)\n",
    "for i in range(len(file_list)):\n",
    "    data_temp = pd.read_csv(path + file_list[i],header=0, sep=',',encoding=\"gbk\")\n",
    "    data_temp = data_temp.sort_values(by='Time')\n",
    "    data_temp['Time'] = pd.to_datetime(data_temp['Time']) \n",
    "    data_temp.index = range(len(data_temp))\n",
    "    delete_list = []\n",
    "    for k in range(len(data_temp)):\n",
    "        if data_temp['Time'].iloc[k].hour < 9:\n",
    "            delete_list.append(k)\n",
    "        elif data_temp['Time'].iloc[k].hour == 9 and data_temp['Time'].iloc[k].minute <= 29:\n",
    "            delete_list.append(k)\n",
    "        elif data_temp['Time'].iloc[k].hour >= 15: \n",
    "            delete_list.append(k)\n",
    "        elif data_temp['MaxPrice'].iloc[k] - data_temp['MinPrice'].iloc[k] > 50:\n",
    "            delete_list.append(k)\n",
    "\n",
    "    for j in range(len(data_temp)):\n",
    "        if j in delete_list:\n",
    "            data_temp.drop(j,inplace=True)\n",
    "    data_temp.index = range(len(data_temp))\n",
    "    data_temp.to_csv(new_path + file_list[i].split('.')[0]+ '_cleaned' + '.csv',sep=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## （1）添加用于训练的诸多特征，如移动平均线、布林带等\n",
    "## （2）添加用于预测标签的诸多特征，如后1-5分钟的涨跌幅度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import datetime\n",
    "from pandas import Series,DataFrame\n",
    "path = '/home/chocolate/LSTM-source/Main_product_cleaned/'\n",
    "new_path = '/home/chocolate/LSTM-source/Main_product_added_features/'\n",
    "file_list = os.listdir(path)\n",
    "ma_list = [5, 12, 26]\n",
    "for j in range(len(file_list)):\n",
    "    newdata1 = pd.read_csv(path + file_list[j], header=0, sep=',',encoding=\"gbk\")\n",
    "    newTime = pd.to_datetime(newdata1['Time']) # 将时间转换成datetime格式\n",
    "#  添加收盘价的移动平均线MA和指数平滑移动平均线EMA\n",
    "    for ma in ma_list:\n",
    "        newdata1['MA_' + str(ma)] = pd.rolling_mean(newdata1['Latestprice'], ma)\n",
    "    # 计算移动平均线之间的距离\n",
    "    newdata1['Dis_MA5_26'] = newdata1['MA_5'] - newdata1['MA_26']\n",
    "    # 计算指数平滑移动平均线\n",
    "    for ma in ma_list:\n",
    "        newdata1['EMA_' + str(ma)] = pd.ewma(newdata1['Latestprice'], span=ma)\n",
    "    # 计算指数平滑移动平均线之间的距离\n",
    "    newdata1['Dis_EMA5_26'] = newdata1['EMA_5'] - newdata1['EMA_26']\n",
    "    \n",
    "# 添加成交量的移动平均线MA和指数平滑移动平均线EMA\n",
    "    for ma in ma_list:\n",
    "        newdata1['Vol_MA_'+str(ma)] =  pd.rolling_mean(newdata1['Volume'], ma)\n",
    "# 计算成交量移动平均线之间的距离\n",
    "    newdata1['Dis_Vol_MA5_26'] = newdata1['Vol_MA_5'] - newdata1['Vol_MA_26']\n",
    "# 添加成交量的指数平滑移动平均线\n",
    "    for ma in ma_list:\n",
    "        newdata1['Vol_EMA_'+str(ma)] = pd.ewma(newdata1['Volume'], span=ma)\n",
    "# 计算指数平滑移动平均线之间的距离\n",
    "    newdata1['Dis_Vol_EMA5_26'] = newdata1['Vol_EMA_5'] - newdata1['Vol_EMA_26']  \n",
    "    newdata1['DIFF_12_26'] = newdata1['EMA_12'] - newdata1['EMA_26']\n",
    "# 计算离差平均值DEA，也就是计算离差值的指数平滑移动平均，设置为5分钟的指数平滑曲线\n",
    "    newdata1['DEA_12_26'] =  pd.ewma(newdata1['DIFF_12_26'],span = 9)\n",
    "# 计算MACD值\n",
    "    newdata1['MACD'] = 2*(newdata1['DIFF_12_26'] - newdata1['DEA_12_26'])\n",
    "    MD = []\n",
    "    std_sum = 0\n",
    "    for i in range(len(newdata1)):\n",
    "        if i < 12:\n",
    "            MD.append(0)\n",
    "        else:\n",
    "            for k in range(12):\n",
    "                std_sum += (newdata1['Latestprice'].iloc[i-k] - newdata1['MA_12'].iloc[i-k])**2\n",
    "            std_sum = np.sqrt(std_sum/12.0)\n",
    "            MD.append(std_sum)\n",
    "            std_sum = 0\n",
    "# 计算上轨线\n",
    "    newdata1['boll_up'] = newdata1['MA_12'] + 2*Series(MD)\n",
    "# 计算下轨线\n",
    "    newdata1['boll_down'] = newdata1['MA_12'] - 2*Series(MD)\n",
    "# 计算%b指标\n",
    "    newdata1['b_index'] = (newdata1['Latestprice'] - newdata1['boll_down'])/(newdata1['boll_up'] - newdata1['boll_down'])\n",
    "# 计算通道宽度\n",
    "    newdata1['channel_width'] = (newdata1['boll_up'] - newdata1['boll_down'])/newdata1['Latestprice']\n",
    "    \n",
    "    data=newdata1.ix[:,3:]\n",
    "    where_are_nan = np.isnan(data)\n",
    "    where_are_inf = np.isinf(data)\n",
    "    data[where_are_nan] = 0\n",
    "    data[where_are_inf] = 0\n",
    "    \n",
    "    data['MeanPrice'] = (data['Latestprice'] +data['MaxPrice']+data['MinPrice'])/3.0\n",
    "    ser_1 = data['MeanPrice'][1:]\n",
    "    ser_1.index = range(len(ser_1))\n",
    "    ser1 = data['MeanPrice'][:-1]\n",
    "    ser1.index = range(len(ser1))\n",
    "    data['RaiseDown_1'] = (ser_1 -ser1)/ser1\n",
    "# 同理，计算当前分钟的后2分钟、3分钟、4分钟、5分钟的涨跌幅(均相对于上一分钟)\n",
    "    ser_2 = data['MeanPrice'][2:]\n",
    "    ser_2.index = range(len(ser_2))\n",
    "    ser2 = ser_1[:-1]\n",
    "    ser2.index = range(len(ser2))\n",
    "    data['RaiseDown_2'] = (ser_2 -ser2)/ser2\n",
    "\n",
    "    ser_3 = data['MeanPrice'][3:]\n",
    "    ser_3.index = range(len(ser_3))\n",
    "    ser3 = ser_2[:-1]\n",
    "    ser3.index = range(len(ser3))\n",
    "    data['RaiseDown_3'] = (ser_3 -ser3)/ser3\n",
    "\n",
    "    ser_4 = data['MeanPrice'][4:]\n",
    "    ser_4.index = range(len(ser_4))\n",
    "    ser4 = ser_3[:-1]\n",
    "    ser4.index = range(len(ser4))\n",
    "    data['RaiseDown_4'] = (ser_4 -ser4)/ser4\n",
    "\n",
    "    ser_5 = data['MeanPrice'][5:]\n",
    "    ser_5.index = range(len(ser_5))\n",
    "    ser5 = ser_4[:-1]\n",
    "    ser5.index = range(len(ser5))\n",
    "    data['RaiseDown_5'] = (ser_5 -ser5)/ser5\n",
    "    \n",
    "    # 添加下一分钟的MA5、MA12、MA26、上轨线、下轨线指标\n",
    "    ser_6 = data['MA_5'][1:]\n",
    "    ser_6.index = range(len(ser_6))\n",
    "    data['MA_5_PRED'] = ser_6\n",
    "\n",
    "    ser_7 = data['MA_12'][1:]\n",
    "    ser_7.index = range(len(ser_7))\n",
    "    data['MA_12_PRED'] = ser_7\n",
    "\n",
    "    ser_8 = data['MA_26'][1:]\n",
    "    ser_8.index = range(len(ser_8))\n",
    "    data['MA_26_PRED'] = ser_8\n",
    "\n",
    "    ser_9 = data['boll_up'][1:]\n",
    "    ser_9.index = range(len(ser_9))\n",
    "    data['boll_up_PRED'] = ser_9\n",
    "\n",
    "    ser_10 = data['boll_down'][1:]\n",
    "    ser_10.index = range(len(ser_10))\n",
    "    data['boll_down_PRED'] = ser_10\n",
    "    \n",
    "    data = data.fillna(method = 'ffill')\n",
    "    data.index = range(len(data))\n",
    "    data.to_csv(new_path + file_list[j].split('.')[0] + '_addfeature' + '.csv',sep=',') # 存储为加标签的数据\n",
    "    \n",
    "#     newdata1.to_csv(new_path + file_list[i].split('.')[0]+ '_features' + '.csv',sep=',')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 制作数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(334420, 25, 34)\n"
     ]
    }
   ],
   "source": [
    "path = '/home/chocolate/LSTM-source/Main_product_added_features/'\n",
    "seq_new_25 = []\n",
    "file_list = os.listdir(path)\n",
    "for j in range(len(file_list)):\n",
    "    data = pd.read_csv(path + file_list[j], header=0, sep=',',encoding=\"gbk\")\n",
    "    data_new = data.drop(data.columns[35:],axis = 1)\n",
    "    data_new = data_new.ix[:, 1:]\n",
    "#   数据归一化\n",
    "    data_array_per = np.array(data_new)\n",
    "    min_max_scaler = preprocessing.MinMaxScaler()\n",
    "    data_array_per = min_max_scaler.fit_transform(data_array_per)\n",
    "#  组时间序列\n",
    "    for k in range(len(data_array_per)):\n",
    "        if k < len(data_array_per)-5  and k >= 25:\n",
    "            seq_new_25.append(data_array_per[k - 25:k])\n",
    "seq_new_25 = np.array(seq_new_25)\n",
    "print seq_new_25.shape\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 为数据集打标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "涨幅3： 数量： 4031 比例： 0.012053704922\n",
      "涨幅2： 数量： 9978 比例： 0.0298367322529\n",
      "涨幅1： 数量： 41399 比例： 0.123793433407\n",
      "平 稳： 数量： 224919 比例： 0.672564439926\n",
      "跌幅1： 数量： 39501 比例： 0.11811793553\n",
      "跌幅2： 数量： 10755 比例： 0.0321601578853\n",
      "跌幅3： 数量： 3837 比例： 0.0114735960768\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(334420, 7)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 打数据集标签,根据归一化后的涨跌情况打标签。\n",
    "#但是不同合约归一化后的涨跌幅分布仍然不同，所以必须按合约分别打标签。\n",
    "seq_label = []\n",
    "\n",
    "# print u'IF合约:'\n",
    "data_IF = pd.read_csv(path + 'IF_main_cleaned_addfeature.csv')\n",
    "data_per_new = data_IF[25:]\n",
    "data_per_new = data_per_new[:-5]   \n",
    " #   数据归一化\n",
    "data_array_per = np.array(data_per_new)\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "data_array_per = min_max_scaler.fit_transform(data_array_per)\n",
    "#     print data_array_per\n",
    "data_per = pd.DataFrame(data_array_per,columns = data_per_new.columns)\n",
    "# print data_per['RaiseDown_1'].describe()\n",
    "    \n",
    "for i in data_per['RaiseDown_1']:\n",
    "    if i > 0.74:\n",
    "        seq_label.append([1,0,0,0,0,0,0])   #大涨\n",
    "    elif i > 0.64:\n",
    "        seq_label.append([0,1,0,0,0,0,0])   #小涨\n",
    "    elif i> 0.52: \n",
    "        seq_label.append([0,0,1,0,0,0,0])   #平稳\n",
    "    elif i > 0.4:\n",
    "        seq_label.append([0,0,0,1,0,0,0])   #小跌\n",
    "    elif i > 0.3:\n",
    "        seq_label.append([0,0,0,0,1,0,0])   #大跌\n",
    "    elif i > 0.2:\n",
    "        seq_label.append([0,0,0,0,0,1,0])   #大跌\n",
    "    else:\n",
    "        seq_label.append([0,0,0,0,0,0,1])   #大跌\n",
    "\n",
    "# print u'涨幅3：',seq_label.count([1,0,0,0,0,0,0]),seq_label.count([1,0,0,0,0,0,0]) /146399.0\n",
    "# print u'涨幅2：',seq_label.count([0,1,0,0,0,0,0]),seq_label.count([0,1,0,0,0,0,0]) /146399.0\n",
    "# print u'涨幅1：',seq_label.count([0,0,1,0,0,0,0]),seq_label.count([0,0,1,0,0,0,0]) /146399.0\n",
    "# print u'平 稳：',seq_label.count([0,0,0,1,0,0,0]),seq_label.count([0,0,0,1,0,0,0]) /146399.0\n",
    "# print u'跌幅1：',seq_label.count([0,0,0,0,1,0,0]),seq_label.count([0,0,0,0,1,0,0]) /146399.0\n",
    "# print u'跌幅2：',seq_label.count([0,0,0,0,0,1,0]),seq_label.count([0,0,0,0,0,1,0]) /146399.0\n",
    "# print u'跌幅3：',seq_label.count([0,0,0,0,0,0,1]),seq_label.count([0,0,0,0,0,0,1]) /146399.0\n",
    "\n",
    "\n",
    "# print u'IC合约:'\n",
    "data_IC = pd.read_csv(path + 'IC_main_cleaned_addfeature.csv')\n",
    "data_per_new = data_IC[25:]\n",
    "data_per_new = data_per_new[:-5]   \n",
    " #   数据归一化\n",
    "data_array_per = np.array(data_per_new)\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "data_array_per = min_max_scaler.fit_transform(data_array_per)\n",
    "#     print data_array_per\n",
    "data_per = pd.DataFrame(data_array_per,columns = data_per_new.columns)\n",
    "# print data_per['RaiseDown_1'].describe()\n",
    "    \n",
    "for i in data_per['RaiseDown_1']:\n",
    "    if i > 0.78:\n",
    "        seq_label.append([1,0,0,0,0,0,0])   #大涨\n",
    "    elif i > 0.67:\n",
    "        seq_label.append([0,1,0,0,0,0,0])   #小涨\n",
    "    elif i> 0.55: \n",
    "        seq_label.append([0,0,1,0,0,0,0])   #平稳\n",
    "    elif i > 0.33:\n",
    "        seq_label.append([0,0,0,1,0,0,0])   #小跌\n",
    "    elif i > 0.23:\n",
    "        seq_label.append([0,0,0,0,1,0,0])   #大跌\n",
    "    elif i > 0.14:\n",
    "        seq_label.append([0,0,0,0,0,1,0])   #大跌\n",
    "    else:\n",
    "        seq_label.append([0,0,0,0,0,0,1])   #大跌\n",
    "\n",
    "# print u'涨幅3：',seq_label.count([1,0,0,0,0,0,0]),seq_label.count([1,0,0,0,0,0,0])/92931.0\n",
    "# print u'涨幅2：',seq_label.count([0,1,0,0,0,0,0]),seq_label.count([0,1,0,0,0,0,0])/92931.0\n",
    "# print u'涨幅1：',seq_label.count([0,0,1,0,0,0,0]),seq_label.count([0,0,1,0,0,0,0])/92931.0\n",
    "# print u'平 稳：',seq_label.count([0,0,0,1,0,0,0]),seq_label.count([0,0,0,1,0,0,0])/92931.0\n",
    "# print u'跌幅1：',seq_label.count([0,0,0,0,1,0,0]),seq_label.count([0,0,0,0,1,0,0])/92931.0\n",
    "# print u'跌幅2：',seq_label.count([0,0,0,0,0,1,0]),seq_label.count([0,0,0,0,0,1,0])/92931.0\n",
    "# print u'跌幅3：',seq_label.count([0,0,0,0,0,0,1]),seq_label.count([0,0,0,0,0,0,1])/92931.0\n",
    "\n",
    "# print u'IH合约:'\n",
    "data_IH = pd.read_csv(path + 'IH_main_cleaned_addfeature.csv')\n",
    "data_per_new = data_IH[25:]\n",
    "data_per_new = data_per_new[:-5]   \n",
    " #   数据归一化\n",
    "data_array_per = np.array(data_per_new)\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "data_array_per = min_max_scaler.fit_transform(data_array_per)\n",
    "#     print data_array_per\n",
    "data_per = pd.DataFrame(data_array_per,columns = data_per_new.columns)\n",
    "# print data_per['RaiseDown_1'].describe()\n",
    "    \n",
    "for i in data_per['RaiseDown_1']:\n",
    "    if i > 0.53:\n",
    "        seq_label.append([1,0,0,0,0,0,0])   #大涨\n",
    "    elif i > 0.463:\n",
    "        seq_label.append([0,1,0,0,0,0,0])   #小涨\n",
    "    elif i> 0.429: \n",
    "        seq_label.append([0,0,1,0,0,0,0])   #平稳\n",
    "    elif i > 0.385:\n",
    "        seq_label.append([0,0,0,1,0,0,0])   #小跌\n",
    "    elif i > 0.355:\n",
    "        seq_label.append([0,0,0,0,1,0,0])   #大跌\n",
    "    elif i > 0.3:\n",
    "        seq_label.append([0,0,0,0,0,1,0])   #大跌\n",
    "    else:\n",
    "        seq_label.append([0,0,0,0,0,0,1])   #大跌\n",
    "\n",
    "# print u'涨幅3：',u'数量：',seq_label.count([1,0,0,0,0,0,0]),u'比例：',seq_label.count([1,0,0,0,0,0,0])/95090.0\n",
    "# print u'涨幅2：',u'数量：',seq_label.count([0,1,0,0,0,0,0]),u'比例：',seq_label.count([0,1,0,0,0,0,0])/95090.0\n",
    "# print u'涨幅1：',u'数量：',seq_label.count([0,0,1,0,0,0,0]),u'比例：',seq_label.count([0,0,1,0,0,0,0])/95090.0\n",
    "# print u'平 稳：',u'数量：',seq_label.count([0,0,0,1,0,0,0]),u'比例：',seq_label.count([0,0,0,1,0,0,0])/95090.0\n",
    "# print u'跌幅1：',u'数量：',seq_label.count([0,0,0,0,1,0,0]),u'比例：',seq_label.count([0,0,0,0,1,0,0])/95090.0\n",
    "# print u'跌幅2：',u'数量：',seq_label.count([0,0,0,0,0,1,0]),u'比例：',seq_label.count([0,0,0,0,0,1,0])/95090.0\n",
    "# print u'跌幅3：',u'数量：',seq_label.count([0,0,0,0,0,0,1]),u'比例：',seq_label.count([0,0,0,0,0,0,1])/95090.0\n",
    "\n",
    "print u'涨幅3：',u'数量：',seq_label.count([1,0,0,0,0,0,0]),u'比例：',seq_label.count([1,0,0,0,0,0,0])/334420.0\n",
    "print u'涨幅2：',u'数量：',seq_label.count([0,1,0,0,0,0,0]),u'比例：',seq_label.count([0,1,0,0,0,0,0])/334420.0\n",
    "print u'涨幅1：',u'数量：',seq_label.count([0,0,1,0,0,0,0]),u'比例：',seq_label.count([0,0,1,0,0,0,0])/334420.0\n",
    "print u'平 稳：',u'数量：',seq_label.count([0,0,0,1,0,0,0]),u'比例：',seq_label.count([0,0,0,1,0,0,0])/334420.0\n",
    "print u'跌幅1：',u'数量：',seq_label.count([0,0,0,0,1,0,0]),u'比例：',seq_label.count([0,0,0,0,1,0,0])/334420.0\n",
    "print u'跌幅2：',u'数量：',seq_label.count([0,0,0,0,0,1,0]),u'比例：',seq_label.count([0,0,0,0,0,1,0])/334420.0\n",
    "print u'跌幅3：',u'数量：',seq_label.count([0,0,0,0,0,0,1]),u'比例：',seq_label.count([0,0,0,0,0,0,1])/334420.0\n",
    "seq_label = np.array(seq_label)\n",
    "seq_label.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 将数据打乱，提高预测准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(334420, 25, 34) (334420, 7)\n"
     ]
    }
   ],
   "source": [
    "from random import shuffle\n",
    "zip_seq = zip(seq_new_25,seq_label)\n",
    "shuffle(zip_seq)\n",
    "seq_new_25,seq_label = zip(*zip_seq)\n",
    "seq_new_25 = np.array(seq_new_25) \n",
    "seq_label = np.array(seq_label)\n",
    "print seq_new_25.shape, seq_label.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 划分训练集和测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((234000, 25, 34), (100420, 7))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = seq_new_25[:234000]\n",
    "train_label = seq_label[:234000]\n",
    "\n",
    "test_data = seq_new_25[234000:]\n",
    "test_label = seq_label[234000:]\n",
    "train_data.shape, test_label.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 保存训练集和测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_new = '/home/chocolate/LSTM-source/train_test_data/'\n",
    "f1 = file(path_new + 'train_data.npy','wb')\n",
    "np.save(f1,train_data)\n",
    "f1.close()\n",
    "f2 = file(path_new + 'test_data.npy','wb')\n",
    "np.save(f2,test_data)\n",
    "f2.close()\n",
    "f3 = file(path_new + 'train_label.npy','wb')\n",
    "np.save(f3,train_label)\n",
    "f3.close()\n",
    "f4 = file(path_new + 'test_label.npy','wb')\n",
    "np.save(f4,test_label)\n",
    "f4.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 每次执行训练从此处开始"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from pandas import Series,DataFrame\n",
    "import numpy as np\n",
    "import os\n",
    "import datetime\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib import learn\n",
    "from tensorflow.python.ops import rnn, rnn_cell\n",
    "import matplotlib.pyplot as plt\n",
    "import cPickle\n",
    "import gzip\n",
    "from sklearn import preprocessing\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取训练集和测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_new = '/home/chocolate/LSTM-source/train_test_data/'\n",
    "# 读取训练集和测试集\n",
    "f1 = file(path_new + 'train_data.npy','rb')\n",
    "train_data = np.load(f1)\n",
    "f2 = file(path_new + 'test_data.npy','rb')\n",
    "test_data = np.load(f2)\n",
    "f3 = file(path_new + 'train_label.npy','rb')\n",
    "train_label = np.load(f3)\n",
    "f4 = file(path_new + 'test_label.npy','rb')\n",
    "test_label = np.load(f4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 构建LSTM神经网络，执行训练并保存模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "learning_rate = 0.001 # 学习速率\n",
    "training_iters = 3000000  # 训练总步长\n",
    "batch_size = 1500 # 每小段步长\n",
    "display_step = 100  # 每隔100小段显示输出\n",
    "\n",
    "# 参数设置\n",
    "n_input = 34 # 特征数量\n",
    "n_steps = 25 # 时间序列长度\n",
    "n_hidden = 500 # 隐藏层神经元个数\n",
    "n_classes = 7 # 分类数量\n",
    "\n",
    "tf.reset_default_graph()  # 重置流图\n",
    "# 设置输入输出格式大小\n",
    "xtr = tf.placeholder(\"float\", [None, n_steps, n_input])\n",
    "ytr = tf.placeholder(\"float\", [None,n_classes])\n",
    "\n",
    "# 定义权重和偏移量\n",
    "weights = {\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden, n_classes]))\n",
    "}\n",
    "biases = {\n",
    "    'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "}\n",
    "\n",
    "# def pred():\n",
    "    \n",
    "def RNN(x, weights, biases):\n",
    "\n",
    "    # 更改数据集的格式，使其满足RNN网络的输入需求\n",
    "    # 现在输入格式: (batch_size, n_steps, n_input)\n",
    "    # 需要的格式: 'n_steps' 个tensor，每个的格式为 (batch_size, n_input)\n",
    "\n",
    "    # 变换batch_size 和 n_steps的位置\n",
    "    x = tf.transpose(x, [1, 0, 2])\n",
    "    # 重新调整格式 (n_steps*batch_size, n_input)\n",
    "    x = tf.reshape(x, [-1, n_input])\n",
    "    # 分割成一系列的 'n_steps' 个tensors，每个的格式为 (batch_size, n_input)\n",
    "    x = tf.split(0, n_steps, x)\n",
    "\n",
    "    # 定义LSTM神经网络结构\n",
    "    lstm_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)\n",
    "    # 获取输出和状态\n",
    "    outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32)\n",
    "    # 返回输出\n",
    "    return tf.matmul(outputs[-1], weights['out']) + biases['out']\n",
    "# 获取预测值\n",
    "pred = RNN(xtr, weights, biases)\n",
    "# train_label = tf.split(0, batch_size, train_label)\n",
    "# 取下一小段的输入\n",
    "def next_batch_tr(batch_size):\n",
    "    step = random.randint(1, len(train_data)/batch_size)\n",
    "    next_batch_data = train_data[(step-1)*batch_size:step*batch_size]\n",
    "    next_batch_label = train_label[(step-1)*batch_size:step*batch_size]   \n",
    "    return next_batch_data, next_batch_label\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 150000, Minibatch Loss= 0.993739, Training Accuracy= 0.65867\n",
      "Iter 300000, Minibatch Loss= 0.960534, Training Accuracy= 0.66200\n",
      "Iter 450000, Minibatch Loss= 0.937401, Training Accuracy= 0.67667\n",
      "Iter 600000, Minibatch Loss= 0.990155, Training Accuracy= 0.63933\n",
      "Iter 750000, Minibatch Loss= 0.915748, Training Accuracy= 0.68200\n",
      "Iter 900000, Minibatch Loss= 0.932645, Training Accuracy= 0.65533\n",
      "Iter 1050000, Minibatch Loss= 0.985310, Training Accuracy= 0.67400\n",
      "Iter 1200000, Minibatch Loss= 0.916208, Training Accuracy= 0.67067\n",
      "Iter 1350000, Minibatch Loss= 0.951309, Training Accuracy= 0.66333\n",
      "Iter 1500000, Minibatch Loss= 0.899215, Training Accuracy= 0.68000\n",
      "Iter 1650000, Minibatch Loss= 0.914494, Training Accuracy= 0.66467\n",
      "Iter 1800000, Minibatch Loss= 0.897618, Training Accuracy= 0.66533\n",
      "Iter 1950000, Minibatch Loss= 0.931017, Training Accuracy= 0.64600\n",
      "Iter 2100000, Minibatch Loss= 0.863796, Training Accuracy= 0.68133\n",
      "Iter 2250000, Minibatch Loss= 0.880785, Training Accuracy= 0.68933\n",
      "Iter 2400000, Minibatch Loss= 0.892960, Training Accuracy= 0.67400\n",
      "Iter 2550000, Minibatch Loss= 0.900846, Training Accuracy= 0.67533\n",
      "Iter 2700000, Minibatch Loss= 0.923086, Training Accuracy= 0.65600\n",
      "Iter 2850000, Minibatch Loss= 0.892495, Training Accuracy= 0.67800\n",
      "Optimization Finished!\n",
      "Model saved.\n"
     ]
    }
   ],
   "source": [
    "# 定义代价函数和最优化方法\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, ytr))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "# 评估模型准确率\n",
    "correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(ytr, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "# 变量初始化\n",
    "init = tf.initialize_all_variables()\n",
    "part_saver = tf.train.Saver()\n",
    "# 创建会话\n",
    "sess = tf.InteractiveSession()\n",
    "sess.run(init)\n",
    "step = 1\n",
    "    # 一直迭代，直到最大步长\n",
    "while step * batch_size < training_iters:\n",
    "    batch_x, batch_y = next_batch_tr(batch_size)\n",
    "    batch_x = batch_x.reshape((-1, n_steps, n_input))\n",
    "        # 利用最优化方法调整权值\n",
    "    sess.run(optimizer, feed_dict={xtr: batch_x, ytr: batch_y})\n",
    "    if step % display_step == 0:\n",
    "            # 计算准确率和损失函数\n",
    "        acc = sess.run(accuracy, feed_dict={xtr: batch_x, ytr: batch_y})\n",
    "        loss = sess.run(cost, feed_dict={xtr: batch_x, ytr: batch_y})\n",
    "        # 每隔一定步长显示训练准确率和损失率\n",
    "        print(\"Iter \" + str(step * batch_size) + \", Minibatch Loss= \" +\n",
    "                \"{:.6f}\".format(loss) + \", Training Accuracy= \" +\n",
    "                \"{:.5f}\".format(acc))\n",
    "    step += 1\n",
    "print(\"Optimization Finished!\")\n",
    "\n",
    "saver_path = part_saver.save(sess, \"/home/chocolate/Model_LSTM-Future_20161123/models/predict_1min.ckpt\")\n",
    "print \"Model saved.\" \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加载模型，并对测试集进行准确率测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tf' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-edb4ad531db8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msaver\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSaver\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0msaver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"/home/chocolate/Model_LSTM-Future_20161123/models/predict_1min.ckpt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0;34m\"Model restored.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mtest_data1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_steps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_input\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# test_label1 = test_label[:test_len]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'tf' is not defined"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver()\n",
    "saver.restore(sess, \"/home/chocolate/Model_LSTM-Future_20161123/models/predict_1min.ckpt\")\n",
    "print \"Model restored.\"\n",
    "test_data1 = test_data.reshape((-1, n_steps, n_input))\n",
    "# test_label1 = test_label[:test_len]\n",
    "print(\"Testing Accuracy:\",\n",
    "        sess.run(accuracy, feed_dict={xtr: test_data1, ytr: test_label}))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}