6.2_Model_XGBoost_Lags_promo_open.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "#from sklearn import cross_validation\n",
    "from sklearn.model_selection import cross_validate\n",
    "import xgboost as xgb\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import validation_curve\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.style.use('ggplot')\n",
    "plt.rcParams['figure.figsize'] = (15, 6)\n",
    "plt.rcParams['figure.figsize'] = [20.0,8.0]\n",
    "plt.rcParams['figure.dpi']=80\n",
    "\n",
    "import modify_dataset as md"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Common error function to minimize: Root Mean Square Percentage Error\n",
    "#We will have to integrate our loss function with xgboost\n",
    "\n",
    "def ToWeight(y):\n",
    "    w = np.zeros(y.shape, dtype=float)\n",
    "    ind = y != 0\n",
    "    w[ind] = 1./(y[ind]**2)\n",
    "    return w\n",
    "\n",
    "\n",
    "def rmspe(yhat, y):\n",
    "    w = ToWeight(y)\n",
    "    rmspe = np.sqrt(np.mean( w * (y - yhat)**2 ))\n",
    "    return rmspe\n",
    "\n",
    "\n",
    "def rmspe_xg(yhat, y):\n",
    "    # y = y.values\n",
    "    y = y.get_label()\n",
    "    y = np.exp(y) - 1\n",
    "    yhat = np.exp(yhat) - 1\n",
    "    w = ToWeight(y)\n",
    "    rmspe = np.sqrt(np.mean(w * (y - yhat)**2))\n",
    "    return \"rmspe\", rmspe\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "_cell_guid": "da09e07b-fdfc-30b1-dfff-ad7dc625d15a"
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"dataset/preprocessed_PCA_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(['NumberOfCustomers'], axis=1)\n",
    "#df = df.drop(['Region','Events'], axis=1)\n",
    "# I could leave events"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.IsOpen == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for now let's drop unusable categorical vars\n",
    "df = df.drop(['StoreType','AssortmentType','Events'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#only for PCA\n",
    "df = df.drop(['Unnamed: 0'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "_cell_guid": "6872b7e5-8b37-6aa9-033c-a03dd5647974"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>StoreID</th>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <td>01/03/2016</td>\n",
       "      <td>02/03/2016</td>\n",
       "      <td>04/03/2016</td>\n",
       "      <td>05/03/2016</td>\n",
       "      <td>07/03/2016</td>\n",
       "      <td>08/03/2016</td>\n",
       "      <td>09/03/2016</td>\n",
       "      <td>10/03/2016</td>\n",
       "      <td>11/03/2016</td>\n",
       "      <td>12/03/2016</td>\n",
       "      <td>14/03/2016</td>\n",
       "      <td>15/03/2016</td>\n",
       "      <td>16/03/2016</td>\n",
       "      <td>17/03/2016</td>\n",
       "      <td>18/03/2016</td>\n",
       "      <td>19/03/2016</td>\n",
       "      <td>21/03/2016</td>\n",
       "      <td>22/03/2016</td>\n",
       "      <td>23/03/2016</td>\n",
       "      <td>24/03/2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IsHoliday</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IsOpen</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HasPromotions</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NearestCompetitor</th>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Region</th>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NumberOfSales</th>\n",
       "      <td>5676</td>\n",
       "      <td>8111</td>\n",
       "      <td>8300</td>\n",
       "      <td>7154</td>\n",
       "      <td>10110</td>\n",
       "      <td>9019</td>\n",
       "      <td>8804</td>\n",
       "      <td>7823</td>\n",
       "      <td>7989</td>\n",
       "      <td>5895</td>\n",
       "      <td>6348</td>\n",
       "      <td>4738</td>\n",
       "      <td>6014</td>\n",
       "      <td>6087</td>\n",
       "      <td>6962</td>\n",
       "      <td>6618</td>\n",
       "      <td>9183</td>\n",
       "      <td>8155</td>\n",
       "      <td>7109</td>\n",
       "      <td>8041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Region_AreaKM2</th>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Region_GDP</th>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Region_PopulationK</th>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CloudCover</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Max_VisibilityKm</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Min_VisibilitykM</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WindDirDegrees</th>\n",
       "      <td>23</td>\n",
       "      <td>56</td>\n",
       "      <td>22</td>\n",
       "      <td>108</td>\n",
       "      <td>68</td>\n",
       "      <td>62</td>\n",
       "      <td>53</td>\n",
       "      <td>54</td>\n",
       "      <td>30</td>\n",
       "      <td>202</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "      <td>272</td>\n",
       "      <td>218</td>\n",
       "      <td>138</td>\n",
       "      <td>190</td>\n",
       "      <td>211</td>\n",
       "      <td>191</td>\n",
       "      <td>331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_Day</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>19</td>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_DayOfYear</th>\n",
       "      <td>61</td>\n",
       "      <td>62</td>\n",
       "      <td>64</td>\n",
       "      <td>65</td>\n",
       "      <td>67</td>\n",
       "      <td>68</td>\n",
       "      <td>69</td>\n",
       "      <td>70</td>\n",
       "      <td>71</td>\n",
       "      <td>72</td>\n",
       "      <td>74</td>\n",
       "      <td>75</td>\n",
       "      <td>76</td>\n",
       "      <td>77</td>\n",
       "      <td>78</td>\n",
       "      <td>79</td>\n",
       "      <td>81</td>\n",
       "      <td>82</td>\n",
       "      <td>83</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_Month</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_Year</th>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_DayOfweek</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StoreType_SuperMarket</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StoreType_HyperMarket</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StoreType_StandardMarket</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StoreType_ShoppingCenter</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AssortmentType_General</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AssortmentType_WithNFDept</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AssortmentType_WithFishDept</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MeanStoreSales</th>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "      <td>6296.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MeanRegionSales</th>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "      <td>3906.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_DayOfWeek_cos</th>\n",
       "      <td>0.62349</td>\n",
       "      <td>-0.222521</td>\n",
       "      <td>-0.900969</td>\n",
       "      <td>-0.222521</td>\n",
       "      <td>1</td>\n",
       "      <td>0.62349</td>\n",
       "      <td>-0.222521</td>\n",
       "      <td>-0.900969</td>\n",
       "      <td>-0.900969</td>\n",
       "      <td>-0.222521</td>\n",
       "      <td>1</td>\n",
       "      <td>0.62349</td>\n",
       "      <td>-0.222521</td>\n",
       "      <td>-0.900969</td>\n",
       "      <td>-0.900969</td>\n",
       "      <td>-0.222521</td>\n",
       "      <td>1</td>\n",
       "      <td>0.62349</td>\n",
       "      <td>-0.222521</td>\n",
       "      <td>-0.900969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_DayOfWeek_sin</th>\n",
       "      <td>0.781831</td>\n",
       "      <td>0.974928</td>\n",
       "      <td>-0.433884</td>\n",
       "      <td>-0.974928</td>\n",
       "      <td>0</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>0.974928</td>\n",
       "      <td>0.433884</td>\n",
       "      <td>-0.433884</td>\n",
       "      <td>-0.974928</td>\n",
       "      <td>0</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>0.974928</td>\n",
       "      <td>0.433884</td>\n",
       "      <td>-0.433884</td>\n",
       "      <td>-0.974928</td>\n",
       "      <td>0</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>0.974928</td>\n",
       "      <td>0.433884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p1</th>\n",
       "      <td>0.0038253</td>\n",
       "      <td>0.00380447</td>\n",
       "      <td>0.00215876</td>\n",
       "      <td>0.00293107</td>\n",
       "      <td>0.00261322</td>\n",
       "      <td>0.00236662</td>\n",
       "      <td>0.00185775</td>\n",
       "      <td>0.000473164</td>\n",
       "      <td>-0.000416869</td>\n",
       "      <td>-0.00021493</td>\n",
       "      <td>0.00454891</td>\n",
       "      <td>0.00485476</td>\n",
       "      <td>0.00590618</td>\n",
       "      <td>0.00619993</td>\n",
       "      <td>0.00620682</td>\n",
       "      <td>0.00389524</td>\n",
       "      <td>0.00180157</td>\n",
       "      <td>0.00183232</td>\n",
       "      <td>0.0010165</td>\n",
       "      <td>0.00392148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p2</th>\n",
       "      <td>0.0014332</td>\n",
       "      <td>0.000903979</td>\n",
       "      <td>-0.000101837</td>\n",
       "      <td>-0.000539355</td>\n",
       "      <td>-0.000531053</td>\n",
       "      <td>-0.000662307</td>\n",
       "      <td>-0.00067526</td>\n",
       "      <td>-0.000457064</td>\n",
       "      <td>-0.000287349</td>\n",
       "      <td>-0.000300094</td>\n",
       "      <td>0.00152879</td>\n",
       "      <td>0.00280597</td>\n",
       "      <td>0.00112061</td>\n",
       "      <td>0.0012244</td>\n",
       "      <td>0.00106947</td>\n",
       "      <td>-0.00052353</td>\n",
       "      <td>-0.00104194</td>\n",
       "      <td>-0.000465838</td>\n",
       "      <td>0.00139047</td>\n",
       "      <td>0.00132576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p3</th>\n",
       "      <td>-0.00076347</td>\n",
       "      <td>-0.000480038</td>\n",
       "      <td>-0.000370168</td>\n",
       "      <td>-0.000204276</td>\n",
       "      <td>-0.000171205</td>\n",
       "      <td>-8.46579e-05</td>\n",
       "      <td>-2.25402e-05</td>\n",
       "      <td>-1.45983e-05</td>\n",
       "      <td>-1.82241e-05</td>\n",
       "      <td>-1.93295e-05</td>\n",
       "      <td>-0.000878183</td>\n",
       "      <td>-0.00144622</td>\n",
       "      <td>-0.00079585</td>\n",
       "      <td>-0.000888527</td>\n",
       "      <td>-0.000800649</td>\n",
       "      <td>-0.000309571</td>\n",
       "      <td>0.000374328</td>\n",
       "      <td>7.00345e-05</td>\n",
       "      <td>-0.00109489</td>\n",
       "      <td>-0.000718271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p4</th>\n",
       "      <td>-0.000912617</td>\n",
       "      <td>-0.000547427</td>\n",
       "      <td>-0.000560225</td>\n",
       "      <td>-0.000257912</td>\n",
       "      <td>-0.00031086</td>\n",
       "      <td>-0.000191952</td>\n",
       "      <td>-0.000306754</td>\n",
       "      <td>-0.000647226</td>\n",
       "      <td>-0.000792144</td>\n",
       "      <td>-0.000840343</td>\n",
       "      <td>-0.000915415</td>\n",
       "      <td>0.00177991</td>\n",
       "      <td>-0.000396647</td>\n",
       "      <td>-6.89417e-06</td>\n",
       "      <td>-0.000271905</td>\n",
       "      <td>-0.000147691</td>\n",
       "      <td>0.00211691</td>\n",
       "      <td>0.00174133</td>\n",
       "      <td>-0.000321022</td>\n",
       "      <td>-0.000839272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p5</th>\n",
       "      <td>-0.00222642</td>\n",
       "      <td>-0.00247007</td>\n",
       "      <td>0.000554448</td>\n",
       "      <td>0.000438362</td>\n",
       "      <td>0.000403522</td>\n",
       "      <td>0.000326159</td>\n",
       "      <td>0.000255091</td>\n",
       "      <td>0.000167626</td>\n",
       "      <td>0.00023278</td>\n",
       "      <td>0.000170166</td>\n",
       "      <td>-0.00210205</td>\n",
       "      <td>-0.000269195</td>\n",
       "      <td>-0.00211805</td>\n",
       "      <td>-0.0019094</td>\n",
       "      <td>-0.00210155</td>\n",
       "      <td>0.000539229</td>\n",
       "      <td>0.000427248</td>\n",
       "      <td>0.00068135</td>\n",
       "      <td>0.00196911</td>\n",
       "      <td>-0.00228311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p6</th>\n",
       "      <td>0.00224159</td>\n",
       "      <td>0.00230143</td>\n",
       "      <td>-0.00110596</td>\n",
       "      <td>-0.00145466</td>\n",
       "      <td>-0.0013466</td>\n",
       "      <td>-0.00120862</td>\n",
       "      <td>-0.00103422</td>\n",
       "      <td>-0.000539841</td>\n",
       "      <td>-0.000105736</td>\n",
       "      <td>-0.0002203</td>\n",
       "      <td>0.00188936</td>\n",
       "      <td>0.00290054</td>\n",
       "      <td>0.00141545</td>\n",
       "      <td>0.00145775</td>\n",
       "      <td>0.00128724</td>\n",
       "      <td>-0.00190977</td>\n",
       "      <td>-0.000453241</td>\n",
       "      <td>-0.000564274</td>\n",
       "      <td>-9.33638e-05</td>\n",
       "      <td>0.00213789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p7</th>\n",
       "      <td>-0.000719875</td>\n",
       "      <td>0.00107936</td>\n",
       "      <td>-0.000526003</td>\n",
       "      <td>0.00139192</td>\n",
       "      <td>0.00129003</td>\n",
       "      <td>0.00148701</td>\n",
       "      <td>0.00139519</td>\n",
       "      <td>-2.56231e-05</td>\n",
       "      <td>-0.000303005</td>\n",
       "      <td>-0.000694627</td>\n",
       "      <td>-0.000661546</td>\n",
       "      <td>-0.000843696</td>\n",
       "      <td>0.00123804</td>\n",
       "      <td>0.00139421</td>\n",
       "      <td>0.00146344</td>\n",
       "      <td>0.00175458</td>\n",
       "      <td>0.000625665</td>\n",
       "      <td>-0.00123484</td>\n",
       "      <td>9.40244e-06</td>\n",
       "      <td>-0.000332149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p8</th>\n",
       "      <td>-0.00116095</td>\n",
       "      <td>-0.00118984</td>\n",
       "      <td>-0.00130027</td>\n",
       "      <td>-0.00117349</td>\n",
       "      <td>-0.00112774</td>\n",
       "      <td>-0.00124582</td>\n",
       "      <td>-0.00119338</td>\n",
       "      <td>-0.00132962</td>\n",
       "      <td>-0.00128861</td>\n",
       "      <td>-0.00124548</td>\n",
       "      <td>-0.00105218</td>\n",
       "      <td>-0.000794234</td>\n",
       "      <td>-0.00107538</td>\n",
       "      <td>-0.00124269</td>\n",
       "      <td>-0.00111452</td>\n",
       "      <td>-0.00113884</td>\n",
       "      <td>-0.00102667</td>\n",
       "      <td>-0.000933344</td>\n",
       "      <td>-0.00122761</td>\n",
       "      <td>-0.00117337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p9</th>\n",
       "      <td>0.000247264</td>\n",
       "      <td>4.99143e-05</td>\n",
       "      <td>-0.000166103</td>\n",
       "      <td>0.000203075</td>\n",
       "      <td>0.00037153</td>\n",
       "      <td>-6.42772e-05</td>\n",
       "      <td>0.000124802</td>\n",
       "      <td>-0.000294708</td>\n",
       "      <td>-0.000151588</td>\n",
       "      <td>2.26767e-05</td>\n",
       "      <td>0.000650834</td>\n",
       "      <td>0.00161685</td>\n",
       "      <td>0.000479975</td>\n",
       "      <td>-0.000106806</td>\n",
       "      <td>0.000340916</td>\n",
       "      <td>0.000335526</td>\n",
       "      <td>0.000759144</td>\n",
       "      <td>0.00120153</td>\n",
       "      <td>0.000106001</td>\n",
       "      <td>0.000198016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p10</th>\n",
       "      <td>-0.00013257</td>\n",
       "      <td>0.000141136</td>\n",
       "      <td>-0.000514758</td>\n",
       "      <td>-0.000551018</td>\n",
       "      <td>-0.00035296</td>\n",
       "      <td>-0.000653497</td>\n",
       "      <td>-0.000468897</td>\n",
       "      <td>-0.00137889</td>\n",
       "      <td>-0.000605416</td>\n",
       "      <td>-0.000104071</td>\n",
       "      <td>-0.000373564</td>\n",
       "      <td>2.09806e-05</td>\n",
       "      <td>-0.000359549</td>\n",
       "      <td>-0.000654383</td>\n",
       "      <td>-0.000817936</td>\n",
       "      <td>-0.0013901</td>\n",
       "      <td>-0.000305311</td>\n",
       "      <td>-0.000471439</td>\n",
       "      <td>-0.000386896</td>\n",
       "      <td>-0.000910045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p11</th>\n",
       "      <td>-0.000344916</td>\n",
       "      <td>-0.000293834</td>\n",
       "      <td>-0.000337692</td>\n",
       "      <td>-0.000326059</td>\n",
       "      <td>5.03901e-05</td>\n",
       "      <td>-4.39029e-05</td>\n",
       "      <td>7.81451e-05</td>\n",
       "      <td>-0.000204166</td>\n",
       "      <td>-0.000212506</td>\n",
       "      <td>0.000304696</td>\n",
       "      <td>-0.000103976</td>\n",
       "      <td>-0.000567888</td>\n",
       "      <td>-0.000204156</td>\n",
       "      <td>0.000866687</td>\n",
       "      <td>-1.55093e-05</td>\n",
       "      <td>-0.000301076</td>\n",
       "      <td>-0.000162035</td>\n",
       "      <td>0.000449306</td>\n",
       "      <td>0.000579355</td>\n",
       "      <td>-0.000281794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p12</th>\n",
       "      <td>-4.3497e-05</td>\n",
       "      <td>0.00016746</td>\n",
       "      <td>0.000414398</td>\n",
       "      <td>0.00025032</td>\n",
       "      <td>0.000291776</td>\n",
       "      <td>7.02017e-06</td>\n",
       "      <td>-0.00010592</td>\n",
       "      <td>-4.79889e-05</td>\n",
       "      <td>-0.000167507</td>\n",
       "      <td>0.000120549</td>\n",
       "      <td>-8.6487e-05</td>\n",
       "      <td>-0.000151292</td>\n",
       "      <td>0.00014157</td>\n",
       "      <td>-9.86859e-05</td>\n",
       "      <td>6.9519e-05</td>\n",
       "      <td>0.000542752</td>\n",
       "      <td>-0.000404629</td>\n",
       "      <td>0.000203928</td>\n",
       "      <td>0.000234422</td>\n",
       "      <td>0.000193493</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      0            1            2   \\\n",
       "StoreID                             1000         1000         1000   \n",
       "Date                          01/03/2016   02/03/2016   04/03/2016   \n",
       "IsHoliday                              0            0            0   \n",
       "IsOpen                                 1            1            1   \n",
       "HasPromotions                          0            0            0   \n",
       "NearestCompetitor                    326          326          326   \n",
       "Region                                 7            7            7   \n",
       "NumberOfSales                       5676         8111         8300   \n",
       "Region_AreaKM2                      9643         9643         9643   \n",
       "Region_GDP                         17130        17130        17130   \n",
       "Region_PopulationK                  2770         2770         2770   \n",
       "CloudCover                             8            8            8   \n",
       "Max_VisibilityKm                       0            1            2   \n",
       "Min_VisibilitykM                       0            1            2   \n",
       "WindDirDegrees                        23           56           22   \n",
       "D_Day                                  1            2            4   \n",
       "D_DayOfYear                           61           62           64   \n",
       "D_Month                                3            3            3   \n",
       "D_Year                              2016         2016         2016   \n",
       "D_DayOfweek                            1            2            4   \n",
       "StoreType_SuperMarket                  0            0            0   \n",
       "StoreType_HyperMarket                  1            1            1   \n",
       "StoreType_StandardMarket               0            0            0   \n",
       "StoreType_ShoppingCenter               0            0            0   \n",
       "AssortmentType_General                 1            1            1   \n",
       "AssortmentType_WithNFDept              0            0            0   \n",
       "AssortmentType_WithFishDept            0            0            0   \n",
       "MeanStoreSales                   6296.18      6296.18      6296.18   \n",
       "MeanRegionSales                  3906.26      3906.26      3906.26   \n",
       "D_DayOfWeek_cos                  0.62349    -0.222521    -0.900969   \n",
       "D_DayOfWeek_sin                 0.781831     0.974928    -0.433884   \n",
       "p1                             0.0038253   0.00380447   0.00215876   \n",
       "p2                             0.0014332  0.000903979 -0.000101837   \n",
       "p3                           -0.00076347 -0.000480038 -0.000370168   \n",
       "p4                          -0.000912617 -0.000547427 -0.000560225   \n",
       "p5                           -0.00222642  -0.00247007  0.000554448   \n",
       "p6                            0.00224159   0.00230143  -0.00110596   \n",
       "p7                          -0.000719875   0.00107936 -0.000526003   \n",
       "p8                           -0.00116095  -0.00118984  -0.00130027   \n",
       "p9                           0.000247264  4.99143e-05 -0.000166103   \n",
       "p10                          -0.00013257  0.000141136 -0.000514758   \n",
       "p11                         -0.000344916 -0.000293834 -0.000337692   \n",
       "p12                          -4.3497e-05   0.00016746  0.000414398   \n",
       "\n",
       "                                      3            5            6   \\\n",
       "StoreID                             1000         1000         1000   \n",
       "Date                          05/03/2016   07/03/2016   08/03/2016   \n",
       "IsHoliday                              0            0            0   \n",
       "IsOpen                                 1            1            1   \n",
       "HasPromotions                          0            1            1   \n",
       "NearestCompetitor                    326          326          326   \n",
       "Region                                 7            7            7   \n",
       "NumberOfSales                       7154        10110         9019   \n",
       "Region_AreaKM2                      9643         9643         9643   \n",
       "Region_GDP                         17130        17130        17130   \n",
       "Region_PopulationK                  2770         2770         2770   \n",
       "CloudCover                             6            0            0   \n",
       "Max_VisibilityKm                       3            5            6   \n",
       "Min_VisibilitykM                       3            5            6   \n",
       "WindDirDegrees                       108           68           62   \n",
       "D_Day                                  5            7            8   \n",
       "D_DayOfYear                           65           67           68   \n",
       "D_Month                                3            3            3   \n",
       "D_Year                              2016         2016         2016   \n",
       "D_DayOfweek                            5            0            1   \n",
       "StoreType_SuperMarket                  0            0            0   \n",
       "StoreType_HyperMarket                  1            1            1   \n",
       "StoreType_StandardMarket               0            0            0   \n",
       "StoreType_ShoppingCenter               0            0            0   \n",
       "AssortmentType_General                 1            1            1   \n",
       "AssortmentType_WithNFDept              0            0            0   \n",
       "AssortmentType_WithFishDept            0            0            0   \n",
       "MeanStoreSales                   6296.18      6296.18      6296.18   \n",
       "MeanRegionSales                  3906.26      3906.26      3906.26   \n",
       "D_DayOfWeek_cos                -0.222521            1      0.62349   \n",
       "D_DayOfWeek_sin                -0.974928            0     0.781831   \n",
       "p1                            0.00293107   0.00261322   0.00236662   \n",
       "p2                          -0.000539355 -0.000531053 -0.000662307   \n",
       "p3                          -0.000204276 -0.000171205 -8.46579e-05   \n",
       "p4                          -0.000257912  -0.00031086 -0.000191952   \n",
       "p5                           0.000438362  0.000403522  0.000326159   \n",
       "p6                           -0.00145466   -0.0013466  -0.00120862   \n",
       "p7                            0.00139192   0.00129003   0.00148701   \n",
       "p8                           -0.00117349  -0.00112774  -0.00124582   \n",
       "p9                           0.000203075   0.00037153 -6.42772e-05   \n",
       "p10                         -0.000551018  -0.00035296 -0.000653497   \n",
       "p11                         -0.000326059  5.03901e-05 -4.39029e-05   \n",
       "p12                           0.00025032  0.000291776  7.02017e-06   \n",
       "\n",
       "                                      7            8            9   \\\n",
       "StoreID                             1000         1000         1000   \n",
       "Date                          09/03/2016   10/03/2016   11/03/2016   \n",
       "IsHoliday                              0            0            0   \n",
       "IsOpen                                 1            1            1   \n",
       "HasPromotions                          1            1            1   \n",
       "NearestCompetitor                    326          326          326   \n",
       "Region                                 7            7            7   \n",
       "NumberOfSales                       8804         7823         7989   \n",
       "Region_AreaKM2                      9643         9643         9643   \n",
       "Region_GDP                         17130        17130        17130   \n",
       "Region_PopulationK                  2770         2770         2770   \n",
       "CloudCover                             0            7            7   \n",
       "Max_VisibilityKm                       7            8            9   \n",
       "Min_VisibilitykM                       7            8            9   \n",
       "WindDirDegrees                        53           54           30   \n",
       "D_Day                                  9           10           11   \n",
       "D_DayOfYear                           69           70           71   \n",
       "D_Month                                3            3            3   \n",
       "D_Year                              2016         2016         2016   \n",
       "D_DayOfweek                            2            3            4   \n",
       "StoreType_SuperMarket                  0            0            0   \n",
       "StoreType_HyperMarket                  1            1            1   \n",
       "StoreType_StandardMarket               0            0            0   \n",
       "StoreType_ShoppingCenter               0            0            0   \n",
       "AssortmentType_General                 1            1            1   \n",
       "AssortmentType_WithNFDept              0            0            0   \n",
       "AssortmentType_WithFishDept            0            0            0   \n",
       "MeanStoreSales                   6296.18      6296.18      6296.18   \n",
       "MeanRegionSales                  3906.26      3906.26      3906.26   \n",
       "D_DayOfWeek_cos                -0.222521    -0.900969    -0.900969   \n",
       "D_DayOfWeek_sin                 0.974928     0.433884    -0.433884   \n",
       "p1                            0.00185775  0.000473164 -0.000416869   \n",
       "p2                           -0.00067526 -0.000457064 -0.000287349   \n",
       "p3                          -2.25402e-05 -1.45983e-05 -1.82241e-05   \n",
       "p4                          -0.000306754 -0.000647226 -0.000792144   \n",
       "p5                           0.000255091  0.000167626   0.00023278   \n",
       "p6                           -0.00103422 -0.000539841 -0.000105736   \n",
       "p7                            0.00139519 -2.56231e-05 -0.000303005   \n",
       "p8                           -0.00119338  -0.00132962  -0.00128861   \n",
       "p9                           0.000124802 -0.000294708 -0.000151588   \n",
       "p10                         -0.000468897  -0.00137889 -0.000605416   \n",
       "p11                          7.81451e-05 -0.000204166 -0.000212506   \n",
       "p12                          -0.00010592 -4.79889e-05 -0.000167507   \n",
       "\n",
       "                                      10           12           13  \\\n",
       "StoreID                             1000         1000         1000   \n",
       "Date                          12/03/2016   14/03/2016   15/03/2016   \n",
       "IsHoliday                              0            0            0   \n",
       "IsOpen                                 1            1            1   \n",
       "HasPromotions                          0            0            0   \n",
       "NearestCompetitor                    326          326          326   \n",
       "Region                                 7            7            7   \n",
       "NumberOfSales                       5895         6348         4738   \n",
       "Region_AreaKM2                      9643         9643         9643   \n",
       "Region_GDP                         17130        17130        17130   \n",
       "Region_PopulationK                  2770         2770         2770   \n",
       "CloudCover                             6            7            7   \n",
       "Max_VisibilityKm                      10           12           13   \n",
       "Min_VisibilitykM                      10           12           13   \n",
       "WindDirDegrees                       202           26           26   \n",
       "D_Day                                 12           14           15   \n",
       "D_DayOfYear                           72           74           75   \n",
       "D_Month                                3            3            3   \n",
       "D_Year                              2016         2016         2016   \n",
       "D_DayOfweek                            5            0            1   \n",
       "StoreType_SuperMarket                  0            0            0   \n",
       "StoreType_HyperMarket                  1            1            1   \n",
       "StoreType_StandardMarket               0            0            0   \n",
       "StoreType_ShoppingCenter               0            0            0   \n",
       "AssortmentType_General                 1            1            1   \n",
       "AssortmentType_WithNFDept              0            0            0   \n",
       "AssortmentType_WithFishDept            0            0            0   \n",
       "MeanStoreSales                   6296.18      6296.18      6296.18   \n",
       "MeanRegionSales                  3906.26      3906.26      3906.26   \n",
       "D_DayOfWeek_cos                -0.222521            1      0.62349   \n",
       "D_DayOfWeek_sin                -0.974928            0     0.781831   \n",
       "p1                           -0.00021493   0.00454891   0.00485476   \n",
       "p2                          -0.000300094   0.00152879   0.00280597   \n",
       "p3                          -1.93295e-05 -0.000878183  -0.00144622   \n",
       "p4                          -0.000840343 -0.000915415   0.00177991   \n",
       "p5                           0.000170166  -0.00210205 -0.000269195   \n",
       "p6                            -0.0002203   0.00188936   0.00290054   \n",
       "p7                          -0.000694627 -0.000661546 -0.000843696   \n",
       "p8                           -0.00124548  -0.00105218 -0.000794234   \n",
       "p9                           2.26767e-05  0.000650834   0.00161685   \n",
       "p10                         -0.000104071 -0.000373564  2.09806e-05   \n",
       "p11                          0.000304696 -0.000103976 -0.000567888   \n",
       "p12                          0.000120549  -8.6487e-05 -0.000151292   \n",
       "\n",
       "                                      14           15           16  \\\n",
       "StoreID                             1000         1000         1000   \n",
       "Date                          16/03/2016   17/03/2016   18/03/2016   \n",
       "IsHoliday                              0            0            0   \n",
       "IsOpen                                 1            1            1   \n",
       "HasPromotions                          0            0            0   \n",
       "NearestCompetitor                    326          326          326   \n",
       "Region                                 7            7            7   \n",
       "NumberOfSales                       6014         6087         6962   \n",
       "Region_AreaKM2                      9643         9643         9643   \n",
       "Region_GDP                         17130        17130        17130   \n",
       "Region_PopulationK                  2770         2770         2770   \n",
       "CloudCover                             5            4            2   \n",
       "Max_VisibilityKm                      14           15           16   \n",
       "Min_VisibilitykM                      14           15           16   \n",
       "WindDirDegrees                        27          272          218   \n",
       "D_Day                                 16           17           18   \n",
       "D_DayOfYear                           76           77           78   \n",
       "D_Month                                3            3            3   \n",
       "D_Year                              2016         2016         2016   \n",
       "D_DayOfweek                            2            3            4   \n",
       "StoreType_SuperMarket                  0            0            0   \n",
       "StoreType_HyperMarket                  1            1            1   \n",
       "StoreType_StandardMarket               0            0            0   \n",
       "StoreType_ShoppingCenter               0            0            0   \n",
       "AssortmentType_General                 1            1            1   \n",
       "AssortmentType_WithNFDept              0            0            0   \n",
       "AssortmentType_WithFishDept            0            0            0   \n",
       "MeanStoreSales                   6296.18      6296.18      6296.18   \n",
       "MeanRegionSales                  3906.26      3906.26      3906.26   \n",
       "D_DayOfWeek_cos                -0.222521    -0.900969    -0.900969   \n",
       "D_DayOfWeek_sin                 0.974928     0.433884    -0.433884   \n",
       "p1                            0.00590618   0.00619993   0.00620682   \n",
       "p2                            0.00112061    0.0012244   0.00106947   \n",
       "p3                           -0.00079585 -0.000888527 -0.000800649   \n",
       "p4                          -0.000396647 -6.89417e-06 -0.000271905   \n",
       "p5                           -0.00211805   -0.0019094  -0.00210155   \n",
       "p6                            0.00141545   0.00145775   0.00128724   \n",
       "p7                            0.00123804   0.00139421   0.00146344   \n",
       "p8                           -0.00107538  -0.00124269  -0.00111452   \n",
       "p9                           0.000479975 -0.000106806  0.000340916   \n",
       "p10                         -0.000359549 -0.000654383 -0.000817936   \n",
       "p11                         -0.000204156  0.000866687 -1.55093e-05   \n",
       "p12                           0.00014157 -9.86859e-05   6.9519e-05   \n",
       "\n",
       "                                      17           19           20  \\\n",
       "StoreID                             1000         1000         1000   \n",
       "Date                          19/03/2016   21/03/2016   22/03/2016   \n",
       "IsHoliday                              0            0            0   \n",
       "IsOpen                                 1            1            1   \n",
       "HasPromotions                          0            1            1   \n",
       "NearestCompetitor                    326          326          326   \n",
       "Region                                 7            7            7   \n",
       "NumberOfSales                       6618         9183         8155   \n",
       "Region_AreaKM2                      9643         9643         9643   \n",
       "Region_GDP                         17130        17130        17130   \n",
       "Region_PopulationK                  2770         2770         2770   \n",
       "CloudCover                             0            6            4   \n",
       "Max_VisibilityKm                      17           19           20   \n",
       "Min_VisibilitykM                      17           19           20   \n",
       "WindDirDegrees                       138          190          211   \n",
       "D_Day                                 19           21           22   \n",
       "D_DayOfYear                           79           81           82   \n",
       "D_Month                                3            3            3   \n",
       "D_Year                              2016         2016         2016   \n",
       "D_DayOfweek                            5            0            1   \n",
       "StoreType_SuperMarket                  0            0            0   \n",
       "StoreType_HyperMarket                  1            1            1   \n",
       "StoreType_StandardMarket               0            0            0   \n",
       "StoreType_ShoppingCenter               0            0            0   \n",
       "AssortmentType_General                 1            1            1   \n",
       "AssortmentType_WithNFDept              0            0            0   \n",
       "AssortmentType_WithFishDept            0            0            0   \n",
       "MeanStoreSales                   6296.18      6296.18      6296.18   \n",
       "MeanRegionSales                  3906.26      3906.26      3906.26   \n",
       "D_DayOfWeek_cos                -0.222521            1      0.62349   \n",
       "D_DayOfWeek_sin                -0.974928            0     0.781831   \n",
       "p1                            0.00389524   0.00180157   0.00183232   \n",
       "p2                           -0.00052353  -0.00104194 -0.000465838   \n",
       "p3                          -0.000309571  0.000374328  7.00345e-05   \n",
       "p4                          -0.000147691   0.00211691   0.00174133   \n",
       "p5                           0.000539229  0.000427248   0.00068135   \n",
       "p6                           -0.00190977 -0.000453241 -0.000564274   \n",
       "p7                            0.00175458  0.000625665  -0.00123484   \n",
       "p8                           -0.00113884  -0.00102667 -0.000933344   \n",
       "p9                           0.000335526  0.000759144   0.00120153   \n",
       "p10                           -0.0013901 -0.000305311 -0.000471439   \n",
       "p11                         -0.000301076 -0.000162035  0.000449306   \n",
       "p12                          0.000542752 -0.000404629  0.000203928   \n",
       "\n",
       "                                      21           22  \n",
       "StoreID                             1000         1000  \n",
       "Date                          23/03/2016   24/03/2016  \n",
       "IsHoliday                              0            0  \n",
       "IsOpen                                 1            1  \n",
       "HasPromotions                          1            1  \n",
       "NearestCompetitor                    326          326  \n",
       "Region                                 7            7  \n",
       "NumberOfSales                       7109         8041  \n",
       "Region_AreaKM2                      9643         9643  \n",
       "Region_GDP                         17130        17130  \n",
       "Region_PopulationK                  2770         2770  \n",
       "CloudCover                             6            6  \n",
       "Max_VisibilityKm                      21           22  \n",
       "Min_VisibilitykM                      21           22  \n",
       "WindDirDegrees                       191          331  \n",
       "D_Day                                 23           24  \n",
       "D_DayOfYear                           83           84  \n",
       "D_Month                                3            3  \n",
       "D_Year                              2016         2016  \n",
       "D_DayOfweek                            2            3  \n",
       "StoreType_SuperMarket                  0            0  \n",
       "StoreType_HyperMarket                  1            1  \n",
       "StoreType_StandardMarket               0            0  \n",
       "StoreType_ShoppingCenter               0            0  \n",
       "AssortmentType_General                 1            1  \n",
       "AssortmentType_WithNFDept              0            0  \n",
       "AssortmentType_WithFishDept            0            0  \n",
       "MeanStoreSales                   6296.18      6296.18  \n",
       "MeanRegionSales                  3906.26      3906.26  \n",
       "D_DayOfWeek_cos                -0.222521    -0.900969  \n",
       "D_DayOfWeek_sin                 0.974928     0.433884  \n",
       "p1                             0.0010165   0.00392148  \n",
       "p2                            0.00139047   0.00132576  \n",
       "p3                           -0.00109489 -0.000718271  \n",
       "p4                          -0.000321022 -0.000839272  \n",
       "p5                            0.00196911  -0.00228311  \n",
       "p6                          -9.33638e-05   0.00213789  \n",
       "p7                           9.40244e-06 -0.000332149  \n",
       "p8                           -0.00122761  -0.00117337  \n",
       "p9                           0.000106001  0.000198016  \n",
       "p10                         -0.000386896 -0.000910045  \n",
       "p11                          0.000579355 -0.000281794  \n",
       "p12                          0.000234422  0.000193493  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(20).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(433958, 43)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = md.get_fake_train(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(397047, 43)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(36911, 43)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = md.get_fake_test(df)\n",
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(\"Load the training, test and store data using pandas\")\n",
    "#train = pd.read_csv(\"train.csv\",low_memory=False)\n",
    "#test = pd.read_csv(\"test.csv\")\n",
    "#store = pd.read_csv(\"store.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Selecting predictive features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features\n",
      "['IsHoliday', 'HasPromotions', 'NearestCompetitor', 'Region', 'Region_AreaKM2', 'Region_GDP', 'Region_PopulationK', 'CloudCover', 'Max_VisibilityKm', 'Min_VisibilitykM', 'WindDirDegrees', 'D_Day', 'D_DayOfYear', 'D_Month', 'D_Year', 'D_DayOfweek', 'StoreType_SuperMarket', 'StoreType_HyperMarket', 'StoreType_StandardMarket', 'StoreType_ShoppingCenter', 'AssortmentType_General', 'AssortmentType_WithNFDept', 'AssortmentType_WithFishDept', 'MeanStoreSales', 'MeanRegionSales', 'D_DayOfWeek_cos', 'D_DayOfWeek_sin', 'p1', 'p2', 'p3', 'p4', 'p5', 'p6', 'p7', 'p8', 'p9', 'p10', 'p11', 'p12']\n"
     ]
    }
   ],
   "source": [
    "features = list(train.columns.values)\n",
    "features.remove('NumberOfSales')\n",
    "features.remove('StoreID')\n",
    "features.remove('Date')\n",
    "features.remove('IsOpen')\n",
    "#features.remove('IsOpen_t-7')\n",
    "#features.remove('IsOpen_t-4')  #Is Open lags worsen a lot\n",
    "#features.remove('IsOpen_t-1')\n",
    "#features.remove('HasPromotions_t-2')\n",
    "#features.remove('HasPromotions_t-1')\n",
    "#features.remove('Region_GDP')\n",
    "#features.remove('Max_Dew_PointC')\n",
    "#features.remove('Min_Dew_PointC')\n",
    "#features.remove('Max_Wind_SpeedKm_h')\n",
    "#features.remove('Max_VisibilityKm')\n",
    "#features.remove('IsOpen')\n",
    "#features.remove('Min_Sea_Level_PressurehPa')\n",
    "#features.remove('Precipitationmm')\n",
    "#features.remove('Mean_Dew_PointC')\n",
    "#features.remove('Min_TemperatureC')\n",
    "print(\"features\")\n",
    "\n",
    "print(features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "features.remove('p6')\n",
    "features.remove('p3')\n",
    "features.remove('p7')\n",
    "features.remove('p9')\n",
    "features.remove('Events_Thunderstorm')\n",
    "features.remove('Events_Fog')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "params = {\"objective\": \"reg:linear\",\n",
    "          \"eta\": 0.2,\n",
    "          \"max_depth\": 8,\n",
    "          \"subsample\": 1.0,\n",
    "          \"colsample_bytree\": 0.3, #0.3\n",
    "          \"silent\": 1,\n",
    "          \"nthread\":-1,\n",
    "          #added as a test\n",
    "          #\"eval_metric\": \"rmse\",\n",
    "          #\"gamma\": 2  #added as a test\n",
    "          }\n",
    "\n",
    "#reg_alpha \n",
    "#reg_lambda\n",
    "num_trees = 400 #400"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#split in train and test (not real test which is called test and is the fake test provided by md)\n",
    "X_train, X_test = train_test_split(train, test_size=0.05)\n",
    "#X_train, X_test = train.head(len(train) - val_size), train.tail(val_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtrain = xgb.DMatrix(X_train[features], np.log(X_train[\"NumberOfSales\"] + 1))\n",
    "dvalid = xgb.DMatrix(X_test[features], np.log(X_test[\"NumberOfSales\"] + 1))\n",
    "#dtest = xgb.DMatrix(test[features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(377194, 43)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(19853, 43)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'params' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-64-a2936e6ca2c9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mwatchlist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdvalid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'test'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'train'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mgbm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxgb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_trees\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevals\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwatchlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mearly_stopping_rounds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrmspe_xg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose_eval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'params' is not defined"
     ]
    }
   ],
   "source": [
    "watchlist = [(dvalid, 'test'), (dtrain, 'train')]\n",
    "gbm = xgb.train(params, dtrain, num_trees, evals=watchlist, early_stopping_rounds=20, feval=rmspe_xg, verbose_eval=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#watchlist = [(dvalid, 'test'), (dtrain, 'train')]\n",
    "#gbm = xgb.train(params, dtrain, num_trees, evals=watchlist, early_stopping_rounds=20, verbose_eval=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Validating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Validating\")\n",
    "train_probs = gbm.predict(xgb.DMatrix(X_test[features]))\n",
    "indices = train_probs < 0\n",
    "train_probs[indices] = 0\n",
    "error = rmspe(np.exp(train_probs) - 1, X_test['NumberOfSales'].values)\n",
    "print('error', error)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Feature importance XGB for all features \n",
    "from xgboost import plot_importance\n",
    "plot_importance(gbm, importance_type=\"gain\")\n",
    "plt.title(\"XGBoost Feature Gain\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Feature importance XGB for all features \n",
    "from xgboost import plot_importance\n",
    "plot_importance(gbm, importance_type=\"weight\")\n",
    "plt.title(\"XGBoost Feature weight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Feature importance XGB for all features \n",
    "from xgboost import plot_importance\n",
    "plot_importance(gbm, importance_type=\"cover\")\n",
    "plt.title(\"XGBoost Feature cover\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict on fake test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_probs = gbm.predict(xgb.DMatrix(test[features]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test['_NumberOfSales'] = np.exp(gbm.predict(xgb.DMatrix(test[features])))-1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BIP Error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from BIP import get_BIP_error\n",
    "error1 = get_BIP_error(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "diff=test['NumberOfSales']-test['_NumberOfSales']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "diff.head(15).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "diff.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_probs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test['predicted'] = pd.Series([train_probs], index=test.index)\n",
    "#test['predicted']=train_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare = test[['NumberOfSales','_NumberOfSales']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test.to_csv('./dataset/XGB5_fake_test.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "error = rmspe(test['_NumberOfSales'].values, test['NumberOfSales'].values)\n",
    "print('error', error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare[1000:1100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare[1000:1100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#REAL\n",
    "start = 500\n",
    "end = 600\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "y = compare.iloc[start:end,0].values\n",
    "x_coordinate = [ 1 * i for i in range(len(y)) ]\n",
    "plt.plot(x_coordinate,y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "yhat = compare.iloc[start:end:1].values\n",
    "x_coordinate = [ 1 * i for i in range(len(yhat)) ]\n",
    "plt.plot(x_coordinate,yhat)\n",
    "plt.legend(['Real', 'Predicted'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "#import matplotlib as mpl\n",
    "\n",
    "#mpl.style.use(\"default\")\n",
    "\n",
    "\n",
    "plt.plot(x_coordinate,y)\n",
    "plt.plot(x_coordinate,yhat)\n",
    "plt.legend(['Predicted', 'Real'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "indices = train_probs < 0\n",
    "train_probs[indices] = 0\n",
    "error = rmspe(np.exp(train_probs) - 1, X_test['NumberOfSales'].values)\n",
    "print('error', error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_probs = gbm.predict(xgb.DMatrix(test[features]))\n",
    "indices = test_probs < 0\n",
    "test_probs[indices] = 0\n",
    "submission = pd.DataFrame({\"Id\": test[\"Id\"], \"Sales\": np.exp(test_probs) - 1})\n",
    "submission.to_csv(\"xgboost_kscript_submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = df.iloc[:, 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}