Santandermelcior.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "import  matplotlib.pyplot as plt\n",
    "import seaborn as sns \n",
    "\n",
    "test_dfcopia = pd.read_csv('C:/Users/joanarino/Documents/santander/test.csv', sep = ',')\n",
    "train_dfcopia = pd.read_csv('C:/Users/joanarino/Documents/santander/train.csv', sep = ',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = train_dfcopia.target\n",
    "y = np.log1p(y)\n",
    "train_id = train_dfcopia.ID\n",
    "test_id = test_dfcopia.ID\n",
    "train_df = train_dfcopia.drop(['ID', 'target'], axis = 1)\n",
    "test_df = test_dfcopia.drop(['ID'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0.9685413111171313\n",
      "0.9859478933508308\n"
     ]
    }
   ],
   "source": [
    "print(np.sum(y == 0))\n",
    "print(np.sum(train_df.values.flatten() == 0)/np.size(train_df))\n",
    "print(np.sum(test_df.values.flatten() == 0)/np.size(test_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4459, 4991)\n",
      "(49342, 4991)\n"
     ]
    }
   ],
   "source": [
    "print(np.shape(train_df))\n",
    "print(np.shape(test_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d9c3087a9f748f29c24b74fece02a93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4991), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "256"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##treure columnes amb valor constant\n",
    "treure =[]\n",
    "for i in tqdm(range(len(train_df.columns))):\n",
    "    if len(np.unique(train_df.values[:,i])) == 1:\n",
    "        treure = treure + [i]\n",
    "len(treure)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "EmptyDataError",
     "evalue": "No columns to parse from file",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mEmptyDataError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-20-2206e7966362>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m#treure_df = pd.DataFrame(treure)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;31m#treure_df.to_csv('C:/Users/joanarino/Documents/santander/treure.csv', index=False)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mtest_dfcopia\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'C:/Users/joanarino/Documents/santander/treure.csv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msep\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m','\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[1;31m#print(test_dfcopia)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, doublequote, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[0;32m    676\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[0;32m    677\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 678\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    679\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    680\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m    438\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    439\u001b[0m     \u001b[1;31m# Create the parser.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 440\u001b[1;33m     \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    441\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    442\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[0;32m    785\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'has_index_names'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'has_index_names'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    786\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 787\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    788\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    789\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[1;34m(self, engine)\u001b[0m\n\u001b[0;32m   1012\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'c'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1013\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'c'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1014\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1015\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1016\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'python'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, src, **kwds)\u001b[0m\n\u001b[0;32m   1706\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'usecols'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1707\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1708\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1709\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1710\u001b[0m         \u001b[0mpassed_names\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnames\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mEmptyDataError\u001b[0m: No columns to parse from file"
     ]
    }
   ],
   "source": [
    "#la cel·la de dalt tarda uns 10 mins a executar-se, guardar-la en local un cop i carregar-la des d'allà \n",
    "#treure_df = pd.DataFrame(treure)\n",
    "#treure_df.to_csv('C:/Users/joanarino/Documents/santander/treure.csv', index=False)\n",
    "test_dfcopia = pd.read_csv('C:/Users/joanarino/Documents/santander/treure.csv', sep = ',')\n",
    "#print(test_dfcopia)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_train_ctt = train_df.columns[treure]\n",
    "cols_test_ctt = test_df.columns[treure]\n",
    "train_df.drop(cols_train_ctt, axis = 1, inplace = True)\n",
    "test_df.drop(cols_test_ctt, axis = 1, inplace = True)\n",
    "del train_dfcopia\n",
    "del test_dfcopia\n",
    "train_dfcopia = train_df.copy()\n",
    "test_dfcopia = test_df .copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4459, 4735)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(train_df.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a79e02d409a4b2b881ea6093e48f637",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4735), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##treure columnes repetides no n'hi han, no executar\n",
    "'''\n",
    "treure =[]\n",
    "for i in tqdm(range(len(train_df.columns))):\n",
    "    for j in range(i+1, len(train_df.columns)):\n",
    "        if np.array_equal(train_df.columns[i], train_df.columns[j]):\n",
    "            treure = treure + [i]\n",
    "len(treure)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_train_ctt = train_df.columns[treure]\n",
    "cols_test_ctt = test_df.columns[treure]\n",
    "train_df.drop(cols_train_ctt, axis = 1, inplace = True)\n",
    "test_df.drop(cols_test_ctt, axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec3f9a4c9f564964bd5051c512279740",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4735), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1041\n"
     ]
    }
   ],
   "source": [
    "#kolmogorov smirnov  and(0,768), or(1230,3165) (sobre 4735 columnes) or(704,649), and(7,53) sobre 1000\n",
    "from scipy.stats import ks_2samp\n",
    "\n",
    "train = train_df\n",
    "test = test_df\n",
    "\n",
    "THRESHOLD_P_VALUE = 0.001 #need tuned\n",
    "THRESHOLD_STATISTIC = 0.1 #need tuned\n",
    "diff_cols = []\n",
    "for col in tqdm(train.columns):\n",
    "    statistic, pvalue = ks_2samp(train[col].values, test[col].values)\n",
    "    if pvalue <= THRESHOLD_P_VALUE: #and np.abs(statistic) > THRESHOLD_STATISTIC:\n",
    "        diff_cols.append(col)\n",
    "    '''\n",
    "    train_aux = train[col].values[train[col] != 0]\n",
    "    test_aux = test[col].values[test[col] != 0]\n",
    "    statistic2, pvalue2 = ks_2samp(train_aux, test_aux)\n",
    "    if pvalue2 <= THRESHOLD_P_VALUE and np.abs(statistic2) > THRESHOLD_STATISTIC:\n",
    "        diff_cols2.append(col)\n",
    "    '''\n",
    "print(len(diff_cols))\n",
    "#print(len(diff_cols2))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.drop(diff_cols, axis = 1, inplace = True)\n",
    "test_df.drop(diff_cols, axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75dfe5d8a72444b2b59ccb1d9252e308",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4735), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-16-14cd200ba36f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mpercent_zeros\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m     \u001b[0mpercent_zeros\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m \u001b[0msns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdistplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpercent_zeros\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mbins\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m50\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkde\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36mvalues\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   4627\u001b[0m         \"\"\"\n\u001b[0;32m   4628\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_consolidate_inplace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4629\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_AXIS_REVERSED\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   4630\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4631\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\internals.py\u001b[0m in \u001b[0;36mas_array\u001b[1;34m(self, transpose, items)\u001b[0m\n\u001b[0;32m   3947\u001b[0m             \u001b[0marr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmgr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mblocks\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3948\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3949\u001b[1;33m             \u001b[0marr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmgr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_interleave\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3950\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3951\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0marr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mtranspose\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0marr\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\internals.py\u001b[0m in \u001b[0;36m_interleave\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   3976\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mblk\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mblocks\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3977\u001b[0m             \u001b[0mrl\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mblk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmgr_locs\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3978\u001b[1;33m             \u001b[0mresult\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mrl\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mblk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3979\u001b[0m             \u001b[0mitemmask\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mrl\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3980\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "##codi per preprocessat\n",
    "\n",
    "percent_zeros = np.zeros(len(train_df.columns))\n",
    "for i in tqdm(range(len(train_df.columns))):\n",
    "    percent_zeros[i] = np.sum(train_df.values[:,i] == 0)/len(train_df.values[:,i])\n",
    "sns.distplot(percent_zeros,bins = 50, kde = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34617605a8594406b2c75972643ce203",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3694), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\melcior.pijoan\\Miniconda3\\lib\\site-packages\\matplotlib\\axes\\_axes.py:6462: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.\n",
      "  warnings.warn(\"The 'normed' kwarg is deprecated, and has been \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x281bdca5a20>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADtRJREFUeJzt3XuQnfVdx/H3l4SAgC2QbBlMQpdK1OIVjEDbaSvEzgDVBi1Yho5NO5nJH0Wt4qWof3SmM8wUL+Uy7TBmiDV1Si8iSsahFgygo0KmS7kT28SoyRKE7XBRyjBt9Osf55ey3WyyZ3PO2bP73fdr5sw+l995zu+bJ/mcJ7/nspGZSJLqOmbYHZAkDZZBL0nFGfSSVJxBL0nFGfSSVJxBL0nFGfSSVJxBL0nFGfSSVNzSYXcAYMWKFTk6OjrsbkjSgvLQQw99MzNHZmo3L4J+dHSUsbGxYXdDkhaUiPjPbto5dCNJxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9Jxc2LO2MlqbrbduyddvlV558x8M/2iF6SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SivN59JLUJ4d75vyweUQvScUZ9JJUXFdBHxG/GRFPRsQTEfH5iDg+Is6MiB0RsSsivhgRy1rb49r87rZ+dJAFSJKObMagj4iVwK8DazPzx4AlwJXA9cANmbkGeAHY2N6yEXghM88CbmjtJElD0u3QzVLg+yJiKXAC8AxwEXB7W78VuKxNr2/ztPXrIiL6011J0mzNGPSZ+TTwx8BeOgH/EvAQ8GJmHmjNxoGVbXolsK+990Brv7y/3ZYkdauboZtT6Bylnwn8AHAicMk0TfPgW46wbvJ2N0XEWESMTUxMdN9jSdKsdDN083PAv2fmRGZ+B7gDeCtwchvKAVgF7G/T48BqgLb+9cDzUzeamZszc21mrh0ZGemxDEnS4XQT9HuBCyLihDbWvg54CrgPuLy12QDc2aa3tXna+nsz85AjeknS3OhmjH4HnZOqXwMeb+/ZDHwUuCYidtMZg9/S3rIFWN6WXwNcO4B+S5K61NUjEDLzY8DHpizeA5w3TdtXgSt675okqR+8M1aSijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16Siusq6CPi5Ii4PSL+NSJ2RsRbIuLUiLgnIna1n6e0thERN0fE7oh4LCLOHWwJkqQj6faI/ibg7zLzR4CfBHYC1wLbM3MNsL3NA1wCrGmvTcAtfe2xJGlWZgz6iHgd8A5gC0BmfjszXwTWA1tbs63AZW16PfDZ7HgQODkiTu97zyVJXenmiP5NwATwmYh4OCJujYgTgdMy8xmA9vMNrf1KYN+k94+3ZZKkIegm6JcC5wK3ZOY5wLd4bZhmOjHNsjykUcSmiBiLiLGJiYmuOitJmr1ugn4cGM/MHW3+djrB/+zBIZn287lJ7VdPev8qYP/UjWbm5sxcm5lrR0ZGjrb/kqQZzBj0mflfwL6I+OG2aB3wFLAN2NCWbQDubNPbgA+0q28uAF46OMQjSZp7S7ts92vA5yJiGbAH+BCdL4kvRcRGYC9wRWt7F3ApsBt4pbWVJA1JV0GfmY8Aa6dZtW6atglc3WO/JEl94p2xklScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klRc10EfEUsi4uGI+Ns2f2ZE7IiIXRHxxYhY1pYf1+Z3t/Wjg+m6JKkbszmi/wiwc9L89cANmbkGeAHY2JZvBF7IzLOAG1o7SdKQdBX0EbEKeDdwa5sP4CLg9tZkK3BZm17f5mnr17X2kqQh6PaI/kbgd4H/a/PLgRcz80CbHwdWtumVwD6Atv6l1l6SNAQzBn1E/DzwXGY+NHnxNE2zi3WTt7spIsYiYmxiYqKrzkqSZq+bI/q3Ae+JiP8AvkBnyOZG4OSIWNrarAL2t+lxYDVAW/964PmpG83MzZm5NjPXjoyM9FSEJOnwZgz6zPy9zFyVmaPAlcC9mfl+4D7g8tZsA3Bnm97W5mnr783MQ47oJUlzo5fr6D8KXBMRu+mMwW9py7cAy9vya4Bre+uiJKkXS2du8prMvB+4v03vAc6bps2rwBV96JskqQ+8M1aSijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SipvV8+glaTG5bcfeaZdfdf4Zc9yT3nhEL0nFGfSSVJxBL0nFGfSSVJxBL0nFGfSSVJxBL0nFGfSSVJw3TEnSLB3uRqr5yqCXtOgttOCeLYduJKk4g16SijPoJak4g16SijPoJak4g16SijPoJak4g16SivOGKUmLRvUbow7HI3pJKs6gl6TiZgz6iFgdEfdFxM6IeDIiPtKWnxoR90TErvbzlLY8IuLmiNgdEY9FxLmDLkKSdHjdHNEfAH4rM98MXABcHRFnA9cC2zNzDbC9zQNcAqxpr03ALX3vtSSpazMGfWY+k5lfa9P/A+wEVgLrga2t2Vbgsja9HvhsdjwInBwRp/e955KkrsxqjD4iRoFzgB3AaZn5DHS+DIA3tGYrgX2T3jbelkmShqDroI+Ik4C/An4jM//7SE2nWZbTbG9TRIxFxNjExES33ZAkzVJXQR8Rx9IJ+c9l5h1t8bMHh2Taz+fa8nFg9aS3rwL2T91mZm7OzLWZuXZkZORo+y9JmkE3V90EsAXYmZmfnLRqG7ChTW8A7py0/APt6psLgJcODvFIkuZeN3fGvg34FeDxiHikLft94BPAlyJiI7AXuKKtuwu4FNgNvAJ8qK89lqQZLNY7YA9nxqDPzH9i+nF3gHXTtE/g6h77JUnqE++MlaTiDHpJKs6gl6TiDHpJKs6gl6TiDHpJKs7fMCVpwfJ6+e54RC9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScQS9JxRn0klScj0CQNG/4SIPB8Ihekooz6CWpOINekooz6CWpOINekooz6CWpOINekooz6CWpOINekooz6CWpOINekooz6CWpOB9qJmnO+fCyuWXQS0UcLjyvOv+MOe6J5huHbiSpOI/opQXGYQ/NlkEvLVKz/cJwCGjhMuileapfR+5zsR2/BOY3x+glqbiBHNFHxMXATcAS4NbM/MQgPkdaSLwqRsPS96CPiCXAp4F3AePAVyNiW2Y+1e/PkoapX8Fd4eSqX2Lz2yCO6M8DdmfmHoCI+AKwHjDotShUCO5+8c9ifhhE0K8E9k2aHwfOH8DnSANnUKmCQQR9TLMsD2kUsQnY1GZfjoivD6Av/bAC+OawO9EHFeqoUAPUqKNCDTAP6nh/b29/YzeNBhH048DqSfOrgP1TG2XmZmDzAD6/ryJiLDPXDrsfvapQR4UaoEYdFWqAOnXMZBCXV34VWBMRZ0bEMuBKYNsAPkeS1IW+H9Fn5oGI+FXgK3Qur/yzzHyy358jSerOQK6jz8y7gLsGse0hmPfDS12qUEeFGqBGHRVqgDp1HFFkHnKeVJJUiI9AkKTiFnXQR8TFEfH1iNgdEddOs/6NEbE9Ih6LiPsjYtWU9a+LiKcj4lNz1+tD+njUNUTE/0bEI+011BPmPdZxRkTcHRE7I+KpiBidy75P6sdR1RARF07aD49ExKsRcdncV/DdfvayL/4wIp5s++LmiJjucuuB67GG6yPiifZ639z2fEAyc1G+6Jwo/jfgTcAy4FHg7Clt/hLY0KYvAv5iyvqbgNuATy3EGoCXh70f+lTH/cC72vRJwAkLrYZJbU4Fnh9GDb3WAbwV+Oe2jSXAA8DPLrAa3g3cQ+f85YnAGPC6YeyLfr4W8xH9dx/VkJnfBg4+qmGys4Htbfq+yesj4qeB04C756Cvh9NTDfPIUdcREWcDSzPzHoDMfDkzX5mbbn+Pfu2Ly4EvD6kG6K2OBI6nE67HAccCzw68x4fqpYazgX/IzAOZ+S06XxIXz0GfB2oxB/10j2pYOaXNo8B72/QvAt8fEcsj4hjgT4DfGXgvj+yoa2jzx0fEWEQ8OMyhAnqr44eAFyPijoh4OCL+qD1Yb671ui8OuhL4/EB62J2jriMzH6ATms+011cyc+eA+zudXvbFo8AlEXFCRKwALuR7bwBdkBZz0HfzqIbfBt4ZEQ8D7wSeBg4AHwbuysx9DFcvNQCckZ27Aq8CboyIHxxYT4+slzqWAm9v63+Gzn/XPziwnh5er/uCiDgd+HE696AMy1HXERFnAW+mczf8SuCiiHjHIDt7GEddQ2beTefS8H+h84X7AJP20UK1mH/D1IyPasjM/cAvAUTEScB7M/OliHgL8PaI+DCdMeFlEfFyZh5y0mfAjrqGSevIzD0RcT9wDp2xzbnWy74YBx7O156W+jfABcCWuej4JD3ti+aXgb/OzO8MuK9H0su+2AQ8mJkvt3VfprMv/nEuOj5Jr/8urgOua+tuA3bNQZ8Ha9gnCYb1ovMltwc4k9dO2PzolDYrgGPa9HXAx6fZzgcZ3snYo64BOAU4blKbXUw5YbVA6ljS2o+0+c8AVy+kGiatfxC4cBj7oE/74n3A37dtHEtnDPwXFlgNS4DlbfongCfonAMa2j7py5/JsDsw1OLhUuAbdI5i/6At+zjwnjZ9eQvAbwC3HgzGKdsYWtD3UgOdKyQeb/8IHgc2LtR9QeeX3DzW6vhzYNkCrGGUzvDBMcPcDz3+nVoC/Cmwk87vn/jkAqzh+Nb3p+h88f7UsPdHP17eGStJxS3mk7GStCgY9JJUnEEvScUZ9JJUnEEvScUZ9JJUnEEvScUZ9JJU3P8DF4Vah95+wiIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "percent_zeros_test = np.zeros(len(test_df.columns))\n",
    "for i in tqdm(range(len(test_df.columns))):\n",
    "    percent_zeros_test[i] = np.sum(test_df.values[:,i] == 0)/len(test_df.values[:,i])\n",
    "sns.distplot(percent_zeros_test,bins = 50, kde = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3695\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\melcior.pijoan\\Miniconda3\\lib\\site-packages\\matplotlib\\axes\\_axes.py:6462: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.\n",
      "  warnings.warn(\"The 'normed' kwarg is deprecated, and has been \"\n",
      "C:\\Users\\melcior.pijoan\\Miniconda3\\lib\\site-packages\\matplotlib\\axes\\_axes.py:6462: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.\n",
      "  warnings.warn(\"The 'normed' kwarg is deprecated, and has been \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x281e78c5f60>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFBtJREFUeJzt3X+sZOV93/H3h1+mju0A3gWhhfU69VoyTRub3sKmUWJjYmehqiEJJLaTsEa0KyW46o80Emr+cOvUkpMocYNiudnGxIvVtUPaOKwiXEy3UNrUu+VSDOZHCWvqLDeLYAk2LUVOgvvtH3OuPb57d+8wc+fMvfd5v6TRnPPMc+d8n3vunc+cH3MmVYUkqT2nzLoASdJsGACS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRp026wJOZtOmTbVt27ZZlyFJ68r999//XFVtXqnfmg6Abdu2MT8/P+syJGldSfIno/RzF5AkNcoAkKRGGQCS1CgDQJLWkH2HjrDv0JFelmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUSsGQJJbkjyb5OGhtnOS3JXkie7+7K49SW5OcjjJQ0kuHvqZXV3/J5Lsms5wJEmjGmUL4FPAziVtNwEHqmo7cKCbB7gC2N7ddgOfgEFgAB8CLgUuAT60GBqSpNlYMQCq6l7g+SXNVwF7u+m9wNVD7bfWwEHgrCTnAz8C3FVVz1fV14C7OD5UJEk9GvcYwHlV9TRAd39u174FeGqo30LXdqJ2SdKMrPZB4CzTVidpP/4Jkt1J5pPMHzt2bFWLkyR927gB8Ey3a4fu/tmufQG4cKjfBcDRk7Qfp6r2VNVcVc1t3rx5zPIkSSsZNwD2A4tn8uwCbh9qv647G2gH8EK3i+hO4N1Jzu4O/r67a5MkzchpK3VI8hngHcCmJAsMzub5KHBbkhuAI8C1Xfc7gCuBw8BLwPUAVfV8kl8C7uv6fbiqlh5YliT1aMUAqKr3neChy5fpW8CNJ3ieW4BbXlF1kqSp8ZPAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkrRH7Dh3pdXkGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1KiJAiDJV5N8OcmXksx3beckuSvJE9392V17ktyc5HCSh5JcvBoDkCSNZzW2AC6rqrdW1Vw3fxNwoKq2Awe6eYArgO3dbTfwiVVYtiRpTNPYBXQVsLeb3gtcPdR+aw0cBM5Kcv4Uli9JGsGkAVDAF5Lcn2R313ZeVT0N0N2f27VvAZ4a+tmFrk2SNAOnTfjzP1BVR5OcC9yV5H+epG+WaavjOg2CZDfA1q1bJyxPknQiE20BVNXR7v5Z4HPAJcAzi7t2uvtnu+4LwIVDP34BcHSZ59xTVXNVNbd58+ZJypMkncTYAZDku5K8dnEaeDfwMLAf2NV12wXc3k3vB67rzgbaAbywuKtIktS/SXYBnQd8Lsni8+yrqv+Q5D7gtiQ3AEeAa7v+dwBXAoeBl4DrJ1i2JGlCYwdAVT0JfN8y7X8GXL5MewE3jrs8SdLq8pPAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJI0A/sOHWHfoSMzrcEAkKRGGQCS1CgDQJIaZQBIUqMm/UYwSdII9h06wvsvPf5bDmd5INgtAElqlAEgST1ZC6d+DjMAJKlRBoAk9WytbAUYAJLUKANAkhplAEhSo/wcgCRN0VrZ378ctwAkqVEGgCQ1ygCQpEYZAJLUqN4DIMnOJI8nOZzkpr6XL0l9WcsHgKHns4CSnAp8HHgXsADcl2R/VT3aZx2SNE1r/YV/Ud9bAJcAh6vqyar6C+CzwFU91yBJov/PAWwBnhqaXwAuHe6QZDewu5t9McnjPdW2Fm0Cnpt1ETPk+B1/s+P/qcnG/4ZROvUdAFmmrb5jpmoPsKefcta2JPNVNTfrOmbF8Tt+xz/d8fe9C2gBuHBo/gLgaM81SJLoPwDuA7YneWOSM4D3Avt7rkGSRM+7gKrq5SQfBO4ETgVuqapH+qxhnWl9V5jjb5vjn7JU1cq9JEkbjp8ElqRGGQCS1CgDYMZWujRGkn+S5NEkDyU5kGSk83vXi1EvDZLkmiSVZMOdFjjK7yDJT3R/B48k2dd3jdM0wv/A1iR3J3mg+z+4chZ1TkOSW5I8m+ThEzyeJDd3v5uHkly8qgVUlbcZ3RgcCP8K8D3AGcCDwEVL+lwGvLqb/lngd2ddd5/j7/q9FrgXOAjMzbruGfwNbAceAM7u5s+ddd09j38P8LPd9EXAV2dd9yqO/4eAi4GHT/D4lcDnGXyGagdwaDWX7xbAbK14aYyquruqXupmDzL47MRGMeqlQX4J+BXgG30W15NRfgd/H/h4VX0NoKqe7bnGaRpl/AW8rpv+bjbQZ4eq6l7g+ZN0uQq4tQYOAmclOX+1lm8AzNZyl8bYcpL+NzB4N7BRrDj+JG8DLqyqP+yzsB6N8jfwZuDNSf4oycEkO3urbvpGGf8/B346yQJwB/AP+iltTXilrxGviN8JPFsrXhrjWx2TnwbmgLdPtaJ+nXT8SU4BPgZ8oK+CZmCUv4HTGOwGegeDLcD/kuR7q+rrU66tD6OM/33Ap6rq15J8P/Dpbvz/b/rlzdzIrxHjcAtgtka6NEaSHwZ+EXhPVf15T7X1YaXxvxb4XuCeJF9lsA90/wY7EDzK38ACcHtV/WVV/S/gcQaBsBGMMv4bgNsAquqLwJkMLpTWgqlePscAmK0VL43R7QL5LQYv/htp3y+sMP6qeqGqNlXVtqraxuAYyHuqan425U7FKJdH+QMGJwOQZBODXUJP9lrl9Iwy/iPA5QBJ3sIgAI71WuXs7Aeu684G2gG8UFVPr9aTuwtohuoEl8ZI8mFgvqr2A78KvAb4vSQAR6rqPTMrehWNOP4NbcTfwZ3Au5M8CnwT+IWq+rPZVb16Rhz/zwP/Jsk/ZrD74wPVnSKz3iX5DINde5u6YxwfAk4HqKp/zeCYx5XAYeAl4PpVXf4G+T1Kkl4hdwFJUqMMAElqlAEgSY1a0weBN23aVNu2bZt1GZK0rtx///3PVdXmlfqt6QDYtm0b8/Mb6Yw/SZq+JH8ySj93AUlSowwASWqUASBJjTIAJGmG9h06MrNlGwCS1CgDQJIaZQBIUqMMAElqlAEgSTO279CRmRwMNgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWrU2AGQ5Mwk/z3Jg0keSfIvuvY3JjmU5Ikkv5vkjK79Vd384e7xbaszBEnSOCbZAvhz4J1V9X3AW4GdSXYAvwx8rKq2A18Dbuj63wB8rareBHys6ydJmpGxA6AGXuxmT+9uBbwT+Hdd+17g6m76qm6e7vHLk2Tc5UuSJjPRdwInORW4H3gT8HHgK8DXq+rlrssCsKWb3gI8BVBVLyd5AXg98NyS59wN7AbYunXrJOVJ0po1y+8BWDTRQeCq+mZVvRW4ALgEeMty3br75d7t13ENVXuqaq6q5jZvXvFL7SVJY1qVs4Cq6uvAPcAO4Kwki1sWFwBHu+kF4EKA7vHvBp5fjeVLkl65Sc4C2pzkrG76rwA/DDwG3A1c03XbBdzeTe/v5uke/09VddwWgCSpH5McAzgf2NsdBzgFuK2q/jDJo8Bnk/xL4AHgk13/TwKfTnKYwTv/906wbEnShMYOgKp6CHjbMu1PMjgesLT9G8C14y5PkrS6/CSwJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNWqS7wS+MMndSR5L8kiSf9i1n5PkriRPdPdnd+1JcnOSw0keSnLxag1CkvTKTbIF8DLw81X1FmAHcGOSi4CbgANVtR040M0DXAFs7267gU9MsGxJ0oTGDoCqerqq/kc3/X+Ax4AtwFXA3q7bXuDqbvoq4NYaOAicleT8sSuXJE1kVY4BJNnG4AviDwHnVdXTMAgJ4Nyu2xbgqaEfW+jalj7X7iTzSeaPHTu2GuVJkpYxcQAkeQ3w74F/VFX/+2Rdl2mr4xqq9lTVXFXNbd68edLyJEknMFEAJDmdwYv/v62q3++an1nctdPdP9u1LwAXDv34BcDRSZYvSevRvkNHZl0CMNlZQAE+CTxWVb8+9NB+YFc3vQu4faj9uu5soB3AC4u7iiRJ/Tttgp/9AeBngC8n+VLX9s+AjwK3JbkBOAJc2z12B3AlcBh4Cbh+gmVLkiY0dgBU1X9l+f36AJcv07+AG8ddniStd2tl188iPwksSY0yACSpUQaAJDXKAJCkRhkAkrRG9H2Q2ACQpEYZAJLUKANAknqw1j4DAAaAJDXLAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVGTfifwLUmeTfLwUNs5Se5K8kR3f3bXniQ3Jzmc5KEkF09avCRpfJNuAXwK2Lmk7SbgQFVtBw508wBXANu7227gExMuW5I0gYkCoKruBZ5f0nwVsLeb3gtcPdR+aw0cBM5Kcv4ky5ckjW8axwDOq6qnAbr7c7v2LcBTQ/0WujZJ0gz0eRB4uS+Qr+M6JbuTzCeZP3bsWA9lSVKbTpvCcz6T5PyqerrbxfNs174AXDjU7wLg6NIfrqo9wB6Aubm54wJCktaTtXgV0EXT2ALYD+zqpncBtw+1X9edDbQDeGFxV5EkrWdr+UX+ZCbaAkjyGeAdwKYkC8CHgI8CtyW5ATgCXNt1vwO4EjgMvARcP8myJUmTmSgAqup9J3jo8mX6FnDjJMuTpFlbfLf//ku3zriSyflJYElq1DQOAkvShrde9/sPcwtAkka0EV70hxkAktQoA0CSpmStbzF4DEBS84ZfqMc9u2c9nh3kFoCkpp3oXfpw+75DR0Z+N7/W3/UPMwAkqVHuApKkIUvf+U/6HGuZASCpCauxn3+jMQAkNWe9vEOfNo8BSFKj3AKQtC7tO3TkW7tyhqcX57UytwAkqVEGgKR1Y+k7++XOz1/v7/5fyWcOJuUuIElr3mqcmqnjuQUgaU3zBX96et8CSLIT+A3gVOC3q+qjfdcgteJkB0fX6rnw415Tx6B45XrdAkhyKvBx4ArgIuB9SS7qswZJoxneFz3KvvbV3nfd577wVvW9BXAJcLiqngRI8lngKuDRnuuQ1pyl79ZH6f9Kn3+Sn1/8meW2KE52CuZa3dIQZPBd7T0tLLkG2FlVf6+b/xng0qr64HL95+bman5+vrf6pNWw9Pz0RZ6n/m3vv3Rr87+DlUwSnEnur6q5lfr1vQWQZdq+I4GS7AZ2d7MvJnl86lWNbxPw3KyLmJBjmIKfGrFtiTU3jjGMNIYRfhezNvN1MeHv6A2jdOo7ABaAC4fmLwCODneoqj3Anj6LGleS+VFSdi1zDGvHRhjHRhgDbJxxrKTv00DvA7YneWOSM4D3Avt7rkGSRM9bAFX1cpIPAncyOA30lqp6pM8aJEkDvX8OoKruAO7oe7lTsi52Va3AMawdG2EcG2EMsHHGcVK9ngUkSVo7vBSEJDXKAFhGkp1JHk9yOMlNyzz+hiQHkjyU5J4kFyx5/HVJ/jTJb/ZX9XE1jj2GJN9M8qXuNtOD9BOOY2uSLyR5LMmjSbb1WftQHWONIcllQ+vhS0m+keTq/kfwrTonWRe/kuSRbl3cnGS5U8KnbsIx/HKSh7vbT/Zb+ZRUlbehG4OD018Bvgc4A3gQuGhJn98DdnXT7wQ+veTx3wD2Ab+5HscAvDjr9bBK47gHeFc3/Rrg1ettDEN9zgGen8UYJh0H8LeBP+qe41Tgi8A71tkY/g5wF4Pjpt8FzAOvm8W6WM2bWwDH+9blKqrqL4DFy1UMuwg40E3fPfx4kr8JnAd8oYdaT2SiMawhY4+ju8bUaVV1F0BVvVhVL/VT9ndYrXVxDfD5GY0BJhtHAWcyeNF9FXA68MzUKz7eJGO4CPjPVfVyVf1fBuGxs4eap8oAON4W4Kmh+YWubdiDwI930z8KvDbJ65OcAvwa8AtTr/Lkxh5DN39mkvkkB2e5y4HJxvFm4OtJfj/JA0l+tbsYYd8mXReL3gt8ZioVjmbscVTVFxm8mD7d3e6sqsemXO9yJlkXDwJXJHl1kk3AZXznh1rXJQPgeCtergL4p8DbkzwAvB34U+Bl4OeAO6rqKWZrkjEAbK3BpyDfD/yrJH91apWe3CTjOA34we7xv8Vgs/8DU6v0xCZdFyQ5H/jrDD4/MytjjyPJm4C3MPjk/xbgnUl+aJrFnsDYY6iqLzA4ff2/MQjiLzK0jtYrvxHseKNcruIo8GMASV4D/HhVvZDk+4EfTPJzDPY5n5Hkxao67mDTlI09hqHHqKonk9wDvI3BvtO+TbIuFoAH6ttXnv0DYAfwyT4KHzLRuuj8BPC5qvrLKdd6MpOsi93Awap6sXvs8wzWxb19FD5k0v+LjwAf6R7bBzzRQ83TNeuDEGvtxiAUnwTeyLcPFP21JX02Aad00x8BPrzM83yA2R0EHnsMwNnAq4b6PMGSA2XrZByndv03d/O/A9y4nsYw9PhB4LJZrINVWhc/CfzH7jlOZ7CP/e+uszGcCry+m/4bwMMMjjHNbJ2syu9k1gWsxRtwJfDHDN71/mLX9mHgPd30Nd0L4x8Dv734grnkOWYWAJOMgcEZG1/u/jm+DNywXtcF8C7goW4cnwLOWIdj2MZgN8Qps1wPE/5NnQr8FvAYg+/++PV1OIYzu9ofZRDIb531+liNm58ElqRGeRBYkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1Kj/D/giBjjg9FevAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "b = np.array(percent_zeros)\n",
    "percent_zeros.sort()\n",
    "b.sort()\n",
    "plt.figure(1)\n",
    "plt.subplot(211)\n",
    "#b = b[b>0.998]\n",
    "print(b.size)\n",
    "sns.distplot(b,bins = 200, kde = False)\n",
    "\n",
    "\n",
    "plt.subplot(212)\n",
    "c = np.array(percent_zeros_test)\n",
    "c.sort()\n",
    "#c = c[c>0.99]\n",
    "sns.distplot(c,bins = 200, kde = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([      0.        ,       0.        ,       0.        , ...,\n",
       "             0.        ,       0.        , 6185422.74052478])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "364\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\melcior.pijoan\\Miniconda3\\lib\\site-packages\\matplotlib\\axes\\_axes.py:6462: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.\n",
      "  warnings.warn(\"The 'normed' kwarg is deprecated, and has been \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x22edccf3e10>"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAE2lJREFUeJzt3X+sZGd93/H3p7YxUaHY2NfU3V2zTrJNcKqyWDe2VaSG2FGx3ahrGlzZJOAiV5tWBkEDbez8UUhVq0Rq4hQpoVqwy4JijEuIvEJOUsc2Qih44RqMY+MQNuDay1rem/gHIBS3Nt/+MWfDaD33zrl3Zu7c++z7JY3uOc955sz3uT8+c+4zZ+akqpAktevvzLsASdJsGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxp087wIAzjzzzNq5c+e8y5CkLeX+++//q6paGNdvUwT9zp07WVpamncZkrSlJPk/ffo5dSNJjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6KXG3HrwsXmXoE3GoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1rnfQJzkpyVeSfKZbPzfJwSTfSPLJJC/p2k/t1g9123fOpnRJUh9rOaJ/F/DI0PpvADdV1S7gaeDarv1a4Omq+nHgpq6fJGlOegV9ku3APwc+0q0HuBj4VNdlP3BFt7ynW6fbfknXX5I0B32P6H8b+I/AD7r1M4Bnqur5bv0wsK1b3gY8DtBtf7brL0mag7FBn+TngaNVdf9w84iu1WPb8H73JllKsrS8vNyrWEnS2vU5on898C+SPArcxmDK5reB05Kc3PXZDhzplg8DOwC67a8Anjp+p1W1r6oWq2pxYWFhokFIklY2Nuir6oaq2l5VO4GrgHuq6heBe4E3d92uAe7olg9063Tb76mqFx3RS5I2xiTn0f8q8CtJDjGYg7+5a78ZOKNr/xXg+slKlCRN4uTxXX6oqj4LfLZb/iZwwYg+fwNcOYXaJElT4DtjJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJalyfa8a+NMkXk3w1ycNJfr1r/2iSbyV5oLvt7tqT5INJDiV5MMn5sx6EJGllfS488hxwcVV9L8kpwOeT/GG37T9U1aeO638ZsKu7XQh8qPsqSZqDPteMrar6Xrd6Sndb7Rqwe4CPdfe7j8FFxM+evFRJ0nr0mqNPclKSB4CjwF1VdbDbdGM3PXNTklO7tm3A40N3P9y1SZLmoFfQV9ULVbUb2A5ckOQfATcAPwn8NPBKBhcLB8ioXRzfkGRvkqUkS8vLy+sqXpI03prOuqmqZxhcHPzSqnqim555Dvif/PBC4YeBHUN32w4cGbGvfVW1WFWLCwsL6ypekjRen7NuFpKc1i3/CPBzwJ8fm3dPEuAK4KHuLgeAt3Vn31wEPFtVT8ykeknSWH3Oujkb2J/kJAZPDLdX1WeS3JNkgcFUzQPAv+363wlcDhwCvg+8ffplS1rNrQcfA+AtF54z50q0GYwN+qp6EHjdiPaLV+hfwHWTlyZJmgbfGStJjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJalyfSwm+NMkXk3w1ycNJfr1rPzfJwSTfSPLJJC/p2k/t1g9123fOdgiSpNX0OaJ/Dri4ql4L7AYu7a4F+xvATVW1C3gauLbrfy3wdFX9OHBT10+SNCdjg74GvtetntLdCrgY+FTXvp/BBcIB9nTrdNsv6S4gLkmag15z9ElOSvIAcBS4C/hL4Jmqer7rchjY1i1vAx4H6LY/C5wxYp97kywlWVpeXp5sFJKkFfUK+qp6oap2A9uBC4DXjOrWfR119F4vaqjaV1WLVbW4sLDQt15J0hqt6aybqnoG+CxwEXBakpO7TduBI93yYWAHQLf9FcBT0yhWkrR2fc66WUhyWrf8I8DPAY8A9wJv7rpdA9zRLR/o1um231NVLzqilyRtjJPHd+FsYH+Skxg8MdxeVZ9J8jXgtiT/BfgKcHPX/2bg40kOMTiSv2oGdUuSehob9FX1IPC6Ee3fZDBff3z73wBXTqU6SdLEfGesJDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4/p8TLGkLeDWg4/NuwRtUh7RS1Lj+lxhakeSe5M8kuThJO/q2t+f5NtJHuhulw/d54Ykh5J8PckbZzkASdLq+kzdPA+8p6q+nOTlwP1J7uq23VRV/224c5LzGFxV6qeAfwD8SZJ/WFUvTLNwSVI/Y4/oq+qJqvpyt/xdBteL3bbKXfYAt1XVc1X1LeAQI65EJWl6nJ/XatY0R59kJ4PLCh7smt6R5MEktyQ5vWvbBjw+dLfDrP7EIEmaod5Bn+RlwO8D766q7wAfAn4M2A08Afzmsa4j7l4j9rc3yVKSpeXl5TUXLknqp1fQJzmFQcj/XlV9GqCqnqyqF6rqB8CH+eH0zGFgx9DdtwNHjt9nVe2rqsWqWlxYWJhkDJKkVfQ56ybAzcAjVfVbQ+1nD3V7E/BQt3wAuCrJqUnOBXYBX5xeyZKktehz1s3rgbcCf5bkga7t14Crk+xmMC3zKPDLAFX1cJLbga8xOGPnOs+4kaT5GRv0VfV5Rs+737nKfW4EbpygLknSlPjOWElqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBLzXMz6kXGPSS1DyDXpIaZ9BLUuMMeklqnEEvSY3rc4WpHUnuTfJIkoeTvKtrf2WSu5J8o/t6eteeJB9Mcqi7cPj5sx6EJGllfY7onwfeU1WvAS4CrktyHnA9cHdV7QLu7tYBLmNw+cBdwF4GFxGXJM3J2KCvqieq6svd8neBR4BtwB5gf9dtP3BFt7wH+FgN3Aecdtz1ZSVJG2hNc/RJdgKvAw4Cr6qqJ2DwZACc1XXbBjw+dLfDXZskaQ56B32SlwG/D7y7qr6zWtcRbTVif3uTLCVZWl5e7luGJGmNegV9klMYhPzvVdWnu+Ynj03JdF+Pdu2HgR1Dd98OHDl+n1W1r6oWq2pxYWFhvfVLksboc9ZNgJuBR6rqt4Y2HQCu6ZavAe4Yan9bd/bNRcCzx6Z4JEkb7+QefV4PvBX4syQPdG2/BnwAuD3JtcBjwJXdtjuBy4FDwPeBt0+1YknSmowN+qr6PKPn3QEuGdG/gOsmrEuSNCW+M1aSGmfQS1LjDHpJapxBL0mN63PWjaRNyksFqg+P6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1rs8Vpm5JcjTJQ0Nt70/y7SQPdLfLh7bdkORQkq8neeOsCpck9dPniP6jwKUj2m+qqt3d7U6AJOcBVwE/1d3nd5OcNK1iJUlrNzboq+pzwFM997cHuK2qnquqbzG4nOAFE9QnSZrQJHP070jyYDe1c3rXtg14fKjP4a5N0hT4aZVaj/UG/YeAHwN2A08Av9m1j7q2bI3aQZK9SZaSLC0vL6+zDEnSOOsK+qp6sqpeqKofAB/mh9Mzh4EdQ123A0dW2Me+qlqsqsWFhYX1lCFJ6mFdQZ/k7KHVNwHHzsg5AFyV5NQk5wK7gC9OVqIkaRJjrzCV5BPAG4AzkxwG3ge8IcluBtMyjwK/DFBVDye5Hfga8DxwXVW9MJvSJfVxbF7/LReeM+dKNC9jg76qrh7RfPMq/W8EbpykKEnS9PjOWElqnEEvSY0z6CWpcWPn6CXNn2+U0iQ8opekxhn0ktQ4g16SGmfQS1LjDHrpBOELuicug16SGmfQS1LjDHpJapxBL0mNM+ilLebWg4/5wqrWxKCXpMYZ9JLUuLFBn+SWJEeTPDTU9sokdyX5Rvf19K49ST6Y5FCSB5OcP8viJUnj9Tmi/yhw6XFt1wN3V9Uu4O5uHeAyBteJ3QXsBT40nTIlSes1Nuir6nPAU8c17wH2d8v7gSuG2j9WA/cBpx13IXFJIxz/Aqsvtmqa1jtH/6qqegKg+3pW174NeHyo3+Gu7UWS7E2ylGRpeXl5nWVI7Vsp9H0yUF/TfjE2I9pqVMeq2ldVi1W1uLCwMOUyJEnHrDfonzw2JdN9Pdq1HwZ2DPXbDhxZf3mSpEmtN+gPANd0y9cAdwy1v607++Yi4NljUzySxpv1m6F8s9WJaew1Y5N8AngDcGaSw8D7gA8Atye5FngMuLLrfidwOXAI+D7w9hnULElag7FBX1VXr7DpkhF9C7hu0qIkSdPjO2MlqXEGvSQ1zqCXpMYZ9NKceRaMZs2gl05APrmcWAx6SWrc2NMrJc2HR92aFo/oJalxBr0kNc6gl6TGGfTSCcoPODtx+GKsNAcGrDaSR/SS1DiDXpIaZ9BLJzinkdo30Rx9kkeB7wIvAM9X1WKSVwKfBHYCjwL/qqqenqxMSdJ6TeOI/merandVLXbr1wN3V9Uu4O5uXZI0J7OYutkD7O+W9wNXzOAxpC3D0xg1b5MGfQH/O8n9SfZ2ba86dkHw7utZEz6GpBnzyahtk55H//qqOpLkLOCuJH/e947dE8NegHPOOWfCMqT5OxaUb7nQ32dtLhMd0VfVke7rUeAPgAuAJ5OcDdB9PbrCffdV1WJVLS4sLExShiRpFesO+iR/N8nLjy0D/wx4CDgAXNN1uwa4Y9Iipc3k2DSHUx3aKiaZunkV8AdJju3n1qr6oyRfAm5Pci3wGHDl5GVKktZr3UFfVd8EXjui/a+BSyYpStqsJjmK9z8AzYvvjJUm4BSOtgKDXpoBnwC0mRj00gwZ9toMDHppCgx0bWYGvSQ1zitMST14xK6tzCN6acq2+pOCLyS3x6CXpMYZ9JLUOINeGuNEmsYYNdYTafytMuglqXEGvTaNeR05jnrx0aPY6X9f/J7Oj6dXasPcevCxv70ox0p/9KPa+1zIY3jfkxh+fINpdcf/PEd9/6f1c9FkDHrN1Eb+ofe5wtNqfQz2lfX53qz0vfX7On8GvWZiVkfGw/vySHFzMtg3n5kFfZJLgf8OnAR8pKo+MKvH0tax3hBY6WyQlcJ+vVNAGm+SJ3GvqzsfM3kxNslJwO8AlwHnAVcnOW8WjzXK8ItIvstvffp+z0ZdVm/a3+/V9reWn6/z75vXVv873ey1z+qI/gLgUHcVKpLcBuwBvjbtBxo+QhgXCB5FrM24OdfNOhe7nuDXxtrM3/sW/+tIVU1/p8mbgUur6t90628FLqyqd4zqv7i4WEtLS+t6rFn9wgz/kNczL7yWX5Zp/WKtdz9r+R6Oe0KVJjXqb2+ls3tWOtAb1X+l39uV/tbX2me476i6+h4wrUWS+6tqcWy/GQX9lcAbjwv6C6rqnUN99gJ7u9WfAL4+9UJm40zgr+ZdxAw5vq3N8W1tax3fq6tqYVynWU3dHAZ2DK1vB44Md6iqfcC+GT3+zCRZ6vMMulU5vq3N8W1tsxrfrN4Z+yVgV5Jzk7wEuAo4MKPHkiStYiZH9FX1fJJ3AH/M4PTKW6rq4Vk8liRpdTM7j76q7gTunNX+52jLTTetkePb2hzf1jaT8c3kxVhJ0ubhp1dKUuMM+hGSXJrk60kOJbl+xPabkjzQ3f4iyTPzqHO9eozvnCT3JvlKkgeTXD6POifRY4yvTnJ3N77PJtk+jzrXI8ktSY4meWiF7UnywW7sDyY5f6NrnESP8f1kki8keS7Jeze6vkn1GN8vdj+3B5P8aZLXTvygVeVt6MbgxeO/BH4UeAnwVeC8Vfq/k8GLzXOvfVrjYzBP+O+65fOAR+dd9wzG+L+Aa7rli4GPz7vuNYzvnwLnAw+tsP1y4A+BABcBB+dd85THdxbw08CNwHvnXe8MxvdPgNO75cum8fPziP7F/vbjG6rq/wLHPr5hJVcDn9iQyqajz/gK+Hvd8is47j0QW0CfMZ4H3N0t3zti+6ZVVZ8Dnlqlyx7gYzVwH3BakrM3prrJjRtfVR2tqi8B/2/jqpqeHuP706p6ulu9j8H7kCZi0L/YNuDxofXDXduLJHk1cC5wzwbUNS19xvd+4JeSHGZw5tQ72Vr6jPGrwC90y28CXp7kjA2obSP0/h3Wpnctg//OJmLQv1hGtK10atJVwKeq6oUZ1jNtfcZ3NfDRqtrOYBrg40m20u9KnzG+F/iZJF8Bfgb4NvD8rAvbIGv5HdYmleRnGQT9r066Ly888mJjP75hyFXAdTOvaLr6jO9a4FKAqvpCkpcy+AyOoxtS4eT6fATHEeBfAiR5GfALVfXshlU4W2v5HdYmlOQfAx8BLquqv550f1vpKG2j9Pr4hiQ/AZwOfGGD65tUn/E9BlwCkOQ1wEuB5Q2tcjJjx5jkzKH/Um4AbtngGmfpAPC27uybi4Bnq+qJeRelfpKcA3waeGtV/cU09ukR/XFqhY9vSPKfgaWqOhYYVwO3VffS+FbRc3zvAT6c5N8z+Jf/X2+lcfYc4xuA/5qkgM+xhf4zS/IJBvWf2b2O8j7gFICq+h8MXle5HDgEfB94+3wqXZ9x40vy94ElBicM/CDJuxmcVfWdOZW8Jj1+fv8JOAP43SQAz9eEH3TmO2MlqXFO3UhS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIa9/8B0WUtO59b40AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ratio_zeros = percent_zeros/percent_zeros_test\n",
    "eliminar = ratio_zeros[ratio_zeros < 0.9]\n",
    "eliminar = np.append(eliminar, ratio_zeros[ratio_zeros > 1.1])\n",
    "print(len(eliminar))\n",
    "sns.distplot(ratio_zeros,bins = 200, kde = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.425812793634984\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(4459, 1000)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import model_selection\n",
    "from sklearn import ensemble\n",
    "NUM_OF_FEATURES = 1000\n",
    "def rmsle(y_aux, pred):\n",
    "    return np.sqrt(np.mean(np.power(y_aux - pred, 2)))\n",
    "\n",
    "x1, x2, y1, y2 = model_selection.train_test_split(\n",
    "    train_df, y.values, test_size=0.20, random_state=5)\n",
    "model = ensemble.RandomForestRegressor(n_jobs=-1, random_state=7)\n",
    "model.fit(x1, y1)\n",
    "print(rmsle(y2, model.predict(x2)))\n",
    "\n",
    "col = pd.DataFrame({'importance': model.feature_importances_, 'feature': train_df.columns}).sort_values(\n",
    "    by=['importance'], ascending=[False])[:NUM_OF_FEATURES]['feature'].values\n",
    "train = train_df[col]\n",
    "test = test_df[col]\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-63-71d6c8288310>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[0mtmp_test\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtest\u001b[0m\u001b[1;33m!=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"weight_count\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtmp_train\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m \u001b[0mtest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"weight_count\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtmp_test\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"count_not0\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtrain\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"count_not0\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtest\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36mstat_func\u001b[1;34m(self, axis, skipna, level, numeric_only, min_count, **kwargs)\u001b[0m\n\u001b[0;32m   9567\u001b[0m                                       skipna=skipna, min_count=min_count)\n\u001b[0;32m   9568\u001b[0m         return self._reduce(f, name, axis=axis, skipna=skipna,\n\u001b[1;32m-> 9569\u001b[1;33m                             numeric_only=numeric_only, min_count=min_count)\n\u001b[0m\u001b[0;32m   9570\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   9571\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mset_function_name\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstat_func\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m_reduce\u001b[1;34m(self, op, name, axis, skipna, numeric_only, filter_type, **kwds)\u001b[0m\n\u001b[0;32m   6859\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   6860\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 6861\u001b[1;33m                 \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   6862\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   6863\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36mf\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m   6848\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   6849\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 6850\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mskipna\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mskipna\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   6851\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   6852\u001b[0m         \u001b[0mlabels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_agg_axis\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\nanops.py\u001b[0m in \u001b[0;36m_f\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     75\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     76\u001b[0m                 \u001b[1;32mwith\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrstate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minvalid\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'ignore'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 77\u001b[1;33m                     \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     78\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     79\u001b[0m                 \u001b[1;31m# we want to transform an object array\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\nanops.py\u001b[0m in \u001b[0;36mnansum\u001b[1;34m(values, axis, skipna, min_count)\u001b[0m\n\u001b[0;32m    328\u001b[0m \u001b[1;33m@\u001b[0m\u001b[0mdisallow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'M8'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    329\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mnansum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mskipna\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmin_count\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 330\u001b[1;33m     \u001b[0mvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype_max\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_get_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mskipna\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    331\u001b[0m     \u001b[0mdtype_sum\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdtype_max\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    332\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mis_float_dtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\nanops.py\u001b[0m in \u001b[0;36m_get_values\u001b[1;34m(values, skipna, fill_value, fill_value_typ, isfinite, copy)\u001b[0m\n\u001b[0;32m    224\u001b[0m             \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    225\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mdtype_ok\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 226\u001b[1;33m             \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mputmask\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    227\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    228\u001b[0m         \u001b[1;31m# promote if needed\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#from sklearn import random_projection\n",
    "ntrain = len(train)\n",
    "ntest = len(test)\n",
    "#tmp = pd.concat([train,test])#RandomProjection\n",
    "weight = ((train != 0).sum()/len(train)).values\n",
    "tmp_train = train[train!=0]\n",
    "tmp_test = test[test!=0]\n",
    "train[\"weight_count\"] = (tmp_train*weight).sum(axis=1)\n",
    "test[\"weight_count\"] = (tmp_test*weight).sum(axis=1)\n",
    "train[\"count_not0\"] = (train != 0).sum(axis=1)\n",
    "test[\"count_not0\"] = (test != 0).sum(axis=1)\n",
    "train[\"sum\"] = train.sum(axis=1)\n",
    "test[\"sum\"] = test.sum(axis=1)\n",
    "train[\"var\"] = tmp_train.var(axis=1)\n",
    "test[\"var\"] = tmp_test.var(axis=1)\n",
    "train[\"median\"] = tmp_train.median(axis=1)\n",
    "test[\"median\"] = tmp_test.median(axis=1)\n",
    "train[\"mean\"] = tmp_train.mean(axis=1)\n",
    "test[\"mean\"] = tmp_test.mean(axis=1)\n",
    "train[\"std\"] = tmp_train.std(axis=1)\n",
    "test[\"std\"] = tmp_test.std(axis=1)\n",
    "train[\"max\"] = tmp_train.max(axis=1)\n",
    "test[\"max\"] = tmp_test.max(axis=1)\n",
    "train[\"min\"] = tmp_train.min(axis=1)\n",
    "test[\"min\"] = tmp_test.min(axis=1)\n",
    "train[\"skew\"] = tmp_train.skew(axis=1)\n",
    "test[\"skew\"] = tmp_test.skew(axis=1)\n",
    "train[\"kurtosis\"] = tmp_train.kurtosis(axis=1)\n",
    "test[\"kurtosis\"] = tmp_test.kurtosis(axis=1)\n",
    "del(tmp_train)\n",
    "del(tmp_test)\n",
    "'''\n",
    "NUM_OF_COM = 100 #need tuned\n",
    "transformer = random_projection.SparseRandomProjection(n_components = NUM_OF_COM)\n",
    "RP = transformer.fit_transform(tmp)\n",
    "rp = pd.DataFrame(RP)\n",
    "columns = [\"RandomProjection{}\".format(i) for i in range(NUM_OF_COM)]\n",
    "rp.columns = columns\n",
    "\n",
    "rp_train = rp[:ntrain]\n",
    "rp_test = rp[ntrain:]\n",
    "rp_test.index = test.index\n",
    "\n",
    "#concat RandomProjection and raw data\n",
    "train = pd.concat([train,rp_train],axis=1)\n",
    "test = pd.concat([test,rp_test],axis=1)\n",
    "\n",
    "del(rp_train)\n",
    "del(rp_test)\n",
    "'''\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\melcior.pijoan\\Miniconda3\\lib\\site-packages\\ipykernel_launcher.py:12: RuntimeWarning: invalid value encountered in log1p\n",
      "  if sys.path[0] == '':\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-53-3f4819ccb2ce>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     45\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     46\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 47\u001b[1;33m \u001b[0mscore\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrmsle_cv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel_xgb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     48\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Xgboost score: {:.4f} ({:.4f})\\n\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     49\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-53-3f4819ccb2ce>\u001b[0m in \u001b[0;36mrmsle_cv\u001b[1;34m(model)\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mrmsle_cv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[0mkf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mKFold\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mNUM_FOLDS\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m42\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_n_splits\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m     \u001b[0mrmse\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mcross_val_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog1p\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscoring\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"neg_mean_squared_error\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m     \u001b[1;32mreturn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrmse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[1;31m#ensemble method: model averaging\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\u001b[0m in \u001b[0;36mcross_val_score\u001b[1;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch)\u001b[0m\n\u001b[0;32m    340\u001b[0m                                 \u001b[0mn_jobs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    341\u001b[0m                                 \u001b[0mfit_params\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 342\u001b[1;33m                                 pre_dispatch=pre_dispatch)\n\u001b[0m\u001b[0;32m    343\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mcv_results\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test_score'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    344\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\u001b[0m in \u001b[0;36mcross_validate\u001b[1;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, return_train_score)\u001b[0m\n\u001b[0;32m    204\u001b[0m             \u001b[0mfit_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreturn_train_score\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mreturn_train_score\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    205\u001b[0m             return_times=True)\n\u001b[1;32m--> 206\u001b[1;33m         for train, test in cv.split(X, y, groups))\n\u001b[0m\u001b[0;32m    207\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    208\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mreturn_train_score\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m    777\u001b[0m             \u001b[1;31m# was dispatched. In particular this covers the edge\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    778\u001b[0m             \u001b[1;31m# case of Parallel used with an exhausted iterator.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 779\u001b[1;33m             \u001b[1;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    780\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    781\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[1;34m(self, iterator)\u001b[0m\n\u001b[0;32m    623\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    624\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 625\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    626\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    627\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    586\u001b[0m         \u001b[0mdispatch_timestamp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    587\u001b[0m         \u001b[0mcb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBatchCompletionCallBack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdispatch_timestamp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 588\u001b[1;33m         \u001b[0mjob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    589\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    590\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[1;34m(self, func, callback)\u001b[0m\n\u001b[0;32m    109\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    110\u001b[0m         \u001b[1;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 111\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    112\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    113\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    330\u001b[0m         \u001b[1;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    331\u001b[0m         \u001b[1;31m# arguments in memory\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 332\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    333\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    334\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    129\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    130\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 131\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    132\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    133\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    129\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    130\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 131\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    132\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    133\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\u001b[0m in \u001b[0;36m_fit_and_score\u001b[1;34m(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, error_score)\u001b[0m\n\u001b[0;32m    456\u001b[0m             \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    457\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 458\u001b[1;33m             \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    459\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    460\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\xgboost\\sklearn.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight, eval_set, eval_metric, early_stopping_rounds, verbose, xgb_model)\u001b[0m\n\u001b[0;32m    291\u001b[0m                               \u001b[0mearly_stopping_rounds\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mearly_stopping_rounds\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    292\u001b[0m                               \u001b[0mevals_result\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevals_result\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 293\u001b[1;33m                               verbose_eval=verbose, xgb_model=xgb_model)\n\u001b[0m\u001b[0;32m    294\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    295\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mevals_result\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks, learning_rates)\u001b[0m\n\u001b[0;32m    202\u001b[0m                            \u001b[0mevals\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    203\u001b[0m                            \u001b[0mobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 204\u001b[1;33m                            xgb_model=xgb_model, callbacks=callbacks)\n\u001b[0m\u001b[0;32m    205\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    206\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36m_train_internal\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks)\u001b[0m\n\u001b[0;32m     72\u001b[0m         \u001b[1;31m# Skip the first update if it is a recovery step.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     73\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mversion\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m2\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 74\u001b[1;33m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     75\u001b[0m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_rabit_checkpoint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     76\u001b[0m             \u001b[0mversion\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\xgboost\\core.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[0;32m    893\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mfobj\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    894\u001b[0m             _check_call(_LIB.XGBoosterUpdateOneIter(self.handle, ctypes.c_int(iteration),\n\u001b[1;32m--> 895\u001b[1;33m                                                     dtrain.handle))\n\u001b[0m\u001b[0;32m    896\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    897\u001b[0m             \u001b[0mpred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold, cross_val_score, train_test_split\n",
    "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "#define evaluation method for a given model. we use k-fold cross validation on the training set. \n",
    "#the loss function is root mean square logarithm error between target and prediction\n",
    "#note: train and y_train are feeded as global variables\n",
    "NUM_FOLDS = 5 #need tuned\n",
    "def rmsle_cv(model):\n",
    "    kf = KFold(NUM_FOLDS, shuffle=True, random_state=42).get_n_splits(train.values)\n",
    "    rmse= np.sqrt(-cross_val_score(model, np.log1p(train), y, scoring=\"neg_mean_squared_error\", cv = kf))\n",
    "    return(rmse)\n",
    "#ensemble method: model averaging\n",
    "class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):\n",
    "    def __init__(self, models):\n",
    "        self.models = models\n",
    "        \n",
    "    # we define clones of the original models to fit the data in\n",
    "    # the reason of clone is avoiding affect the original base models\n",
    "    def fit(self, X, y_aux):\n",
    "        self.models_ = [clone(x) for x in self.models]  \n",
    "        # Train cloned base models\n",
    "        for model in self.models_:\n",
    "            model.fit(X, y_aux)\n",
    "        return self\n",
    "    \n",
    "    #Now we do the predictions for cloned models and average them\n",
    "    def predict(self, X):\n",
    "        predictions = np.column_stack([ model.predict(X) for model in self.models_ ])\n",
    "        return np.mean(predictions, axis=1)\n",
    "\n",
    "model_xgb = xgb.XGBRegressor(colsample_bytree=0.055, colsample_bylevel =0.5, \n",
    "                             gamma=1.5, learning_rate=0.02, max_depth=32, \n",
    "                             objective='reg:linear',booster='gbtree',\n",
    "                             min_child_weight=57, n_estimators=1000, reg_alpha=0, \n",
    "                             reg_lambda = 0,eval_metric = 'rmse', subsample=0.7, \n",
    "                             silent=1, n_jobs = -1, early_stopping_rounds = 14,\n",
    "                             random_state =7, nthread = -1)\n",
    "model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=144,\n",
    "                              learning_rate=0.005, n_estimators=720, max_depth=13,\n",
    "                              metric='rmse',is_training_metric=True,\n",
    "                              max_bin = 55, bagging_fraction = 0.8,verbose=-1,\n",
    "                              bagging_freq = 5, feature_fraction = 0.9) \n",
    "\n",
    "\n",
    "score = rmsle_cv(model_xgb)\n",
    "print(\"Xgboost score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))\n",
    "\n",
    "score = rmsle_cv(model_lgb)\n",
    "print(\"LGBM score: {:.4f} ({:.4f})\\n\" .format(score.mean(), score.std()))\n",
    "\n",
    "averaged_models = AveragingModels(models = (model_xgb, model_lgb))\n",
    "score = rmsle_cv(averaged_models)\n",
    "print(\"averaged score: {:.4f} ({:.4f})\\n\" .format(score.mean(), score.std()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold, cross_val_score, train_test_split\n",
    "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "\n",
    "model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=144,\n",
    "                              learning_rate=0.005, n_estimators=720, max_depth=13,\n",
    "                              metric='rmse',is_training_metric=True,\n",
    "                              max_bin = 55, bagging_fraction = 0.8,verbose=-1,\n",
    "                              bagging_freq = 5, feature_fraction = 0.9) \n",
    "\n",
    "model_xgb = xgb.XGBRegressor(colsample_bytree=0.055, colsample_bylevel =0.5, \n",
    "                             gamma=1.5, learning_rate=0.02, max_depth=32, \n",
    "                             objective='reg:linear',booster='gbtree',\n",
    "                             min_child_weight=57, n_estimators=1000, reg_alpha=0, \n",
    "                             reg_lambda = 0,eval_metric = 'rmse', subsample=0.7, \n",
    "                             silent=1, n_jobs = -1, early_stopping_rounds = 14,\n",
    "                             random_state =7, nthread = -1)\n",
    "model_lgb.fit(train_df,y)\n",
    "model_xgb.fit(train_df,y)\n",
    "lgb_pred = model_lgb.predict(test_df)\n",
    "xgb_pred = model_lgb.predict(test_df)\n",
    "\n",
    "def rmsle_cv(model):\n",
    "    kf = KFold(NUM_FOLDS, shuffle=True, random_state=42).get_n_splits(train_df.values)\n",
    "    rmse= np.sqrt(-cross_val_score(model, train, y_train, scoring=\"neg_mean_squared_error\", cv = kf))\n",
    "    return(rmse)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'kurtosis'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36mget_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   3063\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3064\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3065\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'kurtosis'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-70-eea0fd71ab11>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtrain_df\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'kurtosis'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   2686\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2687\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2688\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2689\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2690\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_getitem_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m_getitem_column\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   2693\u001b[0m         \u001b[1;31m# get column\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2694\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_unique\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2695\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_item_cache\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2696\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2697\u001b[0m         \u001b[1;31m# duplicate columns & possible reduce dimensionality\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36m_get_item_cache\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m   2484\u001b[0m         \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcache\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2485\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2486\u001b[1;33m             \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2487\u001b[0m             \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_box_item_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2488\u001b[0m             \u001b[0mcache\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mres\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\internals.py\u001b[0m in \u001b[0;36mget\u001b[1;34m(self, item, fastpath)\u001b[0m\n\u001b[0;32m   4113\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4114\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4115\u001b[1;33m                 \u001b[0mloc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   4116\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4117\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0misna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36mget_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   3064\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3065\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3066\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3067\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3068\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'kurtosis'"
     ]
    }
   ],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}